fix: send images as top-level ImageData field, not ChatMessage blob

SendUserCascadeMessageRequest proto field layout (from JS bundle analysis):
- Field 6 is 'images' (repeated ImageData) at the REQUEST level
- NOT a Blob sub-message inside ChatMessage (field 2)

ImageData proto uses base64_data (field 1) + mime_type (field 2),
not raw bytes. The LS was silently ignoring our ChatMessage blob
because the field structure didn't match.

Also protect MITM modifier from stripping messages containing
inlineData (image parts in Google API JSON).
This commit is contained in:
Nikketryhard
2026-02-15 17:46:41 -06:00
parent 2ac2016ed4
commit 0a33c1b706
2 changed files with 64 additions and 22 deletions

View File

@@ -85,7 +85,17 @@ pub fn modify_request(body: &[u8], tool_ctx: Option<&ToolContext>) -> Option<Vec
let before = contents.len();
// Remove messages that are pure Antigravity context injection
// IMPORTANT: Never strip messages containing inlineData (images)
contents.retain(|msg| {
// Always keep messages with image/binary data in any part
if let Some(parts) = msg["parts"].as_array() {
for part in parts {
if part.get("inlineData").is_some() {
return true;
}
}
}
if let Some(text) = msg["parts"][0]["text"].as_str() {
// Strip user_information (OS, workspace paths)
if text.starts_with("<user_information>") {
@@ -151,8 +161,16 @@ pub fn modify_request(body: &[u8], tool_ctx: Option<&ToolContext>) -> Option<Vec
}
}
// Remove now-empty messages
// Remove now-empty messages (but preserve messages with non-text parts like images)
contents.retain(|msg| {
if let Some(parts) = msg["parts"].as_array() {
// Keep if any part has inlineData
for part in parts {
if part.get("inlineData").is_some() {
return true;
}
}
}
if let Some(text) = msg["parts"][0]["text"].as_str() {
!text.trim().is_empty()
} else {

View File

@@ -111,34 +111,53 @@ pub fn build_init_metadata(
/// Build the `SendUserCascadeMessageRequest` protobuf binary.
///
/// Produces a byte-exact match to real Antigravity webview traffic.
/// Verified against Chrome DevTools network capture 2026-02-12.
/// See `build_request_with_image` for the full field layout documentation.
///
/// Field layout:
/// Field layout (key fields):
/// 1: cascade_id (string)
/// 2: ChatMessage { 1: text, 6: Blob { 1: mime_type, 2: data } }
/// 2: items (ChatMessage { 1: text })
/// 3: metadata { 1: client_name, 3: oauth_token, 4: "en", 7: version, 12: client_name }
/// 5: PlannerConfig { 1: inner_config, 7: { 1: 1 } }
/// inner_config contains: f2 (conv mode), f13 (tool config), f15 (model), f21 (ephemeral), f32 (knowledge)
/// 11: conversation_history = true
/// 5: cascade_config (PlannerConfig)
/// 6: images (ImageData { 1: base64_data, 2: mime_type }, repeated)
/// 11: client_type
#[allow(dead_code)]
pub fn build_request(cascade_id: &str, text: &str, oauth_token: &str, model_enum: u32) -> Vec<u8> {
build_request_with_image(cascade_id, text, oauth_token, model_enum, None)
}
/// Image data to embed in the ChatMessage protobuf.
/// Image data to attach to a cascade message.
pub struct ImageData {
/// MIME type, e.g. "image/png", "image/jpeg", "image/webp", "image/gif"
pub mime_type: String,
/// Raw image bytes (NOT base64 — already decoded)
/// Raw image bytes (NOT base64 — we base64-encode them into the proto)
pub data: Vec<u8>,
}
/// Build `SendUserCascadeMessageRequest` with optional image attachment.
///
/// When `image` is Some, the ChatMessage includes a Blob field (field 6)
/// alongside the text (field 1). This matches how the real Antigravity
/// webview sends images: `ChatMessage { text, blob: { mime_type, data } }`.
/// Field layout (from JS protobuf-es registration in workbench bundle):
/// 1: cascade_id (string)
/// 2: items (ChatMessage, repeated) — contains the text
/// 3: metadata (auto-populated by LS)
/// 4: experiment_config
/// 5: cascade_config (PlannerConfig)
/// 6: images (ImageData, repeated) — TOP-LEVEL, NOT inside ChatMessage
/// 8: blocking (bool)
/// 9: additional_steps (repeated)
/// 10: artifact_comments (repeated)
/// 11: client_type (enum)
/// 12: file_diff_comments (repeated)
/// 13: file_comments (repeated)
/// 14: media (repeated)
/// 15: agent_script_item
/// 16: propagate_error (bool)
/// 17: planner_response
///
/// ImageData proto (from codeium_common_pb):
/// 1: base64_data (string) — BASE64 encoded, NOT raw bytes
/// 2: mime_type (string)
/// 3: caption (string)
/// 4: uri (string)
pub fn build_request_with_image(
cascade_id: &str,
text: &str,
@@ -151,13 +170,8 @@ pub fn build_request_with_image(
// Field 1: cascade_id
msg.extend(proto_string(1, cascade_id.as_bytes()));
// Field 2: ChatMessage { f1: text, f6?: Blob { f1: mime_type, f2: data } }
let mut chat_msg = proto_string(1, text.as_bytes());
if let Some(img) = image {
let mut blob = proto_string(1, img.mime_type.as_bytes());
blob.extend(proto_string(2, &img.data));
chat_msg.extend(proto_message(6, &blob));
}
// Field 2: items (ChatMessage) — text only, no blob
let chat_msg = proto_string(1, text.as_bytes());
msg.extend(proto_message(2, &chat_msg));
// Field 3: Metadata (Auth + Client ID)
@@ -169,7 +183,7 @@ pub fn build_request_with_image(
meta.extend(proto_string(12, CLIENT_NAME.as_bytes()));
msg.extend(proto_message(3, &meta));
// Field 5: PlannerConfig
// Field 5: PlannerConfig (cascade_config)
let mut inner = Vec::new();
// field 2: conversational mode { f4: 1, f14: 0 }
@@ -204,7 +218,17 @@ pub fn build_request_with_image(
.concat();
msg.extend(proto_message(5, &f5_payload));
// Field 11: conversation history flag
// Field 6: images (ImageData, repeated) — TOP-LEVEL field
// ImageData { 1: base64_data (string), 2: mime_type (string) }
if let Some(img) = image {
use base64::Engine;
let b64 = base64::engine::general_purpose::STANDARD.encode(&img.data);
let mut image_data = proto_string(1, b64.as_bytes());
image_data.extend(proto_string(2, img.mime_type.as_bytes()));
msg.extend(proto_message(6, &image_data));
}
// Field 11: client_type (but we use it for conversation history flag)
msg.extend(bool_field(11, true));
msg