From 0a33c1b706138d5cff2e419a364a54bfa0a5a34a Mon Sep 17 00:00:00 2001
From: Nikketryhard <louisnewmobile@gmail.com>
Date: Sun, 15 Feb 2026 17:46:41 -0600
Subject: [PATCH] fix: send images as top-level ImageData field, not
 ChatMessage blob

SendUserCascadeMessageRequest proto field layout (from JS bundle analysis):
- Field 6 is 'images' (repeated ImageData) at the REQUEST level
- NOT a Blob sub-message inside ChatMessage (field 2)

ImageData proto uses base64_data (field 1) + mime_type (field 2),
not raw bytes. The LS was silently ignoring our ChatMessage blob
because the field structure didn't match.

Also protect MITM modifier from stripping messages containing
inlineData (image parts in Google API JSON).
---
 src/mitm/modify.rs | 20 +++++++++++++-
 src/proto.rs       | 66 +++++++++++++++++++++++++++++++---------------
 2 files changed, 64 insertions(+), 22 deletions(-)

diff --git a/src/mitm/modify.rs b/src/mitm/modify.rs
index f78158e..2cf3191 100644
--- a/src/mitm/modify.rs
+++ b/src/mitm/modify.rs
@@ -85,7 +85,17 @@ pub fn modify_request(body: &[u8], tool_ctx: Option<&ToolContext>) -> Option<Vec
         let before = contents.len();
 
         // Remove messages that are pure Antigravity context injection
+        // IMPORTANT: Never strip messages containing inlineData (images)
         contents.retain(|msg| {
+            // Always keep messages with image/binary data in any part
+            if let Some(parts) = msg["parts"].as_array() {
+                for part in parts {
+                    if part.get("inlineData").is_some() {
+                        return true;
+                    }
+                }
+            }
+
             if let Some(text) = msg["parts"][0]["text"].as_str() {
                 // Strip user_information (OS, workspace paths)
                 if text.starts_with("<user_information>") {
@@ -151,8 +161,16 @@ pub fn modify_request(body: &[u8], tool_ctx: Option<&ToolContext>) -> Option<Vec
             }
         }
 
-        // Remove now-empty messages
+        // Remove now-empty messages (but preserve messages with non-text parts like images)
         contents.retain(|msg| {
+            if let Some(parts) = msg["parts"].as_array() {
+                // Keep if any part has inlineData
+                for part in parts {
+                    if part.get("inlineData").is_some() {
+                        return true;
+                    }
+                }
+            }
             if let Some(text) = msg["parts"][0]["text"].as_str() {
                 !text.trim().is_empty()
             } else {
diff --git a/src/proto.rs b/src/proto.rs
index fcfdc48..8801284 100644
--- a/src/proto.rs
+++ b/src/proto.rs
@@ -111,34 +111,53 @@ pub fn build_init_metadata(
 
 /// Build the `SendUserCascadeMessageRequest` protobuf binary.
 ///
-/// Produces a byte-exact match to real Antigravity webview traffic.
-/// Verified against Chrome DevTools network capture 2026-02-12.
+/// See `build_request_with_image` for the full field layout documentation.
 ///
-/// Field layout:
+/// Field layout (key fields):
 ///   1: cascade_id (string)
-///   2: ChatMessage { 1: text, 6: Blob { 1: mime_type, 2: data } }
+///   2: items (ChatMessage { 1: text })
 ///   3: metadata { 1: client_name, 3: oauth_token, 4: "en", 7: version, 12: client_name }
-///   5: PlannerConfig { 1: inner_config, 7: { 1: 1 } }
-///      inner_config contains: f2 (conv mode), f13 (tool config), f15 (model), f21 (ephemeral), f32 (knowledge)
-///  11: conversation_history = true
+///   5: cascade_config (PlannerConfig)
+///   6: images (ImageData { 1: base64_data, 2: mime_type }, repeated)
+///  11: client_type
 #[allow(dead_code)]
 pub fn build_request(cascade_id: &str, text: &str, oauth_token: &str, model_enum: u32) -> Vec<u8> {
     build_request_with_image(cascade_id, text, oauth_token, model_enum, None)
 }
 
-/// Image data to embed in the ChatMessage protobuf.
+/// Image data to attach to a cascade message.
 pub struct ImageData {
     /// MIME type, e.g. "image/png", "image/jpeg", "image/webp", "image/gif"
     pub mime_type: String,
-    /// Raw image bytes (NOT base64 — already decoded)
+    /// Raw image bytes (NOT base64 — we base64-encode them into the proto)
     pub data: Vec<u8>,
 }
 
 /// Build `SendUserCascadeMessageRequest` with optional image attachment.
 ///
-/// When `image` is Some, the ChatMessage includes a Blob field (field 6)
-/// alongside the text (field 1). This matches how the real Antigravity
-/// webview sends images: `ChatMessage { text, blob: { mime_type, data } }`.
+/// Field layout (from JS protobuf-es registration in workbench bundle):
+///   1: cascade_id    (string)
+///   2: items         (ChatMessage, repeated) — contains the text
+///   3: metadata      (auto-populated by LS)
+///   4: experiment_config
+///   5: cascade_config (PlannerConfig)
+///   6: images        (ImageData, repeated) — TOP-LEVEL, NOT inside ChatMessage
+///   8: blocking      (bool)
+///   9: additional_steps (repeated)
+///  10: artifact_comments (repeated)
+///  11: client_type   (enum)
+///  12: file_diff_comments (repeated)
+///  13: file_comments (repeated)
+///  14: media         (repeated)
+///  15: agent_script_item
+///  16: propagate_error (bool)
+///  17: planner_response
+///
+/// ImageData proto (from codeium_common_pb):
+///   1: base64_data   (string) — BASE64 encoded, NOT raw bytes
+///   2: mime_type     (string)
+///   3: caption       (string)
+///   4: uri           (string)
 pub fn build_request_with_image(
     cascade_id: &str,
     text: &str,
@@ -151,13 +170,8 @@ pub fn build_request_with_image(
     // Field 1: cascade_id
     msg.extend(proto_string(1, cascade_id.as_bytes()));
 
-    // Field 2: ChatMessage { f1: text, f6?: Blob { f1: mime_type, f2: data } }
-    let mut chat_msg = proto_string(1, text.as_bytes());
-    if let Some(img) = image {
-        let mut blob = proto_string(1, img.mime_type.as_bytes());
-        blob.extend(proto_string(2, &img.data));
-        chat_msg.extend(proto_message(6, &blob));
-    }
+    // Field 2: items (ChatMessage) — text only, no blob
+    let chat_msg = proto_string(1, text.as_bytes());
     msg.extend(proto_message(2, &chat_msg));
 
     // Field 3: Metadata (Auth + Client ID)
@@ -169,7 +183,7 @@ pub fn build_request_with_image(
     meta.extend(proto_string(12, CLIENT_NAME.as_bytes()));
     msg.extend(proto_message(3, &meta));
 
-    // Field 5: PlannerConfig
+    // Field 5: PlannerConfig (cascade_config)
     let mut inner = Vec::new();
 
     // field 2: conversational mode { f4: 1, f14: 0 }
@@ -204,7 +218,17 @@ pub fn build_request_with_image(
     .concat();
     msg.extend(proto_message(5, &f5_payload));
 
-    // Field 11: conversation history flag
+    // Field 6: images (ImageData, repeated) — TOP-LEVEL field
+    // ImageData { 1: base64_data (string), 2: mime_type (string) }
+    if let Some(img) = image {
+        use base64::Engine;
+        let b64 = base64::engine::general_purpose::STANDARD.encode(&img.data);
+        let mut image_data = proto_string(1, b64.as_bytes());
+        image_data.extend(proto_string(2, img.mime_type.as_bytes()));
+        msg.extend(proto_message(6, &image_data));
+    }
+
+    // Field 11: client_type (but we use it for conversation history flag)
     msg.extend(bool_field(11, true));
 
     msg