feat: inject images via MITM layer instead of relying on LS
The LS silently ignores the 'images' field from our SendUserCascadeMessageRequest proto — it never forwards image data to Google's API. New approach: store the image in MitmStore, then the MITM request modifier injects it as 'inlineData' directly into the last user message's parts array in the Google API JSON request. Flow: Client → Proxy (decode base64) → MitmStore.set_pending_image() LS → Google API → MITM intercepts → inject inlineData part → Google receives image + text together This works for all three API endpoints (responses, completions, gemini).
This commit is contained in:
@@ -60,6 +60,17 @@ pub struct PendingToolResult {
|
||||
pub result: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A pending image to inject via MITM into the Google API request.
|
||||
/// The LS doesn't forward images from our SendUserCascadeMessage proto,
|
||||
/// so we inject them directly at the MITM layer.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PendingImage {
|
||||
/// Base64-encoded image data (no prefix).
|
||||
pub base64_data: String,
|
||||
/// MIME type, e.g. "image/png".
|
||||
pub mime_type: String,
|
||||
}
|
||||
|
||||
/// Client-specified generation parameters for MITM injection.
|
||||
/// Set by API handlers, consumed by the MITM modify layer.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -137,6 +148,10 @@ pub struct MitmStore {
|
||||
// ── Grounding metadata capture ──────────────────────────────────────
|
||||
/// Captured grounding metadata from Google API responses (search results).
|
||||
captured_grounding: Arc<RwLock<Option<serde_json::Value>>>,
|
||||
|
||||
// ── Pending image for MITM injection ─────────────────────────────────
|
||||
/// Image to inject into the next Google API request via MITM.
|
||||
pending_image: Arc<RwLock<Option<PendingImage>>>,
|
||||
}
|
||||
|
||||
/// Aggregate statistics across all intercepted traffic.
|
||||
@@ -181,6 +196,7 @@ impl MitmStore {
|
||||
response_complete: Arc::new(AtomicBool::new(false)),
|
||||
generation_params: Arc::new(RwLock::new(None)),
|
||||
captured_grounding: Arc::new(RwLock::new(None)),
|
||||
pending_image: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,4 +522,16 @@ impl MitmStore {
|
||||
pub async fn peek_grounding(&self) -> Option<serde_json::Value> {
|
||||
self.captured_grounding.read().await.clone()
|
||||
}
|
||||
|
||||
// ── Pending image for MITM injection ─────────────────────────────────
|
||||
|
||||
/// Store a pending image for MITM injection.
|
||||
pub async fn set_pending_image(&self, image: PendingImage) {
|
||||
*self.pending_image.write().await = Some(image);
|
||||
}
|
||||
|
||||
/// Take (consume) pending image for injection.
|
||||
pub async fn take_pending_image(&self) -> Option<PendingImage> {
|
||||
self.pending_image.write().await.take()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user