From 365630115cecf4620b277070be15c9078b905ed0 Mon Sep 17 00:00:00 2001 From: Bl0ck <36800583+Bl0ck154@users.noreply.github.com> Date: Mon, 14 Sep 2026 01:30:50 +0300 Subject: [PATCH 1/2] fix(flow): restore Omni first-frame video on batch API --- agent/services/omni_flash.py | 99 +++++++++++++++++++++++++---------- docs/OMNI_FLASH.md | 33 ++++++++---- tests/unit/test_omni_flash.py | 97 ++++++++++++++++++++++++++-------- 3 files changed, 167 insertions(+), 62 deletions(-) diff --git a/agent/services/omni_flash.py b/agent/services/omni_flash.py index 796d6a31..dcb8a8b4 100644 --- a/agent/services/omni_flash.py +++ b/agent/services/omni_flash.py @@ -12,10 +12,11 @@ that mapping is deliberately configurable separately so it can be changed without a code release if Google's rollout rotates the wire key. -Important: Omni submit responses may contain operation-looking handles, but -those handles are not compatible with the legacy -``batchCheckAsyncVideoGenerationStatus`` polling endpoint. Omni jobs are -workflow-backed and are polled through Flow's authenticated project data. +Important: on the migrated ``flow.google.com`` batch transport, first-frame +Omni I2V uses the same ``eb1hJf`` operation contract as migrated Veo, with the +wire model switched to ``abra_i2v_s``. Those jobs therefore use the +normal batch operation poller. Legacy Omni and migrated text-to-video keep their +workflow/media polling contracts. """ from __future__ import annotations @@ -33,23 +34,19 @@ _MODELS_FILE = Path(__file__).parent.parent / "models.json" -#: Every Omni surface here rides the pre-migration transports — the REST -#: endpoints on aisandbox-pa and the labs.google tRPC snapshot it polls -#: through. Flow moved to flow.google.com in September 2026 and stopped -#: minting the bearer both of those need, and no Omni payload has been -#: captured off the new frontend, so on the batch path these fail with a -#: name rather than dying on a 401 five retries deep. -_UNSUPPORTED_ON_BATCH = ( - "UNSUPPORTED_ON_BATCH_API: Omni Flash frame/reference generation is not yet " - "ported to flow.google.com batchexecute. Omni text-to-video is supported on " - "the batch path; frame-to-video, start+end and reference-to-video still need " - "their migrated payload captures." +#: First-frame I2V is live on the migrated batch transport. Start+end and +#: reference-image modes still need their current Flow UI payloads captured; +#: keep those explicitly blocked instead of falling through to dead legacy auth. +_UNSUPPORTED_START_END_ON_BATCH = ( + "UNSUPPORTED_ON_BATCH_API: Omni Flash first+last frame generation is not yet " + "ported to flow.google.com batchexecute. First-frame image-to-video and " + "text-to-video are supported on the batch path." +) +_UNSUPPORTED_REFERENCE_ON_BATCH = ( + "UNSUPPORTED_ON_BATCH_API: Omni Flash reference-to-video is not yet ported " + "to flow.google.com batchexecute. First-frame image-to-video and text-to-video " + "are supported on the batch path." ) - - -def _batch_path_blocks_omni() -> dict | None: - """The error to return instead of reaching for auth that is gone.""" - return {"error": _UNSUPPORTED_ON_BATCH} if USE_BATCH_RPC else None OMNI_FLASH_VALID_DURATIONS = (4, 6, 8, 10) OMNI_FLASH_VALID_ASPECTS = { @@ -278,10 +275,13 @@ async def _submit_omni_frame_video( user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, ) -> dict: - """Submit Omni first-frame or First+Last generation.""" - blocked = _batch_path_blocks_omni() - if blocked: - return blocked + """Submit Omni first-frame or First+Last generation. + + Migrated first-frame I2V was recaptured from the live Flow UI on + 2026-09-14: it uses ``eb1hJf`` with the normal image-to-video payload and + ``abra_i2v_s`` as the wire model. Its response is a standard + batch operation, so poll it through ``/api/flow/check-status``. + """ _validate_frame_inputs( start_image_media_id, end_image_media_id, @@ -294,13 +294,55 @@ async def _submit_omni_frame_video( if end_image_media_id is not None else "frame_to_video" ) + model_key = _load_model_key(duration_s, mode=mode) + client = get_flow_client() + + if USE_BATCH_RPC: + if end_image_media_id is not None: + return {"error": _UNSUPPORTED_START_END_ON_BATCH} + try: + pid = client._batch_project_id(project_id) + freq = fb.video_request( + prompt, + pid, + start_image_media_id, + aspect=aspect_ratio, + model=model_key, + ) + payload = await client._batch_payload( + fb.RPC_GEN_VIDEO, + freq, + fb.CAPTCHA_VIDEO, + timeout=120, + ) + operation = fb.read_operation(payload) + client._remember_operation(operation.operation_id, pid) + except Exception as exc: + return {"status": 502, "error": f"{type(exc).__name__}: {exc}"} + + pending = { + "operation": {"name": operation.operation_id}, + "status": "MEDIA_GENERATION_STATUS_PENDING", + } + return { + "status": 200, + "data": { + "operations": [pending], + "model": model_key, + "duration_s": duration_s, + "flowkitPolling": { + "mode": "batch_operation", + "project_id": pid, + "operations": [pending], + }, + }, + } + endpoint = ( "generate_video_start_end" if end_image_media_id is not None else "generate_video" ) - model_key = _load_model_key(duration_s, mode=mode) - client = get_flow_client() ts = int(time.time() * 1000) request_item = { @@ -401,9 +443,8 @@ async def generate_omni_flash_video( the workflow names and primary media IDs required by the Omni polling path. Do not feed Omni operation handles to ``check_video_status``. """ - blocked = _batch_path_blocks_omni() - if blocked: - return blocked + if USE_BATCH_RPC: + return {"error": _UNSUPPORTED_REFERENCE_ON_BATCH} refs = _validate_reference_inputs(reference_media_ids, duration_s, aspect_ratio) model_key = _load_model_key(duration_s, mode="reference_to_video") client = get_flow_client() diff --git a/docs/OMNI_FLASH.md b/docs/OMNI_FLASH.md index 64d3c040..57407be3 100644 --- a/docs/OMNI_FLASH.md +++ b/docs/OMNI_FLASH.md @@ -15,33 +15,44 @@ Expected state on the migrated Flow transport: ```json {"status":"ok","extension_connected":true} -{"connected":true,"transport":"batch"} +{"connected":true,"transport":"batch","authenticated":true,"at_token_present":true} ``` Use `http://127.0.0.1:8100` when the caller runs on the FlowKit host. For a remote integration, set `FLOWKIT_BASE_URL` to the protected HTTPS reverse-proxy URL and allow only the required source IPs or private network. Do not expose Chrome, VNC/noVNC, the extension WebSocket, or port 8100 publicly. ## Supported modes -On `flow.google.com`, Omni **text-to-video** is migrated and live-verified. The older frame/reference implementations still use the pre-migration REST transport and remain fail-fast while `USE_BATCH_RPC=1`. +On the current `flow.google.com` batch transport, Omni **text-to-video** and **first-frame image-to-video** are live and verified. First+Last and multi-reference generation still depend on the pre-migration transport and remain explicitly refused while `USE_BATCH_RPC=1` until their current UI payloads are captured. | Mode | Batch status | Endpoint | Internal model family | |---|---|---|---| | Text to video | **supported** | `POST /api/flow/generate-video-omni-text` | `abra_t2v_s` | -| First frame to video | not yet ported | `POST /api/flow/generate-video` | `abra_i2v_s` (legacy only) | +| First frame to video | **supported** | `POST /api/flow/generate-video` with `model_family=omni_flash` | `abra_i2v_s` | | First + Last frame to video | not yet ported | `POST /api/flow/generate-video` | `abra_i2v_s` (legacy only) | | References to video | not yet ported | `POST /api/flow/generate-video-omni` | `abra_r2v_s` (legacy only) | -Text-to-video supports `4`, `6`, `8`, and `10` seconds, with portrait and landscape aspect ratios. The migrated `YhhmEf` wire was live-verified with `abra_t2v_4s`; the downloaded result was exactly 4.000 seconds at 1280×720/24 fps. Completed media resolves through the migrated `as29s` media lookup. +Text-to-video durations are `4`, `6`, `8`, and `10` seconds. Supported aspect ratios are: + +- `VIDEO_ASPECT_RATIO_PORTRAIT` (`9:16`) +- `VIDEO_ASPECT_RATIO_LANDSCAPE` (`16:9`) + +The migrated `YhhmEf` wire was live-verified with `abra_t2v_4s`; the downloaded result was exactly 4.000 seconds at 1280x720/24 fps. First-frame I2V was re-captured from the live Flow UI on 2026-09-14: it uses RPC `eb1hJf`, the normal migrated I2V payload shape, and `abra_i2v_s`. A live API smoke test completed successfully and resolved its signed video URL through the existing batch operation poller. + +Polling differs by migrated mode: text-to-video returns workflow/media descriptors and uses `/api/flow/check-omni-status`; first-frame I2V returns a `flowkitPolling.mode = batch_operation` descriptor and uses `/api/flow/check-status` with its `operations` array. ## End-to-end integration flow +For migrated Omni, an integration agent should use the polling mode returned by the submit response: + 1. Check `/health` and `/api/flow/status`. -2. Submit `POST /api/flow/generate-video-omni-text` with prompt, project ID, duration and aspect ratio. -3. Persist the returned `flowkitPolling` object. -4. Poll `/api/flow/check-omni-status` every 10–20 seconds with its `project_id` and `workflows`. -5. On `COMPLETED`, immediately download `workflows[].media.url`; the signed URL is short-lived. +2. Submit either text-to-video or first-frame I2V. +3. Persist the complete `flowkitPolling` object returned by the submit. +4. If `mode=batch_media`, poll `/api/flow/check-omni-status` using `project_id` + `workflows`. +5. If `mode=batch_operation`, poll `/api/flow/check-status` using `project_id` + `operations`. +6. On pending state, continue polling; on failure, stop; on success, immediately download the returned signed video URL. +7. Store the downloaded video in durable storage because Google URLs are signed and short-lived. -Example: +Example 4-second submit: ```bash curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni-text" \ @@ -54,11 +65,11 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni-text" \ }' ``` -Do not feed Omni workflow names to the legacy Veo operation poller. +Do not send Omni workflow names to the legacy Veo `batchCheckAsyncVideoGenerationStatus` operation poller. Do not use the obsolete `/v1/media/` polling path. ## Supplying images -This section applies to the legacy frame/reference Omni modes, which are not yet ported to the migrated batch transport. +This section applies to the older frame/reference Omni modes. Those generation modes are currently refused on `USE_BATCH_RPC=1` until their migrated payloads are captured; image upload itself may still be used by supported Veo workflows. `POST /api/flow/upload-image` is not a multipart upload endpoint. Its `file_path` is an absolute path on the **FlowKit server**, not on the calling server. diff --git a/tests/unit/test_omni_flash.py b/tests/unit/test_omni_flash.py index cf3b2fb1..e1a2998a 100644 --- a/tests/unit/test_omni_flash.py +++ b/tests/unit/test_omni_flash.py @@ -1,9 +1,8 @@ """Unit tests for Gemini Omni Flash Flow submissions and workflow polling. -Omni speaks the pre-migration transports — the REST endpoints on aisandbox-pa -and the labs.google tRPC snapshot it polls through — so the wire contracts -asserted here are legacy-path contracts and the module is pinned to that path -for the file. What happens on the batch path is one test at the bottom. +Most legacy Omni wire-contract tests stay pinned to the pre-migration path. +Batch-specific tests opt into ``USE_BATCH_RPC`` explicitly and lock down the +migrated Flow payloads separately. """ from unittest.mock import AsyncMock, MagicMock, patch @@ -27,7 +26,7 @@ @pytest.fixture(autouse=True) def legacy_transport(monkeypatch): - """Omni is only reachable on the pre-migration path; assert it there.""" + """Default to legacy for legacy wire-contract tests; batch tests opt in.""" monkeypatch.setattr(omni_flash, "USE_BATCH_RPC", False) @@ -139,12 +138,17 @@ async def test_batch_text_video_builds_4s_yhhmef_submit(monkeypatch): client = MagicMock() client._batch_project_id.return_value = "11111111-2222-3333-4444-555555555555" client._batch_payload = AsyncMock(return_value=[ - None, 10, [], [[ + None, + 10, + [], + [[ "22222222-3333-4444-5555-666666666666", "11111111-2222-3333-4444-555555555555", - "77777777-8888-9999-aaaa-bbbbbbbbbbbb", "CAE", + "77777777-8888-9999-aaaa-bbbbbbbbbbbb", + "CAE", ]], ]) + with patch("agent.services.omni_flash.get_flow_client", return_value=client): result = await generate_omni_flash_text_video( prompt="A red paper boat drifts across a pond", @@ -152,17 +156,63 @@ async def test_batch_text_video_builds_4s_yhhmef_submit(monkeypatch): duration_s=4, aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE", ) + assert result["status"] == 200 assert result["data"]["model"] == "abra_t2v_4s" assert result["data"]["duration_s"] == 4 assert result["data"]["flowkitPolling"]["mode"] == "batch_media" + assert result["data"]["flowkitPolling"]["workflows"][0]["primary_media_id"] == ( + "22222222-3333-4444-5555-666666666666" + ) rpcid, freq, captcha = client._batch_payload.await_args.args[:3] assert rpcid == omni_flash.fb.RPC_GEN_VIDEO_TEXT assert captcha == omni_flash.fb.CAPTCHA_VIDEO payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) assert payload[0][0][1] == "abra_t2v_4s" + assert payload[0][0][2] == omni_flash.fb.VIDEO_ASPECT_LANDSCAPE + + +@pytest.mark.asyncio +async def test_batch_first_frame_video_uses_eb1hjf_abra_i2v(monkeypatch): + monkeypatch.setattr(omni_flash, "USE_BATCH_RPC", True) + client = MagicMock() + pid = "11111111-2222-3333-4444-555555555555" + client._batch_project_id.return_value = pid + client._batch_payload = AsyncMock(return_value=[ + None, + 50, + [["op-omni-1", pid, "scene-1", None]], + ]) + + with patch("agent.services.omni_flash.get_flow_client", return_value=client): + result = await generate_omni_flash_first_frame_video( + start_image_media_id="media-start", + prompt="Three children clap gently", + project_id=pid, + duration_s=6, + aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE", + ) + + assert result["status"] == 200 + assert result["data"]["model"] == "abra_i2v_6s" + assert result["data"]["duration_s"] == 6 + assert result["data"]["flowkitPolling"]["mode"] == "batch_operation" + assert result["data"]["flowkitPolling"]["project_id"] == pid + assert result["data"]["operations"][0]["operation"]["name"] == "op-omni-1" + client._remember_operation.assert_called_once_with("op-omni-1", pid) + + rpcid, freq, captcha = client._batch_payload.await_args.args[:3] + assert rpcid == omni_flash.fb.RPC_GEN_VIDEO + assert captcha == omni_flash.fb.CAPTCHA_VIDEO + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + request = payload[0][0] + assert request[0][2][0][0][0] == "Three children clap gently" + assert request[1] == "abra_i2v_6s" + assert request[2] == omni_flash.fb.VIDEO_ASPECT_LANDSCAPE + assert request[4][1] == "media-start" + @pytest.mark.asyncio async def test_submit_builds_flow_omni_first_frame_request_and_poll_descriptor(): client = _mock_submit_client() @@ -390,13 +440,24 @@ async def test_batch_omni_poll_uses_as29s_media(monkeypatch): client = MagicMock() client.get_media = AsyncMock(return_value={ "status": 200, - "data": {"video": {"fifeUrl": "https://flow-content.google/video/media-1?Signature=test"}}, + "data": { + "video": { + "fifeUrl": "https://flow-content.google/video/media-1?Signature=test" + } + }, }) + with patch("agent.services.omni_flash.get_flow_client", return_value=client): - result = await check_omni_flash_status([{ - "name": "workflow-1", "primary_media_id": "media-1", "project_id": "project-1", - }]) + result = await check_omni_flash_status([ + { + "name": "workflow-1", + "primary_media_id": "media-1", + "project_id": "project-1", + } + ]) + assert result["done"] is True + assert result["status"] == "COMPLETED" assert result["workflows"][0]["media"]["resolved_via"] == "as29s" client.get_media.assert_awaited_once_with("media-1") @@ -525,10 +586,8 @@ async def test_submit_rejects_empty_reference_set(): ) -class TestUnportedOmniBatchModesAreRefusedRatherThanAttempted: - """Flow stopped minting the bearer these endpoints need, and no Omni - payload has been captured off the new frontend. Saying so beats a 401 - five retries deep.""" +class TestRemainingUnportedOmniBatchModesAreRefused: + """Only start+end/reference remain blocked; first-frame I2V is migrated.""" @pytest.fixture(autouse=True) def batch_transport(self, monkeypatch): @@ -542,12 +601,6 @@ def client(self): factory.return_value = stub yield stub - async def test_first_frame_names_the_gap_and_sends_nothing(self, client): - result = await generate_omni_flash_first_frame_video( - start_image_media_id="mid", prompt="go", project_id="pid") - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - client._send.assert_not_called() - async def test_first_last_names_the_gap_and_sends_nothing(self, client): result = await generate_omni_flash_first_last_video( start_image_media_id="a", end_image_media_id="b", @@ -564,5 +617,5 @@ async def test_reference_to_video_names_the_gap_and_sends_nothing(self, client): async def test_the_message_points_to_supported_text_to_video(self, client): result = await generate_omni_flash_video( reference_media_ids=["a"], prompt="go", project_id="pid") - assert "text-to-video is supported" in result["error"] + assert "image-to-video and text-to-video are supported" in result["error"] assert "reference-to-video" in result["error"] From b1db615757adfb2fdb85a2511a6cc9579119e0c4 Mon Sep 17 00:00:00 2001 From: Bl0ck <36800583+Bl0ck154@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:31:23 +0300 Subject: [PATCH 2/2] feat(flow): restore all Omni reference modes on batch API --- agent/api/flow.py | 17 ++-- agent/services/flow_batch.py | 97 ++++++++++++++++++ agent/services/omni_flash.py | 184 +++++++++++++++++----------------- docs/OMNI_FLASH.md | 121 ++++++++++------------ tests/unit/test_flow_batch.py | 36 +++++++ tests/unit/test_omni_flash.py | 77 ++++++++++---- 6 files changed, 345 insertions(+), 187 deletions(-) diff --git a/agent/api/flow.py b/agent/api/flow.py index 5aeed8fa..99d851de 100644 --- a/agent/api/flow.py +++ b/agent/api/flow.py @@ -39,6 +39,7 @@ class GenerateVideoRequest(BaseModel): # Backward compatible: legacy requests remain Veo unless explicitly set. model_family: Literal["veo", "omni_flash"] = "veo" duration_s: int = 8 + resolution: Literal["360p", "720p"] = "720p" class GenerateVideoRefsRequest(BaseModel): @@ -52,6 +53,7 @@ class GenerateVideoRefsRequest(BaseModel): # explicitly opt into Omni Flash. model_family: Literal["veo", "omni_flash"] = "veo" duration_s: int = 8 + resolution: Literal["360p", "720p"] = "720p" class GenerateOmniFlashVideoRequest(BaseModel): @@ -60,6 +62,7 @@ class GenerateOmniFlashVideoRequest(BaseModel): project_id: str scene_id: str = "" duration_s: int = 8 + resolution: Literal["360p", "720p"] = "720p" aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT" user_paygate_tier: str = "PAYGATE_TIER_ONE" @@ -173,8 +176,8 @@ async def generate_video(body: GenerateVideoRequest): request uses Omni First frame. When ``end_image_media_id`` is also present, it uses Omni First+Last frames. - Omni responses include ``flowkitPolling.workflows`` and must use workflow - media polling rather than legacy operation polling. + On the migrated batch transport, Omni frame-conditioned responses return + ``flowkitPolling.mode=batch_operation`` and are polled through ``/check-status``. """ client = get_flow_client() if not client.connected: @@ -188,6 +191,7 @@ async def generate_video(body: GenerateVideoRequest): project_id=body.project_id, scene_id=body.scene_id, duration_s=body.duration_s, + resolution=body.resolution, aspect_ratio=body.aspect_ratio, user_paygate_tier=body.user_paygate_tier, ) @@ -202,7 +206,7 @@ async def generate_video(body: GenerateVideoRequest): raise HTTPException(400, str(exc)) from exc else: result = await client.generate_video( - **body.model_dump(exclude={"model_family", "duration_s"}, exclude_none=True) + **body.model_dump(exclude={"model_family", "duration_s", "resolution"}, exclude_none=True) ) if result.get("error") or (isinstance(result.get("status"), int) and result["status"] >= 400): @@ -216,8 +220,8 @@ async def generate_video_refs(body: GenerateVideoRefsRequest): Existing requests default to ``model_family=veo``. Set ``model_family=omni_flash`` and ``duration_s`` to 4/6/8/10 to use Omni. - Omni responses include ``flowkitPolling.workflows``; poll those workflows, - not the operation-looking handles in the raw Flow response. + Migrated Omni Ingredients/R2V returns ``flowkitPolling.mode=batch_operation``; + poll its operations through ``/check-status``. """ client = get_flow_client() if not client.connected: @@ -231,6 +235,7 @@ async def generate_video_refs(body: GenerateVideoRefsRequest): project_id=body.project_id, scene_id=body.scene_id, duration_s=body.duration_s, + resolution=body.resolution, aspect_ratio=body.aspect_ratio, user_paygate_tier=body.user_paygate_tier, ) @@ -238,7 +243,7 @@ async def generate_video_refs(body: GenerateVideoRefsRequest): raise HTTPException(400, str(exc)) from exc else: result = await client.generate_video_from_references( - **body.model_dump(exclude={"model_family", "duration_s"}) + **body.model_dump(exclude={"model_family", "duration_s", "resolution"}) ) if result.get("error") or (isinstance(result.get("status"), int) and result["status"] >= 400): diff --git a/agent/services/flow_batch.py b/agent/services/flow_batch.py index f81c1e44..e2f41c00 100644 --- a/agent/services/flow_batch.py +++ b/agent/services/flow_batch.py @@ -37,6 +37,8 @@ RPC_GEN_IMAGE = "ogiZ0b" RPC_GEN_VIDEO = "eb1hJf" RPC_GEN_VIDEO_TEXT = "YhhmEf" +RPC_GEN_VIDEO_FIRST_LAST = "nprQif" +RPC_GEN_VIDEO_REFERENCES = "MZZa6b" RPC_OPERATION = "jwpduf" RPC_PROJECT_MEDIA = "Zzl0ze" RPC_MEDIA = "as29s" @@ -411,6 +413,101 @@ def video_request(prompt: str, project_id: str, source_media_id: str, return build_envelope(RPC_GEN_VIDEO, inner) +def omni_first_frame_request(prompt: str, project_id: str, source_media_id: str, + *, duration_s: int = 8, resolution: str = "720p", + aspect: Any = VIDEO_ASPECT_LANDSCAPE, + crop: Optional[list] = None) -> str: + """Build current Omni first-frame I2V (RPC ``eb1hJf``). + + Live-captured from Flow on 2026-09-14. 720p uses ``abra_i2v_s``; + 360p appends ``_360p`` and carries the UI's low-resolution option slot. + """ + if duration_s not in (4, 6, 8, 10): + raise ValueError("Omni duration must be 4, 6, 8 or 10 seconds") + res = str(resolution).strip().lower() + if res not in {"360p", "720p"}: + raise ValueError("Omni resolution must be 360p or 720p") + model = f"abra_i2v_{duration_s}s" + ("_360p" if res == "360p" else "") + request = [ + [None, None, [[[prompt]]]], + model, + resolve_video_aspect(aspect), + None, + [None, source_media_id, None, None, None, + FULL_FRAME_CROP if crop is None else crop], + [None, None, None, None, _client_uuid(), _client_uuid()], + ] + if res == "360p": + request.extend([None, None, None, [4]]) + return build_envelope(RPC_GEN_VIDEO, [ + [request], + _context(project_id), + [_client_uuid(), 2], + ]) + + +def omni_first_last_request(prompt: str, project_id: str, + start_media_id: str, end_media_id: str, + *, duration_s: int = 8, resolution: str = "720p", + aspect: Any = VIDEO_ASPECT_LANDSCAPE, + start_crop: Optional[list] = None, + end_crop: Optional[list] = None) -> str: + """Build Omni First+Last frames submit (RPC ``nprQif``).""" + if duration_s not in (4, 6, 8, 10): + raise ValueError("Omni duration must be 4, 6, 8 or 10 seconds") + res = str(resolution).strip().lower() + if res not in {"360p", "720p"}: + raise ValueError("Omni resolution must be 360p or 720p") + model = f"omni_flash_i2v_{duration_s}s_first_last" + ("_360p" if res == "360p" else "") + request = [ + [None, None, [[[prompt]]]], + model, + resolve_video_aspect(aspect), + None, + [None, start_media_id, None, None, None, + FULL_FRAME_CROP if start_crop is None else start_crop], + [None, end_media_id, None, None, None, + FULL_FRAME_CROP if end_crop is None else end_crop], + [None, None, None, None, _client_uuid(), _client_uuid()], + ] + return build_envelope(RPC_GEN_VIDEO_FIRST_LAST, [ + [request], + _context(project_id), + [_client_uuid(), 2], + ]) + + +def omni_reference_video_request(prompt: str, project_id: str, + reference_media_ids: list[str], + *, duration_s: int = 8, resolution: str = "720p", + aspect: Any = VIDEO_ASPECT_LANDSCAPE) -> str: + """Build Omni Ingredients/reference-to-video submit (RPC ``MZZa6b``).""" + refs = [str(mid) for mid in reference_media_ids if str(mid)] + if not refs: + raise ValueError("Omni reference-to-video requires at least one reference image") + if duration_s not in (4, 6, 8, 10): + raise ValueError("Omni duration must be 4, 6, 8 or 10 seconds") + res = str(resolution).strip().lower() + if res not in {"360p", "720p"}: + raise ValueError("Omni resolution must be 360p or 720p") + model = f"abra_r2v_{duration_s}s" + ("_360p" if res == "360p" else "") + request = [ + [None, None, [[[prompt]]]], + [[None, mid] for mid in refs], + model, + resolve_video_aspect(aspect), + None, + [None, None, None, None, _client_uuid(), _client_uuid()], + ] + if res == "360p": + request.extend([None, None, None, None, None, [4]]) + return build_envelope(RPC_GEN_VIDEO_REFERENCES, [ + [request], + _context(project_id), + [_client_uuid(), 2], + ]) + + def text_video_request(prompt: str, project_id: str, aspect: Any = VIDEO_ASPECT_LANDSCAPE, model: str = "abra_t2v_4s") -> str: diff --git a/agent/services/omni_flash.py b/agent/services/omni_flash.py index dcb8a8b4..fa078e41 100644 --- a/agent/services/omni_flash.py +++ b/agent/services/omni_flash.py @@ -1,22 +1,15 @@ """Gemini Omni Flash video generation through the Google Flow bridge. -Supported Omni surfaces in this module: - -* first frame -> video via ``batchAsyncGenerateVideoStartImage`` -* first + last frame -> video via ``batchAsyncGenerateVideoStartAndEndImage`` -* reference images -> video via ``batchAsyncGenerateVideoReferenceImages`` - -Omni duration-specific model keys live in ``agent/models.json``. First-frame -Flow requests have been captured with ``abra_i2v_s``. The current -First+Last rollout uses the same Omni I2V family but the StartAndEnd endpoint; -that mapping is deliberately configurable separately so it can be changed -without a code release if Google's rollout rotates the wire key. - -Important: on the migrated ``flow.google.com`` batch transport, first-frame -Omni I2V uses the same ``eb1hJf`` operation contract as migrated Veo, with the -wire model switched to ``abra_i2v_s``. Those jobs therefore use the -normal batch operation poller. Legacy Omni and migrated text-to-video keep their -workflow/media polling contracts. +Migrated ``flow.google.com`` batch surfaces live-verified on 2026-09-14: + +* text -> video: ``YhhmEf`` + ``abra_t2v_s`` (workflow/media polling) +* first frame -> video: ``eb1hJf`` + ``abra_i2v_s`` +* first + last frame -> video: ``nprQif`` + ``omni_flash_i2v_s_first_last`` +* Ingredients/references -> video: ``MZZa6b`` + ``abra_r2v_s`` + +Reference-conditioned modes return normal batch operation receipts and use the +same operation/media poller as migrated Veo. The legacy REST implementations are +kept below for non-batch deployments, but are not used by current production. """ from __future__ import annotations @@ -34,20 +27,6 @@ _MODELS_FILE = Path(__file__).parent.parent / "models.json" -#: First-frame I2V is live on the migrated batch transport. Start+end and -#: reference-image modes still need their current Flow UI payloads captured; -#: keep those explicitly blocked instead of falling through to dead legacy auth. -_UNSUPPORTED_START_END_ON_BATCH = ( - "UNSUPPORTED_ON_BATCH_API: Omni Flash first+last frame generation is not yet " - "ported to flow.google.com batchexecute. First-frame image-to-video and " - "text-to-video are supported on the batch path." -) -_UNSUPPORTED_REFERENCE_ON_BATCH = ( - "UNSUPPORTED_ON_BATCH_API: Omni Flash reference-to-video is not yet ported " - "to flow.google.com batchexecute. First-frame image-to-video and text-to-video " - "are supported on the batch path." -) - OMNI_FLASH_VALID_DURATIONS = (4, 6, 8, 10) OMNI_FLASH_VALID_ASPECTS = { "VIDEO_ASPECT_RATIO_PORTRAIT", @@ -110,6 +89,34 @@ def _validate_aspect(aspect_ratio: str) -> None: ) +def _validate_resolution(resolution: str) -> str: + value = str(resolution or "720p").strip().lower() + if value not in {"360p", "720p"}: + raise ValueError("Omni Flash resolution must be 360p or 720p") + return value + + +def _batch_operation_result(operation, project_id: str, model: str, duration_s: int, resolution: str) -> dict: + pending = { + "operation": {"name": operation.operation_id}, + "status": "MEDIA_GENERATION_STATUS_PENDING", + } + return { + "status": 200, + "data": { + "operations": [pending], + "model": model, + "duration_s": duration_s, + "resolution": resolution, + "flowkitPolling": { + "mode": "batch_operation", + "project_id": project_id, + "operations": [pending], + }, + }, + } + + def _load_model_key(duration_s: int, mode: str = "reference_to_video") -> str: """Resolve a configured Omni Flash model key for ``mode`` + duration.""" _validate_duration(duration_s) @@ -271,80 +278,53 @@ async def _submit_omni_frame_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, ) -> dict: """Submit Omni first-frame or First+Last generation. - Migrated first-frame I2V was recaptured from the live Flow UI on - 2026-09-14: it uses ``eb1hJf`` with the normal image-to-video payload and - ``abra_i2v_s`` as the wire model. Its response is a standard - batch operation, so poll it through ``/api/flow/check-status``. + Batch payloads were live-captured from ``flow.google.com`` on 2026-09-14: + first-frame uses ``eb1hJf``; First+Last uses ``nprQif``. Both return normal + batch operations and are polled through ``/api/flow/check-status``. """ - _validate_frame_inputs( - start_image_media_id, - end_image_media_id, - duration_s, - aspect_ratio, - ) - - mode = ( - "start_end_frame_to_video" - if end_image_media_id is not None - else "frame_to_video" - ) + _validate_frame_inputs(start_image_media_id, end_image_media_id, duration_s, aspect_ratio) + resolution = _validate_resolution(resolution) + mode = "start_end_frame_to_video" if end_image_media_id is not None else "frame_to_video" model_key = _load_model_key(duration_s, mode=mode) client = get_flow_client() if USE_BATCH_RPC: - if end_image_media_id is not None: - return {"error": _UNSUPPORTED_START_END_ON_BATCH} try: pid = client._batch_project_id(project_id) - freq = fb.video_request( - prompt, - pid, - start_image_media_id, - aspect=aspect_ratio, - model=model_key, - ) + if end_image_media_id is None: + freq = fb.omni_first_frame_request( + prompt, pid, start_image_media_id, duration_s=duration_s, + resolution=resolution, aspect=aspect_ratio, + ) + rpcid = fb.RPC_GEN_VIDEO + batch_model = f"abra_i2v_{duration_s}s" + ("_360p" if resolution == "360p" else "") + else: + freq = fb.omni_first_last_request( + prompt, pid, start_image_media_id, end_image_media_id, + duration_s=duration_s, resolution=resolution, aspect=aspect_ratio, + ) + rpcid = fb.RPC_GEN_VIDEO_FIRST_LAST + batch_model = f"omni_flash_i2v_{duration_s}s_first_last" + ( + "_360p" if resolution == "360p" else "" + ) payload = await client._batch_payload( - fb.RPC_GEN_VIDEO, - freq, - fb.CAPTCHA_VIDEO, - timeout=120, + rpcid, freq, fb.CAPTCHA_VIDEO, timeout=120, ) operation = fb.read_operation(payload) client._remember_operation(operation.operation_id, pid) except Exception as exc: return {"status": 502, "error": f"{type(exc).__name__}: {exc}"} + return _batch_operation_result(operation, pid, batch_model, duration_s, resolution) - pending = { - "operation": {"name": operation.operation_id}, - "status": "MEDIA_GENERATION_STATUS_PENDING", - } - return { - "status": 200, - "data": { - "operations": [pending], - "model": model_key, - "duration_s": duration_s, - "flowkitPolling": { - "mode": "batch_operation", - "project_id": pid, - "operations": [pending], - }, - }, - } - - endpoint = ( - "generate_video_start_end" - if end_image_media_id is not None - else "generate_video" - ) + endpoint = "generate_video_start_end" if end_image_media_id is not None else "generate_video" ts = int(time.time() * 1000) - request_item = { "aspectRatio": aspect_ratio, "textInput": {"structuredPrompt": {"parts": [{"text": prompt}]}}, @@ -355,7 +335,6 @@ async def _submit_omni_frame_video( } if end_image_media_id is not None: request_item["endImage"] = {"mediaId": end_image_media_id} - context = client._client_context(project_id, user_paygate_tier) body = { "mediaGenerationContext": {"batchId": str(uuid.uuid4())}, @@ -363,7 +342,6 @@ async def _submit_omni_frame_video( "requests": [request_item], "useV2ModelConfig": True, } - result = await client._send( "api_request", { @@ -384,6 +362,7 @@ async def generate_omni_flash_first_frame_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, @@ -396,6 +375,7 @@ async def generate_omni_flash_first_frame_video( project_id=project_id, scene_id=scene_id, duration_s=duration_s, + resolution=resolution, aspect_ratio=aspect_ratio, user_paygate_tier=user_paygate_tier, seed=seed, @@ -409,6 +389,7 @@ async def generate_omni_flash_first_last_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, @@ -421,6 +402,7 @@ async def generate_omni_flash_first_last_video( project_id=project_id, scene_id=scene_id, duration_s=duration_s, + resolution=resolution, aspect_ratio=aspect_ratio, user_paygate_tier=user_paygate_tier, seed=seed, @@ -433,22 +415,38 @@ async def generate_omni_flash_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, ) -> dict: - """Submit an Omni Flash reference-to-video generation. + """Submit Omni Flash Ingredients/reference-to-video generation. - Successful responses are annotated with ``data.flowkitPolling`` containing - the workflow names and primary media IDs required by the Omni polling path. - Do not feed Omni operation handles to ``check_video_status``. + The migrated UI uses RPC ``MZZa6b`` with ``abra_r2v_s`` (720p) + or ``abra_r2v_s_360p``. Batch jobs use normal operation polling. """ - if USE_BATCH_RPC: - return {"error": _UNSUPPORTED_REFERENCE_ON_BATCH} refs = _validate_reference_inputs(reference_media_ids, duration_s, aspect_ratio) + resolution = _validate_resolution(resolution) model_key = _load_model_key(duration_s, mode="reference_to_video") client = get_flow_client() + if USE_BATCH_RPC: + try: + pid = client._batch_project_id(project_id) + freq = fb.omni_reference_video_request( + prompt, pid, refs, duration_s=duration_s, + resolution=resolution, aspect=aspect_ratio, + ) + payload = await client._batch_payload( + fb.RPC_GEN_VIDEO_REFERENCES, freq, fb.CAPTCHA_VIDEO, timeout=120, + ) + operation = fb.read_operation(payload) + client._remember_operation(operation.operation_id, pid) + except Exception as exc: + return {"status": 502, "error": f"{type(exc).__name__}: {exc}"} + batch_model = f"abra_r2v_{duration_s}s" + ("_360p" if resolution == "360p" else "") + return _batch_operation_result(operation, pid, batch_model, duration_s, resolution) + ts = int(time.time() * 1000) request_item = { "aspectRatio": aspect_ratio, @@ -461,7 +459,6 @@ async def generate_omni_flash_video( for mid in refs ], } - context = client._client_context(project_id, user_paygate_tier) body = { "mediaGenerationContext": { @@ -472,7 +469,6 @@ async def generate_omni_flash_video( "requests": [request_item], "useV2ModelConfig": True, } - url = client._build_url("generate_video_references") result = await client._send( "api_request", diff --git a/docs/OMNI_FLASH.md b/docs/OMNI_FLASH.md index 57407be3..e83bb980 100644 --- a/docs/OMNI_FLASH.md +++ b/docs/OMNI_FLASH.md @@ -22,37 +22,37 @@ Use `http://127.0.0.1:8100` when the caller runs on the FlowKit host. For a remo ## Supported modes -On the current `flow.google.com` batch transport, Omni **text-to-video** and **first-frame image-to-video** are live and verified. First+Last and multi-reference generation still depend on the pre-migration transport and remain explicitly refused while `USE_BATCH_RPC=1` until their current UI payloads are captured. +On the current `flow.google.com` batch transport, every Omni 1.1 Flash video mode exposed by Flow's Video composer is supported and live-verified: -| Mode | Batch status | Endpoint | Internal model family | +| Mode | Batch status | Endpoint | Current wire | |---|---|---|---| -| Text to video | **supported** | `POST /api/flow/generate-video-omni-text` | `abra_t2v_s` | -| First frame to video | **supported** | `POST /api/flow/generate-video` with `model_family=omni_flash` | `abra_i2v_s` | -| First + Last frame to video | not yet ported | `POST /api/flow/generate-video` | `abra_i2v_s` (legacy only) | -| References to video | not yet ported | `POST /api/flow/generate-video-omni` | `abra_r2v_s` (legacy only) | +| Text to video | **supported** | `POST /api/flow/generate-video-omni-text` | `YhhmEf` + `abra_t2v_s` | +| First frame to video | **supported** | `POST /api/flow/generate-video` with `model_family=omni_flash` | `eb1hJf` + `abra_i2v_s` | +| First + Last frame to video | **supported** | `POST /api/flow/generate-video` with `model_family=omni_flash` + `end_image_media_id` | `nprQif` + `omni_flash_i2v_s_first_last` | +| Ingredients / references to video | **supported** | `POST /api/flow/generate-video-omni` or `/generate-video-refs` | `MZZa6b` + `abra_r2v_s` | -Text-to-video durations are `4`, `6`, `8`, and `10` seconds. Supported aspect ratios are: +Reference-conditioned modes support durations `4`, `6`, `8`, and `10` seconds, resolutions `360p` and `720p`, and: - `VIDEO_ASPECT_RATIO_PORTRAIT` (`9:16`) - `VIDEO_ASPECT_RATIO_LANDSCAPE` (`16:9`) -The migrated `YhhmEf` wire was live-verified with `abra_t2v_4s`; the downloaded result was exactly 4.000 seconds at 1280x720/24 fps. First-frame I2V was re-captured from the live Flow UI on 2026-09-14: it uses RPC `eb1hJf`, the normal migrated I2V payload shape, and `abra_i2v_s`. A live API smoke test completed successfully and resolved its signed video URL through the existing batch operation poller. +For 360p Flow uses `_360p` model variants. The 360p first-frame and Ingredients payloads also carry the same low-resolution option slots captured from the live UI. First+Last uses its dedicated `nprQif` payload and model family. -Polling differs by migrated mode: text-to-video returns workflow/media descriptors and uses `/api/flow/check-omni-status`; first-frame I2V returns a `flowkitPolling.mode = batch_operation` descriptor and uses `/api/flow/check-status` with its `operations` array. +The migrated wires were re-captured from the live Flow UI on 2026-09-14. Live API smoke tests completed successfully for First frame, First+Last, and Ingredients/R2V and resolved signed `flow-content.google` video URLs through the batch operation poller. -## End-to-end integration flow +Polling differs only for text-to-video: text-to-video returns `flowkitPolling.mode=batch_media` and uses `/api/flow/check-omni-status`; all image/reference-conditioned modes return `flowkitPolling.mode=batch_operation` and use `/api/flow/check-status` with their `operations` array. -For migrated Omni, an integration agent should use the polling mode returned by the submit response: +## End-to-end integration flow 1. Check `/health` and `/api/flow/status`. -2. Submit either text-to-video or first-frame I2V. -3. Persist the complete `flowkitPolling` object returned by the submit. +2. Submit the desired Omni mode. +3. Persist the complete `flowkitPolling` object returned by the submit before doing anything else. 4. If `mode=batch_media`, poll `/api/flow/check-omni-status` using `project_id` + `workflows`. 5. If `mode=batch_operation`, poll `/api/flow/check-status` using `project_id` + `operations`. -6. On pending state, continue polling; on failure, stop; on success, immediately download the returned signed video URL. +6. Continue on pending state; stop on failure; on success immediately download the signed video URL. 7. Store the downloaded video in durable storage because Google URLs are signed and short-lived. -Example 4-second submit: +Example text-to-video submit: ```bash curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni-text" \ @@ -65,11 +65,11 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni-text" \ }' ``` -Do not send Omni workflow names to the legacy Veo `batchCheckAsyncVideoGenerationStatus` operation poller. Do not use the obsolete `/v1/media/` polling path. +Do not convert workflow names into operation handles, and do not convert batch operation handles into workflow names. Persist and replay the polling descriptor exactly as FlowKit returns it. ## Supplying images -This section applies to the older frame/reference Omni modes. Those generation modes are currently refused on `USE_BATCH_RPC=1` until their migrated payloads are captured; image upload itself may still be used by supported Veo workflows. +Frame, First+Last, and Ingredients/R2V all consume Flow media IDs. Upload or reuse the reference images first, then pass their media IDs to the generation endpoint. `POST /api/flow/upload-image` is not a multipart upload endpoint. Its `file_path` is an absolute path on the **FlowKit server**, not on the calling server. @@ -117,12 +117,13 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video" \ "project_id": "FLOW_PROJECT_ID", "scene_id": "JOB_ID", "duration_s": 4, + "resolution": "720p", "aspect_ratio": "VIDEO_ASPECT_RATIO_PORTRAIT", "user_paygate_tier": "PAYGATE_TIER_ONE" }' ``` -This uses `batchAsyncGenerateVideoStartImage`. +This uses batch RPC `eb1hJf` with `abra_i2v_s` (or the `_360p` variant). ## First + Last frame to video @@ -137,12 +138,13 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video" \ "project_id": "FLOW_PROJECT_ID", "scene_id": "JOB_ID", "duration_s": 4, + "resolution": "720p", "aspect_ratio": "VIDEO_ASPECT_RATIO_PORTRAIT", "user_paygate_tier": "PAYGATE_TIER_ONE" }' ``` -This uses `batchAsyncGenerateVideoStartAndEndImage` and sends both `startImage` and `endImage`. +This uses batch RPC `nprQif` with the dedicated `omni_flash_i2v_s_first_last` model family. ## References to video @@ -157,86 +159,71 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni" \ "project_id": "FLOW_PROJECT_ID", "scene_id": "JOB_ID", "duration_s": 4, + "resolution": "720p", "aspect_ratio": "VIDEO_ASPECT_RATIO_LANDSCAPE", "user_paygate_tier": "PAYGATE_TIER_ONE" }' ``` -The compatible generic endpoint is `POST /api/flow/generate-video-refs` with the same fields plus `"model_family":"omni_flash"`. +The compatible generic endpoint is `POST /api/flow/generate-video-refs` with the same fields plus `"model_family":"omni_flash"`. Both routes use batch RPC `MZZa6b`. ## Submit response and polling -Persist the entire normalized polling descriptor returned by submit: +Image/reference-conditioned Omni submits return normal batch operation polling: ```json { + "operations": [ + {"operation":{"name":"OPERATION_ID"},"status":"MEDIA_GENERATION_STATUS_PENDING"} + ], + "model": "abra_r2v_4s", + "duration_s": 4, + "resolution": "720p", "flowkitPolling": { - "mode": "project_media", + "mode": "batch_operation", "project_id": "FLOW_PROJECT_ID", - "workflows": [ - { - "name": "WORKFLOW_NAME", - "primary_media_id": "PRIMARY_MEDIA_ID", - "project_id": "FLOW_PROJECT_ID" - } + "operations": [ + {"operation":{"name":"OPERATION_ID"},"status":"MEDIA_GENERATION_STATUS_PENDING"} ] } } ``` -Poll using those values without transforming workflow names into operation handles: +Poll that descriptor without transforming it: ```bash -curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/check-omni-status" \ +curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/check-status" \ -H 'Content-Type: application/json' \ -d '{ "project_id": "FLOW_PROJECT_ID", - "workflows": [ - { - "name": "WORKFLOW_NAME", - "primary_media_id": "PRIMARY_MEDIA_ID", - "project_id": "FLOW_PROJECT_ID" - } - ], - "include_encoded_video": false + "operations": [ + {"operation":{"name":"OPERATION_ID"},"status":"MEDIA_GENERATION_STATUS_PENDING"} + ] }' ``` -The generic `POST /api/flow/check-status` endpoint also accepts the same `workflows` and automatically selects Omni project polling. - -Pending response: - -```json -{ - "project_id": "FLOW_PROJECT_ID", - "done": false, - "status": "PENDING", - "workflows": [{"done":false,"status":"PENDING"}] -} -``` - -Successful response: +When complete, the operation contains the generated media ID and signed video URL: ```json { - "project_id": "FLOW_PROJECT_ID", - "done": true, - "status": "COMPLETED", - "workflows": [ + "operations": [ { - "done": true, - "status": "MEDIA_GENERATION_STATUS_SUCCESSFUL", - "media": { - "media_id": "PRIMARY_MEDIA_ID", - "url": "https://flow-content.google/...", - "encoded_video_available": false - } + "operation": { + "name": "OPERATION_ID", + "metadata": { + "video": { + "mediaId": "MEDIA_ID", + "fifeUrl": "https://flow-content.google/video/..." + } + } + }, + "status": "MEDIA_GENERATION_STATUS_SUCCESSFUL" } ] } ``` -Keep `include_encoded_video` set to `false`. FlowKit resolves the signed download URL without buffering the MP4 through Chrome or the extension bridge. +Text-to-video remains the exception: it returns `mode=batch_media` with workflow/media descriptors and is polled through `/api/flow/check-omni-status`. ## Retry and failure policy @@ -278,15 +265,15 @@ Mappings live in `agent/models.json`: } ``` -The mappings can be changed through `PATCH /api/models` if Google rotates internal keys. Treat configured credit-cost estimates as informational only: Google can change pricing, so use `GET /api/flow/credits` and the submit response's `remainingCredits` where available. +The legacy mappings above remain for the pre-migration transport. On the migrated batch transport, current Flow wire names are derived from duration + resolution exactly as live-captured: First frame uses `abra_i2v_*`, First+Last uses `omni_flash_i2v_*_first_last`, and Ingredients uses `abra_r2v_*`, with `_360p` appended for 360p. The mappings can still be changed through `PATCH /api/models` for legacy transport if Google rotates old keys. Treat configured credit-cost estimates as informational only: Google can change pricing, so use `GET /api/flow/credits` and the submit response's `remainingCredits` where available. ## Minimal agent checklist - Use the `/api/flow/...` paths exactly. - Select Omni explicitly with `model_family: "omni_flash"` on shared endpoints. - Upload inputs once and reuse returned Flow media IDs. -- Persist `project_id`, workflow `name`, and `primary_media_id` before polling. -- Poll workflows through project media status, never through legacy Veo operations. +- Persist the entire returned `flowkitPolling` descriptor before polling. +- Use `/check-status` for `batch_operation` and `/check-omni-status` for `batch_media`; never transform one receipt type into the other. - Download signed output URLs immediately into durable project storage. - Do not log Google auth data, extension messages, or complete signed URLs. - Use a stable `scene_id`/job ID for traceability. diff --git a/tests/unit/test_flow_batch.py b/tests/unit/test_flow_batch.py index db1f3457..2a080459 100644 --- a/tests/unit/test_flow_batch.py +++ b/tests/unit/test_flow_batch.py @@ -155,6 +155,42 @@ def test_a_hand_reframed_crop_overrides_the_default(self): crop = [None, 0.1, 1, 0.9] assert inner(fb.video_request("go", self.PID, "mid", crop=crop))[0][0][4][5] == crop + def test_omni_first_frame_360p_matches_live_eb1hjf_shape(self): + payload = inner(fb.omni_first_frame_request( + "move", self.PID, "start-mid", duration_s=4, resolution="360p", + aspect="VIDEO_ASPECT_RATIO_LANDSCAPE", + )) + request = payload[0][0] + assert request[1] == "abra_i2v_4s_360p" + assert request[2] == fb.VIDEO_ASPECT_LANDSCAPE + assert request[4][1] == "start-mid" + assert request[-1] == [4] + assert json.loads(fb.omni_first_frame_request("x", self.PID, "m"))[0][0][0] == fb.RPC_GEN_VIDEO + + def test_omni_first_last_matches_live_nprqif_shape(self): + freq = fb.omni_first_last_request( + "morph", self.PID, "start", "end", duration_s=6, resolution="720p", + aspect="VIDEO_ASPECT_RATIO_PORTRAIT", + ) + assert json.loads(freq)[0][0][0] == fb.RPC_GEN_VIDEO_FIRST_LAST + request = inner(freq)[0][0] + assert request[1] == "omni_flash_i2v_6s_first_last" + assert request[2] == fb.VIDEO_ASPECT_PORTRAIT + assert request[4][1] == "start" + assert request[5][1] == "end" + + def test_omni_reference_matches_live_mzza6b_shape(self): + freq = fb.omni_reference_video_request( + "keep refs", self.PID, ["a", "b"], duration_s=4, resolution="360p", + aspect="VIDEO_ASPECT_RATIO_LANDSCAPE", + ) + assert json.loads(freq)[0][0][0] == fb.RPC_GEN_VIDEO_REFERENCES + request = inner(freq)[0][0] + assert request[1] == [[None, "a"], [None, "b"]] + assert request[2] == "abra_r2v_4s_360p" + assert request[3] == fb.VIDEO_ASPECT_LANDSCAPE + assert request[-1] == [4] + def test_text_video_matches_the_captured_yhhmef_shape(self): payload = inner(fb.text_video_request( "a boat", self.PID, diff --git a/tests/unit/test_omni_flash.py b/tests/unit/test_omni_flash.py index e1a2998a..805418fd 100644 --- a/tests/unit/test_omni_flash.py +++ b/tests/unit/test_omni_flash.py @@ -586,9 +586,7 @@ async def test_submit_rejects_empty_reference_set(): ) -class TestRemainingUnportedOmniBatchModesAreRefused: - """Only start+end/reference remain blocked; first-frame I2V is migrated.""" - +class TestMigratedOmniReferenceBatchModes: @pytest.fixture(autouse=True) def batch_transport(self, monkeypatch): monkeypatch.setattr(omni_flash, "USE_BATCH_RPC", True) @@ -597,25 +595,64 @@ def batch_transport(self, monkeypatch): def client(self): with patch("agent.services.omni_flash.get_flow_client") as factory: stub = MagicMock() - stub._send = AsyncMock() + pid = "11111111-2222-3333-4444-555555555555" + stub._batch_project_id.return_value = pid + stub._batch_payload = AsyncMock(return_value=[ + None, 50, [["op-ref-1", pid, "scene-1", None]], + ]) factory.return_value = stub yield stub - async def test_first_last_names_the_gap_and_sends_nothing(self, client): + async def test_first_last_uses_nprqif_and_batch_operation_polling(self, client): result = await generate_omni_flash_first_last_video( - start_image_media_id="a", end_image_media_id="b", - prompt="go", project_id="pid") - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - client._send.assert_not_called() - - async def test_reference_to_video_names_the_gap_and_sends_nothing(self, client): + start_image_media_id="start", end_image_media_id="end", + prompt="morph", project_id="pid", duration_s=4, + resolution="360p", aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE") + assert result["status"] == 200 + assert result["data"]["model"] == "omni_flash_i2v_4s_first_last_360p" + assert result["data"]["resolution"] == "360p" + assert result["data"]["flowkitPolling"]["mode"] == "batch_operation" + rpcid, freq, captcha = client._batch_payload.await_args.args[:3] + assert rpcid == omni_flash.fb.RPC_GEN_VIDEO_FIRST_LAST + assert captcha == omni_flash.fb.CAPTCHA_VIDEO + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + req = payload[0][0] + assert req[1] == "omni_flash_i2v_4s_first_last_360p" + assert req[4][1] == "start" + assert req[5][1] == "end" + client._remember_operation.assert_called_once_with( + "op-ref-1", "11111111-2222-3333-4444-555555555555") + + async def test_reference_to_video_uses_mzza6b_and_all_references(self, client): result = await generate_omni_flash_video( - reference_media_ids=["a"], prompt="go", project_id="pid") - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - client._send.assert_not_called() - - async def test_the_message_points_to_supported_text_to_video(self, client): - result = await generate_omni_flash_video( - reference_media_ids=["a"], prompt="go", project_id="pid") - assert "image-to-video and text-to-video are supported" in result["error"] - assert "reference-to-video" in result["error"] + reference_media_ids=["a", "b", "c"], prompt="keep all refs", + project_id="pid", duration_s=6, resolution="720p", + aspect_ratio="VIDEO_ASPECT_RATIO_PORTRAIT") + assert result["status"] == 200 + assert result["data"]["model"] == "abra_r2v_6s" + assert result["data"]["flowkitPolling"]["mode"] == "batch_operation" + rpcid, freq, _captcha = client._batch_payload.await_args.args[:3] + assert rpcid == omni_flash.fb.RPC_GEN_VIDEO_REFERENCES + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + req = payload[0][0] + assert req[1] == [[None, "a"], [None, "b"], [None, "c"]] + assert req[2] == "abra_r2v_6s" + assert req[3] == omni_flash.fb.VIDEO_ASPECT_PORTRAIT + + async def test_reference_360p_uses_live_wire_model_and_quality_slot(self, client): + await generate_omni_flash_video( + reference_media_ids=["a", "b"], prompt="refs", project_id="pid", + duration_s=4, resolution="360p", + aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE") + _rpcid, freq, _captcha = client._batch_payload.await_args.args[:3] + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + req = payload[0][0] + assert req[2] == "abra_r2v_4s_360p" + assert req[-1] == [4] + + async def test_invalid_resolution_is_rejected_before_submit(self, client): + with pytest.raises(ValueError, match="resolution must be 360p or 720p"): + await generate_omni_flash_first_frame_video( + start_image_media_id="a", prompt="go", project_id="pid", + resolution="1080p") + client._batch_payload.assert_not_called()