From 9fc2c9dc7d21bb0f65611c5e569e6557ae3bcd73 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 11 Nov 2026 09:32:46 -0500 Subject: [PATCH] Fix native audit citations for supporting documents Admit supporting-document citations through the retained owner's projection while keeping native facts and strict marker validation separate. Version the checking contract for explicit renewal and cover mixed-source delivery, refusal, and recovery. Update fix documentation and application version to 0.261.428. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/functions_native_audit.py | 44 ++-- .../functions_orchestration_composition.py | 9 +- .../NATIVE_AUDIT_MULTI_SOURCE_CITATION_FIX.md | 164 +++++++++++++ .../fixes/NATIVE_AUDIT_RELIABILITY_FIX.md | 6 + .../test_native_audit_answer_delivery.py | 228 +++++++++++++++--- .../test_native_audit_reliability.py | 73 +++++- 7 files changed, 475 insertions(+), 51 deletions(-) create mode 100644 docs/explanation/fixes/NATIVE_AUDIT_MULTI_SOURCE_CITATION_FIX.md diff --git a/application/single_app/config.py b/application/single_app/config.py index fb149585b..eac683437 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.427" +VERSION = "0.261.428" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_native_audit.py b/application/single_app/functions_native_audit.py index 2a0637f4c..ac13ba39d 100644 --- a/application/single_app/functions_native_audit.py +++ b/application/single_app/functions_native_audit.py @@ -10,7 +10,7 @@ NATIVE_AUDIT_VERSION = "native-audit-v1" -NATIVE_AUDIT_ANSWER_VERSION = "native-audit-answer-v3" +NATIVE_AUDIT_ANSWER_VERSION = "native-audit-answer-v4" MAX_AUDIT_FACTS = 1000 MAX_AUDIT_BYTES = 100000 NATIVE_AUDIT_ANSWER_ERRORS = { @@ -53,6 +53,8 @@ "states the correct absence. Preserve whitespace, false and zero as present values. " "Action-model paraphrases cannot override canonical facts. Native logical order, paint-operation " "counts, and complete returned inventory pages do not prove a rendered visible-content review. " + "Other authorized named-input sources support their own claims without becoming native fact " + "authorities. Their returned content is not unavailable merely because they have no native audit. " "An explicit, accurate limitation satisfies an unsupported check alongside useful findings. " "The candidate is customer Markdown, not the preparation JSON. Return the requested assessment " "schema, not a native-answer envelope." @@ -330,8 +332,12 @@ def _native_page_chain_complete(pages): return expected_offset == total and expected_fragment == 0 -def native_audit_contract(inputs): - """Only recognize actual native-reader invocation results, never document prose.""" +def native_audit_contract(inputs, *, authorized_answer_citations=()): + """Native facts come from reader results; only an owner may grant other answer citations. + + ``authorized_answer_citations`` must come from the retained-result owner's reauthorized + named-input projection, not source prose or a model's citation declarations. + """ facts, citations, files, limitations = [], [], [], [] seen, source_hashes, inventory_pages, inventory_files, unfinished = {}, {}, {}, {}, set() source_versions = {} @@ -406,19 +412,27 @@ def native_audit_contract(inputs): complete_chains = {key: _native_page_chain_complete(pages) for key, pages in inventory_pages.items()} if any(not complete_chains.get(key, False) for key in unfinished): limitations.append("A native inventory page has a continuation; only returned records are represented.") + if type(authorized_answer_citations) not in (list, tuple) or any( + type(citation) is not dict + or type(citation.get("document_id")) is not str or not citation["document_id"] + or type(citation.get("citation")) is not str or not citation["citation"] + for citation in authorized_answer_citations + ): + raise ValueError("The authorized answer citation contract is invalid.") answer_citations = list(citations) + supporting_citations = list(authorized_answer_citations) for entry in inputs.values(): - # These are projected by the authorized retained-result owner, never from source prose. for citation in entry.get("document_citations") or []: - if not isinstance(citation, dict) or citation.get("document_id") not in source_hashes: - continue - version = citation.get("version") - native_version = source_versions[citation["document_id"]] - if version is not None and native_version is not None and version != native_version: - raise ValueError("A supporting citation refers to a different native source version.") - marker = citation.get("citation") - if isinstance(marker, str) and marker: - answer_citations.append(marker) + if isinstance(citation, dict) and citation.get("document_id") in source_hashes: + supporting_citations.append(citation) + for citation in supporting_citations: + version = citation.get("version") + native_version = source_versions.get(citation["document_id"]) + if version is not None and native_version is not None and version != native_version: + raise ValueError("A supporting citation refers to a different native source version.") + marker = citation.get("citation") + if isinstance(marker, str) and marker: + answer_citations.append(marker) return { "facts": facts, "citations": list(dict.fromkeys(citations)), "files": list(dict.fromkeys(files)), "answer_citations": list(dict.fromkeys(answer_citations)), @@ -543,7 +557,9 @@ def native_audit_policy(contract): "Disclose that boundary alongside useful native findings, not as an execution error. " "Do not infer rendered compliance or invent source assertions or citations. " "Copy native_audit_contract.answer_citations verbatim where used; supporting chunk citations " - "do not override native property facts. " + "do not override native property facts. Authorized supporting documents remain available " + "answer evidence even when they have no native inspection; do not describe their returned " + "content as unavailable. " "The server appends its immutable factual evidence table; do not copy that table into answer. " "The checker evaluates the rendered customer answer, not this JSON transport format." ) diff --git a/application/single_app/functions_orchestration_composition.py b/application/single_app/functions_orchestration_composition.py index c530c04a2..d5bba8b1e 100644 --- a/application/single_app/functions_orchestration_composition.py +++ b/application/single_app/functions_orchestration_composition.py @@ -1,7 +1,7 @@ # functions_orchestration_composition.py """Explicit content preparation from named authorized result readers. -Version: 0.261.426 +Version: 0.261.428 Refusals for a missing signed-in session reported as their own failure in: 0.261.209 Charts drawn upstream reach the model as their placement tokens in: 0.261.293 No retrieval, file-format inference, upload, publication, or implicit sibling inputs. @@ -689,7 +689,12 @@ def recheck(): transform_context = transform_preparation_context(step, readers, inputs, images) transform_policy = f'{TRANSFORM_POLICY} Required mode: {transform_context["mode"]}.' plain_text = len(outputs) == 1 and outputs[0]['kind'] in ('text-v1', 'markdown-v1') - audit_contract = native_audit_contract(inputs) if plain_text and not images else None + # Only the reauthorized owner projection can extend citations beyond native fact sources. + audit_contract = native_audit_contract( + inputs, authorized_answer_citations=[ + citation for entry in inputs.values() for citation in entry['document_citations'] + ], + ) if plain_text and not images else None unavailable = _missing_optional_inputs(step, context) missing_images = [item for item in unavailable if item['capability_id'] == CAPABILITY_GENERATE_IMAGE] missing = [item for item in unavailable if item not in missing_images] diff --git a/docs/explanation/fixes/NATIVE_AUDIT_MULTI_SOURCE_CITATION_FIX.md b/docs/explanation/fixes/NATIVE_AUDIT_MULTI_SOURCE_CITATION_FIX.md new file mode 100644 index 000000000..2b811dea0 --- /dev/null +++ b/docs/explanation/fixes/NATIVE_AUDIT_MULTI_SOURCE_CITATION_FIX.md @@ -0,0 +1,164 @@ +# Native audit multi-source citation delivery fix (v0.261.428) + +Fixed/Implemented in version: **0.261.428**. +Application version: `application/single_app/config.py`. +Dependencies: existing authorized retained results, Document Search invocations, +native document readers, configured orchestration models, and private checking +checkpoints. No new package, route, setting, browser asset, or service is required. + +## Issue and root cause + +A native document review could successfully inspect one selected file and read a +different supporting file, yet deliver only a partial-result explanation instead +of the requested review. Native-answer validation admitted supporting citations +only when their document had also supplied a native accessibility report. + +That restriction confused two authorities: the source of immutable native file +facts and the sources allowed to support the answer's other claims. A selected +Markdown evidence file, for example, can support an engineering checklist without +establishing any native PowerPoint properties. + +In the investigated saved attempt, both Document Search operations succeeded. +The gather was retained without a partial flag, and composition received two +located document citations. The first and final candidates failed +`native_answer_citation_unknown`. An intervening semantic assessment rejected a +corrected candidate that inaccurately claimed the supplied supporting-file content +was unavailable. After the configured correction allowance, no checked answer was +committed and the executor selected its existing partial-result explanation. + +The exact rejected markers in the private initial and final candidates were not +recovered. The over-narrow citation contract was independently reproduced through +the real HTTP/action/retention path: same-document supporting citations passed, +while valid citations from a separate selected supporting document failed. + +This is distinct from an older analysis dependency being rejected because its +required upstream result was genuinely incomplete. It is not evidence that an +optional metadata field, model quota, or service timeout caused this failure. + +## Technical changes + +### Separate native facts from answer support + +`native_audit_contract()` now accepts an explicit internal +`authorized_answer_citations` argument. Composition supplies it from the existing +retained-result owner's named-input citation projection, after that owner has +checked result authority and recorded source lineage. + +| Authority | Evidence admitted | What it establishes | +| --- | --- | --- | +| Native facts and appendix citations | Actual native-reader invocation results, with the existing source version, hash, and canonical field-state checks | The inspected native records and their stated coverage | +| Answer-support citations | Exact owner-issued citations for the bound named inputs, including other selected documents | Supporting content for the answer's non-native claims | + +The answer registry does not add supporting documents to the native fact set, +native source list, or immutable appendix citations. Native fact digests do not +change merely because another authorized supporting citation is admitted. +Matching native-source version conflicts still fail. + +Embedded source text, action notes, filename fragments, and an arbitrary +`document_citations` dictionary do not grant another document's citation. +Only the explicit owner projection can extend the answer registry. Validation +still rejects invented or altered markers rather than stripping or rewriting +them into apparent success. + +Preparation, correction, and semantic review receive the same answer citation +authority. Their native policy explains that returned supporting content remains +available evidence even when its document has no native inspection; it does not +make supporting prose authoritative for native properties. + +### Preserve retained-result access semantics + +Workspace document authorization remains at its existing fresh-read boundary. +For retained results, the recorded workspace sources are provenance; the owning +conversation/run and current external integration checks govern access. + +This fix does not add workspace source rereads, require a supporting file to have +native metadata, or change retention/revocation policy. The existing owner still +rejects citations naming another source, invalid lineage, and unavailable current +action access. + +### Renew checking under an explicit contract version + +The private preparation/checkpoint contract advances from +`native-audit-answer-v3` to `native-audit-answer-v4`. The customer preparation +transport remains `native_claims` plus a nonempty Markdown `answer`. + +An explicit new checking retry can renew an older failed preparation under the +new citation contract using its still-authorized retained gather. The existing +retry path discards the obsolete checking state instead of applying an old +assessment to a different request identity. Completed native reads and supporting +searches are not repeated. + +No retry is started automatically. Correction limits, deterministic marker +validation, semantic checks, ownership fencing, and refusal to publish an +unchecked candidate remain unchanged. + +### Files and impact + +| File | Change | +| --- | --- | +| `application/single_app/functions_native_audit.py` | Explicit owner-authorized answer citation registry, unchanged native fact authority, v4 checking contract, and consistent preparation/review guidance | +| `application/single_app/functions_orchestration_composition.py` | Pass the verified named-input citation projection into the native answer contract | +| `application/single_app/config.py` | Application patch version 0.261.428 | +| `functional_tests/test_native_audit_reliability.py` | Registry shape, separate supporting inputs, native version fences, unchanged facts, and embedded-marker controls | +| `functional_tests/test_native_audit_answer_delivery.py` | Real mixed-source delivery, correction, action-access refusal, unknown-marker rejection, and v3 check-only recovery | + +No source deck is changed. Absent/inaccessible optional properties and incomplete +native inventory coverage remain honest review limitations, not new execution +failure gates. Genuine failed reads, unsupported claims, strict upstream +dependencies, and explicitly permitted available-information handling keep their +existing distinct outcomes. + +## Validation and before/after comparison + +Before the fix, the two selected-supporting-document HTTP cases failed while the +same-document PDF and PowerPoint controls passed. After the fix, all four save +checked answers with their exact owner-issued citations. The separate supporting +file is not promoted into native fact evidence. + +Additional regressions demonstrate that correction can restore supplied +supporting-file findings without losing their valid citation; invented markers +still exhaust bounded correction without publication; disabling the retained +action during preparation blocks delivery; and an explicit v3 retry with an +obsolete identity renews checking without another native download or supporting +search. + +The following local selections passed. Their coverage overlaps, so the counts +are not a combined unique-test total. + +| Selection | Result | +| --- | --- | +| Native contracts/delivery, real action HTTP path and lineage, completeness engine/runtime, and available-information controls | 218 passed | +| Optimized Python: native contracts/delivery, source-backed native pipeline, and inspection request contracts | 127 passed | +| Final application-version and focused citation-contract selection | 38 passed | +| Cosmos boundary inventory | 1,094 owners, 425 doubles, zero review gaps | +| Documentation surface coverage / site quality | 7/7 and 6/6 checks passed | +| Changed Python syntax / filename headers | 5 files passed | +| Whitespace | Passed | + +Commands used the existing network guard and external-I/O/provider doubles: + +```powershell +$env:PYTHONDONTWRITEBYTECODE='1' +$env:PYTHONPATH="$PWD\application\single_app;$PWD\functional_tests" +python -B -m pytest .\functional_tests\test_native_audit_reliability.py .\functional_tests\test_native_audit_answer_delivery.py .\functional_tests\test_orchestration_document_action_end_to_end.py .\functional_tests\test_orchestration_document_action_lineage.py .\functional_tests\test_task_completeness_engine.py .\functional_tests\test_task_completeness_runtime.py .\functional_tests\test_orchestration_available_information.py -q --tb=short -p test_support.cosmos_network_guard -p no:cacheprovider --disable-warnings +python -O -B -m pytest .\functional_tests\test_native_audit_reliability.py .\functional_tests\test_native_audit_answer_delivery.py .\functional_tests\test_native_document_fidelity_pipeline.py .\functional_tests\test_native_inspection_request_contract.py -q --tb=short -p test_support.cosmos_network_guard -p no:cacheprovider --disable-warnings +python -B -m pytest .\functional_tests\test_native_audit_reliability.py .\functional_tests\test_native_audit_answer_delivery.py::test_native_audit_delivery_version -q --tb=short -p test_support.cosmos_network_guard -p no:cacheprovider --disable-warnings +python -B .\scripts\check_cosmos_response_boundaries.py +python -B .\functional_tests\test_docs_app_surface_coverage.py +python -B .\functional_tests\test_docs_site_quality.py +git --no-pager diff --check +``` + +Existing dependency and SWIG deprecation warnings remain. These cases prove +application contract, correction, and publication behavior with controlled +provider replies; they do not prove that an arbitrary live model always produces +a valid or complete answer. + +No production settings, logging configuration, service state, deployment, or +saved attempt was changed, and no live retry was submitted. Live acceptance +requires a separately authorized deployment with verified serving-build identity +and an explicit new checking attempt; a mutable image tag is not that proof. + +Related background: +[Native audit reliability and truthful coverage](NATIVE_AUDIT_RELIABILITY_FIX.md) +and [Native audit answer delivery](NATIVE_AUDIT_ANSWER_DELIVERY_FIX.md). diff --git a/docs/explanation/fixes/NATIVE_AUDIT_RELIABILITY_FIX.md b/docs/explanation/fixes/NATIVE_AUDIT_RELIABILITY_FIX.md index 0e20e4c6e..e720e132e 100644 --- a/docs/explanation/fixes/NATIVE_AUDIT_RELIABILITY_FIX.md +++ b/docs/explanation/fixes/NATIVE_AUDIT_RELIABILITY_FIX.md @@ -6,6 +6,12 @@ Dependencies: existing native readers, authorized retained results, configured orchestration models, and private checking checkpoints. No dependency, service, setting, route, browser asset, or extraction-schema change is introduced. +**Multi-source follow-up (0.261.428):** Native-audit answers can also cite other +authorized named-input documents without treating them as native fact evidence. +See the [multi-source citation delivery fix](NATIVE_AUDIT_MULTI_SOURCE_CITATION_FIX.md) +for the v4 checking contract. The v3 contract below describes the original +0.261.426 change. + ## Issue and root cause A successful native read could still be followed by an answer-format failure or diff --git a/functional_tests/test_native_audit_answer_delivery.py b/functional_tests/test_native_audit_answer_delivery.py index 65fc89779..b1a783c60 100644 --- a/functional_tests/test_native_audit_answer_delivery.py +++ b/functional_tests/test_native_audit_answer_delivery.py @@ -1,8 +1,9 @@ # test_native_audit_answer_delivery.py """Selected-file native audits through Auto routing, checked correction and delivery. -Version: 0.261.426 +Version: 0.261.428 Implemented in: 0.261.425 +Multi-source citation authority added in: 0.261.428 Only external storage/configuration and provider replies are isolated. Real ownership, native readers, routing, retention, checking and publication execute. """ @@ -26,6 +27,7 @@ ) from test_orchestration_document_action_end_to_end import LEFT, TURN, action_proposal, frames from test_native_inspection_request_contract import change_cursor +from test_orchestration_source_kind_planning import RIGHT from test_support.versioning import assert_app_version_at_least @@ -77,7 +79,7 @@ def test_native_audit_delivery_version(): - assert_app_version_at_least("0.261.425") + assert_app_version_at_least("0.261.428") @pytest.fixture @@ -102,7 +104,7 @@ def delivery(native_http, native_source, inventory_store, monkeypatch, name): def download(container, blob_path, current): downloads.append(current["id"]) - return data + return harness.blobs.records[(container, blob_path)]["data"] monkeypatch.setattr(importlib.import_module("content_screening.access"), "_download_available_blob", download) if path.suffix == ".xlsx": @@ -123,7 +125,8 @@ def download(container, blob_path, current): state=state, runtime=runtime, harness=harness, path=path, name=name, request=PROMPTS[path.suffix], downloads=downloads, contract=None, executions=[], candidate_payloads=[], original_fact_digests=[], execution_frames=[], raw_facts={}, retained_facts={}, - read_chunks=False, + read_chunks=False, chunk_document_id=LEFT, selected_document_ids=(LEFT,), + supporting_source=None, supporting_citation=None, ) prepare = harness.execution.prepare_harness_execution @@ -191,7 +194,7 @@ async def native_reply(self, history, settings): id="native-supporting-chunk", name="document_search-retrieve_document_chunks", arguments=json.dumps({ - "document_id": LEFT, "doc_scope": "personal", + "document_id": result.chunk_document_id, "doc_scope": "personal", "window_number": 1, "window_size": 1, "window_unit": "pages", }), )])] @@ -341,7 +344,7 @@ def assessment_reply(delivery, *, status="satisfied"): def submit(delivery, replies): - proposal = action_proposal((LEFT,), action=delivery.state.action) + proposal = action_proposal(delivery.selected_document_ids, action=delivery.state.action) proposal["steps"][0]["arguments"]["task"] = delivery.request proposal["steps"][1]["arguments"]["instruction"] = delivery.request delivery.harness.replies = [ @@ -354,7 +357,7 @@ def submit(delivery, replies): response = delivery.runtime.client.post("/api/v2/orchestration/plan", json={ "conversation_id": "conversation-1", "turn_id": TURN, "message": delivery.request, "planning_submission_id": "native-answer-delivery", "approval_mode": "auto", - "model_routing": "auto", "selected_document_ids": [LEFT], + "model_routing": "auto", "selected_document_ids": list(delivery.selected_document_ids), }, buffered=True) events = frames(response) plans = [event["plan"] for event in events if event.get("type") == "orchestration_plan"] @@ -443,20 +446,43 @@ def test_native_audit_correction_can_change_explanation_without_changing_facts(d assert delivery.harness.replies == [] -@pytest.mark.parametrize("name", ["SC-C05.pdf"]) -def test_native_audit_with_real_supporting_chunk_citation_reaches_saved_answer(delivery, name, monkeypatch): +def configure_supporting_chunks(delivery, monkeypatch, supporting_document_id=RIGHT): delivery.read_chunks = True - supporting = [] + delivery.chunk_document_id = supporting_document_id search_calls = [] document_owner = importlib.import_module("functions_documents") monkeypatch.setattr( document_owner, "cosmos_user_documents_container", delivery.harness.config.cosmos_user_documents_container, ) + supporting_name = delivery.name + supporting_text = "Cedar delivery review. Fictional pilot: two sites, one release." + if supporting_document_id != LEFT: + delivery.selected_document_ids = (LEFT, supporting_document_id) + delivery.request = ( + "Check the selected document's title, author, language, reading order and unusually small text. " + "Use the selected closure evidence to separate engineering questions from presentation fixes, " + "state review limits, and do not modify the source deck." + ) + supporting_name = "closure-evidence.md" + supporting_text = "Synthetic closure register. The shift-lead signature is not recorded." + document = delivery.harness.config.cosmos_user_documents_container.read_item( + supporting_document_id, supporting_document_id, + ) + document.update({ + "file_name": supporting_name, "blob_path": "synthetic-retained-source/" + supporting_name, + "source_file_available": True, "enhanced_citations": False, + }) + delivery.harness.config.cosmos_user_documents_container.upsert_item(document) + delivery.harness.blobs.records[(document["blob_container"], document["blob_path"])] = { + "data": supporting_text.encode(), "metadata": {"document_id": supporting_document_id}, + "etag": "supporting-source-etag", + } chunk = { - "id": f"{LEFT}_1", "chunk_id": f"{LEFT}_1", "document_id": LEFT, - "file_name": name, "user_id": "owner", "version": 1, "page_number": 1, - "chunk_sequence": 1, "chunk_text": "Cedar delivery review. Fictional pilot: two sites, one release.", + "id": f"{supporting_document_id}_1", "chunk_id": f"{supporting_document_id}_1", + "document_id": supporting_document_id, + "file_name": supporting_name, "user_id": "owner", "version": 1, "page_number": 1, + "chunk_sequence": 1, "chunk_text": supporting_text, } def search(**kwargs): @@ -466,30 +492,161 @@ def search(**kwargs): monkeypatch.setattr(document_owner, "CLIENTS", { **document_owner.CLIENTS, "search_client_user": SimpleNamespace(search=search), }) + delivery.supporting_source = SimpleNamespace( + name=supporting_name, text=supporting_text, search_calls=search_calls, + ) + return delivery.supporting_source - def answer_with_chunk(): - response = json.loads(prepared_reply(delivery)) - payload = current_payload(delivery) - citation = next( - citation for entry in payload["inputs"].values() - for citation in entry["document_citations"] if citation.get("chunk_id") - ) - supporting.append(citation) - response["answer"] += f'\nSupporting page text. {citation["citation"]}' + +def supporting_reply(delivery, *, missing=False, unknown_citation=False): + response = json.loads(prepared_reply(delivery)) + payload = current_payload(delivery) + inputs = payload.get("inputs") or payload["evidence"] + citation = next( + citation for entry in inputs.values() if isinstance(entry, dict) + for citation in entry.get("document_citations", []) + if citation.get("chunk_id") and citation["document_id"] == delivery.chunk_document_id + ) + delivery.supporting_citation = citation + assert delivery.supporting_source.text in json.dumps(inputs) + if missing: + response["answer"] += "\nThe supporting file contents are unavailable." return json.dumps(response) + response["answer"] += f'\nSupporting page text. {citation["citation"]}' + if delivery.chunk_document_id != LEFT: + language = [ + fact for fact in delivery.contract["facts"] if fact["field"] in ("language", "lang") + ] + response["answer"] += "\n### Language\n" + ( + json.dumps(language, ensure_ascii=True) if language else + "The returned native fact set does not establish a document-level language." + ) + response["answer"] += ( + "\n## Review checklist\n### Engineering questions\n" + f'- Confirm the missing shift-lead signature. {citation["citation"]}\n' + "### Presentation fixes\n" + "Review the returned reading-order and exact text-size records; native structure " + "does not establish rendered reading order or legibility.\n" + "### Limits of review\n" + "No rendered review, language detection, or accessibility certification was performed. " + "The source deck was not modified." + ) + if unknown_citation: + response["answer"] += f"\nPRIVATE_UNAUTHORIZED_CITATION [#unselected-document_{unknown_citation}]" + return json.dumps(response) - saved, _ = submit(delivery, [answer_with_chunk, lambda: assessment_reply(delivery)]) + +@pytest.mark.parametrize("name", ["SC-C05.pdf", "SC-C10-R3.pptx"]) +@pytest.mark.parametrize("supporting_document_id", [LEFT, RIGHT]) +def test_native_audit_with_real_supporting_chunk_citation_reaches_saved_answer( + delivery, name, monkeypatch, supporting_document_id, +): + source = configure_supporting_chunks(delivery, monkeypatch, supporting_document_id) + saved, _ = submit(delivery, [lambda: supporting_reply(delivery), lambda: assessment_reply(delivery)]) answers = delivery.harness.assistant_messages() + citation = delivery.supporting_citation assert saved["status"] == "completed", (saved.get("failure"), delivery.state.failures) - assert len(supporting) == 1 and supporting[0]["document_id"] == LEFT - assert supporting[0]["citation"] in answers[0]["content"] - assert supporting[0]["citation"] in delivery.contract["answer_citations"] - assert search_calls and all(call["filter"] == f"document_id eq '{LEFT}'" for call in search_calls) + assert citation["document_id"] == supporting_document_id + assert citation["citation"] in answers[0]["content"] + assert citation["citation"] in delivery.contract["answer_citations"] + assert source.search_calls and all( + call["filter"] == f"document_id eq '{supporting_document_id}'" for call in source.search_calls + ) + assert saved["plan"]["steps"][0]["arguments"]["document_ids"] == list(delivery.selected_document_ids) + if supporting_document_id != LEFT: + text = answers[0]["content"] + assert all(section in text for section in ( + "### Native document properties", "### Engineering questions", + "### Presentation fixes", "### Limits of review", + )) + assert "shift-lead signature" in text and "not modified" in text + assert "The request was only partially completed" not in text + assert citation["citation"] not in delivery.contract["citations"] + assert delivery.contract["files"] == [name] + assert all(source.name not in fact["location"] for fact in delivery.contract["facts"]) functions = {citation["function_name"] for citation in answers[0]["agent_citations"]} assert {"get_document_accessibility_report", "retrieve_document_chunks"} <= functions assert delivery.harness.replies == [] +@pytest.mark.parametrize("name", ["SC-C10-R3.pptx"]) +def test_multi_source_native_correction_restores_available_evidence_without_losing_its_citation( + delivery, name, monkeypatch, +): + source = configure_supporting_chunks(delivery, monkeypatch) + + def reject_false_missing_source(): + payload = current_payload(delivery) + report = json.loads(assessment_reply(delivery)) + assert source.text in json.dumps(payload["evidence"]["native_findings"]) + assert "The supporting file contents are unavailable." in payload["candidate"] + for identifier in ("supported_claims", "information_limits"): + report["requirements"][identifier].update( + status="unsupported", answer_quote="", + reason="The supporting content was returned; do not claim it is unavailable.", + evidence_refs=["native_findings"], + ) + return json.dumps(report) + + saved, _ = submit(delivery, [ + lambda: supporting_reply(delivery, missing=True), reject_false_missing_source, + lambda: supporting_reply(delivery), lambda: assessment_reply(delivery), + ]) + assert saved["status"] == "completed", (saved.get("failure"), delivery.state.failures) + steps = delivery.harness.run_store.list_run_steps(saved["id"], "owner", "conversation-1") + assert steps[1]["task_completeness"]["corrections"] == 1 + assert steps[1]["task_completeness"]["deterministic_status"] == "passed" + text = delivery.harness.assistant_messages()[0]["content"] + assert delivery.supporting_citation["citation"] in text + assert "Confirm the missing shift-lead signature." in text + assert "The supporting file contents are unavailable." not in text + assert text.count("## Native file evidence") == 1 + assert delivery.harness.replies == [] + + +@pytest.mark.parametrize("name", ["SC-C10-R3.pptx"]) +def test_multi_source_native_answer_still_rejects_an_unselected_citation(delivery, name, monkeypatch): + configure_supporting_chunks(delivery, monkeypatch) + sequence = [] + + def invalid(): + sequence.append(len(sequence) + 1) + return supporting_reply(delivery, unknown_citation=sequence[-1]) + + saved, _ = submit(delivery, [invalid, invalid, invalid]) + assert saved["status"] == "failed" + assert saved["failure"]["code"] == "completeness_incomplete" + steps = delivery.harness.run_store.list_run_steps(saved["id"], "owner", "conversation-1") + assert steps[1]["task_completeness"]["corrections"] == 2 + assert steps[1]["task_completeness"]["deterministic_status"] == "failed" + text = delivery.harness.assistant_messages()[0]["content"] + assert "PRIVATE_UNAUTHORIZED_CITATION" not in text + assert "[#unselected-document_1]" not in text + assert delivery.harness.replies == [] + + +@pytest.mark.parametrize("name", ["SC-C10-R3.pptx"]) +def test_multi_source_native_answer_rechecks_retained_action_access_after_preparation( + delivery, name, monkeypatch, +): + configure_supporting_chunks(delivery, monkeypatch) + + def revoke_action(): + response = supporting_reply(delivery) + action = delivery.state.actions.read_item("native-action", "owner") + action["is_enabled"] = False + delivery.state.actions.upsert_item(action) + return response + + saved, _ = submit(delivery, [revoke_action]) + assert saved["status"] == "failed" + assert saved["failure"]["code"] == "result_unavailable" + text = delivery.harness.assistant_messages()[0]["content"] + assert "Confirm the missing shift-lead signature." not in text + assert delivery.supporting_citation["citation"] not in text + assert delivery.harness.replies == [] + + @pytest.mark.parametrize("name", ["SC-C05.pdf"]) @pytest.mark.parametrize(("problem", "code"), [ ("invalid_json", "native_answer_json_invalid"), @@ -628,13 +785,17 @@ def independent_review(): @pytest.mark.parametrize("name", ["SC-C04.pdf"]) -@pytest.mark.parametrize("legacy_checkpoint", [False, True, "native-audit-answer-v2", "invalid_preparation"]) +@pytest.mark.parametrize("legacy_checkpoint", [ + False, True, "native-audit-answer-v2", "native-audit-answer-v3", "invalid_preparation", +]) def test_explicit_native_check_retry_reuses_gathered_source_without_repeating_reads(delivery, monkeypatch, name, legacy_checkpoint): + source = configure_supporting_chunks(delivery, monkeypatch) if legacy_checkpoint == "native-audit-answer-v3" else None + answer = lambda: supporting_reply(delivery) if source else prepared_reply(delivery) invalid_preparation = legacy_checkpoint == "invalid_preparation" original, _ = submit(delivery, [ "Invalid native envelope 1", "Invalid native envelope 2", "Invalid native envelope 3", ] if invalid_preparation else [ - lambda: prepared_reply(delivery), '{"requirements":[]}', '{"requirements":[]}', + answer, '{"requirements":[]}', '{"requirements":[]}', ]) assert original["status"] == "failed" assert original["failure"]["code"] == ( @@ -654,6 +815,8 @@ def load_legacy(store, step_id, *, binding): state.pop("native_audit_preparation", None) else: state["native_audit_preparation"]["version"] = legacy_checkpoint + if legacy_checkpoint == "native-audit-answer-v3": + state["identity"] = "0" * 64 return state monkeypatch.setattr(store_type, "load_completeness", load_legacy) @@ -667,9 +830,10 @@ def load_legacy(store, step_id, *, binding): child_id = retry_payload["run"]["run_id"] child_record = harness.runs.read_item(child_id, "conversation-1") before_reads, before_calls = len(delivery.downloads), len(harness.model_calls) + before_searches = len(source.search_calls) if source else 0 harness.run_id = child_id harness.replies = [ - *([lambda: prepared_reply(delivery)] if legacy_checkpoint else []), + *([answer] if legacy_checkpoint else []), lambda: assessment_reply(delivery), ] response = delivery.runtime.client.post("/api/v2/orchestration/run", json={ @@ -682,6 +846,10 @@ def load_legacy(store, step_id, *, binding): assert saved["status"] == "completed", (saved.get("failure"), delivery.state.failures, events[-2:]) assert len(delivery.downloads) == before_reads assert len(harness.model_calls) == before_calls + (2 if legacy_checkpoint else 1) + if source: + assert len(source.search_calls) == before_searches + text = harness.assistant_messages()[-1]["content"] + assert delivery.supporting_citation["citation"] in text assert harness.replies == [] diff --git a/functional_tests/test_native_audit_reliability.py b/functional_tests/test_native_audit_reliability.py index bfb63ce81..b13ba11fa 100644 --- a/functional_tests/test_native_audit_reliability.py +++ b/functional_tests/test_native_audit_reliability.py @@ -1,8 +1,9 @@ # test_native_audit_reliability.py """Functional regressions for native-answer contracts and canonical field states. -Version: 0.261.426 +Version: 0.261.428 Implemented in: 0.261.426 +Multi-source citation authority added in: 0.261.428 These bootstrap-free cases exercise real contract/projection code. The companion answer-delivery suite covers authorized readers, Auto routing, retention and HTTP. """ @@ -113,7 +114,7 @@ def preparation(contract, answer="Supported findings and precise inspection limi def test_native_audit_reliability_version(): - assert_app_version_at_least("0.261.426") + assert_app_version_at_least("0.261.428") def test_native_and_retained_chunk_owner_citations_are_accepted_without_changing_fact_authority(): @@ -129,6 +130,66 @@ def test_native_and_retained_chunk_owner_citations_are_accepted_without_changing assert rendered.count("## Native file evidence") == 1 +@pytest.mark.parametrize("separate_input", [False, True]) +def test_authorized_supporting_document_citations_do_not_become_native_fact_authority(separate_input): + inputs = native_inputs() + supporting = { + "document_id": "selected-supporting-document", "version": 2, + "citation": build_inline_citation_marker("supporting.md", "Page", "1", "selected-supporting-document_1"), + } + entry = inputs["inspection"] + if separate_input: + entry = { + "kind": "structured-v1", + "value": {"capability_id": "action_invoke", "citations": []}, + "document_citations": [], + } + inputs["supporting"] = entry + entry["document_citations"].append(supporting) + entry["value"]["citations"].append({ + "function_name": "retrieve_document_chunks", + "function_result": { + "document": {"id": supporting["document_id"], "version": 2}, + "chunks": [{"chunk_text": "The signature has not been recorded."}], + }, + }) + original = deepcopy(inputs) + native_only = native_audit_contract(inputs) + contract = native_audit_contract(inputs, authorized_answer_citations=[supporting]) + response = preparation(contract, f"Confirm the missing signature. {supporting['citation']}") + decoded, rendered = decode_native_audit_answer(json.dumps(response), contract, require_answer=True) + assert inputs == original and decoded == response + assert supporting["citation"] not in native_only["answer_citations"] + assert supporting["citation"] in contract["answer_citations"] + assert supporting["citation"] not in contract["citations"] + assert contract["facts"] == native_only["facts"] + assert contract["fact_set_sha256"] == native_only["fact_set_sha256"] + assert contract["files"] == native_only["files"] == ["control.xlsx"] + assert rendered.count(supporting["citation"]) == 1 + + +@pytest.mark.parametrize("citations", [None, {}, ["[#source]"], [ + {"document_id": None, "citation": "[#source]"}, +], [ + {"document_id": "source", "citation": ""}, +]]) +def test_authorized_answer_citation_registry_requires_owner_projection_shape(citations): + with pytest.raises(ValueError, match="authorized answer citation contract is invalid"): + native_audit_contract(native_inputs(), authorized_answer_citations=citations) + + +def test_authorized_supporting_citation_label_cannot_grant_an_embedded_marker(): + supporting = { + "document_id": "selected-supporting-document", + "citation": "(Source: name [#filename-spoof].md, Page: 1) [#selected-supporting-document_1]", + } + contract = native_audit_contract(native_inputs(), authorized_answer_citations=[supporting]) + response = preparation(contract, "A forged reference. [#filename-spoof]") + with pytest.raises(NativeAuditAnswerError) as caught: + decode_native_audit_answer(json.dumps(response), contract, require_answer=True) + assert caught.value.code == "native_answer_citation_unknown" + + @pytest.mark.parametrize("marker", [ "[#forged-in-source]", "[#forged-in-action-notes]", "[#unselected-document_1]", ]) @@ -156,11 +217,15 @@ def test_marker_literal_in_source_filename_does_not_grant_an_extra_citation(): assert caught.value.code == "native_answer_citation_unknown" -def test_supporting_chunk_from_a_different_source_version_is_rejected(): +@pytest.mark.parametrize("authorized_registry", [False, True]) +def test_supporting_chunk_from_a_different_source_version_is_rejected(authorized_registry): inputs = native_inputs() inputs["inspection"]["document_citations"][0]["version"] = 2 + citations = inputs["inspection"]["document_citations"] if authorized_registry else () + if authorized_registry: + inputs["inspection"]["document_citations"] = [] with pytest.raises(ValueError, match="different native source version"): - native_audit_contract(inputs) + native_audit_contract(inputs, authorized_answer_citations=citations) def test_native_pages_from_different_versions_cannot_share_a_fact_contract():