From e7667ccd1a2466758d7f0d99f8589b2146c9b715 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Mon, 9 Nov 2026 09:58:53 -0500 Subject: [PATCH] Keep chat analysis details in Sources Move saved analysis browsing out of classic and V2 answers while preserving evidence, follow-up actions, and genuine limitations. Use retained native coverage to distinguish complete empty comment inventories from missing evidence. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_orchestration_result_runtime.py | 55 ++++- .../static/js/chat/chat-analysis-results.js | 84 +++++++- .../static/js/chat/chat-messages.js | 31 ++- .../src/components/chat/AnalysisResult.tsx | 16 +- .../src/components/chat/EvidenceStatus.tsx | 23 ++- .../src/components/chat/MessageActions.tsx | 2 + .../src/components/chat/MessageInspector.tsx | 39 +++- .../v2_ui/src/components/chat/MessageList.tsx | 36 ++-- application/v2_ui/src/lib/savedAnalysis.ts | 24 +++ docs/explanation/features/ANALYZE_RESULTS.md | 25 ++- .../V2_MODEL_TOOLS_AND_EVIDENCE_STATUS.md | 17 +- ..._ANALYSIS_PRESENTATION_AND_EVIDENCE_FIX.md | 134 ++++++++++++ docs/guides/analyze-results.md | 63 ++++-- docs/reference/chat-controls.md | 18 +- ...rchestration_attachment_native_evidence.py | 49 ++++- .../test_orchestration_search_citations.py | 73 ++++++- ui_tests/test_chat_saved_analysis.py | 195 +++++++++++++++++- ui_tests/test_v2_evidence_status.py | 42 ++-- 19 files changed, 814 insertions(+), 114 deletions(-) create mode 100644 docs/explanation/fixes/CHAT_ANALYSIS_PRESENTATION_AND_EVIDENCE_FIX.md diff --git a/application/single_app/config.py b/application/single_app/config.py index 4e90e6aef..77f9f7c89 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.423" +VERSION = "0.261.424" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_orchestration_result_runtime.py b/application/single_app/functions_orchestration_result_runtime.py index 2a860bef4..e268320eb 100644 --- a/application/single_app/functions_orchestration_result_runtime.py +++ b/application/single_app/functions_orchestration_result_runtime.py @@ -1,7 +1,7 @@ # functions_orchestration_result_runtime.py """Typed runtime handoffs; no clients, model calls, publication, or new storage. -Version: 0.261.365 +Version: 0.261.424 """ from copy import deepcopy @@ -274,13 +274,39 @@ def read_result_source_coverage(context, reference): return None +def _native_inventory_evidence_state(prepared, document_id): + """An inspected empty inventory is evidence of absence, not missing excerpts.""" + coverage = prepared.get('coverage') + inventories = coverage.get('native_inspection') if type(coverage) is dict else None + if type(inventories) is not list: + return 'unknown' + matches = [ + item for item in inventories + if type(item) is dict and item.get('document_id') == document_id + ] + if len(matches) != 1: + return 'unknown' + inventory = matches[0] + if inventory.get('execution_mode') != 'native_inspection' or inventory.get('section') != 'comments': + return 'unknown' + total, returned = inventory.get('total_records'), inventory.get('returned_records') + if type(total) is not int or type(returned) is not int or not 0 <= returned <= total: + return 'unknown' + flags = ('partial', 'has_more', 'limit_reached') + if returned < total or any(inventory.get(flag) is True for flag in flags): + return 'partial' + if inventory.get('inventory_complete') is True and all(inventory.get(flag) is False for flag in flags): + return 'complete' + return 'unknown' + + def _read_result_document_citations(service, reference, *, evidence_status_out=None): root = service.open_result(reference, allow_partial=True, require_current_sources=True) if root.metadata()['source_count'] == 0: return [] sources, citations, visited, expanded = {}, {}, set(), set() coverage_values = {} - search_counts, source_names = {}, {} + search_counts, source_names, source_outcomes = {}, {}, {} citation_fields = ( 'citation_id', 'chunk_id', 'file_name', 'title', 'page_number', 'chunk_sequence', 'sheet_name', 'location_label', 'location_value', 'classification', 'document_classification', 'score', @@ -327,6 +353,7 @@ def visit(current): visit(ResultRef.from_dict(parent)) retained_citations = [] native_reference = None + producer_counts = {} if ( producer.capability_id == 'document_search' and (current.output_name, current.kind) in ( @@ -360,7 +387,9 @@ def visit(current): if prepared['content_scope'] in ('native_excel_annotations', 'native_document_annotations'): native_reference = prepared_reader.reference.to_dict() for source in lineage['sources']: - search_counts.setdefault(source_key(source), set()) + identity = source_key(source) + search_counts.setdefault(identity, set()) + producer_counts[identity] = set() elif producer.capability_id == CAPABILITY_ACTION_INVOKE and lineage['sources']: prepared = read_complete_input(reader) if ( @@ -419,14 +448,21 @@ def visit(current): producer.capability_id == 'document_search' and identity_key in search_counts and citation.get('evidence_kind') != 'native_annotation_coverage' ): - search_counts[identity_key].add(( + evidence_key = ( producer, current.manifest_sha256, canonical_digest({name: value for name, value in projected.items() if name != 'evidence_id'}), - )) + ) + search_counts[identity_key].add(evidence_key) + producer_counts[identity_key].add(evidence_key) identity = {key: value for key, value in projected.items() if key != 'evidence_id'} citations.setdefault(source_key(matches[0]), {}).setdefault( canonical_digest(identity), projected, ) + for identity, counts in producer_counts.items(): + state = _native_inventory_evidence_state(prepared, identity[2]) if native_reference is not None else ( + 'complete' if counts else 'missing' + ) + source_outcomes.setdefault(identity, {})[key] = state reader.recheck() visit(reference) @@ -440,15 +476,20 @@ def visit(current): rows = [] for index, (identity, source) in enumerate(sources.items(), start=1): count = len(search_counts[identity]) if identity in search_counts else None + outcome = evidence_status_from_sources([ + {'state': state} for state in source_outcomes.get(identity, {}).values() + ]) row = { "name": str(source_names.get(identity) or source.get("file_name") or f"Selected source {index}")[:160], "document_id": source["document_id"], - "state": "missing" if count == 0 else "complete" if count is not None else "unknown", + "state": outcome["state"] if outcome is not None else "unknown", } if count is not None: row["evidence_count"] = count rows.append(row) - evidence_status_out["value"] = evidence_status_from_sources(rows) + evidence_status_out["value"] = evidence_status_from_sources( + rows, partial=root.completeness.status == 'partial', + ) return projected diff --git a/application/single_app/static/js/chat/chat-analysis-results.js b/application/single_app/static/js/chat/chat-analysis-results.js index 5996caa98..3f37db6bd 100644 --- a/application/single_app/static/js/chat/chat-analysis-results.js +++ b/application/single_app/static/js/chat/chat-analysis-results.js @@ -1,5 +1,5 @@ // chat-analysis-results.js -// Saved Analyze findings supplement the answer; only an authorized complete-record page is read. +// Saved Analyze findings live in Sources; only an authorized complete-record page is read. const pageSize = 25; const validationNotices = { @@ -96,6 +96,56 @@ export function readSavedAnalyses(metadata) { return results; } +function savedAnalysisLimitation(descriptors, unavailable) { + if (unavailable || descriptors.some(descriptor => !descriptor || descriptor.available === false)) { + return 'Saved analysis unavailable'; + } + const labels = { + invalid: 'Analysis validation failed', + partial: 'Partial analysis', + pending: 'Analysis validation pending', + not_validated: 'Analysis not validated', + }; + for (const status of Object.keys(labels)) { + if (descriptors.some(descriptor => descriptor?.validation_status === status)) { + return labels[status]; + } + } + return null; +} + +function renderAnalysisWarning(message) { + const label = savedAnalysisLimitation(message._savedAnalysisDescriptors, + message.dataset.savedAnalysisUnavailable === 'true'); + let warning = message._savedAnalysisWarning; + if (!label) { + warning?.remove(); + message._savedAnalysisWarning = null; + return; + } + if (!warning) { + warning = node('div', 'saved-analysis-warning mt-2'); + warning.setAttribute('role', 'status'); + const button = node('button', 'btn btn-sm alert alert-warning py-1 px-2 mb-0 text-start'); + button.type = 'button'; + button.addEventListener('click', () => { + const toggle = message._savedAnalysisSources.parentElement.querySelector( + ':scope > .message-footer .citation-toggle-btn', + ); + if (toggle?.getAttribute('aria-expanded') !== 'true') { + toggle?.click(); + } + }); + warning.append(button); + message._savedAnalysisSources.parentElement.querySelector(':scope > .message-text')?.after(warning); + message._savedAnalysisWarning = warning; + } + const button = warning.querySelector('button'); + button.textContent = `${label}. View sources`; + button.setAttribute('aria-controls', message._savedAnalysisSources.id); + warning.classList.toggle('d-none', message.dataset.savedAnalysisMasked === 'true'); +} + export function syncSavedAnalysisConversation(conversationId = activeConversationId()) { const next = String(conversationId || '').trim() || null; if (contextConversationId !== next) { @@ -304,8 +354,9 @@ export function disposeSavedAnalysisViews() { function hideUnavailableActions(message) { message.dataset.savedAnalysisUnavailable = 'true'; - message.querySelectorAll('.saved-analysis-downloads, .generated-tabular-outputs-container, .assistant-inline-export-actions, .message-footer') + message.querySelectorAll('.saved-analysis-downloads, .generated-tabular-outputs-container, .assistant-inline-export-actions, .message-footer .message-actions') .forEach(element => element.classList.add('d-none')); + renderAnalysisWarning(message); } export function hydrateSavedAnalysisResult(message, fullMessage) { @@ -315,18 +366,32 @@ export function hydrateSavedAnalysisResult(message, fullMessage) { } message.dataset.savedAnalysis = 'true'; message._savedAnalysisDescriptors = descriptors; - let anchor = message.querySelector('.message-text'); + const sources = message.querySelector(':scope > .message-content > .message-bubble > .message-sources'); + message._savedAnalysisSources = sources; for (const descriptor of descriptors) { - anchor = hydrateAnalysisResult(message, descriptor, anchor); + hydrateAnalysisResult(message, descriptor, sources); } + renderAnalysisWarning(message); + message.addEventListener('chat:sources-toggled', event => { + if (!event.detail.open) { + views.forEach(view => { + if (message.contains(view.root)) { + view.dispose(); + } + }); + sources.querySelectorAll('.saved-analysis-result details[open]').forEach(details => { + details.open = false; + }); + } + }); } -function hydrateAnalysisResult(message, descriptor, anchor) { - const root = node('section', 'saved-analysis-result border rounded p-3 mt-3'); +function hydrateAnalysisResult(message, descriptor, sources) { + const root = node('section', 'saved-analysis-result border-top pt-3 mt-3'); root.setAttribute('aria-label', descriptor?.label ? `Saved analysis: ${descriptor.label}` : 'Saved analysis'); const heading = node('h3', 'h6 mb-1 text-break', descriptor?.label || 'Saved analysis'); root.append(heading); - anchor?.after(root); + sources.append(root); const controllers = new Set(); const view = { root, dispose: () => { controllers.forEach(controller => controller.abort()); @@ -380,7 +445,7 @@ function hydrateAnalysisResult(message, descriptor, anchor) { } }; const status = node('p', 'small text-muted mb-1', - `0 of ${descriptor.record_count} records displayed · ${descriptor.source_count} sources in the saved analysis.`); + `${descriptor.record_count} saved findings · ${descriptor.source_count} sources.`); status.setAttribute('role', 'status'); status.setAttribute('aria-live', 'polite'); const validationNotice = status => descriptor.semantics?.mode === 'source_observations' @@ -544,7 +609,7 @@ function hydrateAnalysisResult(message, descriptor, anchor) { } else { view.dispose(); content.replaceChildren(); - status.textContent = `0 of ${descriptor.record_count} records displayed · ${descriptor.source_count} sources in the saved analysis.`; + status.textContent = `${descriptor.record_count} saved findings · ${descriptor.source_count} sources.`; } }); retry.addEventListener('click', () => void loadPage()); @@ -582,6 +647,7 @@ export function setSavedAnalysisMasked(message, metadata) { element.classList.toggle('d-none', masked || (element.classList.contains('saved-analysis-downloads') && message.dataset.savedAnalysisUnavailable === 'true')); }); + message._savedAnalysisWarning?.classList.toggle('d-none', masked); if (masked && message._savedAnalysisDescriptors.some(descriptor => sameAnalysis(selectedContext, descriptor))) { clearSavedAnalysisContext(); } diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index a9bfaa142..84858d06f 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -6144,13 +6144,11 @@ export function appendMessage( `Condition check (Boolean(citationsButtonsHtml) && hasAnyCitations): ${shouldShowCitations}` ); - if (shouldShowCitations) { + const hasSavedAnalysis = readSavedAnalyses(fullMessageObject?.metadata).length > 0; + if (shouldShowCitations || hasSavedAnalysis) { console.log(">>> Will generate and include citation elements."); - const citationsContainerId = `citations-${messageId || Date.now()}`; - citationToggleHtml = ``; - // citationsButtonsHtml already contains a
wrapper - // Just add ID and display style by wrapping minimally - citationContentContainerHtml = ``; + citationToggleHtml = ''; + citationContentContainerHtml = '
'; } else { console.log(">>> Will NOT generate citation elements."); } @@ -6185,6 +6183,16 @@ export function appendMessage(
`; + const sourcesContainer = messageDiv.querySelector(':scope > .message-content > .message-bubble > .message-sources'); + const sourcesToggle = messageDiv.querySelector( + ':scope > .message-content > .message-bubble > .message-footer .citation-toggle-btn', + ); + if (sourcesContainer) { + sourcesContainer.id = `citations-${messageId || Date.now()}`; + sourcesContainer.innerHTML = sanitizeHtmlSafely(citationsButtonsHtml); + sourcesToggle.setAttribute('aria-controls', sourcesContainer.id); + } + messageDiv.dataset.replySenderName = stripHtmlTags(senderLabel).replace(/\s+/g, " ").trim() || "AI"; messageDiv.dataset.replyPreviewText = buildPlainTextPreview(renderedAiContent.previewMarkdown); @@ -6381,24 +6389,25 @@ export function appendMessage( showToast("Failed to copy text.", "warning"); }); }); - const toggleBtn = messageDiv.querySelector(".citation-toggle-btn"); + const toggleBtn = sourcesToggle; if (toggleBtn) { toggleBtn.addEventListener("click", () => { /* ... toggle logic ... */ - const targetId = toggleBtn.getAttribute("aria-controls"); - const citationsContainer = messageDiv.querySelector(`#${targetId}`); + const citationsContainer = sourcesContainer; if (!citationsContainer) return; // Store current scroll position to maintain user's view const currentScrollTop = document.getElementById('chat-messages-container')?.scrollTop || window.pageYOffset; - const isExpanded = citationsContainer.style.display !== "none"; - citationsContainer.style.display = isExpanded ? "none" : "block"; + const isExpanded = !citationsContainer.classList.contains('d-none'); + citationsContainer.classList.toggle('d-none', isExpanded); toggleBtn.setAttribute("aria-expanded", !isExpanded); toggleBtn.title = isExpanded ? "Show sources" : "Hide sources"; + toggleBtn.setAttribute("aria-label", toggleBtn.title); toggleBtn.innerHTML = isExpanded ? '' : ''; + messageDiv.dispatchEvent(new CustomEvent('chat:sources-toggled', { detail: { open: !isExpanded } })); // Note: Removed scrollChatToBottom() to prevent jumping when expanding citations // Restore scroll position after DOM changes diff --git a/application/v2_ui/src/components/chat/AnalysisResult.tsx b/application/v2_ui/src/components/chat/AnalysisResult.tsx index 7c876800f..5ae7ea84d 100644 --- a/application/v2_ui/src/components/chat/AnalysisResult.tsx +++ b/application/v2_ui/src/components/chat/AnalysisResult.tsx @@ -1,5 +1,5 @@ // AnalysisResult.tsx -// Lazy, complete-record pages supporting the readable answer above them. +// Lazy, complete-record pages in a message's Sources disclosure. import { useCallback, useEffect, useState } from 'react'; import { fetchAnalysisEvidence, fetchAnalysisRecords } from '../../lib/endpoints'; @@ -99,9 +99,11 @@ function Evidence({ export function AnalysisResult({ descriptor, onUnavailable, + knownUnavailable, }: { descriptor: SavedAnalysisDescriptor; - onUnavailable: (descriptor: SavedAnalysisDescriptor) => void; + onUnavailable?: (descriptor: SavedAnalysisDescriptor, message: string) => void; + knownUnavailable?: string; }) { const [open, setOpen] = useState(false); const [offset, setOffset] = useState(0); @@ -113,12 +115,12 @@ export function AnalysisResult({ const activeConversationId = useChatStore((state) => state.activeConversationId); const selectAnalysisResult = useChatStore((state) => state.selectAnalysisResult); const belongsHere = descriptor.conversation_id === activeConversationId; - const blocked = descriptor.available === false || !belongsHere || Boolean(unavailable); + const blocked = descriptor.available === false || !belongsHere || Boolean(unavailable || knownUnavailable); const reportUnavailable = useCallback((message: string) => { setUnavailable(message); setPage(null); - onUnavailable(descriptor); + onUnavailable?.(descriptor, message); const store = useChatStore.getState(); if (sameAnalysis(store.analysisResultContext, descriptor)) { store.clearAnalysisResultContext(); @@ -158,11 +160,11 @@ export function AnalysisResult({ return (
-

{descriptor.label || 'Saved analysis'}

+ className="min-w-0 border-b border-edge pb-4"> +

{descriptor.label || 'Saved analysis'}

{blocked ? (

- {unavailable || analysisUnavailableMessage()} + {unavailable || knownUnavailable || analysisUnavailableMessage()}

) : ( <> diff --git a/application/v2_ui/src/components/chat/EvidenceStatus.tsx b/application/v2_ui/src/components/chat/EvidenceStatus.tsx index ee57e5141..665970954 100644 --- a/application/v2_ui/src/components/chat/EvidenceStatus.tsx +++ b/application/v2_ui/src/components/chat/EvidenceStatus.tsx @@ -14,20 +14,23 @@ export function EvidenceSourceBadge({ source }: { source: EvidenceSource }) { ); } -export function EvidenceStatusFooter({ status, expanded, onOpen, controlsId }: { +export function EvidenceStatusFooter({ status, analysisNotice, expanded, onOpen, controlsId }: { status: EvidenceStatus | null; expanded: boolean; onOpen: () => void; controlsId?: string; + analysisNotice?: string | null; }) { - if (!status || status.state === 'complete' || status.state === 'unknown') return null; + const state = status && status.state !== 'complete' && status.state !== 'unknown' ? status.state : null; + const label = state ? EVIDENCE_LABELS[state] : analysisNotice; + if (!label) return null; return (
); @@ -47,7 +50,11 @@ export function EvidenceStatusDetails({ status }: { status: EvidenceStatus | nul {source.name} - {source.evidence_count !== undefined && {source.evidence_count} evidence item{source.evidence_count === 1 ? '' : 's'} returned} + {source.evidence_count !== undefined && + {source.state === 'complete' && source.evidence_count === 0 + ? 'Inspection completed with no matching records' + : `${source.evidence_count} evidence item${source.evidence_count === 1 ? '' : 's'} returned`} + } ))} diff --git a/application/v2_ui/src/components/chat/MessageActions.tsx b/application/v2_ui/src/components/chat/MessageActions.tsx index 90ed03459..b590d4f73 100644 --- a/application/v2_ui/src/components/chat/MessageActions.tsx +++ b/application/v2_ui/src/components/chat/MessageActions.tsx @@ -668,6 +668,8 @@ export function MessageActions({ : 'Show sources' } active={inspector === 'sources'} + expanded={inspector === 'sources'} + controls={`message-inspector-${message.id}`} onClick={() => inspect('sources')} > diff --git a/application/v2_ui/src/components/chat/MessageInspector.tsx b/application/v2_ui/src/components/chat/MessageInspector.tsx index 64c0c0718..e27beedfe 100644 --- a/application/v2_ui/src/components/chat/MessageInspector.tsx +++ b/application/v2_ui/src/components/chat/MessageInspector.tsx @@ -4,7 +4,7 @@ // One panel with three sections rather than three separate panels: a user comparing what a // response cited against how it was generated should not have to close one to open another. -import { useEffect, useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { clsx } from 'clsx'; import { Braces, @@ -31,7 +31,9 @@ import { import { normalizeOrchestrationAttempt } from '../../lib/orchestration'; import { openOrchestrationRecovery } from '../../lib/orchestrationController'; import { m365ChipLabel, m365SecondaryLine } from '../../lib/m365Citations'; -import type { ChatMessage, Json, PersistedThought } from '../../lib/types'; +import type { ChatMessage, Json, PersistedThought, SavedAnalysisDescriptor } from '../../lib/types'; +import { analysisUnavailableMessage, readSavedAnalyses } from '../../lib/savedAnalysis'; +import { AnalysisResult } from './AnalysisResult'; import { buildToolResultView, type RowMode } from '../../lib/agentCitationRows'; import { GlassButton } from '../ui/primitives'; import { M365KindIcon, M365OpenLink } from './M365CitationChip'; @@ -170,15 +172,32 @@ function DetailsSection({ message }: { message: ChatMessage }) { ); } -function SourcesSection({ sources, evidence }: { sources: MessageSources; evidence: EvidenceStatus | null }) { - if (sources.total === 0 && !evidence) { +function SourcesSection({ sources, evidence, analyses, onAnalysisUnavailable, unavailableAnalyses }: { + sources: MessageSources; + evidence: EvidenceStatus | null; + analyses: (SavedAnalysisDescriptor | null)[]; + onAnalysisUnavailable?: (descriptor: SavedAnalysisDescriptor, message: string) => void; + unavailableAnalyses?: ReadonlyMap; +}) { + if (sources.total === 0 && !evidence && !analyses.length) { return This response did not cite any sources.; } return (
- {sources.total === 0 && This response did not cite any sources.} + {analyses.map((descriptor, index) => descriptor ? ( + + ) : ( +

+ {analysisUnavailableMessage()} +

+ ))} + {sources.total === 0 && !analyses.length && This response did not cite any sources.} {sources.conversations.length > 0 && (

@@ -454,15 +473,21 @@ export function MessageInspector({ section, onSection, onClose, + onAnalysisUnavailable, + unavailableAnalyses, }: { message: ChatMessage; section: InspectorSection; onSection: (next: InspectorSection) => void; onClose: () => void; + onAnalysisUnavailable?: (descriptor: SavedAnalysisDescriptor, message: string) => void; + unavailableAnalyses?: ReadonlyMap; }) { const sources = readSources(message as unknown as Json); const masks = readMaskState(message); const evidence = masks.hasAnyMask ? null : readEvidenceStatus(message.metadata?.evidence_status); + const analyses = useMemo(() => masks.hasAnyMask ? [] : readSavedAnalyses(message.metadata), + [message.metadata, masks.hasAnyMask]); // An uploaded file cites nothing and has no reasoning. Its sources are what ingestion // extracted from it, which is the material any later citation of the file comes from. const upload = message.role === 'file'; @@ -527,7 +552,9 @@ export function MessageInspector({ This upload has no stored extraction to show. ) )} - {active === 'sources' && !upload && } + {active === 'sources' && !upload && } {active === 'reasoning' && }

diff --git a/application/v2_ui/src/components/chat/MessageList.tsx b/application/v2_ui/src/components/chat/MessageList.tsx index b8713b663..f445d11ad 100644 --- a/application/v2_ui/src/components/chat/MessageList.tsx +++ b/application/v2_ui/src/components/chat/MessageList.tsx @@ -43,7 +43,6 @@ import { import { InlineMapCards } from './InlineMapCard'; import { ChatFilePreview } from './ChatFilePreview'; import { GeneratedArtifactCard } from './GeneratedArtifactCard'; -import { AnalysisResult } from './AnalysisResult'; import type { SavedAnalysisDescriptor } from '../../lib/types'; import { IconButton, MessageActions } from './MessageActions'; import { isCheckingFailureNotice, OrchestrationMessageRecovery } from './OrchestrationRecoveryNotice'; @@ -116,7 +115,7 @@ import { orchestrationStartedWorkflow } from '../../lib/orchestrationWorkflowRun import { orchestrationHandedOffWorkflow } from '../../lib/workflowHandoffs'; import { readWorkflowDelivery } from '../../lib/workflowDelivery'; import { workflowRunTrackerShouldRun } from '../../lib/workflowRunTracker'; -import { analysisUnavailableMessage, readSavedAnalyses, sameAnalysis } from '../../lib/savedAnalysis'; +import { readSavedAnalyses, sameAnalysis, savedAnalysisLimitation } from '../../lib/savedAnalysis'; import { readMessagePrompt } from '../../lib/messagePrompt'; import { PromptCard } from './PromptCard'; import { @@ -916,14 +915,14 @@ function MessageBubbleInner({ const managedOutputs = orchestration.outputs !== undefined || liveOutputs !== undefined; const savedAnalyses = useMemo(() => readSavedAnalyses(message.metadata), [message.metadata]); const hasSavedAnalysis = savedAnalyses.length > 0; - const [unavailableAnalysisKeys, setUnavailableAnalysisKeys] = useState>(() => new Set()); - const markAnalysisUnavailable = useCallback((descriptor: SavedAnalysisDescriptor) => { + const [unavailableAnalyses, setUnavailableAnalyses] = useState>(() => new Map()); + const markAnalysisUnavailable = useCallback((descriptor: SavedAnalysisDescriptor, message: string) => { const key = `${descriptor.conversation_id}:${descriptor.message_id}:${descriptor.result_sha256}`; - setUnavailableAnalysisKeys((keys) => new Set(keys).add(key)); + setUnavailableAnalyses((entries) => new Map(entries).set(key, message)); }, []); const analysisUnavailable = hasSavedAnalysis && savedAnalyses.some((descriptor) => !descriptor || descriptor.available === false || descriptor.conversation_id !== activeConversationId || - unavailableAnalysisKeys.has(`${descriptor.conversation_id}:${descriptor.message_id}:${descriptor.result_sha256}`)); + unavailableAnalyses.has(`${descriptor.conversation_id}:${descriptor.message_id}:${descriptor.result_sha256}`)); const canRepeatPlan = message.role === 'assistant' && !analysisUnavailable && planReplayAvailable && Boolean(orchestration.run_id) && orchestration.outcome === 'completed' && !masks.fullyMasked && masks.ranges.length === 0 && personalConversation @@ -950,10 +949,10 @@ function MessageBubbleInner({ const selected = savedAnalyses.find((descriptor) => sameAnalysis(store.analysisResultContext, descriptor)); if (selected && (masks.fullyMasked || masks.ranges.length > 0 || selected.available === false || selected.conversation_id !== activeConversationId || - unavailableAnalysisKeys.has(`${selected.conversation_id}:${selected.message_id}:${selected.result_sha256}`))) { + unavailableAnalyses.has(`${selected.conversation_id}:${selected.message_id}:${selected.result_sha256}`))) { store.clearAnalysisResultContext(); } - }, [unavailableAnalysisKeys, activeConversationId, masks.fullyMasked, masks.ranges.length, + }, [unavailableAnalyses, activeConversationId, masks.fullyMasked, masks.ranges.length, savedAnalyses, message.id, message.role, message.conversation_id]); const artifactCards = artifacts.filter((artifact) => !managedOutputs || !isOrchestrationOutputArtifact(artifact)).map((artifact, index) => ( @@ -1262,16 +1261,6 @@ function MessageBubbleInner({ ) : null} {/* Inside the bubble, because a generated file belongs to the reply that produced it rather than sitting loose in the thread. */} - {hasSavedAnalysis && (masks.ranges.length > 0 ? ( -

Saved findings are hidden while this message is masked.

- ) : savedAnalyses.map((descriptor, index) => descriptor ? ( - - ) : ( -

- {analysisUnavailableMessage()} -

- )))} {!analysisUnavailable && (!hasSavedAnalysis || masks.ranges.length === 0) && ( hasSavedAnalysis ? artifactCards.length > 0 && (
@@ -1292,6 +1281,7 @@ function MessageBubbleInner({ ) : null} {masks.ranges.length === 0 && inspectMessage('sources')} @@ -1319,7 +1309,13 @@ function MessageBubbleInner({ inspector || replayExpanded ? 'opacity-100' : 'opacity-0', )} > - {!analysisUnavailable && inspectMessage(inspector === 'sources' ? null : 'sources')}> + + + ) : setEditing(true) : undefined} inspector={inspector} @@ -1354,6 +1350,8 @@ function MessageBubbleInner({ section={inspector} onSection={inspectMessage} onClose={() => setInspector(null)} + onAnalysisUnavailable={markAnalysisUnavailable} + unavailableAnalyses={unavailableAnalyses} /> )} diff --git a/application/v2_ui/src/lib/savedAnalysis.ts b/application/v2_ui/src/lib/savedAnalysis.ts index 14f3e5709..18ad8e705 100644 --- a/application/v2_ui/src/lib/savedAnalysis.ts +++ b/application/v2_ui/src/lib/savedAnalysis.ts @@ -108,6 +108,30 @@ export function readSavedAnalyses(metadata: unknown): (SavedAnalysisDescriptor | return results; } +export function savedAnalysisLimitation( + analyses: readonly (SavedAnalysisDescriptor | null)[], + unavailable = false, +): string | null { + if (!analyses.length) { + return null; + } + if (unavailable || analyses.some((descriptor) => !descriptor || descriptor.available === false)) { + return 'Saved analysis unavailable'; + } + const labels = { + invalid: 'Analysis validation failed', + partial: 'Partial analysis', + pending: 'Analysis validation pending', + not_validated: 'Analysis not validated', + } as const; + for (const status of ['invalid', 'partial', 'pending', 'not_validated'] as const) { + if (analyses.some((descriptor) => descriptor?.validation_status === status)) { + return labels[status]; + } + } + return null; +} + export function analysisResultContext(value: AnalysisResultContext): AnalysisResultContext { return { conversation_id: value.conversation_id, diff --git a/docs/explanation/features/ANALYZE_RESULTS.md b/docs/explanation/features/ANALYZE_RESULTS.md index a31fc2336..0cf9548db 100644 --- a/docs/explanation/features/ANALYZE_RESULTS.md +++ b/docs/explanation/features/ANALYZE_RESULTS.md @@ -1,6 +1,6 @@ # Saved Analyze results -**Version: 0.261.232** +**Version: 0.261.424** Implemented in version: **0.261.109**, recorded in `application/single_app/config.py`. @@ -33,6 +33,29 @@ from the narrative default. The user-facing workflow is documented in [Read and discuss saved Analyze results](../../guides/analyze-results.md). +## Answer-first chat presentation + +Implemented in version: **0.261.424**, recorded in +`application/single_app/config.py`. + +Classic chat and React V2 keep the answer and its inline citations primary. +The existing response-level Sources disclosure contains the saved-result +browser: validation details, complete-record pages, evidence, diagnostics, and +selection of a particular result for follow-up. Sources remains available even +when a saved result contains no findings or the answer has no ordinary citations. +The normal chat view no longer appends a result card for every Analyze step. + +Material limitations retain a compact warning that opens Sources. A routine +structural-success disclaimer is not shown under every answer. Downloads remain +supporting outputs in the conversation, and the Plan panel is unchanged. +Closing Sources cancels pending findings/evidence reads; masking and conversation +changes retain their existing context and visibility guards. + +Native comment inventory counts are not saved Analyze finding counts. A +completed native inventory with zero comments remains evidence for that absence, +whereas partial inventories and unavailable reads cannot establish it. See +[chat presentation and evidence correction](../fixes/CHAT_ANALYSIS_PRESENTATION_AND_EVIDENCE_FIX.md). + ## Dependencies and implementation This extends the existing document-analysis runner, workflow result store, source diff --git a/docs/explanation/features/V2_MODEL_TOOLS_AND_EVIDENCE_STATUS.md b/docs/explanation/features/V2_MODEL_TOOLS_AND_EVIDENCE_STATUS.md index 6ce2ebf54..ed4826cc3 100644 --- a/docs/explanation/features/V2_MODEL_TOOLS_AND_EVIDENCE_STATUS.md +++ b/docs/explanation/features/V2_MODEL_TOOLS_AND_EVIDENCE_STATUS.md @@ -1,4 +1,4 @@ -# V2 model tools and evidence status (v0.261.389) +# V2 model tools and evidence status (v0.261.424) ## Overview @@ -86,7 +86,7 @@ authorized source ledger or verified retained Search lineage: | State | Meaning | |---|---| -| Complete | Evidence was returned for the recorded sources; no warning footer is needed. | +| Complete | Evidence is available for the recorded sources, including a verified completed native inventory with no comments; no evidence warning footer is needed. | | Partial | Some document evidence is available, but another recorded source is missing, partial, unavailable, or unknown. | | Missing | No supporting document evidence was returned from the sources actually searched. This is not an exhaustive-corpus absence claim. | | Unavailable | Source work failed, was skipped/cancelled, or could not be authorized. This is not evidence that the fact does not exist. | @@ -97,6 +97,14 @@ General chat and historical responses without this metadata do not acquire a warning simply because they have no citation chips. The status describes document evidence, not the success or relevance of unrelated YAMCS/database tools. +The empty-inventory distinction was implemented in version **0.261.424**, +coordinated with `application/single_app/config.py`. Native comment coverage is +read from the exact authorized retained producer, not inferred from citation +count or answer wording. Zero annotation records do not become missing evidence +when that coverage establishes a completed inspection. Partial inventories stay +partial, and a completed comment inspection does not erase a separate Search +evidence gap. Counts are not increased to manufacture a successful result. + ### Using the V2 controls Partial evidence has a yellow indicator; missing document evidence has a red @@ -116,6 +124,11 @@ specific clarification, document, time range, or authorized source. Routine generic coverage paragraphs are discouraged, not removed with text matching. Evidence validation and strict partial-input contracts remain enforced. +Saved Analyze details now live in the response's Sources disclosure rather than +an always-visible analysis card. Material analysis-validation limitations can +also use the compact warning; a normal structural-success notice remains in +Sources. See [saved Analyze results](ANALYZE_RESULTS.md). + Copying an answer preserves a short material evidence notice. **Copy with sources** also includes recorded per-source states; it never includes private reasoning items. diff --git a/docs/explanation/fixes/CHAT_ANALYSIS_PRESENTATION_AND_EVIDENCE_FIX.md b/docs/explanation/fixes/CHAT_ANALYSIS_PRESENTATION_AND_EVIDENCE_FIX.md new file mode 100644 index 000000000..6582c32a3 --- /dev/null +++ b/docs/explanation/fixes/CHAT_ANALYSIS_PRESENTATION_AND_EVIDENCE_FIX.md @@ -0,0 +1,134 @@ +# Chat analysis presentation and evidence status fix (v0.261.424) + +Fixed/Implemented in version: **0.261.424**, coordinated with +`application/single_app/config.py`. + +## Issue + +A cited chat answer could be followed by an Analyze step's result card containing +saved-finding counts, structural-validation language, a follow-up action, and a +findings disclosure. This duplicated the answer's visual hierarchy and exposed +processing detail before the user asked to inspect it. + +An empty native comment inventory could also produce "No supporting document +evidence returned" even though its complete inventory was itself cited to +support "no comments were found." + +## Root cause + +Both chat renderers placed every saved-analysis descriptor directly after the +answer. The browser was useful for inspecting all saved records, but its default +placement made its counts look like a second verdict about the answer. + +Retained citation projection separately excluded native inventory coverage from +the annotation-record count, then interpreted zero records as missing evidence. +That lost the difference between a completed empty inspection and a search +that returned no supporting passages. A saved Analyze finding count was a third, +independent count, not the number of comments. + +## Technical changes + +| Area | Change | +| --- | --- | +| `application/v2_ui/src/components/chat/MessageList.tsx`, `MessageInspector.tsx`, `AnalysisResult.tsx` | Move saved-result browsing into the existing message Sources section without adding a replacement result card. | +| `application/v2_ui/src/components/chat/MessageActions.tsx`, `EvidenceStatus.tsx`, `lib/savedAnalysis.ts` | Keep Sources keyboard state explicit, preserve access for unavailable results, and show compact material limitations rather than routine success details. | +| `application/single_app/static/js/chat/chat-analysis-results.js`, `chat-messages.js` | Apply the same Sources placement in classic chat, including saved-result-only messages, cancellation on disclosure close, and masking. | +| `application/single_app/functions_orchestration_result_runtime.py` | Classify native inspection from its retained per-source coverage, keeping annotation counts and separate producer outcomes intact. | +| `application/single_app/config.py` | Record the application patch version. | + +Saved results still use the existing authorized readers and immutable result +references. Pagination, evidence, diagnostics, explicit follow-up selection, +downloads, and multiple results from an interrupted plan remain available. +There is no new route, setting, storage resource, or dependency. + +Classic chat binds Sources and warning controls to UI-owned nodes rather than +matching class names inside sanitized answer HTML. A denied or stale V2 result +also keeps its safe explanation when Sources is closed and reopened, without +briefly presenting it as an available result again. + +Native coverage must identify the inspected document and comments section, +provide valid counts, and explicitly establish a complete inventory without +partial, continuation, or limit flags. Uncertain coverage cannot certify an +empty inventory. Partial native inspection remains partial even if no complete +annotation record fits the retained subset. + +The projection still checks the exact producer and retained digest. Source and +conversation authorization boundaries are unchanged; status classification does +not reread the original file. + +## Before and after + +| Scenario | Before | After | +| --- | --- | --- | +| Successful Analyze answer | A second result card competes with the answer. | The answer and inline citations remain primary; saved findings open through Sources. | +| Complete empty native comment inventory | A cited inventory can still receive a missing-evidence warning. | Completed empty coverage supports the absence claim without inflating the annotation count. | +| Truncated native inventory | Zero retained annotation records can look like missing evidence. | Explicit partial coverage remains partial. | +| Separate Search gap | Counts can be combined without preserving the producer's outcome. | A completed inventory does not conceal another retained Search outcome. | +| Unavailable saved analysis | Hiding the entire action footer can hide the explanation entry point. | Unsafe result actions stay unavailable while Sources remains reachable. | + +## Validation + +The original regressions were reproduced with the real retained-result readers: +completed empty Word/Excel inventories were labelled `missing`, and a +resource-limited inventory could also be labelled `missing` rather than `partial`. + +Regression coverage includes: + +- `functional_tests/test_orchestration_attachment_native_evidence.py`: real + Word/Excel readers, empty/nonempty inventories, reopened evidence, and partial + or unavailable inspections. +- `functional_tests/test_orchestration_search_citations.py`: separate producer + outcomes and conservative handling of uncertain native coverage. +- `functional_tests/test_evidence_status.py`: existing aggregation, sanitization, + export notices, and per-answer scoping. +- `ui_tests/test_chat_saved_analysis.py`: both clients' Sources placement, + zero-record results, pagination, evidence, follow-ups, downloads, retries, + masking, stream/history behavior, cancellation on closing Sources, and + preservation of answer HTML containing control-like class names. +- `ui_tests/test_v2_evidence_status.py`: complete-empty and genuinely limited + evidence states across narrow/wide and light/dark layouts. + +The browser tests use the shared local/Azure Playwright connection and +deterministic transport; they do not qualify a live model or deployment. +The supplied live conversation required sign-in, so this fix does not claim +to have independently inspected that original file. + +### Verified checks + +For the native harness suites, set the application and functional-test import +paths before running pytest from the repository root: + +```powershell +$env:PYTHONPATH = "$PWD\application\single_app;$PWD\functional_tests;$PWD" +``` + +| Check | Result | +| --- | --- | +| `python -m pytest .\functional_tests\test_orchestration_attachment_native_evidence.py .\functional_tests\test_orchestration_search_citations.py .\functional_tests\test_evidence_status.py -q` | 116 passed | +| `python -m pytest .\ui_tests\test_chat_saved_analysis.py .\ui_tests\test_v2_evidence_status.py -q` | 96 passed | +| `python -m pytest .\ui_tests\test_chat_saved_analysis.py -k classic -q` | 39 passed after the final classic DOM-ownership hardening, including its additional regression | +| `npm --prefix .\application\v2_ui run typecheck` and `npm --prefix .\application\v2_ui run build` | Passed | +| `python .\functional_tests\test_docs_app_surface_coverage.py` and `python .\functional_tests\test_docs_site_quality.py` | 7 coverage checks and 6 quality checks passed | +| `python .\scripts\check_cosmos_response_boundaries.py` | No review gaps | + +Both changed classic JavaScript modules also passed `node --check`. The +repository XSS scanner's added-line inspection found no issues in the ten +changed application files. Desktop/mobile and light/dark browser inspection +confirmed that normal answers have no analysis card and that saved findings +remain usable in Sources. The full-file design detector's existing typing-dot +animation and dynamic-image/comment matches were reviewed without changing +unrelated interface behavior or adding suppressions. + +## Compatibility and limitations + +The Sources placement applies when existing messages are rendered again. +Stored historical evidence metadata is not rewritten or dismissed based on +citation presence. Corrected status classification applies when the retained +result lineage is projected; it does not retroactively rerun old conversations. + +A complete comment inventory does not prove that the body text was available +or that a broader analysis found every issue. Actual evidence gaps and material +validation limitations remain visible. + +See [saved Analyze results](../features/ANALYZE_RESULTS.md) and +[reading and discussing results](../../guides/analyze-results.md). diff --git a/docs/guides/analyze-results.md b/docs/guides/analyze-results.md index b78a2cd79..09cfa0b4c 100644 --- a/docs/guides/analyze-results.md +++ b/docs/guides/analyze-results.md @@ -4,15 +4,21 @@ title: "Read and discuss saved Analyze results" description: "Read the answer first, browse saved findings and evidence, and ask follow-up questions without requesting another source pass." section: "Guides" audience: user -version: "0.261.398" +version: "0.261.424" --- ## What this does Analyze reviews selected documents according to your question and saves the accepted findings for later review. Both classic chat and React V2 keep the -readable answer in the conversation, with a compact **Saved analysis** section -under it. A supporting file becoming ready does not replace that answer. +readable answer and inline citations in the conversation. Open that response's +**Sources** control to inspect its saved findings; an analysis card is not +appended to every answer. A supporting file becoming ready does not replace +the answer. + +The Sources placement was implemented in version: **0.261.424**, tracked in +`application/single_app/config.py`. A compact warning remains visible when an +analysis is partial, pending, not validated, invalid, or unavailable. {% include media.html id="slide-analyze-and-compare" %} @@ -38,7 +44,8 @@ Use the existing source-selection and Analyze flow: **Analyze** in **Action**, rather than **Search**. - In React V2, **Documents** selects context. When an **Orchestrate** plan includes Analyze, use the existing plan-review and approval controls; the saved-result - view appears with its answer. **Documents** is not a separate Analyze button. + view is available through the answer's **Sources** control. **Documents** is + not a separate Analyze button. Since **0.261.115**, pinning documents does not also require a Search step. Ask for Analyze in the message and review the proposed operation. All selected @@ -72,20 +79,23 @@ The assistant's answer is the readable overview. It can explain the main themes without displaying every saved finding. Do not treat that overview, or a file preview, as the complete set of records. -1. Read the answer and the validation notice under **Saved analysis**. -2. Expand **Findings and limitations** to load a page of accepted findings. -3. Use **Next findings** and **Previous findings** to move through the saved set. +1. Read the answer and its inline citations. +2. Open **Sources** on that response and read the saved analysis's validation notice. +3. Expand **Findings and limitations** to load a page of accepted findings. +4. Use **Next findings** and **Previous findings** to move through the saved set. Pages contain up to **25 complete records**; the last page may be shorter. -4. Compare the displayed count with the total. For example, +5. Compare the displayed count with the total. For example, `Showing 1–25 of 60 records` means that 35 records are on other pages. -A closed findings section reports zero records displayed, not zero records saved. -Source counts are separate: the sources represented on one page can be fewer than -the sources in the saved analysis. One source can contribute several findings. -Neither count proves that every possible issue was discovered. +A closed findings section shows the total number of saved findings. An expanded +section shows the displayed range and the full total. Source counts are separate: +the sources represented on one page can be fewer than the sources in the saved +analysis. One source can contribute several findings. These are not counts of +native comments or citation chips, and they do not prove that every possible +issue was discovered. Since **0.261.398**, a new orchestration reply can show several retained analyses -when a later step could not finish. Their headings identify the producing +when a later step could not finish. Open **Sources** to see them. Their headings identify the producing steps; each has its own count, findings, evidence, and diagnostic link. Read their validation notices separately. A usable source review does not mean the overall answer was delivered. @@ -127,6 +137,10 @@ The **Limitations and validation issues** section appears when the saved result includes those notices. Read it alongside the findings, particularly when the analysis is incomplete. +Routine structural-success notices stay in Sources rather than competing with +the answer. A material limitation still gets a compact warning below the answer; +select **View sources** to inspect it. + | Notice | How to interpret it | | --- | --- | | **Structural checks passed.** | Structural checks completed successfully. This is not independent factual verification or proof that the findings are exhaustive. | @@ -139,14 +153,29 @@ Findings remain model judgments. Inspecting a saved quotation helps you understa its support; it does not reread the original source or independently verify the conclusion. -For an audit, **View diagnostics (JSON)** opens the separately saved processing +For an audit, **View diagnostics (JSON)** in Sources opens the separately saved processing data in a new tab. It is not the final findings or a readable report. The response is bounded; `next_offset` identifies another byte page when the audit is larger than one response. Ordinary reading and follow-up questions do not use this data. +In V2, expand **Findings and limitations**, then **Advanced analysis details** +to reach this link. + +### An empty comment inventory is not an evidence failure + +A successfully completed native Word or Excel comment inspection can find zero +comments. Its saved inventory is still citable evidence for that absence. Since +**0.261.424**, that result does not receive a missing-evidence warning merely +because no annotation records were returned. + +An unavailable or incomplete inspection cannot establish that there are no +comments. Missing body-text evidence also remains separate: checking comments +does not establish the document's current wording. Likewise, an Analyze result +with zero saved findings does not, by itself, establish a native comment count. ## Ask about the saved analysis -1. Select **Ask about this analysis** on the result you want to discuss. +1. Open **Sources** on the response and select **Ask about this analysis** on + the result you want to discuss. 2. Check the composer notice: > Saved analysis selected. Explaining the saved analysis — not running a new pass over the original sources. @@ -208,3 +237,7 @@ Local functional and browser-fixture coverage is separate from live acceptance. Use a matching backend and V2 build, a new conversation, and currently authorized sources when evaluating a deployment. Reopening an old conversation does not re-analyze its documents or retrofit new saved-result metadata. + +The Sources placement applies when existing saved results are displayed again. +Stored historical evidence warnings are not rewritten or dismissed merely +because an answer has citations. diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index 4ec6952c6..e37c832b3 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -4,7 +4,7 @@ title: "Chat interface controls" description: "Reference for every documented control in the SimpleChat chat interface." section: "Reference" audience: user -version: "0.261.423" +version: "0.261.424" --- ## How to use this reference @@ -694,8 +694,11 @@ also depends on configured storage and current source access. Implemented in version **0.261.109**. These response-specific controls appear in classic chat and React V2 when an assistant message has a saved Analyze result. -They are rendered with the response rather than added to the always-visible -toolbar. See [Read and discuss saved Analyze results]({{ '/guides/analyze-results/' | relative_url }}). +Since **0.261.424** (`application/single_app/config.py`), open the response's +**Sources** control to read them instead of finding a separate analysis card +under every answer. Sources remains available for saved analyses with no +ordinary citations, zero records, or an unavailable-result explanation. +See [Read and discuss saved Analyze results]({{ '/guides/analyze-results/' | relative_url }}). The answer remains the readable overview, even after a supporting export finishes. The overview and download previews are not the full findings set. Ordinary @@ -710,7 +713,8 @@ failed answer or promote partial findings to complete input. | Control or notice | What it does | Why you would use it | Available when | | --- | --- | --- | --- | -| **Saved analysis** | Identifies the saved result and shows displayed-versus-total record counts, separate source counts, and its validation notice. | Distinguish a readable overview or one page from the full saved findings set. | The assistant message carries a saved result; an unavailable result shows a notice instead of result controls | +| **Saved analysis** | Identifies the result inside Sources, with total saved findings, displayed-versus-total counts when expanded, separate source counts, and validation details. | Distinguish a readable overview or one page from the full saved findings set. | Sources is open and the assistant message carries a saved result; an unavailable result shows an explanation instead of result controls | +| **Analysis limitation / View sources** | Keeps a compact warning below the answer for partial, pending, unvalidated, invalid, or unavailable saved analysis. | Notice a material qualification without displaying processing details under every successful answer. | A saved result has a limitation; ordinary structural success does not add a warning | | **Findings and limitations** | Expands accepted findings and any **Limitations and validation issues**, loading up to 25 complete records at a time. | Inspect details without downloading a file or displaying the whole result at once. | The saved result is available | | **Previous findings** / **Next findings** | Reads the preceding or following page of the same saved result. | Review records outside the currently displayed range. A page's source count may be smaller than the result's total source count. | A corresponding page exists and the current page has loaded | | **Evidence for finding …** | Loads that finding's saved supporting passages and any saved filename, page, or chunk location. The label includes the finding's identifier. | Understand what supports a finding without treating it as independently verified. | The finding has saved evidence references | @@ -733,6 +737,12 @@ and **Not validated** remain distinct notices; execution completion does not tur them into successful validation. Saved evidence and explanations do not imply a new independent check of the original sources. +A completed native comment inventory can return zero comments and still support +a cited absence claim. Since **0.261.424**, that is not classified as missing +document evidence merely because the annotation count is zero. Incomplete +inspection and missing body-text evidence remain separate limitations. A +saved-finding count is not a native comment count. + Since **0.261.232**, these results, their evidence, derived explanations, and original exports take their access from the originating conversation. A contributing document being deleted, replaced, held, or no longer shared does diff --git a/functional_tests/test_orchestration_attachment_native_evidence.py b/functional_tests/test_orchestration_attachment_native_evidence.py index e51f3f40f..7a084f47b 100644 --- a/functional_tests/test_orchestration_attachment_native_evidence.py +++ b/functional_tests/test_orchestration_attachment_native_evidence.py @@ -1,9 +1,10 @@ # test_orchestration_attachment_native_evidence.py """Real planning, inventory paging, retained evidence and composition regressions. -Version: 0.261.423 +Version: 0.261.424 Implemented in: 0.261.416 Word-native comment routing and evidence delivery: 0.261.422 +Complete empty native inventory evidence status: 0.261.424 Only external storage/model I/O is replaced; no provider calls are made. """ @@ -440,6 +441,9 @@ def test_word_comments_use_the_real_word_reader_and_reach_retained_answers( payload = json.loads(case.model.calls[0][0][-1]["content"]) assert "Raise cap to 19500." in json.dumps(payload) and "Keep cap at 15000." in json.dumps(payload) reader = importlib.import_module("functions_orchestration_result_runtime") + evidence_status = reader.read_result_evidence_status(case.context, task.output("prepared")) + assert evidence_status["state"] == "complete" + assert evidence_status["sources"][0]["evidence_count"] == 1 citations = reader.read_result_document_citations(case.context, task.output("prepared")) citation = next(item for item in citations if item.get("evidence_kind") == "native_word_annotation") contracts = importlib.import_module("functions_orchestration_result_contracts") @@ -469,6 +473,43 @@ def test_word_comments_use_the_real_word_reader_and_reach_retained_answers( assert answer == candidate +@pytest.mark.parametrize("source_format", ["docx", "xlsx"]) +@pytest.mark.parametrize("output", ["prepared", "evidence"]) +def test_complete_empty_native_inventory_remains_supporting_evidence( + runtime, monkeypatch, inventory_store, tmp_path, source_format, output, +): + if source_format == "docx": + step, calls, _ = configure_word_inventory(runtime, monkeypatch, word_comment_package(comments=False)) + else: + workbook = Workbook() + workbook.active["A1"] = "No review comments" + path = tmp_path / "empty-comments.xlsx" + workbook.save(path) + workbook.close() + step, calls, _ = configure_inventory(runtime, monkeypatch, inventory_store, path) + case = runtime.make([ + step, compose("answer", inputs={"comments": source_input("comments", output)}), + ], ["The complete native inventory contains no comments."], final_response=binding("answer")) + case.fixture.sources["document-1"] = {**source(), "file_name": f"review.{source_format}"} + result = execute(runtime, case) + assert result["status"] == "completed", result["steps"] + reader = importlib.import_module("functions_orchestration_result_runtime") + reopened = SimpleNamespace(result_service=case.fixture.restart()) + for reference in ( + case.context.task_results["comments"].output(output), + case.context.task_results["answer"].output("answer"), + ): + status = reader.read_result_evidence_status(reopened, reference) + citations = reader.read_result_document_citations(reopened, reference) + assert status["state"] == "complete" + assert status["sources"][0]["state"] == "complete" + assert status["sources"][0]["evidence_count"] == 0 + assert len(citations) == 1 + assert citations[0]["evidence_kind"] == "native_annotation_coverage" + assert citations[0]["native_evidence"] + assert len(calls) == 1, "Reading saved coverage must not inspect the original file again." + + def test_word_evidence_does_not_bypass_repeated_unsupported_answer_findings(runtime, monkeypatch): step, _, _ = configure_word_inventory(runtime, monkeypatch, word_comment_package()) candidate = "The author accepted the higher cap of 19500." @@ -601,6 +642,9 @@ def test_native_workbook_comments_reach_retained_evidence_and_composition( assert all(citation["location_label"] == "Location" for citation in annotated) assert all(citation["document_id"] == "document-1" and citation["native_evidence"] for citation in annotated) reader = importlib.import_module("functions_orchestration_result_runtime") + evidence_status = reader.read_result_evidence_status(case.context, task.output("prepared")) + assert evidence_status["state"] == "complete" + assert evidence_status["sources"][0]["evidence_count"] == 5 contracts = importlib.import_module("functions_orchestration_result_contracts") for citation in citations: evidence = reader.read_native_citation_evidence( @@ -765,6 +809,9 @@ def read(*args, **kwargs): assert coverage["partial"] and coverage["has_more"] assert coverage["returned_records"] < coverage["total_records"] assert task.output("prepared").completeness.status == "partial" + reader = importlib.import_module("functions_orchestration_result_runtime") + status = reader.read_result_evidence_status(case.context, task.output("prepared")) + assert status["state"] == "partial" else: assert result["status"] == "failed" assert "comments" not in case.context.task_results diff --git a/functional_tests/test_orchestration_search_citations.py b/functional_tests/test_orchestration_search_citations.py index b105a73ef..7f48f8e56 100644 --- a/functional_tests/test_orchestration_search_citations.py +++ b/functional_tests/test_orchestration_search_citations.py @@ -1,8 +1,9 @@ # test_orchestration_search_citations.py """Search content handoffs retain exact passage citations, not just document identities. -Version: 0.261.389 +Version: 0.261.424 Implemented in: 0.261.365; per-answer evidence status in 0.261.388 +Native empty-inventory and mixed inspection status covered in: 0.261.424 The real adapters, retained readers, lineage projection and Compose execute with offline search/model/storage transport. No external qualification harness is changed. @@ -245,6 +246,76 @@ def test_historical_evidence_without_a_prepared_sidecar_remains_provenance_only( assert case.model.calls == [] +@pytest.mark.parametrize('body_matches', [False, True]) +def test_complete_native_inventory_does_not_replace_a_separate_search_outcome(runtime, body_matches): + native_step = {**search_step(), 'step_id': 'comments'} + case = runtime.make([ + native_step, search_step(), + compose('answer', inputs={ + 'comments': source_input('comments'), 'body': source_input('search'), + }), + ]) + source = case.fixture.sources['document-1'] + inventory = { + 'document_id': 'document-1', 'execution_mode': 'native_inspection', 'section': 'comments', + 'inventory_complete': True, 'total_records': 0, 'returned_records': 0, + 'partial': False, 'has_more': False, 'limit_reached': False, + } + coverage_citation = { + 'document_id': 'document-1', 'citation_id': 'document-1:native-comments:coverage', + 'file_name': 'Review.docx', 'evidence_kind': 'native_annotation_coverage', + } + body_citation = { + 'document_id': 'document-1', 'citation_id': 'document-1_1', + 'file_name': 'Review.docx', 'page_number': 1, + } + parents = [] + for step in case.plan['steps'][:2]: + native = step['step_id'] == 'comments' + prepared = { + 'version': 'orchestration-gathered-content-v1', 'capability_id': 'document_search', + 'content_scope': 'native_document_annotations' if native else 'returned_excerpts', + 'citations': [coverage_citation] if native else [body_citation] if body_matches else [], + 'coverage': {'native_inspection': [inventory]} if native else {'returned_excerpt_count': int(body_matches)}, + } + task = case.fixture.service.persist_task_result( + producer=case.context.result_producer(step), role='gather', status='complete', + outputs=[runtime.NamedOutput('prepared', 'structured-v1', prepared, runtime.complete(1))], + sources=[source], origin='grounded', + guard_token=case.context.result_guard_token_for_step(step['step_id']), + ) + parents.append(task.output('prepared')) + task = case.fixture.service.persist_task_result( + producer=case.context.result_producer(case.plan['steps'][2]), role='reason', status='complete', + outputs=[runtime.NamedOutput('answer', 'markdown-v1', 'A source-grounded answer.', runtime.complete(1))], + sources=[], upstream=parents, origin='grounded', + guard_token=case.context.result_guard_token_for_step('answer'), + ) + reopened = SimpleNamespace(result_service=case.fixture.restart()) + status = runtime.result_runtime.read_result_evidence_status(reopened, task.output('answer')) + assert status['state'] == ('complete' if body_matches else 'partial') + assert status['sources'][0]['state'] == status['state'] + assert status['sources'][0]['evidence_count'] == int(body_matches) + assert case.model.calls == [] + + +@pytest.mark.parametrize('change', [ + {'inventory_complete': False}, {'returned_records': True}, {'total_records': -1}, + {'partial': None}, {'section': 'body'}, {'document_id': 'another-document'}, +]) +def test_unknown_native_coverage_cannot_certify_an_empty_inventory(runtime, change): + inventory = { + 'document_id': 'document-1', 'execution_mode': 'native_inspection', 'section': 'comments', + 'inventory_complete': True, 'total_records': 0, 'returned_records': 0, + 'partial': False, 'has_more': False, 'limit_reached': False, + **change, + } + state = runtime.result_runtime._native_inventory_evidence_state( + {'coverage': {'native_inspection': [inventory]}}, 'document-1', + ) + assert state == 'unknown' + + def test_evidence_sidecar_verifies_stored_content_digest_without_fallback(search_content): case = search_content.make() result = execute(search_content.runtime, case) diff --git a/ui_tests/test_chat_saved_analysis.py b/ui_tests/test_chat_saved_analysis.py index 86c11875b..4be2e6241 100644 --- a/ui_tests/test_chat_saved_analysis.py +++ b/ui_tests/test_chat_saved_analysis.py @@ -1,9 +1,10 @@ # test_chat_saved_analysis.py """ Saved Analyze findings, evidence, and explanation context in both chat interfaces. -Version: 0.261.398 +Version: 0.261.424 Implemented in: 0.261.109 Retained analyses on failed orchestration replies covered in: 0.261.398 +Answer-first presentation and on-demand Sources covered in: 0.261.424 Runs the real classic message/stream modules and React MessageList/Composer/store. Record/evidence requests cross the production Flask reader and serialized result @@ -230,6 +231,12 @@ def handle(self, route): if path == "/api/v2/orchestration/runs": route.fulfill(json={"runs": []}) return + if path == "/api/v2/orchestration/questions" and request.method == "GET": + route.fulfill(json={"questions": [], "csrf_token": "fixture-csrf"}) + return + if path == "/api/v2/user-input/pending" and request.method == "GET": + route.fulfill(json={"requests": [], "csrf_token": "fixture-csrf"}) + return if path == "/api/action-credentials/pending" and request.method == "GET": route.fulfill(json={"requests": [], "csrf_token": "fixture-csrf"}) return @@ -386,6 +393,17 @@ def result(ui): return ui.page.get_by_role("region", name="Saved analysis").first +def sources_button(ui): + return ui.page.get_by_role("button", name=re.compile(r"^(Show sources(?: \(\d+\))?|Hide sources)$")).first + + +def open_sources(ui): + button = sources_button(ui) + expect(button).to_be_visible() + if button.get_attribute("aria-expanded") != "true": + button.click() + + def context(ui): return ui.page.evaluate("() => window.analysisUI.context()") @@ -406,6 +424,119 @@ def choose_new_source(ui): ui.page.get_by_title("Documents", exact=True).click() +@pytest.mark.parametrize("width,theme", [(390, "light"), (390, "dark"), (1440, "light"), (1440, "dark")]) +def test_saved_findings_are_disclosed_through_sources_not_the_answer(analysis_ui, width, theme): + ui = analysis_ui + ui.page.set_viewport_size({"width": width, "height": 1000}) + ui.mount() + ui.page.evaluate("""theme => { + document.documentElement.classList.toggle('dark', theme === 'dark'); + document.documentElement.setAttribute('data-bs-theme', theme); + }""", theme) + expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() + expect(result(ui)).to_have_count(0) + expect(ui.page.get_by_text("Structural checks passed.", exact=False)).not_to_be_visible() + expect(ui.page.get_by_role("button", name="Ask about this analysis")).to_have_count(0) + assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] + capture = (width, theme) in ((1440, "light"), (390, "dark")) + if capture: + ui.page.screenshot(path=str(BUILD / f"analysis-{ui.renderer}-{width}-{theme}-answer.png"), full_page=True) + button = sources_button(ui) + button.focus() + button.press("Enter") + expect(button).to_have_attribute("aria-expanded", "true") + expect(result(ui)).to_contain_text("60 saved findings") + assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] + result(ui).locator("summary").filter(has_text="Findings and limitations").click() + expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") + if capture: + ui.page.screenshot(path=str(BUILD / f"analysis-{ui.renderer}-{width}-{theme}-sources.png"), full_page=True) + sources_button(ui).click() + expect(result(ui)).to_have_count(0) + expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() + open_sources(ui) + expect(result(ui)).to_contain_text("60 saved findings") + expect(result(ui).get_by_text("Control 0", exact=True)).to_have_count(0) + + +def test_zero_saved_findings_remain_accessible_without_an_error_or_citations(analysis_ui): + ui = analysis_ui + fixture = ui.api.fixture + manifest, _, _ = saved.load_saved_analysis( + "reader", saved.saved_analysis_context(fixture["descriptor"]), **read_options(fixture), + ) + descriptor = saved.save_chat_analysis( + {"reply": "The completed review saved no findings.", + "analysis_result": { + "analysis_result_version": "analyze-final-v1", + "source_manifest": manifest["analysis_access"]["sources"], + "authoritative_result": {"kind": "records", "value": []}, + "analysis_evidence": [], + "analysis_validation": {"status": "valid"}, + }}, + user_id="owner", conversation_id=CONVERSATION, message_id="assistant-1", + authorize_conversation=lambda user, conversation: fixture["message_loader"]( + user, conversation, "assistant-1", + ), + save_result=fixture["store"].save, + ) + fixture["descriptor"] = descriptor + fixture["message"]["metadata"] = {"saved_analysis": descriptor} + ui.api.message.update({ + "content": "The completed review saved no findings.", + "metadata": {"saved_analysis": descriptor}, "hybrid_citations": [], + }) + ui.mount() + expect(result(ui)).to_have_count(0) + expect(ui.page.get_by_text("No supporting document evidence returned", exact=False)).to_have_count(0) + expect(ui.page.get_by_role("button", name=re.compile("^Partial analysis"))).to_have_count(0) + open_sources(ui) + expect(result(ui)).to_contain_text("0 saved findings") + result(ui).locator("summary").filter(has_text="Findings and limitations").click() + expect(result(ui)).to_contain_text("No accepted findings were saved.") + expect(result(ui).get_by_role("button", name="Next findings", exact=True)).to_be_disabled() + expect(result(ui).get_by_role("button", name="Previous findings", exact=True)).to_be_disabled() + selected = context(ui) + assert selected == expected_context(ui) + + +def test_closing_sources_cancels_pending_findings_and_reopens_cleanly(analysis_ui): + ui = analysis_ui + ui.api.hold_page = True + ui.mount() + open_sources(ui) + with ui.page.expect_request("**/api/analysis_results?*"): + result(ui).locator("summary").filter(has_text="Findings and limitations").click() + sources_button(ui).click() + for pending in ui.api.pending_pages: + ui.api.records(pending) + expect(result(ui)).to_have_count(0) + expect(ui.page.get_by_text("Control 0", exact=True)).to_have_count(0) + ui.api.hold_page = False + open_sources(ui) + expect(result(ui)).to_contain_text("60 saved findings") + result(ui).locator("summary").filter(has_text="Findings and limitations").click() + expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") + + +@pytest.mark.parametrize("analysis_ui", ["classic"], indirect=True) +def test_answer_html_cannot_be_mistaken_for_owned_sources_or_warning_controls(analysis_ui): + ui = analysis_ui + ui.api.message["content"] = ( + 'Keep this answer.\n\n
KEEP-SOURCE-ANSWER
\n\n' + '
KEEP-WARNING-ANSWER
' + ) + ui.mount() + expect(ui.page.get_by_text("KEEP-SOURCE-ANSWER", exact=True)).to_be_visible() + expect(ui.page.get_by_text("KEEP-WARNING-ANSWER", exact=True)).to_be_visible() + expect(result(ui)).to_have_count(0) + open_sources(ui) + result(ui).locator("summary").filter(has_text="Findings and limitations").click() + expect(result(ui)).to_contain_text("Control 0") + expect(ui.page.get_by_text("KEEP-SOURCE-ANSWER", exact=True)).to_be_visible() + expect(ui.page.get_by_text("KEEP-WARNING-ANSWER", exact=True)).to_be_visible() + + def retain_failed_analyses(ui): fixture = ui.api.fixture manifest, _, _ = saved.load_saved_analysis( @@ -452,6 +583,7 @@ def test_failed_run_keeps_each_saved_analysis_and_requires_explicit_selection(an ui = analysis_ui descriptors = retain_failed_analyses(ui) ui.mount() + open_sources(ui) deck = ui.page.get_by_role("region", name="Saved analysis: Deck claims", exact=True) reviewer = ui.page.get_by_role("region", name="Saved analysis: Reviewer comments ", exact=True) expect(deck).to_be_visible() @@ -475,6 +607,9 @@ def test_failed_run_keeps_each_saved_analysis_and_requires_explicit_selection(an ui.page.evaluate("() => window.analysisUI.reload()") after_reload = context(ui) assert after_reload == expected + open_sources(ui) + if ui.renderer == "v2": + reviewer.locator("summary").filter(has_text="Advanced analysis details").click() diagnostic_url = reviewer.get_by_role("link", name="View diagnostics as JSON (opens a new tab)").get_attribute("href") assert parse_qs(urlsplit(diagnostic_url).query)["result_sha256"] == [descriptors[1]["result_sha256"]] ui.page.set_viewport_size({"width": 390, "height": 844}) @@ -490,6 +625,7 @@ def test_failed_live_completion_keeps_plural_analyses_without_auto_selecting(ana ui.api.messages = [] ui.mount(history=False) ui.page.evaluate("() => window.analysisUI.send()") + open_sources(ui) expect(ui.page.get_by_role("region", name="Saved analysis: Deck claims", exact=True)).to_be_visible() expect(ui.page.get_by_role( "region", name="Saved analysis: Reviewer comments ", exact=True, @@ -503,7 +639,8 @@ def test_malformed_plural_entry_is_visible_without_hiding_valid_findings(analysi retain_failed_analyses(ui) ui.api.message["metadata"]["saved_analyses"][1]["semantics"]["mode"] = "unknown-contract" ui.mount() - expect(ui.page.get_by_text("Saved analysis unavailable.", exact=False).first).to_be_visible() + expect(ui.page.get_by_role("button", name=re.compile("^Saved analysis unavailable"))).to_be_visible() + open_sources(ui) deck = ui.page.get_by_role("region", name="Saved analysis: Deck claims", exact=True) deck.locator("summary").filter(has_text="Findings and limitations").click() expect(deck).to_contain_text("Control 0") @@ -515,6 +652,7 @@ def test_masking_a_plural_reply_hides_every_result_and_clears_explicit_selection ui = analysis_ui retain_failed_analyses(ui) ui.mount() + open_sources(ui) reviewer = ui.page.get_by_role( "region", name="Saved analysis: Reviewer comments ", exact=True, ) @@ -535,9 +673,17 @@ def test_saved_analysis_page_two_evidence_download_and_safe_text(analysis_ui): ui.api.unsafe_values = True ui.mount() expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() - expect(result(ui)).to_contain_text("0 of 60 records displayed") + expect(result(ui)).to_have_count(0) + open_sources(ui) + expect(result(ui)).to_contain_text("60 saved findings") assert context(ui) == expected_context(ui) assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] + disclosure = result(ui).locator("summary").filter(has_text="Findings and limitations") + disclosure.focus() + disclosure.press("Enter") + expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") + if ui.renderer == "v2": + result(ui).locator("summary").filter(has_text="Advanced analysis details").click() diagnostics = result(ui).get_by_role("link", name="View diagnostics as JSON (opens a new tab)") expect(diagnostics).to_be_visible() diagnostic_url = urlsplit(diagnostics.get_attribute("href")) @@ -547,10 +693,6 @@ def test_saved_analysis_page_two_evidence_download_and_safe_text(analysis_ui): "representation": ["diagnostics"], } expect(diagnostics).to_have_attribute("rel", "noopener noreferrer") - disclosure = result(ui).locator("summary").filter(has_text="Findings and limitations") - disclosure.focus() - disclosure.press("Enter") - expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") expect(result(ui)).to_contain_text("1 of 1 sources on this page") expect(result(ui)).to_contain_text("") assert result(ui).locator("img, script").count() == 0 @@ -585,12 +727,16 @@ def test_completion_and_reconnect_keep_answer_and_select_followup(analysis_ui): ui.mount(history=False) ui.page.evaluate("() => window.analysisUI.send()") expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() + expect(result(ui)).to_have_count(0) + open_sources(ui) expect(result(ui)).to_be_visible() assert context(ui) == expected_context(ui) ui.api.messages = [] ui.api.reconnect = True ui.mount() expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() + expect(result(ui)).to_have_count(0) + open_sources(ui) expect(result(ui)).to_be_visible() assert context(ui) == expected_context(ui) assert len([item for item in ui.api.requests if item["path"].startswith("/api/chat/stream/reattach/")]) == 1 @@ -646,6 +792,7 @@ def test_analysis_with_expanded_navigation_has_only_local_table_scrolling(analys dimensions = ui.page.evaluate("({viewport: innerWidth, document: document.documentElement.scrollWidth})") assert dimensions["document"] <= width + 1, dimensions close.click() + open_sources(ui) result(ui).locator("summary").filter(has_text="Findings and limitations").click() expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") result(ui).locator("summary").filter(has_text="Evidence for finding record-0").click() @@ -675,6 +822,7 @@ def test_download_prefers_the_server_utf8_filename(analysis_ui): def test_saved_explanation_request_bypasses_sources_and_orchestration(analysis_ui): ui = analysis_ui ui.mount(orchestration=True) + open_sources(ui) result(ui).get_by_role("button", name="Ask about this analysis", exact=True).click() expect(ui.page.get_by_text("Explaining the saved analysis", exact=False)).to_be_visible() draft = ui.page.get_by_role("textbox", name="Message", exact=True) @@ -689,7 +837,8 @@ def test_saved_explanation_request_bypasses_sources_and_orchestration(analysis_u assert body[key] is False assert not body["selected_document_ids"] assert "document_action" not in body and "analyze" not in body - assert not [item for item in ui.api.requests if item["path"].startswith("/api/v2/orchestration/")] + assert not [item for item in ui.api.requests + if item["method"] == "POST" and item["path"].startswith("/api/v2/orchestration/")] assert context(ui) == expected_context(ui), "Explanation messages keep the original result pointer." ui.page.get_by_role("button", name="Remove saved analysis context").click() assert context(ui) is None @@ -697,6 +846,7 @@ def test_saved_explanation_request_bypasses_sources_and_orchestration(analysis_u assert context(ui) is None, "History hydration must not undo the user's removal." ui.page.evaluate("() => window.analysisUI.open()") assert context(ui) is None, "Reselecting the same conversation must preserve explicit context removal." + open_sources(ui) result(ui).get_by_role("button", name="Ask about this analysis", exact=True).click() choose_new_source(ui) assert context(ui) is None @@ -716,12 +866,20 @@ def test_denied_missing_and_stale_results_are_visible_not_empty_success(analysis else: ui.api.message["metadata"]["saved_analysis"]["result_sha256"] = "0" * 64 ui.mount() + open_sources(ui) result(ui).locator("summary").filter(has_text="Findings and limitations").click() expect(result(ui)).to_contain_text("stale" if status == 409 else "unavailable") expect(result(ui).get_by_role("button", name="Ask about this analysis")).to_have_count(0) expect(download_button(ui)).not_to_be_visible() expect(result(ui)).not_to_contain_text("No accepted findings") + expect(sources_button(ui)).to_be_visible() assert context(ui) is None + reads = len([item for item in ui.api.requests if item["path"] == "/api/analysis_results"]) + sources_button(ui).click() + open_sources(ui) + expect(result(ui)).to_contain_text("stale" if status == 409 else "unavailable") + expect(result(ui).get_by_role("button", name="Ask about this analysis")).to_have_count(0) + assert len([item for item in ui.api.requests if item["path"] == "/api/analysis_results"]) == reads @pytest.mark.parametrize("status,notice", [ @@ -735,6 +893,15 @@ def test_validation_state_is_not_execution_completion(analysis_ui, status, notic ui.api.validation_status = status ui.api.message["metadata"]["saved_analysis"]["validation_status"] = status ui.mount() + expect(result(ui)).to_have_count(0) + compact = { + "pending": "Analysis validation pending", + "partial": "Partial analysis", + "invalid": "Analysis validation failed", + "not_validated": "Analysis not validated", + }[status] + expect(ui.page.get_by_role("button", name=re.compile(f"^{compact}"))).to_be_visible() + open_sources(ui) expect(result(ui)).to_contain_text(notice) result(ui).locator("summary").filter(has_text="Findings and limitations").click() expect(result(ui)).to_contain_text("Showing 1–25") @@ -747,6 +914,7 @@ def test_unavailable_or_masked_history_exposes_no_saved_result_controls(analysis ui.api.message["content"] = "This saved analysis is unavailable." ui.api.message["metadata"]["saved_analysis"]["available"] = False ui.mount() + open_sources(ui) expect(result(ui)).to_contain_text("unavailable") expect(ui.page.get_by_role("button", name="Ask about this analysis")).to_have_count(0) assert context(ui) is None @@ -770,6 +938,7 @@ def test_explicit_source_choice_wins_over_late_completion(analysis_ui): assert context(ui) is None assert len(ui.api.pending_streams) == 1 ui.api.stream(ui.api.pending_streams.pop()) + open_sources(ui) expect(result(ui)).to_be_visible() assert context(ui) is None ui.page.evaluate("() => window.analysisUI.reload()") @@ -780,6 +949,7 @@ def test_late_findings_response_cannot_follow_conversation_navigation(analysis_u ui = analysis_ui ui.api.hold_page = True ui.mount() + open_sources(ui) with ui.page.expect_request("**/api/analysis_results?*"): result(ui).locator("summary").filter(has_text="Findings and limitations").click() ui.page.evaluate("() => window.analysisUI.change('another-conversation')") @@ -795,6 +965,7 @@ def test_supporting_export_completion_never_replaces_the_answer(analysis_ui): artifact = ui.api.message["metadata"]["generated_analysis_artifacts"][0] artifact.update({"background_export": True, "export_run_id": "supporting-export", "status": "running"}) ui.mount() + open_sources(ui) expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() ui.page.locator("summary").filter(has_text="Downloads").click() expect(download_button(ui)).to_be_visible(timeout=10000) @@ -806,6 +977,7 @@ def test_supporting_export_completion_never_replaces_the_answer(analysis_ui): def test_evidence_rechecks_access_and_removes_the_previously_loaded_page(analysis_ui): ui = analysis_ui ui.mount() + open_sources(ui) result(ui).locator("summary").filter(has_text="Findings and limitations").click() expect(result(ui)).to_contain_text("Control 0") ui.api.fixture["state"]["conversation_allowed"] = False @@ -819,6 +991,7 @@ def test_evidence_rechecks_access_and_removes_the_previously_loaded_page(analysi def test_source_revocation_does_not_withdraw_conversation_owned_findings(analysis_ui): ui = analysis_ui ui.mount() + open_sources(ui) result(ui).locator("summary").filter(has_text="Findings and limitations").click() expect(result(ui)).to_contain_text("Control 0") ui.api.fixture["state"]["source_allowed"] = False @@ -833,13 +1006,14 @@ def test_transient_page_failure_can_retry_without_exposing_diagnostics(analysis_ ui = analysis_ui ui.api.page_failures = 1 ui.mount() + open_sources(ui) result(ui).locator("summary").filter(has_text="Findings and limitations").click() expect(result(ui).get_by_role("button", name="Retry findings")).to_be_visible() expect(result(ui)).not_to_contain_text("PRIVATE_DIAGNOSTIC") result(ui).get_by_role("button", name="Retry findings").click() expect(result(ui)).to_contain_text("Showing 1–25") result(ui).locator("summary").filter(has_text="Findings and limitations").click() - expect(result(ui)).to_contain_text("0 of 60 records displayed") + expect(result(ui)).to_contain_text("60 saved findings") def test_late_stream_completion_cannot_reselect_another_conversations_analysis(analysis_ui): @@ -875,6 +1049,8 @@ def test_orchestrated_completion_adopts_only_its_saved_public_pointer(analysis_u ui.api.message, ) expect(ui.page.get_by_text(ANSWER, exact=True)).to_be_visible() + expect(result(ui)).to_have_count(0) + open_sources(ui) expect(result(ui)).to_be_visible() assert context(ui) == expected_context(ui) @@ -893,6 +1069,7 @@ def test_saved_records_are_not_misinterpreted_as_answer_mask_offsets(analysis_ui }""" ui.page.get_by_text(ANSWER, exact=True).evaluate(select_text) expect(ui.page.get_by_role("button", name="Mask selection", exact=True)).to_be_visible() + open_sources(ui) result(ui).locator("summary").filter(has_text="Findings and limitations").click() result(ui).get_by_text("Control 0", exact=True).evaluate(select_text) expect(ui.page.get_by_role("button", name="Mask selection", exact=True)).not_to_be_visible() diff --git a/ui_tests/test_v2_evidence_status.py b/ui_tests/test_v2_evidence_status.py index 1a6f769fd..9c221b510 100644 --- a/ui_tests/test_v2_evidence_status.py +++ b/ui_tests/test_v2_evidence_status.py @@ -1,8 +1,9 @@ # test_v2_evidence_status.py """Rendered V2 evidence status, source scoping, and raw citation regressions. -Version: 0.261.389 +Version: 0.261.424 Implemented in: 0.261.388 +Complete empty native inventory presentation covered in: 0.261.424 Uses the shared local/Azure Playwright connection and real bundled components. HTTP responses are synthetic; no production conversations or models are used. @@ -32,21 +33,26 @@ def reply(state, identifier="evidence-answer"): - sources = [{"name": "Guide.md", "document_id": DOCUMENT, + name = "Review.docx" if state == "complete" else "Guide.md" + sources = [{"name": name, "document_id": DOCUMENT, "state": "complete" if state == "partial" else state, **({"evidence_count": 1 if state == "partial" else 0} if state != "unavailable" else {})}] if state == "partial": sources.append({"name": "Additional notes.md", "document_id": "other-document", "state": "missing", "evidence_count": 0}) return { "id": identifier, "conversation_id": CONVERSATION, "role": "assistant", - "content": f"The supported retention period is 30 days. \u3010{CITATION}\u3011" if state == "partial" - else "The selected sources did not provide enough information to answer.", + "content": f"The supported retention period is 30 days. \u3010{CITATION}\u3011" if state == "partial" else + f"The complete native inventory contains no comments. \u3010{CITATION}\u3011" if state == "complete" else + "The selected sources did not provide enough information to answer.", "augmented": True, "hybrid_citations": [{"citation_id": CITATION, "document_id": DOCUMENT, - "file_name": "Guide.md", "page_number": 1}] if state == "partial" else [], + "file_name": name, "page_number": 1, + **({"evidence_kind": "native_annotation_coverage", + "location_label": "Location", "location_value": "Native annotation inventory"} + if state == "complete" else {})}] if state in ("partial", "complete") else [], "metadata": {"evidence_status": { "version": 1, "scope": "returned_evidence", "state": state, "sources": sources, - "suggestions": ["Add a relevant document or select another authorized source."], + "suggestions": [] if state == "complete" else ["Add a relevant document or select another authorized source."], }}, } @@ -54,7 +60,7 @@ def reply(state, identifier="evidence-answer"): @pytest.mark.ui @pytest.mark.parametrize("width", [390, 1440]) @pytest.mark.parametrize("theme", ["light", "dark"]) -@pytest.mark.parametrize("state", ["partial", "missing", "unavailable"]) +@pytest.mark.parametrize("state", ["complete", "partial", "missing", "unavailable"]) def test_evidence_notice_opens_scoped_source_details_and_document_pane(page, width, theme, state): harness_build.ensure_bundle() errors = [] @@ -85,18 +91,27 @@ def test_evidence_notice_opens_scoped_source_details_and_document_pane(page, wid }); H.mount('mount-a', 'MessageList'); }""", {"message": reply(state), "conversation": CONVERSATION}) - button = page.get_by_role("button", name=f"{LABELS[state]}. View source coverage", exact=True) + if state == "complete": + expect(page.get_by_text("No supporting document evidence returned", exact=False)).to_have_count(0) + button = page.get_by_role("button", name="Show sources (1)", exact=True) + else: + button = page.get_by_role("button", name=f"{LABELS[state]}. View source coverage", exact=True) expect(button).to_be_visible() page.evaluate("(dark) => document.documentElement.classList.toggle('dark', dark)", theme == "dark") - expect(button).to_have_attribute("data-evidence-state", state) + if state != "complete": + expect(button).to_have_attribute("data-evidence-state", state) expect(button).to_have_attribute("aria-controls", "message-inspector-evidence-answer") button.focus() page.keyboard.press("Enter") expect(button).to_have_attribute("aria-expanded", "true") details = page.locator("#mount-a").get_by_role("region", name="Evidence for this answer") expect(details).to_be_visible() - expect(details).to_contain_text("Guide.md") - expect(details).to_contain_text("Add a relevant document") + expect(details).to_contain_text("Review.docx" if state == "complete" else "Guide.md") + if state == "complete": + expect(details).to_contain_text("Inspection completed with no matching records") + expect(details).not_to_contain_text("0 evidence items returned") + else: + expect(details).to_contain_text("Add a relevant document") if (state, width, theme) == ("missing", 390, "dark"): artifacts = ROOT / "ui_tests" / "artifacts" artifacts.mkdir(exist_ok=True) @@ -116,7 +131,7 @@ def test_evidence_notice_opens_scoped_source_details_and_document_pane(page, wid }""") drawer = page.get_by_role("complementary", name="Conversation details") expect(drawer.get_by_role("region", name="Evidence for this answer")).to_be_visible() - expect(drawer).to_contain_text("Guide.md") + expect(drawer).to_contain_text("Review.docx" if state == "complete" else "Guide.md") if state == "partial": expect(page.locator("#message-evidence-answer")).not_to_contain_text(CITATION) expect(drawer).to_contain_text("Additional notes.md") @@ -140,7 +155,8 @@ def test_evidence_notice_opens_scoped_source_details_and_document_pane(page, wid }""", mask) expect(page.locator("#mount-a").get_by_role("region", name="Evidence for this answer")).to_have_count(0) expect(drawer.get_by_role("region", name="Evidence for this answer")).to_have_count(0) - expect(button).to_have_count(0) + if state != "complete": + expect(button).to_have_count(0) page.evaluate("""() => { const H = window.OrchHarness; H.stores.chat.useChatStore.setState({