diff --git a/.github/workflows/cosmos-response-contract-check.yml b/.github/workflows/cosmos-response-contract-check.yml index 33d0bd068..f1e7caa85 100644 --- a/.github/workflows/cosmos-response-contract-check.yml +++ b/.github/workflows/cosmos-response-contract-check.yml @@ -40,5 +40,12 @@ jobs: run: python -c "import tiktoken; tiktoken.get_encoding('cl100k_base'); tiktoken.get_encoding('o200k_base')" - name: Require reviewed storage owners and doubles run: python scripts/check_cosmos_response_boundaries.py + - name: Verify real plugin discovery and document-action runtime + run: >- + python -m pytest -q -p no:cacheprovider -p test_support.cosmos_network_guard + --disable-warnings --tb=short + functional_tests/test_plugin_discovery_identity.py + functional_tests/test_orchestration_document_action_end_to_end.py + functional_tests/test_orchestration_document_loader_reliability.py - name: Run contracts with external network blocked before collection run: python scripts/check_cosmos_response_boundaries.py --run-contracts diff --git a/application/single_app/config.py b/application/single_app/config.py index b64e3d74a..4e90e6aef 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.422" +VERSION = "0.261.423" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_action_manifest.py b/application/single_app/functions_action_manifest.py index bdf031bf9..ae558fff4 100644 --- a/application/single_app/functions_action_manifest.py +++ b/application/single_app/functions_action_manifest.py @@ -52,6 +52,14 @@ def is_mcp_action(manifest): return resolve_action_type(manifest) == "mcp" +def is_document_search_action(manifest): + """Only registered native Document Search types can bind workspace sources.""" + declared = resolve_action_type(manifest) + return re.sub(r"[\s_-]", "", declared).lower() in { + "documentsearch", "documentsearchplugin", "search", + } + + def is_retired_mcp_stdio(manifest): """Inspect historical records without running active-config normalization.""" if not isinstance(manifest, dict): diff --git a/application/single_app/functions_appinsights.py b/application/single_app/functions_appinsights.py index caa1392de..1c023410a 100644 --- a/application/single_app/functions_appinsights.py +++ b/application/single_app/functions_appinsights.py @@ -135,12 +135,14 @@ LOGGER_WORKFLOW_HASH_KEYS = frozenset({ "conversationidhash", "turnidhash", "runidhash", "stepidhash", "producerstepidhash", "documentidhash", "plannerbindinghash", "documentsethash", + "consumerrunidhash", }) # Application-owned snake_case codes; any other text under these keys is dropped. LOGGER_WORKFLOW_CODE_KEYS = frozenset({ "validationcode", "validationrule", "responsefailure", "executioncode", "outputcode", "durablestatus", "failurecode", "authorityreason", "capabilityid", "outputformat", "producercapabilityid", "executionpath", "outcome", + "schemakeyword", "argumentfield", "resultkind", "sourceformat", }) diff --git a/application/single_app/functions_office_media.py b/application/single_app/functions_office_media.py index 13713b355..d327076a0 100644 --- a/application/single_app/functions_office_media.py +++ b/application/single_app/functions_office_media.py @@ -754,7 +754,7 @@ def _extract_docx_comment_paragraph_text(paragraph_elem): return '\n'.join(line for line in lines if line).strip() -def build_docx_comment_anchors_and_offsets(archive): +def build_docx_comment_anchors_and_offsets(archive, *, strict=False, anchor_completeness=None): """Return ``(comment_word_offsets, comment_anchors, total_body_words)`` from ``word/document.xml``. Walking ``word/document.xml`` in document order captures both the reading-order word offset @@ -768,16 +768,22 @@ def build_docx_comment_anchors_and_offsets(archive): archive, DOCX_MAIN_DOCUMENT_PART, OFFICE_DOCUMENT_PART_MAX_BYTES ) if document_bytes is None: + if strict: + raise ValueError('word_comment_anchors_unavailable') return comment_word_offsets, completed_anchors, 0 - try: document_root = defused_fromstring(document_bytes) except (DefusedParseError, ValueError): + if strict: + raise ValueError('word_comment_anchors_invalid') from None return comment_word_offsets, completed_anchors, 0 + if strict and _xml_local_name(document_root.tag) != 'document': + raise ValueError('word_comment_anchors_invalid') word_count = 0 active_ranges = {} active_range_lengths = {} + clipped_ranges = set() for element in document_root.iter(): local_name = _xml_local_name(element.tag) @@ -788,6 +794,8 @@ def build_docx_comment_anchors_and_offsets(archive): if comment_id not in completed_anchors and comment_id not in active_ranges: active_ranges[comment_id] = [] active_range_lengths[comment_id] = 0 + if anchor_completeness is not None: + anchor_completeness[comment_id] = False continue if local_name == 'commentRangeEnd': @@ -799,6 +807,9 @@ def build_docx_comment_anchors_and_offsets(archive): completed_anchors.setdefault( comment_id, anchor_text[:OFFICE_COMMENT_ANCHOR_MAX_CHARS] ) + if anchor_completeness is not None: + anchor_completeness[comment_id] = comment_id not in clipped_ranges + clipped_ranges.discard(comment_id) continue if local_name == 'commentReference': @@ -814,6 +825,8 @@ def build_docx_comment_anchors_and_offsets(archive): for comment_id, parts in active_ranges.items(): current_len = active_range_lengths.get(comment_id, 0) remaining = OFFICE_COMMENT_ANCHOR_MAX_CHARS - current_len + if len(text_value) > remaining: + clipped_ranges.add(comment_id) if remaining > 0: snippet = text_value[:remaining] parts.append(snippet) @@ -826,6 +839,8 @@ def build_docx_comment_anchors_and_offsets(archive): if current_len < OFFICE_COMMENT_ANCHOR_MAX_CHARS: parts.append(' ') active_range_lengths[comment_id] = current_len + 1 + else: + clipped_ranges.add(comment_id) for comment_id, parts in active_ranges.items(): anchor_text = re.sub(r'\s+', ' ', ''.join(parts)).strip() @@ -835,19 +850,25 @@ def build_docx_comment_anchors_and_offsets(archive): return comment_word_offsets, completed_anchors, word_count -def _parse_docx_comments_extended(archive): +def _parse_docx_comments_extended(archive, *, strict=False): """Parse ``word/commentsExtended.xml`` for reply parent links and resolved state.""" extended_by_para_id = {} extended_bytes = _read_zip_entry_bounded( archive, DOCX_COMMENTS_EXTENDED_PART, OFFICE_COMMENTS_PART_MAX_BYTES ) if extended_bytes is None: + if strict and DOCX_COMMENTS_EXTENDED_PART in archive.namelist(): + raise ValueError('word_comment_threads_unavailable') return extended_by_para_id try: extended_root = defused_fromstring(extended_bytes) except (DefusedParseError, ValueError): + if strict: + raise ValueError('word_comment_threads_invalid') from None return extended_by_para_id + if strict and _xml_local_name(extended_root.tag) != 'commentsEx': + raise ValueError('word_comment_threads_invalid') for elem in extended_root.iter(): if _xml_local_name(elem.tag) != 'commentEx': @@ -860,28 +881,38 @@ def _parse_docx_comments_extended(archive): extended_by_para_id[para_id] = { 'parent_para_id': parent_para_id or None, 'done': done_raw in ('1', 'true', 'on'), + 'done_available': done_raw in ('1', 'true', 'on', '0', 'false', 'off'), } return extended_by_para_id -def extract_docx_comments(file_path): +def extract_docx_comments(file_path, *, strict=False): """Extract structured review comments from an OOXML Word package (``.docx``, ``.docm``, ``.dotx``). Returns ``(comments, total_body_words)`` where ``comments`` is a list of top-level comment dicts (with nested ``replies`` when ``word/commentsExtended.xml`` links reply threads). + Native inspection uses ``strict`` to distinguish an inspected empty package from + unavailable, malformed or over-limit parts; legacy ingestion behavior is unchanged. """ if not file_path or not zipfile.is_zipfile(file_path): + if strict: + raise ValueError('word_comment_package_invalid') return [], 0 try: with zipfile.ZipFile(file_path) as archive: entry_names = archive.namelist() if len(entry_names) > OFFICE_ZIP_MAX_ENTRIES: + if strict: + raise ValueError('word_comment_package_limit') return [], 0 + anchor_completeness = {} comment_word_offsets, comment_anchors, total_body_words = ( - build_docx_comment_anchors_and_offsets(archive) + build_docx_comment_anchors_and_offsets( + archive, strict=strict, anchor_completeness=anchor_completeness, + ) ) if DOCX_COMMENTS_PART not in entry_names: return [], total_body_words @@ -890,14 +921,20 @@ def extract_docx_comments(file_path): archive, DOCX_COMMENTS_PART, OFFICE_COMMENTS_PART_MAX_BYTES ) if comments_bytes is None: + if strict: + raise ValueError('word_comments_unavailable') return [], total_body_words try: comments_root = defused_fromstring(comments_bytes) except (DefusedParseError, ValueError): + if strict: + raise ValueError('word_comments_invalid') from None return [], total_body_words + if strict and _xml_local_name(comments_root.tag) != 'comments': + raise ValueError('word_comments_invalid') - extended_by_para_id = _parse_docx_comments_extended(archive) + extended_by_para_id = _parse_docx_comments_extended(archive, strict=strict) raw_comments = [] comments_by_id = {} @@ -907,11 +944,17 @@ def extract_docx_comments(file_path): if _xml_local_name(element.tag) != 'comment': continue if len(raw_comments) >= OFFICE_COMMENTS_MAX_COUNT: + if strict: + raise ValueError('word_comment_inventory_limit') break comment_id = _xml_attr_by_local_name(element, 'id').strip() if not comment_id: + if strict: + raise ValueError('word_comment_identity_invalid') comment_id = str(len(raw_comments) + 1) + if strict and comment_id in comments_by_id: + raise ValueError('word_comment_identity_invalid') author = _xml_attr_by_local_name(element, 'author').strip() initials = _xml_attr_by_local_name(element, 'initials').strip() @@ -936,7 +979,7 @@ def extract_docx_comments(file_path): paragraph_texts.append(fallback_text) comment_text = '\n'.join(paragraph_texts).strip() - if not comment_text: + if not comment_text and not strict: continue word_offset = comment_word_offsets.get(comment_id) @@ -950,9 +993,14 @@ def extract_docx_comments(file_path): 'word_offset': word_offset, 'position_known': word_offset is not None, 'parent_id': None, - 'done': False, + 'done': None if strict else False, 'replies': [], '_para_ids': para_ids, + **({ + 'resolution_available': False, + 'anchor_complete': bool(comment_anchors.get(comment_id)) and anchor_completeness.get(comment_id, False), + 'anchor_character_limit': OFFICE_COMMENT_ANCHOR_MAX_CHARS, + } if strict else {}), } raw_comments.append(comment_record) comments_by_id[comment_id] = comment_record @@ -962,7 +1010,10 @@ def extract_docx_comments(file_path): ext_info = extended_by_para_id.get(para_id) if not ext_info: continue - if ext_info.get('done'): + if strict and ext_info.get('done_available'): + comment_record['done'] = ext_info['done'] + comment_record['resolution_available'] = True + elif not strict and ext_info.get('done'): comment_record['done'] = True parent_para_id = ext_info.get('parent_para_id') if parent_para_id: @@ -976,6 +1027,8 @@ def _resolve_root_comment(record): while current.get('parent_id') and current['parent_id'] in comments_by_id: parent_id = current['parent_id'] if parent_id in visited: + if strict: + raise ValueError('word_comment_thread_cycle') break visited.add(parent_id) current = comments_by_id[parent_id] @@ -994,15 +1047,20 @@ def _resolve_root_comment(record): 'date': comment_record['date'], 'text': comment_record['text'], 'done': comment_record['done'], + **({ + 'resolution_available': comment_record['resolution_available'], + } if strict else {}), }) - if root_comment.get('word_offset') is None and comment_record.get('word_offset') is not None: + if not strict and root_comment.get('word_offset') is None and comment_record.get('word_offset') is not None: root_comment['word_offset'] = comment_record['word_offset'] root_comment['position_known'] = True - if not root_comment.get('anchor_text') and comment_record.get('anchor_text'): + if not strict and not root_comment.get('anchor_text') and comment_record.get('anchor_text'): root_comment['anchor_text'] = comment_record['anchor_text'] return top_level_comments, total_body_words except (zipfile.BadZipFile, FileNotFoundError, OSError): + if strict: + raise ValueError('word_comment_package_unavailable') from None return [], 0 diff --git a/application/single_app/functions_orchestration_actions.py b/application/single_app/functions_orchestration_actions.py index 0206edac1..ec2184865 100644 --- a/application/single_app/functions_orchestration_actions.py +++ b/application/single_app/functions_orchestration_actions.py @@ -5,13 +5,15 @@ kernel holds only the built-in chart tools, never the action's own functions, so saved visual preferences can be applied there without reaching calls to the integration. -Version: 0.261.321 +Version: 0.261.421 Microsoft 365 actions authorized for their own step in: 0.261.238 SharePoint and OneDrive steps bind their model's token budget and analysis model in: 0.261.300 Explicit operation intent and named input consumption added in: 0.261.321 +Canonical plugin identity and safe action-stage failures added in: 0.261.421 """ import asyncio +import hashlib import inspect import json import logging @@ -27,7 +29,8 @@ agent_execution, ) from functions_action_catalog import resolve_action_manifest -from functions_appinsights import log_event +from functions_action_manifest import is_document_search_action +from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from functions_chart_operations import ( CHART_PLUGIN_TYPE, CORE_CHART_PLUGIN_NAME, @@ -57,11 +60,15 @@ resolve_available_capability_ids, ) from functions_orchestration_visuals import gathering_visual_addendum, instruction_memory_messages +from functions_workflow_document_selection import selected_document_sources RETRIEVED_CHARTS_PLUGIN_NAME = 'retrieved_data_charts' CHART_STEP_CALL_LIMIT = 4 CHART_STEP_RESULT_SUMMARY_LIMIT = 8000 +DOCUMENT_SOURCE_READ_FUNCTIONS = frozenset({ + 'search_documents', 'retrieve_document_chunks', 'get_document_accessibility_report', 'summarize_document', +}) RETRIEVED_ROWS_CHART_KINDS = ('line', 'area', 'bar', 'stacked_line', 'stacked_bar') CHART_STEP_INSTRUCTIONS = ( 'Create the chart or charts this knowledge-collection step needs, using only the supplied chart ' @@ -81,6 +88,10 @@ class ActionExecutionError(RuntimeError): """A user-safe failure of the selected action.""" + def __init__(self, message="", *, failure_code=None): + super().__init__(message) + self.orchestration_failure_code = failure_code + def _check_access(settings, catalog, action_ref): # The invocation's own source preflight owns the runtime services; this checks the @@ -517,7 +528,7 @@ def _step_chart_titles(invocations): async def invoke_action( action_ref, task, context, *, settings, user_id, cancel_requested, invocation_capture=None, visual_request=None, m365_request_key=None, m365_origin=None, - execution_intent='gather', named_inputs=None, + execution_intent='gather', named_inputs=None, document_sources=None, document_source_check=None, ): """Run only this action's enabled functions and return its findings and invocation scope. @@ -598,6 +609,33 @@ async def invoke_action( user_groups=getattr(context, 'active_group_ids', None) or None, user_roles=list(identity.roles), ) + bound_ids = [] + if document_sources is not None: + if ( + not is_document_search_action(manifest) + or not isinstance(document_sources, list) or not document_sources + or not callable(document_source_check) + or any( + not isinstance(source, dict) or source.get('scope') not in ('personal', 'group', 'public') + or not isinstance(source.get('document_id'), str) or not source['document_id'] + for source in document_sources + ) + ): + raise ActionExecutionError('The document action source binding is unavailable.') + bound_ids = [source['document_id'] for source in document_sources] + if len(set(bound_ids)) != len(bound_ids): + raise ActionExecutionError('The document action source binding is ambiguous.') + document_source_check() + m365_scope.enter_context(selected_document_sources([ + {'document': { + 'document_id': source['document_id'], 'scope_type': source['scope'], + 'scope_id': source.get('scope_id') or user_id, + }} for source in document_sources + ], user_id, restrict_ids=True)) + # Native readers must surface authority failures rather than use legacy fallbacks. + from content_screening.access import strict_source_authority + + m365_scope.enter_context(strict_source_authority()) if invocation_capture is not None: current_settings = deepcopy(current_settings) manifest = deepcopy(manifest) @@ -640,6 +678,32 @@ def capture_manifest(value, observed_settings): selector=action_ref, ) + def trace_document_read(invocation, outcome, call_index, *, payload=None, error=None): + if not is_debug_enabled(): + return + operation = getattr(invocation.function, 'name', '') + value = payload if isinstance(payload, dict) else {} + document_id = invocation.arguments.get('document_id') + log_event( + '[ORCHESTRATION_ADAPTERS] Document action read checkpoint.', + extra={ + **workflow_log_context( + conversation_id=getattr(context, 'conversation_id', None), + turn_id=getattr(context, 'turn_id', None), run_id=getattr(context, 'run_id', None), + step_id=m365_origin.get('step_id') if isinstance(m365_origin, dict) else None, + ), + 'stage': 'document_action_read', 'outcome': outcome, + 'operation': operation if operation in DOCUMENT_SOURCE_READ_FUNCTIONS else 'unknown', + 'function_call_index': call_index, 'bound_source_count': len(bound_ids), + 'result_is_object': isinstance(payload, dict), + 'native_metadata_present': isinstance(value.get('accessibility_metadata'), dict), + 'citation_present': isinstance(value.get('citation'), str) and bool(value['citation']), + 'error_type': type(error).__name__ if error is not None else None, + **({'document_id_hash': hashlib.sha256(document_id.encode('utf-8', errors='replace')).hexdigest()} + if isinstance(document_id, str) and document_id in bound_ids else {}), + }, + ) + async def guard_function(invocation, next): nonlocal calls, failure # Providers can return a batch despite parallel_tool_calls=False. Serialize it @@ -654,6 +718,9 @@ async def guard_function(invocation, next): if calls >= limit: failure = ActionExecutionError('The action function-call limit was reached.') raise failure + call_index = calls + 1 + if bound_ids: + trace_document_read(invocation, 'started', call_index) try: current = get_settings() _check_access(current, catalog, action_ref) @@ -666,6 +733,19 @@ async def guard_function(invocation, next): raise ActionExecutionError('The selected action changed during execution.') if invocation_capture is not None: capture_manifest(fresh, current) + if bound_ids: + document_source_check() + function = getattr(invocation.function, 'name', '') + if function == 'search_documents': + requested = _split_list(invocation.arguments.get('document_ids')) + if set(requested) - set(bound_ids): + raise ActionExecutionError('The action requested an unselected document.') + invocation.arguments['document_ids'] = ','.join(requested or bound_ids) + elif function in DOCUMENT_SOURCE_READ_FUNCTIONS: + if invocation.arguments.get('document_id') not in bound_ids: + raise ActionExecutionError('The action requested an unselected document.') + else: + raise ActionExecutionError('This function does not support the document source binding.') calls += 1 for reader in getattr(context, 'integration_input_readers', {}).values(): reader.recheck() @@ -681,6 +761,8 @@ async def guard_function(invocation, next): failure = step_error(exc) raise failure from exc except Exception as exc: + if bound_ids: + trace_document_read(invocation, 'failed', call_index, error=exc) credential_stop = find_credential_stop(exc) if credential_stop is not None: failure = credential_stop @@ -704,6 +786,21 @@ async def guard_function(invocation, next): # A refused Microsoft 365 call is not data: the model would report it as findings. failure = refusal raise failure + document_error = ( + bound_ids and invocation.result is not None + and isinstance(invocation.result.value, dict) and invocation.result.value.get('error') + ) + if bound_ids: + trace_document_read( + invocation, 'failed' if document_error else 'completed', call_index, + payload=invocation.result.value if invocation.result is not None else None, + ) + if document_error: + failure = ActionExecutionError( + 'The selected document could not be read by this action.', + failure_code='document_action_read_failed', + ) + raise failure if invocation.result is not None: outputs.append(invocation.result.value) retrieved.append({ @@ -740,10 +837,25 @@ async def stop_after_failure(invocation, next): } capture_manifest(manifest, current_settings) if not loader.load_plugin_from_manifest(prepared, user_id): - raise ActionExecutionError('The selected action could not be loaded.') + raise ActionExecutionError( + 'The selected action could not be loaded.', + failure_code='action_initialization_failed', + ) + if bound_ids: + # The actual native reader, not a configured name or description, owns these reads. + from semantic_kernel_plugins.document_search_plugin import DocumentSearchPlugin + + if not any(isinstance(plugin, DocumentSearchPlugin) for plugin in loader.plugin_instances): + raise ActionExecutionError( + 'The action does not provide the native document reader.', + failure_code='document_reader_identity_mismatch', + ) functions = kernel.get_list_of_function_metadata({}) if not functions: - raise ActionExecutionError('The selected action has no enabled functions.') + raise ActionExecutionError( + 'The selected action has no enabled functions.', + failure_code='action_initialization_failed', + ) if cancel_requested(): raise AgentExecutionCancelled('Action execution was cancelled.') kernel.add_filter(FilterTypes.FUNCTION_INVOCATION, guard_function) @@ -758,7 +870,10 @@ async def stop_after_failure(invocation, next): service = _build_action_model(current_settings, context, user_id) kernel.add_service(service) if not getattr(service, 'SUPPORTS_FUNCTION_CALLING', False): - raise ActionExecutionError('The selected model does not support action functions.') + raise ActionExecutionError( + 'The selected model does not support action functions.', + failure_code='action_initialization_failed', + ) if file_source: # Chat binds these through the selected agent; this step's own model stands in. kernel.add_filter(FilterTypes.AUTO_FUNCTION_INVOCATION, file_step_filter( @@ -789,6 +904,10 @@ async def stop_after_failure(invocation, next): 'task': task, 'execution_intent': execution_intent, 'named_inputs': named_inputs, + **({'document_sources': [ + {key: source.get(key) for key in ('document_id', 'scope', 'scope_id')} + for source in document_sources + ]} if bound_ids else {}), })) require_integration_model_budget( model_budget, [message.to_dict() for message in history.messages], @@ -839,7 +958,10 @@ async def stop_after_failure(invocation, next): if invocation_capture is not None: invocation_capture.require_valid(captured=True) if not calls or not outputs: - raise ActionExecutionError('The action did not return any function results.') + raise ActionExecutionError( + 'The action did not return any function results.', + failure_code='action_results_missing', + ) findings = '\n\n'.join(str(reply) for reply in replies or [] if str(reply).strip()) if not findings: findings = '\n\n'.join( diff --git a/application/single_app/functions_orchestration_adapters.py b/application/single_app/functions_orchestration_adapters.py index 321010986..ebb2cd5b9 100644 --- a/application/single_app/functions_orchestration_adapters.py +++ b/application/single_app/functions_orchestration_adapters.py @@ -58,7 +58,7 @@ import logging from copy import deepcopy -from functions_appinsights import log_event +from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from functions_orchestration_context import build_elicitation_user_request, execution_conversation_messages from functions_orchestration_execution_policy import orchestration_file_policy from functions_orchestration_invocation_capture import ( @@ -666,7 +666,7 @@ def validate_source(): def run_document_search(step, context, *, settings, user_id, emit, cancel_requested): arguments = _arguments(step) if arguments.get('native_section') == 'comments': - return _run_excel_comments( + return _run_native_comments( step, context, settings=settings, user_id=user_id, emit=emit, cancel_requested=cancel_requested, ) @@ -834,17 +834,23 @@ def run_document_search(step, context, *, settings, user_id, emit, cancel_reques # document_analyze -> functions_document_analysis.run_document_analysis # -------------------------------------------------------------------------------------- -def _run_excel_comments(step, context, *, settings, user_id, emit, cancel_requested): +def _run_native_comments(step, context, *, settings, user_id, emit, cancel_requested): # Search/storage initialization belongs at execution, not registry discovery. - from functions_orchestration_excel import collect_excel_comments + from functions_orchestration_excel import collect_excel_comments, collect_word_comments from functions_search_service import get_document_accessibility_payload + correlation = workflow_log_context( + conversation_id=_ctx(context, 'conversation_id'), run_id=_ctx(context, 'run_id'), + turn_id=_ctx(context, 'turn_id'), step_id=step.get('step_id'), + ) + failure_rule, source_format = 'native_source_binding', None try: identifiers = _resolve_step_document_ids( step, context, settings=settings, capability_id=CAPABILITY_DOCUMENT_SEARCH, ) if not identifiers: - raise ValueError('Native annotation inspection requires selected workbooks.') + raise ValueError('Native annotation inspection requires selected documents.') + failure_rule = 'native_source_authorization' manifest = resolve_context_source_manifest( context, identifiers, settings=settings, user_id=user_id, cancel_requested=cancel_requested, ) @@ -852,22 +858,42 @@ def _run_excel_comments(step, context, *, settings, user_id, emit, cancel_reques [source.get('document_id') for source in manifest] != identifiers or any(source.get('authorization_status') != 'authorized' for source in manifest) ): - raise PermissionError('The selected workbooks are unavailable.') - _emit(emit, _progress(step, CAPABILITY_DOCUMENT_SEARCH, 'Reviewing workbook comments')) + raise PermissionError('The selected documents are unavailable.') + _emit(emit, _progress(step, CAPABILITY_DOCUMENT_SEARCH, 'Reviewing native document comments')) results, coverage, citations = [], [], [] for source in manifest: - if not _text(source.get('file_name')).lower().endswith(('.xlsx', '.xlsm', '.xls')): - raise ValueError('Native annotation inspection requires an Excel workbook.') - records, source_coverage = collect_excel_comments( + failure_rule = 'native_comments_format_unsupported' + extension = _text(source.get('file_name')).lower().rsplit('.', 1)[-1] + source_format = extension if extension in ('xlsx', 'xlsm', 'xls', 'docx', 'docm', 'dotx') else 'unsupported' + if source_format == 'unsupported': + raise ValueError('Native comments require a supported Word or Excel document.') + word = source_format in ('docx', 'docm', 'dotx') + collector = collect_word_comments if word else collect_excel_comments + log_event('[ORCHESTRATION_ADAPTERS] Native comment read checkpoint.', extra={ + **correlation, 'stage': 'native_comment_read', 'outcome': 'started', + 'source_format': source_format, 'selected_document_count': len(manifest), + }) + failure_rule = 'native_comment_inventory_invalid' + records, source_coverage = collector( source['document_id'], user_id, read_page=get_document_accessibility_payload, - cancel_check=lambda: raise_if_mixed_source_cancelled(cancel_requested, 'excel_comments'), + cancel_check=lambda: raise_if_mixed_source_cancelled(cancel_requested, 'native_comments'), source_count=len(manifest), conversation_id=_ctx(context, 'conversation_id'), doc_scope=source['scope'], active_group_ids=[source['scope_id']] if source['scope'] == 'group' else None, active_public_workspace_id=source['scope_id'] if source['scope'] == 'public' else None, ) + source_coverage['format'] = source_format + log_event('[ORCHESTRATION_ADAPTERS] Native comment read checkpoint.', extra={ + **correlation, 'stage': 'native_comment_read', 'outcome': 'completed', + 'source_format': source_format, 'returned_record_count': len(records), + 'total_record_count': source_coverage.get('total_records'), + **({ + 'returned_reply_count': sum(len(record['replies']) for record in records), + } if word and all(type(record.get('replies')) is list for record in records) else {}), + 'partial': bool(source_coverage.get('partial')), + }) coverage.append({'document_id': source['document_id'], **source_coverage}) - # Include an explicit coverage record even for an inspected, empty workbook. + # An inspected empty inventory is evidence; an unavailable read never reaches retention. results.append({ 'document_id': source['document_id'], 'file_name': source.get('file_name'), 'id': f"{source['document_id']}:native-comments:coverage", @@ -883,21 +909,24 @@ def _run_excel_comments(step, context, *, settings, user_id, emit, cancel_reques results.append({ 'document_id': source['document_id'], 'file_name': source.get('file_name'), 'id': f"{source['document_id']}:native-comments:{index}", - 'chunk_text': json.dumps({'excel_annotation': record}, ensure_ascii=False), + 'chunk_text': json.dumps({'word_annotation' if word else 'excel_annotation': record}, ensure_ascii=False), 'sheet': record.get('sheet'), 'cell': record.get('cell'), }) citations.append({ 'source_type': 'document', 'document_id': source['document_id'], 'file_name': source.get('file_name'), 'sheet': record.get('sheet'), 'cell': record.get('cell'), 'annotation_id': record.get('id'), - 'evidence_kind': 'native_excel_annotation', + 'evidence_kind': 'native_word_annotation' if word else 'native_excel_annotation', 'citation_id': results[-1]['id'], 'sheet_name': record.get('sheet'), 'location_label': 'Location', 'location_value': ( - f"{record['sheet']}!{record['cell']}" - if record.get('sheet') and record.get('cell') else 'Workbook annotation' + f"Word comment {record['id']}" if word else ( + f"{record['sheet']}!{record['cell']}" + if record.get('sheet') and record.get('cell') else 'Workbook annotation' + ) ), }) + failure_rule = 'native_source_changed' current_manifest = resolve_context_source_manifest( context, identifiers, settings=settings, user_id=user_id, cancel_requested=cancel_requested, ) @@ -905,20 +934,26 @@ def _run_excel_comments(step, context, *, settings, user_id, emit, cancel_reques if any(source.get('authorization_status') != 'authorized' for source in current_manifest) or [ tuple(source.get(key) for key in identity_fields) for source in current_manifest ] != [tuple(source.get(key) for key in identity_fields) for source in manifest]: - raise PermissionError('The selected workbook changed during inspection.') + raise PermissionError('The selected document changed during inspection.') + failure_rule = 'native_result_retention' task = retain_document_search_result( step, context, results, source_manifest=manifest, citations=citations, search_scope={'native_inspection': coverage}, ) return build_step_result( status=STEP_STATUS_PARTIAL if task.status == 'partial' else STEP_STATUS_COMPLETED, - summary=f'Retained native annotation evidence from {len(manifest)} workbook(s).', + summary=f'Retained native annotation evidence from {len(manifest)} document(s).', task_result=task, citations=citations, ) except MixedSourceCancellationError: - return _cancelled_result('Workbook inspection was cancelled.') + return _cancelled_result('Native comment inspection was cancelled.') except Exception as exc: - return _failed_result('The workbook annotation inventory could not be read.', exc) + log_event('[ORCHESTRATION_ADAPTERS] Native comment read checkpoint.', level=logging.WARNING, extra={ + **correlation, 'stage': 'native_comment_read', 'outcome': 'failed', + 'source_format': source_format, 'validation_rule': failure_rule, + 'error_type': type(exc).__name__, + }) + return _failed_result('The native document annotation inventory could not be read.', exc) def _resolve_step_document_ids(step, context, *, settings=None, capability_id=None): @@ -2266,6 +2301,62 @@ def run_action_invoke(step, context, *, settings, user_id, emit, cancel_requeste settings = deepcopy(settings) access('action', settings=settings, selector=action_ref) invocation_kwargs = {} + if 'document_ids' in arguments: + document_ids = arguments['document_ids'] + binding_log_context = workflow_log_context( + conversation_id=_ctx(context, 'conversation_id'), turn_id=_ctx(context, 'turn_id'), + run_id=_ctx(context, 'run_id'), step_id=step['step_id'], + ) + if is_debug_enabled(): + log_event('[ORCHESTRATION_ADAPTERS] Document action binding started.', extra={ + **binding_log_context, 'stage': 'document_action_binding', + 'selected_source_count': len(document_ids), + }) + sources = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, + cancel_requested=cancel_requested, + ) + if ( + not document_ids or [source.get('document_id') for source in sources] != document_ids + or any(source.get('authorization_status') != 'authorized' for source in sources) + ): + raise PermissionError('The selected document sources are unavailable.') + if is_debug_enabled(): + log_event('[ORCHESTRATION_ADAPTERS] Document action binding verified.', extra={ + **binding_log_context, 'stage': 'document_action_binding', + 'selected_source_count': len(document_ids), 'source_count': len(sources), + 'source_authorized': True, + }) + + def recheck_document_sources(): + fresh = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, + cancel_requested=cancel_requested, + ) + same_count = len(fresh) == len(sources) + same_identity = same_count and all( + all(current.get(key) == original.get(key) for key in ('document_id', 'scope', 'scope_id')) + for current, original in zip(fresh, sources) + ) + same_revision = same_count and all( + all(current.get(key) == original.get(key) for key in ('source_version', 'source_revision')) + for current, original in zip(fresh, sources) + ) + authorized = all(source.get('authorization_status') == 'authorized' for source in fresh) + valid = same_identity and same_revision and authorized + if is_debug_enabled(): + log_event('[ORCHESTRATION_ADAPTERS] Document action binding rechecked.', extra={ + **binding_log_context, 'stage': 'document_action_recheck', + 'source_count': len(fresh), 'expected_source_count': len(sources), + 'source_identity_matches': same_identity, 'source_revision_matches': same_revision, + 'source_authorized': authorized, 'outcome': 'completed' if valid else 'failed', + }) + if not valid: + raise PermissionError('The selected document sources changed or are unavailable.') + + invocation_kwargs.update( + document_sources=sources, document_source_check=recheck_document_sources, + ) if visual_request: invocation_kwargs['visual_request'] = visual_request invocation_kwargs['m365_request_key'], invocation_kwargs['m365_origin'] = _m365_step_identity(step, context) @@ -2301,13 +2392,30 @@ def run_action_invoke(step, context, *, settings, user_id, emit, cancel_requeste raise except Exception as exc: # This is the boundary for arbitrary plugin/provider code; never expose its exceptions. + failed_result = _failed_result('The action could not complete.', exc) + failure_code = failed_result['failure']['code'] + failure_stage = { + 'action_initialization_failed': 'action_initialization', + 'document_reader_identity_mismatch': 'action_initialization', + 'document_action_read_failed': 'document_action_read', + 'action_results_missing': 'action_results', + }.get(failure_code, 'action_invocation') log_event( f'{_LOG_PREFIX} Direct action execution failed.', level=logging.ERROR, - extra={'action_ref': action_ref, 'step_id': (step or {}).get('step_id'), - 'error_type': type(exc).__name__}, + extra={ + **workflow_log_context( + conversation_id=_ctx(context, 'conversation_id'), turn_id=_ctx(context, 'turn_id'), + run_id=_ctx(context, 'run_id'), step_id=step['step_id'], + ), + 'capability_id': CAPABILITY_ACTION_INVOKE, + 'stage': failure_stage, 'failure_code': failure_code, + 'error_type': type(exc).__name__, + **({'function_calls_attempted': failure_stage == 'document_action_read'} + if failure_stage in ('action_initialization', 'document_action_read') else {}), + }, ) - return _failed_result('The action could not complete.', exc) + return failed_result citations = _agent_citations( None, user_id, _ctx(context, 'conversation_id'), set(), root_id=result['root_id'], scoped_invocations=result['invocations'], diff --git a/application/single_app/functions_orchestration_composition.py b/application/single_app/functions_orchestration_composition.py index 1def7141e..791461bde 100644 --- a/application/single_app/functions_orchestration_composition.py +++ b/application/single_app/functions_orchestration_composition.py @@ -30,7 +30,7 @@ from jsonschema import Draft202012Validator from content_screening.contracts import ScreeningError -from functions_appinsights import log_event, workflow_log_context +from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from functions_chart_operations import INLINE_CHART_BLOCK_LANGUAGE, normalize_inline_chart_markdown from functions_diagram_operations import MERMAID_BLOCK_LANGUAGE from functions_document_citations import build_inline_citation_marker @@ -184,6 +184,16 @@ } +def _log_completeness_diagnostic(step, context, diagnostic, message): + log_event(message, level=logging.WARNING, extra={ + **workflow_log_context( + conversation_id=context.conversation_id, run_id=context.run_id, + turn_id=getattr(context, 'turn_id', None), step_id=step['step_id'], + ), + **diagnostic, 'stage': 'task_completeness_assessment', + }) + + def _quoted(value): return json.dumps(value, ensure_ascii=False) @@ -537,9 +547,9 @@ def checkpoint(value): validate=lambda value: [], checkpoint=checkpoint, saved_state=saved, recheck=recheck, owner_step_id=step['step_id'], evidence_basis='mixed' if legacy_evidence else 'request', legacy_evidence=legacy_evidence, - on_error=lambda error_type: log_event( + on_diagnostic=lambda diagnostic: _log_completeness_diagnostic( + step, context, diagnostic, '[ORCHESTRATION_EXECUTOR] Retained final-answer verification unavailable.', - level=logging.WARNING, extra={'error_type': error_type, 'step_id': step['step_id']}, ), ) except MixedSourceCancellationError: @@ -547,7 +557,11 @@ def checkpoint(value): except Exception as error: log_event( '[ORCHESTRATION_EXECUTOR] Retained final-answer verification could not complete.', - level=logging.WARNING, extra={'error_type': type(error).__name__, 'step_id': step['step_id']}, + level=logging.WARNING, extra={ + **workflow_log_context( + conversation_id=context.conversation_id, run_id=context.run_id, step_id=step['step_id'], + ), 'error_type': type(error).__name__, 'stage': 'task_completeness_assessment', + }, ) checked = {**result, 'task_completeness': public_completeness_report(state)} if state['status'] != 'complete': @@ -624,6 +638,17 @@ def recheck(): if source_cov is not None: entry['source_coverage'] = source_cov inputs[name] = entry + if is_debug_enabled(): + log_event('[ORCHESTRATION_EXECUTOR] Composition input contract verified.', extra={ + **workflow_log_context( + conversation_id=producer.conversation_id, turn_id=getattr(context, 'turn_id', None), + run_id=producer.run_id, step_id=producer.step_id, + ), + 'stage': 'compose_inputs', 'input_count': len(readers), + 'document_citation_count': sum(len(entry['document_citations']) for entry in inputs.values()), + 'declared_output_count': len(step['outputs']), + 'available_information': uses_available_information(step), + }) # A saved workflow's result reaches the model only as fenced, untrusted notes, read again # for this call; the retained value it replaces never does. That includes a run the plan # waited for, which only a setting-gated plan can have compose read. @@ -933,9 +958,9 @@ def save_completeness(state): ) else 'request' ), legacy_evidence=legacy_evidence, - on_error=lambda error_type: log_event( + on_diagnostic=lambda diagnostic: _log_completeness_diagnostic( + step, context, diagnostic, '[ORCHESTRATION_EXECUTOR] Task completeness verification unavailable.', - level=logging.WARNING, extra={'error_type': error_type, 'step_id': step['step_id']}, ), ) completeness_report = public_completeness_report(completeness_state) @@ -1061,6 +1086,16 @@ def save_completeness(state): ) if completeness_report: result['task_completeness'] = completeness_report + if is_debug_enabled(): + log_event('[ORCHESTRATION_EXECUTOR] Prepared content retained.', extra={ + **workflow_log_context( + conversation_id=producer.conversation_id, turn_id=getattr(context, 'turn_id', None), + run_id=producer.run_id, step_id=producer.step_id, + ), + 'stage': 'compose_retained', 'output_count': len(named), 'input_count': len(readers), + 'completeness_check_enabled': check_policy['enabled'], + 'available_information': uses_available_information(step), 'partial': partial, + }) return result except MixedSourceCancellationError: raise diff --git a/application/single_app/functions_orchestration_events.py b/application/single_app/functions_orchestration_events.py index a2a0515eb..c30829bef 100644 --- a/application/single_app/functions_orchestration_events.py +++ b/application/single_app/functions_orchestration_events.py @@ -34,6 +34,7 @@ """ import json +import re # Lane key matched by the client's activity lane table. ORCHESTRATION_LANE_KEY = 'orchestration' @@ -342,10 +343,19 @@ def build_content_event(content): return serialize_sse({'content': content}) -def build_error_event(message, conversation_id=None): +def build_error_event(message, conversation_id=None, *, code=None, stage=None, rule=None, attempt=None): payload = {'error': str(message or 'The request could not be completed.')} if conversation_id: payload['conversation_id'] = conversation_id + for key, value in (('code', code), ('error_stage', stage), ('error_rule', rule)): + if value is not None: + if not isinstance(value, str) or re.fullmatch(r'[a-z][a-z0-9_]{0,79}', value) is None: + raise ValueError('Invalid orchestration error classification.') + payload[key] = value + if attempt is not None: + if type(attempt) is not int or attempt < 1: + raise ValueError('Invalid orchestration planning attempt.') + payload['error_attempt'] = attempt return serialize_sse(payload) diff --git a/application/single_app/functions_orchestration_excel.py b/application/single_app/functions_orchestration_excel.py index 019e8406b..8b43ce841 100644 --- a/application/single_app/functions_orchestration_excel.py +++ b/application/single_app/functions_orchestration_excel.py @@ -1,11 +1,46 @@ # functions_orchestration_excel.py -"""Read authorized Excel annotation inventories, not worksheet-row previews.""" +"""Read exact native annotation inventories, preserving Excel paging and Word threads.""" import json from functions_orchestration_results import MAX_VALUE_BYTES +def collect_word_comments(document_id, user_id, *, read_page, cancel_check, source_count=1, **scope): + cancel_check() + report = read_page(document_id, user_id, sections="comments", **scope) + cancel_check() + inventory = report.get("word_inspection") + if type(inventory) is not dict or inventory.get("section") != "comments": + raise LookupError("The native Word comment inventory is unavailable.") + values, total = inventory.get("records"), inventory.get("total_records") + if ( + inventory.get("inventory_complete") is not True or type(values) is not list + or type(total) is not int or total != len(values) + or type(inventory.get("source_sha256")) is not str + or any(type(record) is not dict for record in values) + ): + raise ValueError("The native Word comment inventory has invalid coverage.") + budget = MAX_VALUE_BYTES // (4 * max(1, source_count)) + records, used = [], 0 + for record in values: + cancel_check() + size = len(json.dumps(record, ensure_ascii=False).encode("utf-8")) + if used + size > budget: + break + records.append(record) + used += size + partial = len(records) != total + return records, { + "execution_mode": "native_inspection", "section": "comments", "format": report.get("format"), + "source_sha256": inventory["source_sha256"], "total_records": total, + "returned_records": len(records), "total_replies": inventory.get("total_replies"), + "inventory_complete": True, "partial": partial, "pages_read": 1, + "has_more": partial, "limit_reached": partial, "next_cursor": None, + "limitations": inventory.get("limitations") or [], + } + + def collect_excel_comments(document_id, user_id, *, read_page, cancel_check, source_count=1, **scope): """Keep exact native records, including paginated large-record fragments.""" cursor = None diff --git a/application/single_app/functions_orchestration_executor.py b/application/single_app/functions_orchestration_executor.py index 878c5cdef..2e260ea27 100644 --- a/application/single_app/functions_orchestration_executor.py +++ b/application/single_app/functions_orchestration_executor.py @@ -48,7 +48,7 @@ from content_screening.access import assert_current_request_sources_available from content_screening.contracts import ScreeningError -from functions_appinsights import log_event, workflow_log_context +from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from public_chat_scope import public_chat_execution_scope from functions_mixed_source_orchestration import ( AUTHORIZATION_STATUS_AUTHORIZED, @@ -1069,7 +1069,18 @@ def _run_dependency_step( entry['document_id'] for entry in [*(result.get('evidence') or []), *(result.get('citations') or [])] if isinstance(entry, dict) and entry.get('document_id') )) if step['capability_id'] == 'document_search' else [] + bound_document_action = ( + step['capability_id'] == 'action_invoke' and bool(runtime_step['arguments'].get('document_ids')) + ) + if bound_document_action: + gathered_ids = list(document_ids) manifest = _dependency_source_manifest(context, gathered_ids, settings, after_return) + if is_debug_enabled(): + log_event('[ORCHESTRATION_EXECUTOR] Gather source handoff verified.', extra={ + **_step_log_context(context, step), 'stage': 'gather_source_handoff', + 'planned_source_count': len(document_ids), 'source_count': len(manifest), + 'bound_document_action': bound_document_action, + }) task = retain_gather_result(step, scoped, result, source_manifest=manifest) validate_task_outputs(step, context, task) if task.status == 'pending': @@ -1430,6 +1441,7 @@ def _execute_dependency_plan( plan, settings=settings, available_capability_ids=available, agent_names=[agent.get('name') for agent in context.agent_catalog or []], action_refs=[action.get('action_ref') for action in context.action_catalog], + actions=context.action_catalog, existing_results=context.result_aliases, composition_profiles=context.composition_profiles, export_catalog=context.export_catalog, contract_version=DEPENDENCY_PLAN_CONTRACT_VERSION, seeds=getattr(context, 'seeds', None), diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index dbebc25ba..0722b8237 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -42,6 +42,7 @@ from azure.identity import DefaultAzureCredential, get_bearer_token_provider from config import cognitive_services_scope +from functions_action_manifest import is_document_search_action from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from functions_document_analysis_results import narrative_analysis_options from functions_orchestration_context import conversation_reference_messages, resolve_elicitation_candidates @@ -101,8 +102,8 @@ PLANNER_MAX_TOKENS = 4000 PLANNER_TEMPERATURE = 0.1 -# Known declaration and source-binding failures share one correction round. -PLAN_REPAIR_ATTEMPTS = 1 +# All correctable draft failures share this allowance; the initial proposal is separate. +PLAN_REPAIR_ATTEMPTS = 2 PLANNER_DEBUG_MAX_ITEMS = 32 # A workflow proposal gets the same correction round; one that still fails is dropped from the # plan, and the rest of the plan runs without it. @@ -121,12 +122,23 @@ # A workflow proposal, run or results step whose check could not run is dropped at once: no # correction can fix it. WORKFLOW_CONTEXT_UNAVAILABLE_RULE = 'workflow_context_unavailable' +DRAFT_CONTRACT_REPAIR_CODES = frozenset({ + 'plan_structure_invalid', 'plan_arguments_invalid', 'plan_outputs_invalid', 'plan_inputs_invalid', + 'result_kind_incompatible', 'result_output_missing', 'result_producer_missing', + 'result_binding_cycle', 'result_duplicate_binding', 'result_duplicate_dependency', + 'result_duplicate_step', 'result_producer_disabled', +}) REPAIRABLE_PLAN_CODES = frozenset({ 'deliverables_invalid', 'source_kind_invalid', 'source_binding_required', 'analysis_options_invalid', 'selected_requirements_missing', 'conversation_retrieval_conflict', *WORKFLOW_REPAIR_CODES, *WORKFLOW_RUN_REPAIR_CODES, *WORKFLOW_RESULTS_REPAIR_CODES, + *DRAFT_CONTRACT_REPAIR_CODES, }) +PLAN_CONTRACT_FAILURE_MESSAGE = ( + 'The plan could not connect its steps or validate their inputs and outputs. ' + 'Your request and selected sources are unchanged. Please retry.' +) DELIVERABLES_FAILURE_MESSAGE = ( 'The plan could not account for everything you asked to receive. Please retry, or ' 'rephrase what you would like delivered.' @@ -227,10 +239,14 @@ def _conversation_transformation(message): class PlannerError(RuntimeError): """Raised when the planner could not be reached or configured.""" - def __init__(self, message, *, reason=None): + def __init__(self, message, *, reason=None, code=None, stage=None, rule=None, attempt=None): super().__init__(message) self.message = message self.reason = reason + self.code = code or reason + self.stage = stage + self.rule = rule + self.attempt = attempt class PlannerResponseError(PlannerError): @@ -592,9 +608,15 @@ def resolve_planner_client(settings): Do not put raw result references, producer identity, storage handles, callbacks, or file permissions in a plan. -Fixed producers expose their declared result_outputs. For a fixed producer, omit outputs -to use those defaults, or declare every required name/kind pair from result_outputs, even -if a later step will not consume all of them. You may also declare offered optional outputs. +Fixed producers expose their declared result_outputs. Prefer omitting outputs for fixed +producers so the server supplies their canonical declarations. If declaring them, include +every required name/kind pair, even if a later step will not consume all of them. You may +also declare offered optional outputs. default_arguments describes semantics the server +applies when an argument is omitted. Where result_output_variants is present, use the +outputs for the actual argument variant, not a different mode's outputs. In Analyze, +omitted options mean source_observations: findings are available, but the optional records +projection is not. If the task requires that records projection, explicitly declare +analysis-options-v2 with reconciled_records and preserve its required fields/calculations. compose must explicitly declare one or more supported named outputs. records-v1 requires columns:[{"name":"field", "value_type":"string","nullable":false}] in exact order. structured-v1 can specify a @@ -658,6 +680,13 @@ def resolve_planner_client(settings): Gather placement describes planning purpose, not a read-only promise; charting retrieved rows remains part of that knowledge step. +An action whose catalog says document_binding_supported can read selected workspace +documents directly, including their available native metadata. Bind its exact source IDs +in arguments.document_ids and bind its prepared output to compose. Do not add a redundant +Search or Analyze step just to declare source use. The server confines the action's +document reads to that binding and current permissions. Other actions cannot declare +document_ids, and mentioning an ID or filename in task text is not a source binding. + Grounding must use named authorized inputs, not incidental notes from an earlier task. Only name a document ID that appears in candidate_documents or that the user selected; never invent one. If the user already selected documents, plan around those documents. @@ -1380,6 +1409,9 @@ def plan_repair_message(error, *, handoff_present=False): 'server-verified identities and types, not permission to add unselected sources. ' 'Retain every selected document through compatible, enabled source-reading work with its ' 'exact authorized document ID, not merely a filename in an instruction. ' + 'For an action whose catalog says document_binding_supported, bind those IDs directly ' + 'in arguments.document_ids; keep the native inspection action instead of inserting ' + 'redundant Search or Analyze work solely to declare source use. ' 'Use narrative Analyze only for narrative sources; use offered spreadsheet operations for ' 'tabular sources, including the available native inspection/evidence path when appropriate. ' 'Do not widen retrieval, remove a selection, invent a source, or replace grounded work with ' @@ -1394,6 +1426,23 @@ def plan_repair_message(error, *, handoff_present=False): 'If the target is genuinely unclear, return a focused elicitation instead. ' 'Return the complete corrected plan as one JSON object.' ) + if error.code in DRAFT_CONTRACT_REPAIR_CODES: + return ( + f'The server rejected the step contract ({error.code}, {error.rule}). ' + f'Content-free diagnostics: {json.dumps(error.diagnostics, sort_keys=True)}. ' + 'Recheck every step against the supplied capability arguments, required named inputs, ' + 'result_outputs, optional_result_outputs and argument-dependent result_output_variants. ' + 'Prefer omitted outputs for fixed producers so the server supplies their required outputs. ' + 'Analyze defaults to source_observations, whose optional records projection is unavailable. ' + 'If the requested work needs records, explicitly choose analysis-options-v2 with ' + 'reconciled_records and retain its required fields/calculations; do not drop requested data. ' + 'Bind only actual declared producer outputs to inputs accepting that exact kind. ' + 'Use unique step/output names, list dependencies, and an acyclic graph. ' + 'Only the server-provided retained-result aliases are authorized. ' + 'An ordering dependency does not provide data, and a source set is not prepared findings. ' + 'Preserve every selected document, requested operation and deliverable. ' + 'Return the complete corrected plan as one JSON object. Do not execute anything.' + ) if getattr(error, 'code', None) in WORKFLOW_REPAIR_CODES: return ( f'The server rejected the workflow proposal in that plan: {error}\n' @@ -1584,6 +1633,57 @@ def _with_analysis_semantics(raw, edit_context=None): return plan +def _selected_source_default(selected_sources, selected_document_ids, authorized_document_ids): + if ( + not isinstance(selected_document_ids, (list, tuple)) or len(selected_document_ids) != 1 + or not isinstance(selected_document_ids[0], str) + or authorized_document_ids is None + or selected_document_ids[0] not in authorized_document_ids + ): + return None + matches = [ + source for source in selected_sources or () + if isinstance(source, dict) and source.get('document_id') == selected_document_ids[0] + ] + if ( + len(matches) != 1 or matches[0].get('availability') != 'available' + or matches[0].get('source_kind') not in ('narrative', 'tabular') + ): + return None + return matches[0] + + +def _with_selected_source_bindings( + raw, *, selected_sources, selected_document_ids, authorized_document_ids, actions, +): + """Fill only absent, unambiguous reader wiring; explicit and dynamic work is immutable.""" + source = _selected_source_default(selected_sources, selected_document_ids, authorized_document_ids) + if source is None or not isinstance(raw, dict) or not isinstance(raw.get('steps'), list): + return raw, [] + plan, repairs = deepcopy(raw), [] + for step in plan['steps']: + if ( + not isinstance(step, dict) or step.get('enabled', True) is not True + or step.get('inputs') or step.get('depends_on') + or not isinstance(step.get('arguments'), dict) + or 'document_ids' in step['arguments'] + ): + continue + capability_id, arguments = step.get('capability_id'), step['arguments'] + eligible = capability_id == 'document_search' or ( + capability_id == CAPABILITY_DOCUMENT_ANALYZE and source['source_kind'] == 'narrative' + ) + if capability_id == 'action_invoke': + eligible = any( + isinstance(action, dict) and action.get('action_ref') == arguments.get('action_ref') + and is_document_search_action(action) for action in actions or () + ) + if eligible: + arguments['document_ids'] = [source['document_id']] + repairs.append(f'Connected the selected document to step "{step.get("step_id")}".') + return plan, repairs + + def plan_request( user_message, planner_context, @@ -1703,6 +1803,10 @@ def plan_request( agent.get('name') for agent in context.get('agents') or () if isinstance(agent, dict) ] actions = context.get('actions') or [] + context['actions'] = [ + {**action, 'document_binding_supported': is_document_search_action(action)} + for action in actions if isinstance(action, dict) + ] correlation = workflow_log_context(conversation_id=conversation_id, turn_id=turn_id) deployment, current_proposal = None, None @@ -1719,12 +1823,19 @@ def _failure(reason, error=None, *, stage=None, message=None, attempt=None): 'response_failure': error.reason if isinstance(error, PlannerResponseError) else None, 'validation_code': getattr(error, 'code', None) if isinstance(error, PlanValidationError) else None, 'validation_rule': getattr(error, 'rule', None) if isinstance(error, PlanValidationError) else None, + **(error.diagnostics if isinstance(error, PlanValidationError) else {}), }, ) raise PlannerError( 'The requested change could not be planned. Your previous plan is unchanged.' if edit_context is not None else message or 'The request could not be planned. Please retry.', - reason=reason, + reason=reason, code=error.code if isinstance(error, PlanValidationError) else reason, + stage=stage or 'planning_failed', + rule=( + error.rule if isinstance(error, PlanValidationError) + else error.reason if isinstance(error, PlannerResponseError) else None + ), + attempt=attempt, ) required = required_capability_ids(seeds) @@ -1734,6 +1845,27 @@ def _failure(reason, error=None, *, stage=None, message=None, attempt=None): for candidate in context.get('candidate_documents') or [] if isinstance(candidate, dict) and candidate.get('source_kind') } + source_candidates = context.get('selected_sources') or context.get('candidate_documents') or [] + source_default = _selected_source_default( + source_candidates, (seeds or {}).get('document_ids'), authorized_document_ids, + ) + if source_default is not None: + for capability in context['capabilities']: + if capability['id'] == 'document_search' or ( + capability['id'] == CAPABILITY_DOCUMENT_ANALYZE and source_default['source_kind'] == 'narrative' + ): + capability['default_arguments'] = { + **capability.get('default_arguments', {}), + 'document_ids': [source_default['document_id']], + } + context['actions'] = [ + {**action, 'default_arguments': {'document_ids': [source_default['document_id']]}} + if is_document_search_action(action) else action for action in context['actions'] + ] + document_source_formats = { + candidate['document_id']: str(candidate.get('file_name') or '').rsplit('.', 1)[-1].lower() + for candidate in source_candidates if isinstance(candidate, dict) and candidate.get('file_name') + } def _diagnose(stage, proposal=None, *, attempt=None, error=None): if not is_debug_enabled(): @@ -1785,6 +1917,7 @@ def _diagnose(stage, proposal=None, *, attempt=None, error=None): } if isinstance(error, PlanValidationError): details.update(validation_code=error.code, validation_rule=error.rule) + details.update(error.diagnostics) source_ids = sorted(selected | used, key=lambda value: (value not in missing, value)) details['omitted_document_count'] = max(0, len(source_ids) - PLANNER_DEBUG_MAX_ITEMS) log_event('[ORCHESTRATION_PLANNER] Planning diagnostic.', extra=details) @@ -1859,6 +1992,19 @@ def _correction_messages(error, stage, attempt, reply, *, handoff_present=False) def _normalize(raw, capability_ids, availability, selected_image, **options): raw = _with_analysis_semantics(raw, edit_context) _diagnose('analysis_defaults', raw, attempt=attempt) + local_repairs = [] + if edit_context is None and not conversation_only: + raw, local_repairs = _with_selected_source_bindings( + raw, selected_sources=source_candidates, + selected_document_ids=(seeds or {}).get('document_ids'), + authorized_document_ids=authorized_document_ids, actions=actions, + ) + if local_repairs: + log_event('[ORCHESTRATION_PLANNER] Selected source wiring repaired.', extra={ + **correlation, 'stage': 'selected_source_defaults', 'attempt': attempt, + 'source_binding_repair_count': len(local_repairs), 'selected_document_count': 1, + }) + _diagnose('selected_source_defaults', raw, attempt=attempt) if conversation_only and any( step.get('enabled', True) and step.get('capability_id') in CONVERSATION_RETRIEVAL_CAPABILITIES for step in raw.get('steps') or [] if isinstance(step, dict) @@ -1911,7 +2057,10 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): image_selected=selected_image, **options, ) - validate_plan_document_source_kinds(normalized, document_source_kinds) + validate_plan_document_source_kinds( + normalized, document_source_kinds, document_source_formats=document_source_formats, + ) + normalized['validation']['repairs'].extend(local_repairs) return normalized log_event( @@ -1961,6 +2110,7 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): } reasoning_metadata = build_model_reasoning_metadata(planner_model, 'planner') token_usage, usage_seen = {}, False + elicitation_plan_correction = False for attempt in range(1, PLAN_REPAIR_ATTEMPTS + 2): try: reply, usage = _call_planner(client, deployment, messages, require_complete_response=True) @@ -2073,6 +2223,7 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): if attempt > PLAN_REPAIR_ATTEMPTS: return _failure('invalid_elicitation', exc, stage='plan_clarification', attempt=attempt) allow_elicitation = False + elicitation_plan_correction = True messages = [ *messages, {'role': 'assistant', 'content': reply}, {'role': 'user', 'content': ( @@ -2083,6 +2234,24 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): continue if kind == 'elicitation': + if elicitation_plan_correction and attempt <= PLAN_REPAIR_ATTEMPTS: + log_event( + '[ORCHESTRATION_PLANNER] Discarding an unrenderable question set.', + level=logging.WARNING, extra={ + **correlation, 'stage': 'plan_clarification', 'attempt': attempt, + 'validation_code': 'plan_structure_invalid', 'validation_rule': 'elicitation_not_allowed', + }, + ) + messages = [ + *messages, {'role': 'assistant', 'content': reply}, + {'role': 'user', 'content': ( + 'The rejected clarification is still not an executable plan. Use the remaining ' + 'shared correction allowance to return the complete plan for the unchanged request, ' + 'authorized sources and requested deliverables, not another elicitation. ' + 'Do not invent choices, missing facts or source access.' + )}, + ] + continue return _failure('repeated_elicitation', stage='plan_clarification', attempt=attempt) raw_steps = parsed.get('steps') @@ -2139,6 +2308,7 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): 'source_kind_invalid': SOURCE_KIND_FAILURE_MESSAGE, 'source_binding_required': SOURCE_BINDING_FAILURE_MESSAGE, 'analysis_options_invalid': ANALYSIS_OPTIONS_FAILURE_MESSAGE, + **{code: PLAN_CONTRACT_FAILURE_MESSAGE for code in DRAFT_CONTRACT_REPAIR_CODES}, 'conversation_retrieval_conflict': ( 'This request needs the conversation content, not a workspace search. ' 'Select or paste the target text and create a new plan.' diff --git a/application/single_app/functions_orchestration_planning.py b/application/single_app/functions_orchestration_planning.py index 88e7daa40..f5c7dbf73 100644 --- a/application/single_app/functions_orchestration_planning.py +++ b/application/single_app/functions_orchestration_planning.py @@ -326,7 +326,7 @@ def planning_projection(question, state, *, include_request=False): "turn_id": state["turn_id"], "submission_id": state["submission_id"], "status": state["status"], "updated_at": state["updated_at"], } - for field in ("run_id", "error", "error_code"): + for field in ("run_id", "error", "error_code", "error_stage", "error_rule", "error_attempt"): if state.get(field): result[field] = state[field] if include_request: diff --git a/application/single_app/functions_orchestration_registry.py b/application/single_app/functions_orchestration_registry.py index 38945d336..b1c8db985 100644 --- a/application/single_app/functions_orchestration_registry.py +++ b/application/single_app/functions_orchestration_registry.py @@ -764,9 +764,10 @@ def _document_transform_descriptor(capability_id, label, instruction): "The question asks about information likely held in the user's own documents, " "and no particular document has been named. Prefer this over analysing a whole " "document when a few passages would answer the question." - " For Excel Notes or threaded comments, use native_section=comments with explicit " - "document_ids. This reads the native inventory across pages, including available " - "authors, replies, dates, resolution, worksheet/cell and saved cell values. Bind " + " For Word review comments (.docx, .docm, .dotx), or Excel Notes/threaded comments, " + "use native_section=comments with document_ids. This reads the native inventory, " + "including available anchors, authors, replies, dates and resolution; Excel also " + "includes worksheet/cell and saved cell values. Bind " "prepared or evidence to compose; row analysis alone cannot read annotations." ), 'settings_gates': (), @@ -798,7 +799,7 @@ def _document_transform_descriptor(capability_id, label, instruction): 'top_n': {'type': 'integer', 'minimum': 1, 'maximum': 50, 'default': 12}, 'native_section': { 'type': 'string', 'enum': ['comments'], - 'description': 'Read actual Excel Notes/threaded comments instead of search excerpts.', + 'description': 'Read actual Word review comments or Excel Notes/threaded comments, not search excerpts.', }, }, 'required': ['query'], @@ -1472,6 +1473,9 @@ def _document_transform_descriptor(capability_id, label, instruction): "The step can use the selected action's functions within execution limits. " "Use execution_intent operate for requested operations, following the action's policies. " "Bind earlier research or prepared content as named inputs. Do not duplicate agent work." + " When the action catalog says document_binding_supported, bind selected workspace " + "documents in arguments.document_ids. Native metadata inspection can then satisfy " + "the selected-source requirement without a redundant Search or Analyze step." ), 'settings_gates': ( 'enable_chat_orchestration', @@ -1498,6 +1502,15 @@ def _document_transform_descriptor(capability_id, label, instruction): 'type': 'string', 'enum': ['gather', 'operate'], 'default': 'gather', 'description': 'Gather information, or perform the requested operation under existing policies.', }, + 'document_ids': { + 'type': 'array', 'minItems': 1, 'maxItems': 100, 'uniqueItems': True, + 'items': {'type': 'string', 'minLength': 1}, + 'description': ( + 'Exact selected workspace document IDs. Only an action whose catalog says ' + 'document_binding_supported may use this field. Every read is confined to ' + 'these sources and the action/user permissions.' + ), + }, 'visuals': { 'type': 'array', 'items': {'type': 'string', 'enum': [VISUAL_CHART]}, 'uniqueItems': True, 'maxItems': 1, @@ -2421,6 +2434,28 @@ def build_planner_capability_projection(capabilities): """ projection = [] for capability in capabilities or (): + outputs = capability['result_outputs'] + optional_outputs = capability.get('optional_result_outputs', {}) + variants = capability.get('result_output_variants') + defaults = {} + if capability['id'] == CAPABILITY_DOCUMENT_ANALYZE: + # The analysis owner reaches bootstrap dependencies; load it only for planning, + # not while registering capabilities or reading their administrative projection. + from functions_document_analysis_results import narrative_analysis_options + + defaults = {'analysis_options': narrative_analysis_options()} + outputs, optional_outputs = get_capability_result_outputs(capability, defaults) + variants = [] + for options in ( + {key: defaults['analysis_options'][key] for key in ('version', 'mode')}, + {'version': 'analysis-options-v2', 'mode': 'reconciled_records'}, + {'version': 'analysis-options-v1', 'mode': 'narrative_findings'}, + ): + required, optional = get_capability_result_outputs(capability, {'analysis_options': options}) + variants.append({ + 'analysis_options': options, + 'result_outputs': required, 'optional_result_outputs': optional, + }) projection.append({ 'id': capability['id'], 'label': capability['label'], @@ -2441,11 +2476,11 @@ def build_planner_capability_projection(capabilities): **({'optional_inputs_supported': True} if capability.get('optional_inputs_supported') else {}), **({'required_result_inputs': list(capability['required_result_inputs'])} if capability.get('required_result_inputs') else {}), - 'result_outputs': capability['result_outputs'], - 'optional_result_outputs': capability.get('optional_result_outputs', {}), + 'result_outputs': outputs, + 'optional_result_outputs': optional_outputs, 'result_output_kinds': list(capability.get('result_output_kinds') or ()), - **({'result_output_variants': capability['result_output_variants']} - if 'result_output_variants' in capability else {}), + **({'default_arguments': defaults} if defaults else {}), + **({'result_output_variants': variants} if variants is not None else {}), }) return projection diff --git a/application/single_app/functions_orchestration_result_contracts.py b/application/single_app/functions_orchestration_result_contracts.py index c4eadad11..076849342 100644 --- a/application/single_app/functions_orchestration_result_contracts.py +++ b/application/single_app/functions_orchestration_result_contracts.py @@ -54,8 +54,10 @@ class ResultContractError(ValueError): """Invalid data is never repaired into a different successful result.""" - def __init__(self, code="result_contract_invalid"): + def __init__(self, code="result_contract_invalid", *, rule=None, diagnostics=None): self.code = code + self.rule = rule + self.diagnostics = diagnostics or {} super().__init__("The orchestration result or input binding is invalid.") @@ -665,14 +667,23 @@ def validate_input_bindings(steps, *, existing_results=None, max_steps=MAX_BINDI else: producer = by_id.get(binding.step_id) if producer is None: - raise ResultContractError("result_producer_missing") + raise ResultContractError("result_producer_missing", rule="input_producer_missing", diagnostics={ + "step_id_hash": hashlib.sha256(step.step_id.encode()).hexdigest(), + }) kind = next((output.kind for output in producer.outputs if output.name == binding.output_name), None) if kind is None: - raise ResultContractError("result_output_missing") + raise ResultContractError("result_output_missing", rule="input_output_missing", diagnostics={ + "step_id_hash": hashlib.sha256(step.step_id.encode()).hexdigest(), + "producer_step_id_hash": hashlib.sha256(producer.step_id.encode()).hexdigest(), + }) if binding.step_id not in required: required.append(binding.step_id) if kind not in spec.kinds: - raise ResultContractError("result_kind_incompatible") + raise ResultContractError("result_kind_incompatible", rule="input_kind_mismatch", diagnostics={ + "step_id_hash": hashlib.sha256(step.step_id.encode()).hexdigest(), + "result_kind": kind.replace("-", "_"), + **{f"accepts_{value.replace('-', '_')}": True for value in spec.kinds}, + }) for dependency in required: if dependency not in by_id: raise ResultContractError("result_producer_missing") diff --git a/application/single_app/functions_orchestration_result_runtime.py b/application/single_app/functions_orchestration_result_runtime.py index cac8c7c21..2a860bef4 100644 --- a/application/single_app/functions_orchestration_result_runtime.py +++ b/application/single_app/functions_orchestration_result_runtime.py @@ -9,13 +9,15 @@ import json from content_screening.contracts import DocumentHeldError, ScreeningError +from functions_action_manifest import is_document_search_action +from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from functions_analysis_access import analysis_source_snapshot from functions_evidence_status import evidence_status_from_sources from functions_orchestration_invocation_capture import ( OrchestrationInvocationCancelledError, OrchestrationInvocationControlError, OrchestrationInvocationServiceError, ) from functions_orchestration_registry import ( - DEPENDENCY_PLAN_CONTRACT_VERSION, get_capability, get_capability_result_outputs, + CAPABILITY_ACTION_INVOKE, DEPENDENCY_PLAN_CONTRACT_VERSION, get_capability, get_capability_result_outputs, ) from functions_orchestration_result_contracts import ( IMAGE_ASSET_KIND, MAX_DESCRIPTOR_BYTES, MAX_EXTERNAL_SOURCES, Completeness, Coverage, ExternalSourceRef, @@ -160,7 +162,23 @@ def read_result_document_citations(context, reference, *, evidence_status_out=No """Project document provenance from the exact final result, never ambient work.""" service = require_result_service(context) with service.read_operation(): - return _read_result_document_citations(service, reference, evidence_status_out=evidence_status_out) + citations = _read_result_document_citations(service, reference, evidence_status_out=evidence_status_out) + if is_debug_enabled(): + producer = reference.producer + consumer = workflow_log_context(run_id=getattr(context, 'run_id', None)) + log_event('[ORCHESTRATION_EXECUTOR] Retained document citation projection verified.', extra={ + **workflow_log_context( + conversation_id=producer.conversation_id, run_id=producer.run_id, step_id=producer.step_id, + ), + **({'consumer_run_id_hash': consumer['run_id_hash']} if consumer else {}), + 'stage': 'retained_citation_projection', 'capability_id': producer.capability_id, + 'citation_count': len(citations), + 'located_citation_count': sum( + bool(citation.get('citation_id')) and citation.get('location_value', citation.get('page_number')) + not in (None, '') for citation in citations + ), + }) + return citations def read_result_evidence_status(context, reference): @@ -329,7 +347,9 @@ def visit(current): or type(prepared) is not dict or prepared.get('version') != 'orchestration-gathered-content-v1' or prepared.get('capability_id') != 'document_search' - or prepared.get('content_scope') not in ('returned_excerpts', 'native_excel_annotations') + or prepared.get('content_scope') not in ( + 'returned_excerpts', 'native_excel_annotations', 'native_document_annotations', + ) or type(prepared.get('citations')) is not list ): raise ResultContractError('result_citations_invalid') @@ -337,10 +357,24 @@ def visit(current): _search_evidence_citations(reader, prepared) if current.output_name == 'evidence' else prepared['citations'] ) - if prepared['content_scope'] == 'native_excel_annotations': + if prepared['content_scope'] in ('native_excel_annotations', 'native_document_annotations'): native_reference = prepared_reader.reference.to_dict() for source in lineage['sources']: search_counts.setdefault(source_key(source), set()) + elif producer.capability_id == CAPABILITY_ACTION_INVOKE and lineage['sources']: + prepared = read_complete_input(reader) + if ( + current.output_name != 'prepared' or current.kind != 'structured-v1' + or type(prepared) is not dict + or prepared.get('version') != 'orchestration-gathered-content-v1' + or prepared.get('capability_id') != CAPABILITY_ACTION_INVOKE + or type(prepared.get('citations')) is not list + ): + raise ResultContractError('result_citations_invalid') + # The tool citation adapter reads config defaults; keep it out of bootstrap. + from functions_agent_document_citations import build_document_citations_from_agent_citations + + retained_citations = build_document_citations_from_agent_citations(prepared['citations']) elif producer.capability_id in ('document_analyze', 'document_compare'): # Coverage is this producer's citation sidecar, not an implicit input from # another task. Reopen the exact committed digest, including after recovery. @@ -422,7 +456,7 @@ def read_native_citation_evidence(service, reference, *, citation_id, document_i """Resolve an exact committed annotation, with current source access and version. Unlike ordinary retained-result inspection, a source citation also authorizes the - workbook it offers to preview. Never search a current inventory by cell or decode + document it offers to preview. Never search a current inventory by location or decode a document identity from the synthetic citation ID. """ if ( @@ -433,7 +467,9 @@ def read_native_citation_evidence(service, reference, *, citation_id, document_i reader = service.open_result(reference, allow_partial=True, require_current_sources=True) prepared = read_complete_input(reader) if ( - type(prepared) is not dict or prepared.get('content_scope') != 'native_excel_annotations' + type(prepared) is not dict or prepared.get('content_scope') not in ( + 'native_excel_annotations', 'native_document_annotations', + ) or type(prepared.get('citations')) is not list or type(prepared.get('search_results')) is not list ): raise ResultContractError('native_citation_reference_invalid') @@ -457,6 +493,7 @@ def read_native_citation_evidence(service, reference, *, citation_id, document_i if type(value) is not dict: raise ResultContractError('native_citation_evidence_invalid') annotation = value.get('excel_annotation') + word_annotation = value.get('word_annotation') coverage = value.get('native_coverage') if type(annotation) is dict: citation = citations[0] @@ -465,6 +502,13 @@ def read_native_citation_evidence(service, reference, *, citation_id, document_i )): raise ResultContractError('native_citation_evidence_invalid') response = {'native_annotation': deepcopy(annotation), 'cited_text': annotation.get('text') or ''} + elif type(word_annotation) is dict: + if ( + citations[0].get('annotation_id') != word_annotation.get('id') + or citations[0].get('evidence_kind') != 'native_word_annotation' + ): + raise ResultContractError('native_citation_evidence_invalid') + response = {'native_annotation': deepcopy(word_annotation), 'cited_text': word_annotation.get('text') or ''} elif type(coverage) is dict: response = {'native_coverage': deepcopy(coverage), 'cited_text': ''} else: @@ -474,7 +518,10 @@ def read_native_citation_evidence(service, reference, *, citation_id, document_i return { **response, 'citation_id': citation_id, 'document_id': document_id, 'file_name': hits[0].get('file_name') or '', 'source_version': sources[0]['source_version'], - 'evidence_kind': 'native_excel_annotation' if annotation is not None else 'native_annotation_coverage', + 'evidence_kind': ( + 'native_word_annotation' if type(word_annotation) is dict + else 'native_excel_annotation' if type(annotation) is dict else 'native_annotation_coverage' + ), } @@ -595,8 +642,8 @@ def retain_document_search_result(step, context, results, *, source_manifest, ci partial = bool(native_coverage and any(item.get('partial') for item in native_coverage)) if native_coverage: limitations = ( - 'Native annotation evidence contains saved workbook records, not recalculated worksheet data.', - 'Reviewer feedback is distinct from the saved cell value; missing metadata must not be inferred.', + 'Native annotation evidence contains saved document records, not inferred or recalculated data.', + 'Reviewer feedback is distinct from document text or a saved cell value; missing metadata must not be inferred.', ) if partial: limitations += ('The native annotation inventory is partial; remaining records were not inspected.',) @@ -634,7 +681,11 @@ def retain_document_search_result(step, context, results, *, source_manifest, ci }, } if native_coverage: - prepared['content_scope'] = 'native_excel_annotations' + prepared['content_scope'] = ( + 'native_excel_annotations' if all( + item.get('format') in (None, 'xlsx', 'xlsm', 'xls') for item in native_coverage + ) else 'native_document_annotations' + ) prepared['coverage']['native_inspection'] = deepcopy(native_coverage) return service.persist_task_result( producer=context.result_producer(step), role='gather', status='partial' if partial else 'complete', @@ -675,6 +726,17 @@ def retain_gather_result(step, context, result, *, source_manifest): service = require_result_service(context) capability_id = step['capability_id'] sources = analysis_source_snapshot(source_manifest) + producer = context.result_producer(step) + trace = workflow_log_context( + conversation_id=producer.conversation_id, turn_id=getattr(context, 'turn_id', None), + run_id=producer.run_id, step_id=producer.step_id, + ) + if is_debug_enabled(): + log_event('[ORCHESTRATION_EXECUTOR] Gather retention started.', extra={ + **trace, 'stage': 'gather_retention', 'capability_id': capability_id, + 'workspace_source_count': len(sources), + 'document_binding_declared': 'document_ids' in (step.get('arguments') or {}), + }) evidence = deepcopy(result.get('evidence') or []) citations = deepcopy(result.get('citations') or []) notes = deepcopy(result.get('notes') or []) @@ -723,8 +785,21 @@ def retain_gather_result(step, context, result, *, source_manifest): )) else: admission = getattr(context, 'external_source_admission', None) + arguments = step.get('arguments') or {} + bound_action = ( + capability_id == CAPABILITY_ACTION_INVOKE + and bool(arguments.get('document_ids')) + and arguments['document_ids'] == [source['document_id'] for source in sources] + and all(source.get('scope') in ('personal', 'group', 'public') for source in sources) + and any( + action.get('action_ref') == arguments.get('action_ref') and is_document_search_action(action) + for action in (getattr(context, 'action_catalog', None) or ()) if isinstance(action, dict) + ) + ) if ( - sources or not callable(admission) + ((sources or (capability_id == CAPABILITY_ACTION_INVOKE and 'document_ids' in arguments)) + and not bound_action) + or not callable(admission) or not callable(service.access.external_source_authorizer) ): raise ResultContractError('result_external_lineage_unsupported') @@ -744,11 +819,18 @@ def retain_gather_result(step, context, result, *, source_manifest): service.access.external_source_catalog.update(admitted) external_sources = tuple(admitted) outputs.append(NamedOutput('prepared', 'structured-v1', prepared, _complete(1, limitations, partial=partial))) - return service.persist_task_result( - producer=context.result_producer(step), role='gather', status='partial' if partial else 'complete', + task = service.persist_task_result( + producer=producer, role='gather', status='partial' if partial else 'complete', outputs=outputs, sources=sources, origin='grounded' if sources or external_sources else 'generated', guard_token=context.result_guard_token_for_step(step['step_id']), input_fingerprint=context.result_input_fingerprint_for_step(step['step_id']), upstream=tuple(dict.fromkeys(reader.reference for reader in readers.values())), **({'external_sources': external_sources} if external_sources else {}), ) + if is_debug_enabled(): + log_event('[ORCHESTRATION_EXECUTOR] Gather result retained.', extra={ + **trace, 'stage': 'gather_retained', 'capability_id': capability_id, + 'workspace_source_count': len(sources), 'external_source_count': len(external_sources), + 'output_count': len(outputs), 'upstream_input_count': len(readers), 'partial': partial, + }) + return task diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index 19e7f1e6d..0635ceddb 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -56,6 +56,7 @@ from functions_m365_operations import M365_SOURCES from functions_model_catalog import ModelCatalogError from functions_document_analysis_results import AnalysisOptionsError, normalize_analysis_options +from functions_action_manifest import is_document_search_action from functions_web_iq import WebIQError, get_web_search_provider, resolve_web_iq_types from functions_orchestration_registry import ( APPROVAL_FLOOR_MANUAL, @@ -229,12 +230,23 @@ class PlanValidationError(ValueError): """Raised when a plan cannot be repaired into something safe to run.""" - def __init__(self, message, *, code='plan_invalid', rule=None): + def __init__(self, message, *, code='plan_invalid', rule=None, diagnostics=None): self.code = code self.rule = rule + self.diagnostics = diagnostics or {} super().__init__(message) +def _step_contract_error(message, step, *, code, rule, **diagnostics): + step_id = step.get('step_id') + return PlanValidationError(message, code=code, rule=rule, diagnostics={ + **({'step_id_hash': hashlib.sha256(step_id.encode('utf-8')).hexdigest()} + if isinstance(step_id, str) else {}), + 'capability_id': step.get('capability_id'), + **diagnostics, + }) + + class LegacyPlanError(PlanValidationError): """A saved plan from the removed legacy contract. It is never interpreted.""" @@ -424,32 +436,44 @@ def validate_inline_output_schema(schema): def step_input_specs(step): capability = get_capability(step.get('capability_id'), contract_version=DEPENDENCY_PLAN_CONTRACT_VERSION) if capability is None or type(step.get('inputs', {})) is not dict: - raise PlanValidationError('The step has invalid named inputs.') + raise _step_contract_error('The step has invalid named inputs.', step, + code='plan_inputs_invalid', rule='inputs_object_required') accepted = capability['result_input_kinds'] if not set(capability.get('required_result_inputs') or ()).issubset(step.get('inputs', {})): - raise PlanValidationError('The capability requires an explicit named input.') + raise _step_contract_error('The capability requires an explicit named input.', step, + code='plan_inputs_invalid', rule='required_input_missing') specs = [] for name, value in step.get('inputs', {}).items(): if type(value) is not dict or set(value) - {'binding', 'allow_partial', 'optional'} or 'binding' not in value: - raise PlanValidationError('Each named input requires an explicit result binding.') + raise _step_contract_error('Each named input requires an explicit result binding.', step, + code='plan_inputs_invalid', rule='input_binding_required') kinds = accepted.get(name, accepted.get('*')) if not kinds: - raise PlanValidationError('This capability does not accept that named input.') + raise _step_contract_error('This capability does not accept that named input.', step, + code='plan_inputs_invalid', rule='input_name_unsupported') if value.get('allow_partial') is True and not capability['partial_inputs_supported']: - raise PlanValidationError('This capability requires complete named inputs.') + raise _step_contract_error('This capability requires complete named inputs.', step, + code='plan_inputs_invalid', rule='input_partial_unsupported') if ( value.get('allow_partial') is True and step.get('capability_id') == 'render_file' and step.get('arguments', {}).get('profile') not in NARRATIVE_EXPORT_PROFILES ): - raise PlanValidationError('This file profile requires complete source data.') + raise _step_contract_error('This file profile requires complete source data.', step, + code='plan_inputs_invalid', rule='input_partial_unsupported') if type(value.get('optional', False)) is not bool: - raise PlanValidationError('Input optionality must be a boolean.') + raise _step_contract_error('Input optionality must be a boolean.', step, + code='plan_inputs_invalid', rule='input_optionality_invalid') if value.get('optional') is True and not capability.get('optional_inputs_supported'): - raise PlanValidationError('This capability cannot proceed without a named input.') - specs.append(InputSpec( - name, InputBinding.from_dict(value['binding']), tuple(kinds), value.get('allow_partial', False), - value.get('optional', False), - )) + raise _step_contract_error('This capability cannot proceed without a named input.', step, + code='plan_inputs_invalid', rule='input_optional_unsupported') + try: + specs.append(InputSpec( + name, InputBinding.from_dict(value['binding']), tuple(kinds), value.get('allow_partial', False), + value.get('optional', False), + )) + except ResultContractError as exc: + raise _step_contract_error('The named input binding is malformed.', step, + code='plan_inputs_invalid', rule='input_binding_invalid') from exc return tuple(specs) @@ -513,20 +537,29 @@ def _dependency_outputs(raw, capability, composition_profiles, arguments): return prepared if capability['role'] == 'render': if outputs != []: - raise PlanValidationError('Render delivers files, not named retained data outputs.') + raise _step_contract_error('Render delivers files, not named retained data outputs.', raw, + code='plan_outputs_invalid', rule='render_outputs_must_be_empty') return [] if type(outputs) is not list or not 1 <= len(outputs) <= 32: - raise PlanValidationError('Each step must declare its supported named outputs.') + raise _step_contract_error('Each step must declare its supported named outputs.', raw, + code='plan_outputs_invalid', rule='outputs_list_required') if capability['id'] != CAPABILITY_COMPOSE: allowed = {**required, **optional} for output in outputs: if type(output) is not dict or set(output) != {'name', 'kind'}: - raise PlanValidationError('The output declaration does not match the producing capability.') - OutputSpec(output['name'], output['kind']) + raise _step_contract_error('The output declaration does not match the producing capability.', raw, + code='plan_outputs_invalid', rule='fixed_output_shape_invalid') + try: + OutputSpec(output['name'], output['kind']) + except ResultContractError as exc: + raise _step_contract_error('The output declaration is invalid.', raw, + code='plan_outputs_invalid', rule='output_name_or_kind_invalid') from exc if allowed.get(output['name']) != output['kind']: - raise PlanValidationError('The output declaration does not match the producing capability.') + raise _step_contract_error('The output declaration does not match the producing capability.', raw, + code='plan_outputs_invalid', rule='fixed_output_not_supported') if not set(required).issubset({output['name'] for output in outputs}): - raise PlanValidationError('The output declaration does not match the producing capability.') + raise _step_contract_error('The output declaration omits a required producing-capability output.', raw, + code='plan_outputs_invalid', rule='fixed_output_missing') return deepcopy(outputs) supported = capability['result_output_kinds'] for item in outputs: @@ -534,14 +567,21 @@ def _dependency_outputs(raw, capability, composition_profiles, arguments): type(item) is not dict or set(item) - {'name', 'kind', 'columns', 'schema', 'profile'} or not {'name', 'kind'}.issubset(item) ): - raise PlanValidationError('The composition output declaration is invalid.') - OutputSpec(item['name'], item['kind']) + raise _step_contract_error('The composition output declaration is invalid.', raw, + code='plan_outputs_invalid', rule='compose_output_shape_invalid') + try: + OutputSpec(item['name'], item['kind']) + except ResultContractError as exc: + raise _step_contract_error('The composition output declaration is invalid.', raw, + code='plan_outputs_invalid', rule='output_name_or_kind_invalid') from exc if item['kind'] not in supported: - raise PlanValidationError('The composition output kind is unsupported.') + raise _step_contract_error('The composition output kind is unsupported.', raw, + code='plan_outputs_invalid', rule='compose_output_kind_unsupported') if item['kind'] == 'records-v1': columns = item.get('columns') if type(columns) is not list or not columns: - raise PlanValidationError('Records require an explicit ordered column schema.') + raise _step_contract_error('Records require an explicit ordered column schema.', raw, + code='plan_outputs_invalid', rule='record_columns_required') parsed = tuple(RecordColumn.from_dict(column) for column in columns) if len(parsed) > 256 or len({column.name for column in parsed}) != len(parsed): raise PlanValidationError('The record column schema is invalid.') @@ -942,7 +982,7 @@ def validate_dependency_plan( plan, *, settings=None, authorized_document_ids=None, available_capability_ids=None, agent_names=None, action_refs=None, existing_results=None, composition_profiles=None, export_catalog=None, deliverable_availability=None, image_selected=False, seeds=None, - workflow_planning=None, + workflow_planning=None, actions=None, ): """Compile a plan without dropping required work, arguments, outputs, or dependencies. @@ -1001,24 +1041,42 @@ def validate_dependency_plan( 'delivers', 'deliverable_context', } try: - for raw in raw_steps: + for step_index, raw in enumerate(raw_steps): if type(raw) is not dict or set(raw) - fields: - raise PlanValidationError('The plan contains an invalid step field.') - output_name(raw.get('step_id')) + raise PlanValidationError( + 'The plan contains an invalid step field.', + code='plan_structure_invalid', rule='step_shape_invalid', + diagnostics={'step_index': step_index}, + ) + try: + output_name(raw.get('step_id')) + except ResultContractError as exc: + raise PlanValidationError( + 'Each step requires a valid unique identifier.', + code='plan_structure_invalid', rule='step_id_invalid', + diagnostics={'step_index': step_index}, + ) from exc capability = get_capability(raw.get('capability_id'), contract_version=DEPENDENCY_PLAN_CONTRACT_VERSION) if capability is None or capability['id'] not in available or capability.get('runtime_unavailable_reason'): raise PlanValidationError('A required capability is unknown or disabled.', code='capability_unavailable') capability_id = capability['id'] if 'role' in raw and raw['role'] != capability['role']: - raise PlanValidationError('Capability purpose is server-owned.') + raise _step_contract_error('Capability purpose is server-owned.', raw, + code='plan_structure_invalid', rule='step_role_invalid') if any(type(raw.get(key, default)) is not bool for key, default in (('enabled', True), ('optional', False))): - raise PlanValidationError('Step enablement and optionality must be booleans.') + raise _step_contract_error('Step enablement and optionality must be booleans.', raw, + code='plan_structure_invalid', rule='step_flags_invalid') if type(raw.get('depends_on', [])) is not list: - raise PlanValidationError('Dependencies must be a list of step IDs.') + raise _step_contract_error('Dependencies must be a list of step IDs.', raw, + code='plan_structure_invalid', rule='dependencies_list_required') if 'model_task' in raw and (type(raw['model_task']) is not str or not raw['model_task'].strip()): - raise PlanValidationError('A model task must be a named task category.') + raise _step_contract_error('A model task must be a named task category.', raw, + code='plan_structure_invalid', rule='model_task_invalid') if 'model_binding' in raw and type(raw['model_binding']) is not dict: - raise PlanValidationError('A model binding is server-owned structured data.') + raise _step_contract_error( + 'A model binding is server-owned structured data.', raw, + code='plan_structure_invalid', rule='model_binding_object_required', + ) if capability_id == CAPABILITY_WORKFLOW_RUN: # Loaded only when a plan starts a workflow. It runs before the generic limit and # contract checks, so each problem with the step is reported with its own rule. @@ -1041,7 +1099,8 @@ def validate_dependency_plan( raise PlanValidationError('The plan exceeds a capability work limit.', code='result_step_limit') arguments = raw.get('arguments', {}) if type(arguments) is not dict: - raise PlanValidationError('Step arguments must be an object.') + raise _step_contract_error('Step arguments must be an object.', raw, + code='plan_arguments_invalid', rule='arguments_object_required') canonical_bytes(arguments) analysis_options = None if capability_id == 'document_analyze' and 'analysis_options' in arguments: @@ -1059,8 +1118,34 @@ def validate_dependency_plan( 'The declared Analyze options do not match their contract.', code='analysis_options_invalid', rule='analysis_options_schema_invalid', ) - if not Draft202012Validator(capability['inputs']).is_valid(arguments): - raise PlanValidationError('Step arguments do not match the capability contract.') + argument_problems = [] + for argument_error in Draft202012Validator(capability['inputs']).iter_errors(arguments): + field = next(iter(argument_error.absolute_path), None) + if field is None and argument_error.validator == 'required': + field = next(( + name for name in capability['inputs'].get('required', ()) if name not in arguments + ), None) + argument_problems.append({ + 'argument_field': field if field in capability['inputs']['properties'] else 'unknown', + 'schema_keyword': argument_error.validator.lower(), + }) + if len(argument_problems) == 8: + break + if argument_problems: + if capability_id in ( + CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RUN, + CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_HANDOFF, + ): + # These owners have their own correction/drop policy; preserve its boundary. + raise PlanValidationError('Step arguments do not match the capability contract.') + field = argument_problems[0]['argument_field'] + keyword = argument_problems[0]['schema_keyword'] + raise _step_contract_error( + 'Step arguments do not match the capability contract.', raw, + code='plan_arguments_invalid', + rule='unknown_argument' if keyword == 'additionalproperties' else 'argument_schema_invalid', + schema_keyword=keyword, argument_field=field, argument_problems=argument_problems, + ) arguments = deepcopy(arguments) if analysis_options is not None: arguments['analysis_options'] = analysis_options @@ -1128,21 +1213,42 @@ def validate_dependency_plan( ): raise PlanValidationError('A destination conversation cannot reference itself as a source.') if any(isinstance(value, str) and not value.strip() for value in arguments.values()): - raise PlanValidationError('String arguments must not be empty or whitespace.') + raise _step_contract_error( + 'String arguments must not be empty or whitespace.', raw, + code='plan_arguments_invalid', rule='argument_string_empty', + ) for name, rule in capability['inputs']['properties'].items(): if name not in arguments and 'default' in rule: arguments[name] = deepcopy(rule['default']) if 'agent_name' in arguments and arguments['agent_name'] not in (agent_names or ()): - raise PlanValidationError('The selected agent is unavailable.') + raise PlanValidationError('The selected agent is unavailable.', + code='capability_unavailable', rule='agent_unavailable') if 'action_ref' in arguments and arguments['action_ref'] not in (action_refs or ()): - raise PlanValidationError('The selected action is unavailable.') + raise PlanValidationError('The selected action is unavailable.', + code='capability_unavailable', rule='action_unavailable') + if capability_id == CAPABILITY_ACTION_INVOKE and 'document_ids' in arguments: + action = next(( + item for item in actions or () + if isinstance(item, dict) and item.get('action_ref') == arguments['action_ref'] + ), None) + if action is None or not is_document_search_action(action): + raise _step_contract_error( + 'The selected action cannot bind workspace documents.', raw, + code='source_binding_invalid', rule='action_document_reader_required', + ) limit = get_capability_document_limit(capability, settings=settings) for name in ('document_ids', 'right_document_ids'): if name in arguments: if any(not value or value != value.strip() for value in arguments[name]): - raise PlanValidationError('Document IDs must be exact, nonempty identifiers.') + raise _step_contract_error( + 'Document IDs must be exact, nonempty identifiers.', raw, + code='plan_arguments_invalid', rule='document_id_invalid', argument_field=name, + ) if len(set(arguments[name])) != len(arguments[name]): - raise PlanValidationError('Document selections must not contain duplicates.') + raise _step_contract_error( + 'Document selections must not contain duplicates.', raw, + code='plan_arguments_invalid', rule='document_selection_duplicate', argument_field=name, + ) if limit and len(arguments[name]) > limit: if handoff_offered: raise PlanValidationError( @@ -1150,9 +1256,15 @@ def validate_dependency_plan( 'plan, or hand the work off with one workflow_handoff step.', code=PLAN_BUDGET_EXCEEDED_CODE, rule='plan_document_budget', ) - raise PlanValidationError('The complete source selection exceeds the document limit.') + raise _step_contract_error( + 'The complete source selection exceeds the document limit.', raw, + code='source_binding_invalid', rule='document_limit_exceeded', argument_field=name, + ) if authorized_document_ids is not None and set(arguments[name]) - set(authorized_document_ids): - raise PlanValidationError('A required source is unavailable.') + raise _step_contract_error( + 'A required source is unavailable.', raw, + code='source_unavailable', rule='document_authorization_required', + ) if 'reference_document_ids' in arguments: _validate_reference_id_list(arguments['reference_document_ids'], 'Reference document IDs') if authorized_document_ids is not None and set(arguments['reference_document_ids']) - set(authorized_document_ids): @@ -1204,7 +1316,10 @@ def validate_dependency_plan( code='source_binding_required', rule='document_sources_required', ) if capability_id == 'document_analyze' and arguments.get('document_ids') and 'sources' in step['inputs']: - raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + raise _step_contract_error( + 'Use either explicit document IDs or a named source-set input.', raw, + code='plan_inputs_invalid', rule='source_binding_conflict', + ) if capability_id == CAPABILITY_TABULAR_MERGE: if not arguments.get('document_ids') and 'sources' not in step['inputs']: raise PlanValidationError( @@ -1311,7 +1426,10 @@ def validate_dependency_plan( availability=deliverable_availability, image_selected=image_selected, ) except ResultContractError as exc: - raise PlanValidationError('The plan has an invalid or unavailable result binding.', code=exc.code) from exc + raise PlanValidationError( + 'The plan has an invalid or unavailable result binding.', + code=exc.code, rule=exc.rule or exc.code, diagnostics=exc.diagnostics, + ) from exc except DeliverableError as exc: raise PlanValidationError(exc.message, code=exc.code, rule=exc.rule) from exc compiled = deepcopy(plan) @@ -1344,7 +1462,7 @@ def validate_plan( deliverable_availability=None, image_selected=False, seeds=None, - workflow_planning=None, + workflow_planning=None, actions=None, ): """Make a planner-authored plan safe to run, or refuse it. @@ -1372,7 +1490,7 @@ def validate_plan( composition_profiles=composition_profiles, export_catalog=export_catalog, deliverable_availability=deliverable_availability, image_selected=image_selected, - seeds=seeds, workflow_planning=workflow_planning, + seeds=seeds, workflow_planning=workflow_planning, actions=actions, ) @@ -1408,11 +1526,26 @@ def effective_plan_document_ids(plan, seeds=None): return list(dict.fromkeys(document_ids)) -def validate_plan_document_source_kinds(plan, document_source_kinds): +def validate_plan_document_source_kinds(plan, document_source_kinds, *, document_source_formats=None): """Check known source kinds without treating model-supplied labels as authority.""" if not isinstance(document_source_kinds, dict): raise PlanValidationError('Document type metadata is unavailable.', code='source_metadata_invalid') for step in plan.get('steps') or []: + if ( + step.get('enabled', True) and step.get('capability_id') == 'document_search' + and (step.get('arguments') or {}).get('native_section') == 'comments' + and any( + (document_source_formats or {}).get(document_id) not in ( + None, 'xlsx', 'xlsm', 'xls', 'docx', 'docm', 'dotx', + ) + for document_id in plan_document_ids({'steps': [step]}) + ) + ): + raise _step_contract_error( + 'Native comments require a supported Word or Excel source. Use another offered ' + 'reader for this format without dropping the selected document.', step, + code='source_kind_invalid', rule='native_comments_format_unsupported', + ) if step.get('enabled', True) and step.get('capability_id') in ( CAPABILITY_TABULAR_MERGE, CAPABILITY_TABULAR_INSPECT, ): @@ -1758,6 +1891,7 @@ def normalize_plan( action_refs=[ action.get('action_ref') for action in actions or () if isinstance(action, dict) ], + actions=actions, existing_results=existing_results, composition_profiles=composition_profiles, export_catalog=export_catalog, @@ -1891,7 +2025,10 @@ def apply_plan_edits( plan.get('deliverables'), plan['steps'], final_response=plan.get('final_response'), ) except ResultContractError as exc: - raise PlanValidationError('This edit leaves a required result unavailable.', code=exc.code) from exc + raise PlanValidationError( + 'This edit leaves a required result unavailable.', + code=exc.code, rule=exc.rule, diagnostics=exc.diagnostics, + ) from exc except DeliverableError as exc: raise PlanValidationError(exc.message, code=exc.code, rule=exc.rule) from exc original.update(plan) @@ -1945,6 +2082,22 @@ def summarize_plan(plan): "This step's agent or action isn't available to you right now, so it didn't run. Check " 'that it still exists, is enabled and is shared with you, then retry.' ), + 'action_initialization_failed': ( + "The action's tools or model could not be initialized. Ask an admin to check the " + 'action and model configuration, then retry.' + ), + 'document_reader_identity_mismatch': ( + "The document-reading tool did not match the application's expected tool. No " + 'document-reading function ran. Ask an admin to repair or update the application, then retry.' + ), + 'document_action_read_failed': ( + 'The action could not complete the selected document read. Check the file availability ' + 'and requested inspection, then retry.' + ), + 'action_results_missing': ( + 'The action returned no usable function results. No findings from this step were used. ' + 'Ask an admin to check the action and model tool support if it keeps failing.' + ), 'operation_recovery_required': ( 'An operation may already have run, or its material changed. Check its saved outgoing ' 'action and the external service before creating a new plan. It was not repeated.' @@ -2121,6 +2274,8 @@ def summarize_plan(plan): # Failures an exception may carry by code, through its ``orchestration_failure_code``. EXCEPTION_FAILURE_CODES = M365_STEP_FAILURE_CODES | { 'external_session_required', 'operation_recovery_required', 'integration_input_unavailable', + 'action_initialization_failed', 'document_reader_identity_mismatch', + 'document_action_read_failed', 'action_results_missing', STORAGE_FAILURE_CODE, } # Steps that hand work to an integration; input-reading failures have their own code. diff --git a/application/single_app/functions_orchestration_source_context.py b/application/single_app/functions_orchestration_source_context.py index 461afde39..df18ebd90 100644 --- a/application/single_app/functions_orchestration_source_context.py +++ b/application/single_app/functions_orchestration_source_context.py @@ -159,10 +159,11 @@ def validate_source_clarification(context, selected_sources): Multiple plausible images require a focused source_selection question. Use normal presentation defaults rather than blocking on optional formatting preferences. -For Excel Notes or threaded comments, use document_search native_section="comments" -with explicit document_ids, then compose from prepared/evidence. This native inventory -contains available authors, replies, dates, resolution and saved worksheet/cell values, -including blank/out-of-table cells. Do not replace it with row previews or tabular_analyze. +For Word review comments (.docx, .docm, .dotx), or Excel Notes/threaded comments, use +document_search native_section="comments" with document_ids, then compose from +prepared/evidence. This native inventory contains available anchors, authors, replies, +dates and resolution; Excel also includes saved worksheet/cell values, including +blank/out-of-table cells. Do not replace it with row previews or tabular_analyze. Use tabular computation additionally only for actual calculations. Reviewer feedback is not a replacement worksheet value. Respect native coverage and unavailable metadata. diff --git a/application/single_app/functions_search_service.py b/application/single_app/functions_search_service.py index 4dc812229..549fd29cb 100644 --- a/application/single_app/functions_search_service.py +++ b/application/single_app/functions_search_service.py @@ -1,6 +1,7 @@ # functions_search_service.py """Shared search, retrieval, and summarization services for documents.""" +import hashlib import io import json import logging @@ -46,6 +47,7 @@ ) from functions_group import get_user_groups from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_office_media import extract_docx_comments from model_endpoint_clients import build_openai_client_timeout_options from public_source_access import get_public_source_workspace_ids from public_source_policy import uses_all_public_sources @@ -1109,6 +1111,7 @@ def get_document_chunks_payload( ACCESSIBILITY_REPORT_SECTIONS = ("accessibility", "style", "issues") +NATIVE_WORD_COMMENT_EXTENSIONS = (".docx", ".docm", ".dotx") ACCESSIBILITY_REPORT_STYLE_KEY = "style_inventory" ACCESSIBILITY_REPORT_ISSUES_KEY = "issues" @@ -1118,10 +1121,32 @@ def _normalize_accessibility_sections(sections): requested = [part.strip().lower() for part in sections.split(",")] else: requested = [str(part or "").strip().lower() for part in (sections or [])] - requested = [part for part in requested if part in ACCESSIBILITY_REPORT_SECTIONS] + requested = [part for part in requested if part in (*ACCESSIBILITY_REPORT_SECTIONS, "comments")] return requested or list(ACCESSIBILITY_REPORT_SECTIONS) +def _read_word_comment_inspection(document_context, user_id): + """Reuse the ingestion parser over the existing authorized original-file reader.""" + document, source_bytes = screening_access.read_available_document_bytes( + document_context["document"], user_id=user_id, + group_id=document_context.get("group_id"), + public_workspace_id=document_context.get("public_workspace_id"), purpose="accessibility", + ) + document_context["document"] = document + comments, total_body_words = extract_docx_comments(io.BytesIO(source_bytes), strict=True) + return { + "section": "comments", "source_sha256": hashlib.sha256(source_bytes).hexdigest(), + "records": comments, "total_records": len(comments), + "total_replies": sum(len(comment["replies"]) for comment in comments), + "total_body_words": total_body_words, "inventory_complete": True, + "limitations": [ + "Anchored text is limited to 300 characters; anchor_complete identifies a bounded or unavailable anchor.", + "Resolution is unknown unless resolution_available is true; a missing flag is not an unresolved comment.", + "Reply relationships come from the available Word commentsExtended part, not inferred from comment text.", + ], + } + + def _read_excel_inspection( document_context, user_id, *, native_section="comments", native_cursor=None, native_page_size=100, sheet_name=None, cell_range=None, @@ -1242,6 +1267,16 @@ def get_document_accessibility_payload( stored_metadata = current_document.get("accessibility_metadata") file_name = str(current_document.get("file_name") or "") extension = "." + file_name.rsplit(".", 1)[-1].lower() if "." in file_name else "" + requested_sections = _normalize_accessibility_sections(sections) + word_inspection = None + if "comments" in requested_sections and extension in NATIVE_WORD_COMMENT_EXTENSIONS: + if native_cursor or cell_range or sheet_name or native_section != "comments": + raise ValueError("Word comments do not accept Excel sections, cursors or cell filters.") + word_inspection = _read_word_comment_inspection(document_context, user_id) + current_document = document_context["document"] + stored_metadata = current_document.get("accessibility_metadata") + if not stored_metadata and all(section == "comments" for section in requested_sections): + stored_metadata = {"format": extension.lstrip(".")} if not stored_metadata and extension in (".xlsx", ".xlsm", ".xls") and ( current_document.get("source_file_available") or current_document.get("enhanced_citations") or current_document.get("blob_path") or current_document.get("archived_blob_path") @@ -1270,8 +1305,11 @@ def get_document_accessibility_payload( elif native_cursor or cell_range: raise LookupError("The original workbook or full native inventory is unavailable for this inspection.") - requested_sections = _normalize_accessibility_sections(sections) + if "comments" in requested_sections and word_inspection is None and excel_inspection is None: + raise LookupError("The native comment inventory is unavailable for this document format.") report = {} + if word_inspection is not None: + report["word_inspection"] = word_inspection if "accessibility" in requested_sections: report["accessibility_metadata"] = { key: value diff --git a/application/single_app/functions_task_completeness.py b/application/single_app/functions_task_completeness.py index bcc6b0d28..57e6c5e7e 100644 --- a/application/single_app/functions_task_completeness.py +++ b/application/single_app/functions_task_completeness.py @@ -190,14 +190,22 @@ def context_only_review_step_ids(steps): return frozenset((step["step_id"],)) -def assessment_response_format(previous=()): - """Close known obligation identities in the transport, not in model-authored prose.""" +def assessment_response_format(previous=(), *, evidence=None): + """Close obligation identities and supplied evidence labels in the transport.""" + refs = {"type": "array", "items": {"type": "string"}} + if evidence is not None: + if type(evidence) is not dict or any(type(key) is not str for key in evidence): + raise CompletenessError("completeness_input_invalid") + if evidence: + refs["items"]["enum"] = list(evidence) + else: + refs["maxItems"] = 0 judgment = { "type": "object", "properties": { "status": {"type": "string", "enum": sorted(REQUIREMENT_STATUSES)}, "reason": {"type": "string"}, - "evidence_refs": {"type": "array", "items": {"type": "string"}}, + "evidence_refs": refs, "answer_quote": {"type": "string"}, }, "required": ["status", "reason", "evidence_refs", "answer_quote"], @@ -356,12 +364,19 @@ def _evidence_aliases(evidence): def _resolve_evidence_refs(refs, evidence, aliases): + def invalid(rule): + raise CompletenessError("completeness_evidence_invalid", details={ + "evidence_rule": rule, "allowed_evidence_refs": list(evidence), + "evidence_ref_count": len(refs) if type(refs) is list else 0, + "evidence_refs_list": type(refs) is list, + }) + if type(refs) is not list or len(refs) > 100: - raise CompletenessError("completeness_evidence_invalid") + invalid("evidence_reference_list_invalid") resolved = [] for ref in refs: if type(ref) is not str: - raise CompletenessError("completeness_evidence_invalid") + invalid("evidence_reference_string_required") cleaned = ref.strip() if not cleaned: continue @@ -378,7 +393,7 @@ def _resolve_evidence_refs(refs, evidence, aliases): if owners: resolved.extend(key for key in evidence if key in owners and key not in resolved) continue - raise CompletenessError("completeness_evidence_invalid") + invalid("evidence_reference_unknown") return resolved @@ -504,7 +519,7 @@ def public_completeness_report(state): def check_and_correct( candidate, *, instruction, contract, evidence, policy, invoke, validate, checkpoint=None, saved_state=None, recheck=None, on_error=None, owner_step_id=None, - evidence_basis=None, legacy_evidence=None, + evidence_basis=None, legacy_evidence=None, on_diagnostic=None, ): """Check every candidate, with one persisted allowance shared by all validators. @@ -581,6 +596,30 @@ def stop(reason): save() return state["candidate"], state + def assessment_error(error): + code = str(error) if isinstance(error, CompletenessError) else type(error).__name__ + if callable(on_error): + on_error(code) + if callable(on_diagnostic): + known = code in _REPAIRABLE_REPORT_CODES or code in _OPTIONAL_CHECKER_ERRORS + diagnostic = { + "error_type": code if known else type(error).__name__, + "validation_code": code if known else "completeness_verification_unavailable", + "report_repair_count": state.get("report_repairs", 0), + "evidence_key_count": len(evidence), + } + if isinstance(error, CompletenessError): + for key in ("evidence_ref_count", "evidence_refs_list"): + if type(error.details.get(key)) in (int, bool): + diagnostic[key] = error.details[key] + rule = error.details.get("evidence_rule") + if rule in ( + "evidence_reference_list_invalid", "evidence_reference_string_required", + "evidence_reference_unknown", + ): + diagnostic["validation_rule"] = rule + on_diagnostic(diagnostic) + if state["status"] in ("complete", "incomplete", "verification_unavailable"): return state["candidate"], state if state["status"] == "correcting": @@ -637,9 +676,13 @@ def skip_optional_error(reason): return skipped state["semantic_status"] = "running" save() - response_format = assessment_response_format(state["requirements"]) + response_format = assessment_response_format(state["requirements"], evidence=evidence) payload = { **request, "candidate": current, + "evidence_reference_rule": ( + "Use only the top-level evidence labels permitted by assessment_schema. " + "Use [] when an obligation needs no source reference; never invent a label." + ), "assessment_schema": response_format["json_schema"]["schema"], "previous_requirements": [ {"id": item["id"], "requirement": item["requirement"]} @@ -661,8 +704,7 @@ def assess(): try: requirements = assess() except Exception as error: - if callable(on_error): - on_error(str(error) if isinstance(error, CompletenessError) else type(error).__name__) + assessment_error(error) if isinstance(error, CompletenessError): skipped = skip_optional_error(str(error)) if skipped is not None: @@ -679,8 +721,7 @@ def assess(): try: requirements = assess() except Exception as repair_error: - if callable(on_error): - on_error(str(repair_error) if isinstance(repair_error, CompletenessError) else type(repair_error).__name__) + assessment_error(repair_error) return stop( str(repair_error) if isinstance(repair_error, CompletenessError) else "completeness_verification_unavailable" diff --git a/application/single_app/requirements.txt b/application/single_app/requirements.txt index e74aba6fe..5587903a7 100644 --- a/application/single_app/requirements.txt +++ b/application/single_app/requirements.txt @@ -53,6 +53,8 @@ pillow==13.0.0 ffmpeg-binaries-compat==1.0.1 ffmpeg-python==0.2.0 semantic-kernel==1.39.4 +# Required by Semantic Kernel's Handlebars template parser. +PyMeta3==0.5.1 snowflake-connector-python[pandas]==4.7.3 tableauserverclient==0.40 yamcs-client==2.1.0 diff --git a/application/single_app/route_backend_orchestration.py b/application/single_app/route_backend_orchestration.py index 802fc7db2..a88a2ef4d 100644 --- a/application/single_app/route_backend_orchestration.py +++ b/application/single_app/route_backend_orchestration.py @@ -420,7 +420,7 @@ def fail(message, code='planning_failed', **fields): '[ORCHESTRATION] Planning failure could not update its saved question.', level=logging.WARNING, extra={'stage': 'planning_finalize', 'error_type': type(exc).__name__}, ) - emit(serialize_sse({'error': message, 'code': code, 'done': True})) + emit(serialize_sse({'error': message, 'code': code, 'done': True, **fields})) @copy_current_request_context def worker(): @@ -450,7 +450,11 @@ def worker(): if holder.get('owned'): _require_planning_owner(holder) if payload.get('error'): - fail(payload['error'], payload.get('code') or 'planning_failed') + fail( + payload['error'], payload.get('code') or 'planning_failed', + **{key: payload[key] for key in ('error_stage', 'error_rule', 'error_attempt') + if key in payload}, + ) terminal = True continue if payload.get('type') == 'orchestration_elicitation': @@ -3557,7 +3561,12 @@ def generate(): 'Conversation context could not be used. Retry or create a new plan.', conversation_id, ) - except (PlannerError, OrchestrationMemoryError) as exc: + except PlannerError as exc: + yield build_error_event( + exc.message, resolved_conversation_id, + code=exc.code, stage=exc.stage, rule=exc.rule, attempt=exc.attempt, + ) + except OrchestrationMemoryError as exc: yield build_error_event(exc.message, resolved_conversation_id) except ModelCatalogError as exc: log_event('[ORCHESTRATION] Model routing failed.', level=logging.WARNING, extra={'code': exc.code}) diff --git a/application/single_app/semantic_kernel_plugins/document_search_plugin.py b/application/single_app/semantic_kernel_plugins/document_search_plugin.py index 0a19ca864..def5a7261 100644 --- a/application/single_app/semantic_kernel_plugins/document_search_plugin.py +++ b/application/single_app/semantic_kernel_plugins/document_search_plugin.py @@ -448,6 +448,9 @@ def retrieve_document_chunks( 'Use it to build custom style-guide or accessibility checks. This is extracted metadata, not a ' 'WCAG or PDF/UA conformance result. Use sections to limit the output and page_number to also ' 'return that page\'s stored chunk text. ' + 'For Word .docx/.docm/.dotx review comments, request sections=comments; word_inspection ' + 'contains exact comment threads, anchors and available resolution metadata from the authorized ' + 'original. Missing originals or malformed comment parts are errors, not empty inventories. ' 'For Excel, inspect native_section and follow excel_inspection.page.next_cursor until has_more ' 'is false when the question requires every record. Compatibility notes are not extra comments. ' 'Use sheet_name and cell_range for actual saved values and formula caches; do not infer them ' @@ -459,7 +462,7 @@ def get_document_accessibility_report( self, document_id: Annotated[str, 'Document id to read accessibility and formatting metadata for.'], doc_scope: Annotated[str, 'all, personal, group, or public.'] = '', - sections: Annotated[str, 'Optional comma-separated sections: accessibility, style, issues. Defaults to all.'] = '', + sections: Annotated[str, 'Optional comma-separated sections: accessibility, style, issues, comments. Defaults to accessibility/style/issues; Word comments are opt-in.'] = '', page_number: Annotated[int, 'Optional 1-based stored page/slide or Excel evidence segment number, not an Excel printed page.'] = 0, active_group_ids: Annotated[str, 'Optional comma-separated group ids when resolving group content.'] = '', active_public_workspace_id: Annotated[str, 'Optional public workspace id when resolving public content.'] = '', diff --git a/application/single_app/semantic_kernel_plugins/plugin_loader.py b/application/single_app/semantic_kernel_plugins/plugin_loader.py index 9e897c4ed..b4759f4bd 100644 --- a/application/single_app/semantic_kernel_plugins/plugin_loader.py +++ b/application/single_app/semantic_kernel_plugins/plugin_loader.py @@ -1,13 +1,19 @@ -import os -import importlib.util +# plugin_loader.py +"""Discover installed plugins without duplicating their runtime class identities.""" + +import importlib import inspect import logging -from functions_appinsights import log_event +import os from typing import Dict, Type, List + +from functions_appinsights import log_event from semantic_kernel_plugins.base_plugin import BasePlugin + PLUGIN_DIR = os.path.dirname(__file__) + def discover_plugins() -> Dict[str, Type[BasePlugin]]: """ Dynamically discover all BasePlugin subclasses in the semantic_kernel_plugins directory. @@ -18,24 +24,26 @@ def discover_plugins() -> Dict[str, Type[BasePlugin]]: for filename in os.listdir(PLUGIN_DIR): if filename.endswith('.py') and not filename.startswith('__') and filename != 'base_plugin.py' and filename != 'plugin_loader.py': module_name = filename[:-3] - module_path = os.path.join(PLUGIN_DIR, filename) - + try: - spec = importlib.util.spec_from_file_location(f"semantic_kernel_plugins.{module_name}", module_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - + # Runtime guards and other loaders must see the same imported classes. + module = importlib.import_module(f"semantic_kernel_plugins.{module_name}") + for name, obj in inspect.getmembers(module, inspect.isclass): if issubclass(obj, BasePlugin) and obj is not BasePlugin: plugins[name] = obj - - except Exception as e: - # Log the error but continue with other plugins - log_event(f"Failed to load plugin module {module_name}: {str(e)}") + + except Exception as exc: + log_event( + "[PLUGIN_LOADER] Plugin module could not be loaded.", + level=logging.WARNING, + extra={"plugin_name": module_name, "error_type": type(exc).__name__}, + ) continue - + return plugins + def get_all_plugin_metadata() -> List[dict]: """ Instantiate each discovered plugin and return a list of their metadata dicts. diff --git a/application/single_app/static/js/chat/chat-citations.js b/application/single_app/static/js/chat/chat-citations.js index 9cabf5e35..d37079d50 100644 --- a/application/single_app/static/js/chat/chat-citations.js +++ b/application/single_app/static/js/chat/chat-citations.js @@ -421,7 +421,9 @@ export function fetchCitedText(citationId, citationContext = {}) { // Check for expected data fields explicitly if (data.native_annotation || data.native_coverage) { const record = data.native_annotation; - const text = record ? [ + const word = data.evidence_kind === 'native_word_annotation' + || ['docx', 'docm', 'dotx'].includes(data.native_coverage?.format); + const text = record && word ? nativeWordCommentText(record, data.cited_text) : record ? [ record.kind === 'note' ? 'Workbook note' : record.parent_id ? 'Threaded comment reply' : 'Threaded comment', `Author: ${record.author || 'Unavailable'}`, `Worksheet / cell: ${record.sheet || 'Unavailable'} / ${record.cell || 'Unavailable'}`, @@ -436,8 +438,8 @@ export function fetchCitedText(citationId, citationContext = {}) { 'Inventory coverage is not a worksheet cell or reviewer comment.', ].join('\n'); showCitedTextPopup(text, data.file_name, '', { - title: `Native workbook evidence: ${data.file_name}`, - workbookPreview: () => showEnhancedCitationModal( + title: `Native ${word ? 'Word' : 'workbook'} evidence: ${data.file_name}`, + workbookPreview: word ? undefined : () => showEnhancedCitationModal( data.document_id, record?.sheet || '', null, record?.sheet || null, ), }); @@ -459,6 +461,32 @@ export function fetchCitedText(citationId, citationContext = {}) { }); } +function nativeWordCommentText(record, citedText) { + const resolution = (comment) => comment.resolution_available && typeof comment.done === 'boolean' + ? comment.done ? 'Yes' : 'No' : 'Unavailable'; + const lines = [ + 'Word review comment', + `Author: ${record.author || 'Unavailable'}`, + `Date: ${record.date || 'Unavailable'}`, + `Anchored text: ${record.anchor_text || 'Unavailable'}`, + `Anchor coverage: ${record.anchor_complete ? 'Complete' : 'Bounded or unavailable'}`, + `Resolved: ${resolution(record)}`, + '', 'Reviewer text', citedText || 'No stored reviewer text.', + ]; + if (Array.isArray(record.replies)) { + for (const reply of record.replies) { + lines.push( + '', `Reply ${reply.id}`, `Author: ${reply.author || 'Unavailable'}`, + `Date: ${reply.date || 'Unavailable'}`, `Resolved: ${resolution(reply)}`, + reply.text || 'No stored reply text.', + ); + } + } else { + lines.push('', 'Reply metadata unavailable.'); + } + return lines.join('\n'); +} + function renderMarkdownIntoCitationElement(contentElement, markdownText) { if (typeof marked === 'undefined' || typeof DOMPurify === 'undefined') { contentElement.textContent = String(markdownText || ''); diff --git a/application/v2_ui/src/components/chat/CitationChip.tsx b/application/v2_ui/src/components/chat/CitationChip.tsx index 8e4203d5b..7ffbefa09 100644 --- a/application/v2_ui/src/components/chat/CitationChip.tsx +++ b/application/v2_ui/src/components/chat/CitationChip.tsx @@ -88,6 +88,9 @@ function CitationDetail({ return () => document.removeEventListener('keydown', onKeyDown); }, [onClose, preview]); + const wordAnnotation = data?.evidence_kind === 'native_word_annotation' + || ['docx', 'docm', 'dotx'].includes(data?.native_coverage?.format ?? ''); + if (preview) return
- {data?.native_annotation &&
+ {data?.native_annotation &&

- {data.native_annotation.kind === 'note' ? 'Workbook note' + {wordAnnotation ? 'Word review comment' : data.native_annotation.kind === 'note' ? 'Workbook note' : data.native_annotation.parent_id ? 'Threaded comment reply' : 'Threaded comment'}

Author: {data.native_annotation.author || 'Unavailable'}

-

Worksheet / cell: {data.native_annotation.sheet || 'Unavailable'} / {data.native_annotation.cell || 'Unavailable'}

+ {wordAnnotation + ? <> +

Anchored text: {data.native_annotation.anchor_text || 'Unavailable'}

+

Anchor coverage: {data.native_annotation.anchor_complete ? 'Complete' : 'Bounded or unavailable'}

+ + :

Worksheet / cell: {data.native_annotation.sheet || 'Unavailable'} / {data.native_annotation.cell || 'Unavailable'}

}

Date: {data.native_annotation.date || 'Unavailable'}

{data.native_annotation.parent_id &&

Reply to: {data.native_annotation.parent_id}

} -

Resolved: {typeof data.native_annotation.resolved === 'boolean' - ? data.native_annotation.resolved ? 'Yes' : 'No' : 'Unavailable'}

-

Saved cell value: {data.native_annotation.cell_value_status === 'blank' ? 'Blank' +

Resolved: {wordAnnotation + ? data.native_annotation.resolution_available && typeof data.native_annotation.done === 'boolean' + ? data.native_annotation.done ? 'Yes' : 'No' : 'Unavailable' + : typeof data.native_annotation.resolved === 'boolean' + ? data.native_annotation.resolved ? 'Yes' : 'No' : 'Unavailable'}

+ {!wordAnnotation &&

Saved cell value: {data.native_annotation.cell_value_status === 'blank' ? 'Blank' : data.native_annotation.cell_value == null ? 'Unavailable (not recalculated)' - : JSON.stringify(data.native_annotation.cell_value)}

+ : JSON.stringify(data.native_annotation.cell_value)}

}

Reviewer text

} {data?.native_coverage &&
@@ -151,7 +162,7 @@ function CitationDetail({

{data.native_coverage.returned_records ?? 'Unknown'} of {data.native_coverage.total_records ?? 'unknown'} records retained.

{data.native_coverage.partial || !data.native_coverage.inventory_complete ? 'Partial inventory; remaining annotations were not inspected.' : 'The saved annotation inventory was inspected completely.'}

-

This is inventory coverage, not a cited worksheet cell or reviewer comment.

+

This is inventory coverage, not a cited {wordAnnotation ? 'document passage' : 'worksheet cell'} or reviewer comment.

} {notice && (

@@ -178,17 +189,32 @@ function CitationDetail({ )} {data?.cited_text && ( -

+

{data.cited_text}

)} + {wordAnnotation && data?.native_annotation?.replies && data.native_annotation.replies.length > 0 && ( +
+

Replies

+ {data.native_annotation.replies.map((reply) => ( +
+

Author: {reply.author || 'Unavailable'}

+

Date: {reply.date || 'Unavailable'}

+

Resolved: {reply.resolution_available && typeof reply.done === 'boolean' + ? reply.done ? 'Yes' : 'No' : 'Unavailable'}

+

{reply.text || 'No stored reply text.'}

+
+ ))} +
+ )} + {!loading && !error && !data?.cited_text && (

This citation has no stored passage to show.

)} - {!loading && !error && citation.nativeEvidence && }
diff --git a/application/v2_ui/src/lib/types.ts b/application/v2_ui/src/lib/types.ts index 28d80713e..7634c34f7 100644 --- a/application/v2_ui/src/lib/types.ts +++ b/application/v2_ui/src/lib/types.ts @@ -1447,7 +1447,9 @@ export interface Citation { cited_text?: string; file_name?: string; page_number?: number | string; + evidence_kind?: string; native_annotation?: { + id?: string; kind?: string; author?: string; date?: string; @@ -1457,8 +1459,21 @@ export interface Citation { resolved?: boolean; cell_value?: unknown; cell_value_status?: string; + anchor_text?: string; + anchor_complete?: boolean; + done?: boolean | null; + resolution_available?: boolean; + replies?: Array<{ + id: string; + author?: string; + date?: string; + text?: string; + done?: boolean | null; + resolution_available?: boolean; + }>; }; native_coverage?: { + format?: string; returned_records?: number; total_records?: number; partial?: boolean; diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index 9c3cd9aab..941343805 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -684,7 +684,8 @@ without calling the integration again. Since **0.261.293**, what a plan gathers always reaches the user. Every gather step's results must be read by a step that writes the answer or prepares a file or image, and a chart or diagram must end up in the answer or a file. Starting a saved workflow is the one -exception. A plan that breaks the rule gets the planner's usual single correction. A chart +exception. Since **0.261.422**, recognized plan errors share at most two planner +corrections without widening source permissions. A chart an action drew is always shown, and one the answer doesn't show is listed in the delivery notes and makes the run partially completed, so **Retry from failed step** can draw it again. Retrying runs the action again, so the user confirms it first. diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md index ba397cb7c..d54135094 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION.md @@ -1,6 +1,6 @@ # Chat Orchestration -**Version: 0.261.417** (tracked in `application/single_app/config.py`) +**Version: 0.261.423** (tracked in `application/single_app/config.py`) **Implemented in version: 0.261.086** **Knowledge phase added in version: 0.261.089** @@ -32,6 +32,7 @@ **Selected-document planning recovery and debug diagnostics fixed/implemented in version: 0.261.413** **Initial-turn persistence and browser-independent Auto planning implemented in version: 0.261.415** **Confirmed progress, exact native workbook citations, and saved-analysis cleanup fixed in version: 0.261.417** +**Conservative source wiring, two shared planning corrections, native Word comments, and evidence-label checking fixed in version: 0.261.422** ## Overview @@ -101,9 +102,14 @@ Only a validated, approved plan reaches the execution adapters. Automatic approv does not bypass validation. Since **0.261.413**, a proposal that omits a selected file or declares incompatible -Analyze options receives targeted correction feedback. It shares the existing -single correction with other repairable planning failures: a second, different -failure does not start another retry budget. The server keeps source access, +Analyze options receives targeted correction feedback. Since **0.261.422**, eligible +failures share two model corrections: at most three proposals, not a new budget +for each error. When exactly one selected source has verified authorization and +availability, the server can fill an absent document binding on an existing, +enabled independent reader without another model call. It does not invent steps, +replace explicit IDs or dynamic inputs, narrow multiple selections, or restore +disabled work. Plan editing keeps its explicit source choices. +The server keeps source access, source-kind checks, every required selection, and the selected planner model. An explicit later plan edit can still narrow the selection, with a review warning. @@ -114,12 +120,27 @@ version/mode combinations are checked together; saved legacy meanings remain unchanged. An uncorrectable proposal fails clearly rather than becoming an ungrounded answer or silently excluding a file. +Native annotation requests distinguish Word from Excel. Word `.docx`, `.docm`, +and `.dotx` comments use the existing Word parser over authorized original bytes, +not the workbook reader or a search preview. Parsed reviewer text, anchored text, +available authors/dates/resolution, and linked replies travel through the retained +result and its citations. Missing metadata is unknown; missing originals and +malformed or over-limit packages are not successful empty inventories. + +Prepared-answer checking advertises only the evidence labels supplied to that +check. An invalid checker reference receives one report repair with usable labels; +it does not by itself prove the answer is factually wrong. A genuinely unsupported +answer still needs correction, and required review still blocks when it cannot +finish. This report repair is separate from the planner's two corrections. + The existing debug logging switch enables content-free proposal and source-binding diagnostics in console and Application Insights. These distinguish a model omission from a normalization change using hashes, counts, and rejection rules, without recording document contents or the model reply. See the [document planning recovery fix](../fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md) and [logging reference](../../reference/logging-tags.md#selected-document-planning-diagnostics). +See the [pipeline reliability fix](../fixes/ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md) +for native Word evidence, checker boundaries, and validation limitations. ## Internal dependency contract (M4) @@ -1484,7 +1505,7 @@ to the front. | `functional_tests/test_v2_orchestration_deliverables.mjs` | Browser normalization of deliverables and `delivers`, deliverable states that follow their steps, labels, generated-image helpers, a terminal frame read through the real run stream client, and a retry's image grouped under every answer that lists it, in React V2 and the classic client | | `ui_tests/test_v2_orchestration_generated_images.py` | The live chat loading an answer's generated images after the run, planned image cards that never offer Approve or Approve all while their image loads, and a reused image shown under both the earlier and the retried answer | | `functional_tests/test_orchestration_context_picker.py` | Picked tags reach the seeds and both search paths under the parameter `hybrid_search` really takes; a tag scopes the probe rather than replacing it; a picked document reaches the planner and the approval card by name; a browser-supplied name cannot widen access; search citations carry the workspace a document came from; a step can read what an earlier step found, an unusable binding is refused, and a run-time document still respects the configured ceiling | -| `functional_tests/test_orchestration_document_planning_recovery.py` | Picked-document HTTP planning across five file formats, preserved previews and Analyze availability, strict option semantics, source/operation retention, one shared correction including malformed questions, editor compatibility, Auto routing, no execution during planning, and private-content-free debug diagnostics in both sinks | +| `functional_tests/test_orchestration_document_planning_recovery.py` | Picked-document HTTP planning across five file formats, preserved previews and Analyze availability, strict option semantics, source/operation retention, conservative local wiring, two shared corrections including malformed questions, editor compatibility, Auto routing, no execution during planning, and private-content-free debug diagnostics in both sinks | | `functional_tests/test_orchestration_conversation_context.py` | Message eligibility, bounds, snapshot validation, nullable unused clarifications, strict response validation, bounded repair, token accounting, provider/refusal handling, contextualized adapters, synthesis roles, and URL provenance | | `functional_tests/test_orchestration_conversation_context_routes.py` | New and existing conversations across HTTP/SSE planning and execution, all approval modes, null clarifications, bounded recovery, model selection and attribution, revocation, completion failures, stream cleanup, stale sources and legacy cutoffs | | `ui_tests/test_v2_orchestration_conversation_context.py` | Matching clarification/model transport, cancellation, original-turn continuity, all approval modes, visible answer model names and navigation | diff --git a/docs/explanation/features/CHAT_ORCHESTRATION_ACTIONS.md b/docs/explanation/features/CHAT_ORCHESTRATION_ACTIONS.md index a2ad4a63b..6a1e39f83 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION_ACTIONS.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION_ACTIONS.md @@ -1,6 +1,6 @@ # Chat Orchestration Action Access -Version: **0.261.321** +Version: **0.261.423** Implemented in version: **0.261.098** @@ -8,6 +8,12 @@ Retrieved-data charts added in version: **0.261.132** Operation intent and named inputs implemented in version: **0.261.321** +Selected-document action binding implemented in version: **0.261.419** + +Canonical plugin discovery and action-stage diagnostics implemented in version: **0.261.421** + +Source defaults and shared corrections fixed/implemented in version: **0.261.422** + Application version tracking remains in `application/single_app/config.py`. ## Purpose @@ -100,6 +106,49 @@ and binds their arguments. Avoiding agent setup does **not** mean avoiding model Conversational follow-ups retain their resolved request and bounded, authorized conversation reference when handed to an action, including accepted clarification answers. +Shared discovery imports each installed plugin through its registered package module. +The native-reader guard therefore sees the same class as the logged and classic loaders, +without sharing plugin instances or their user-specific configuration. +Loading/identity failures, failed document reads, and missing function results retain +distinct safe failure codes through saved step details and the final failure explanation. +They are not successful findings or automatic transient-retry conditions. + +See the [plugin discovery identity fix](../fixes/ORCHESTRATION_PLUGIN_DISCOVERY_IDENTITY_FIX.md) +for the real-loader regression and recovery boundaries. + +### Selected documents and native inspection + +Since **0.261.419**, a registered Document Search action can bind exact workspace +`document_ids` in its plan step. Its native metadata inspection then counts as work +on those sources without an extra Search or Analyze step added just for source +accounting. A filename or ID mentioned only in the task text is not that binding. + +The server checks the action's actual type, current user access, workspace +restrictions, and source snapshots. Native report, chunk and summary calls cannot +read an unbound document; a search with no model-supplied ID filter receives the +bound selection. The existing Run view shows the source chips and narrowing +controls before approval. + +Authorized workspace provenance and the action reference survive retention and +restart. Composition receives citations from those exact retained tool results, +not unrelated work in the conversation. Input reads still require current +authority; retained findings keep their existing conversation/run and integration +access policies. No source binding makes missing native metadata available, +establishes conformance, or permits an outgoing operation. + +Since **0.261.422**, the planner's two shared corrections can repair classified contract declarations +without changing its model or executing tools. Invalid source permissions and +unsupported integrations remain explicit failures. See the +[planner contract recovery explanation](../fixes/ORCHESTRATION_PLANNER_CONTRACT_RECOVERY_FIX.md) +for defaulted Analyze output variants, diagnostics, and full-path coverage. + +For exactly one verified available selection, an existing independent Document +Search step can receive an absent binding locally before model correction. +Explicit IDs, dynamic inputs, disabled work, and edited selections remain intact. +Word comment-only metadata calls opt into the authorized original reader with +`sections="comments"`; they do not send Word files to an Excel collector. See the +[pipeline reliability fix](../fixes/ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md). + ### Charting retrieved data Since **0.261.132**, an action step can chart what it retrieves. When the user's current diff --git a/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md b/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md index 9b20f48a6..1e735bf46 100644 --- a/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md +++ b/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md @@ -3,7 +3,8 @@ **Implemented in version: 0.261.391** (formatting, style inventory, and programmatic access added in **0.261.392**) **Excel implemented in version: 0.261.399** **Excel native content/table improvements implemented in version: 0.261.408** -**Related config.py version: `0.261.408`** +**Native Word comment inspection fixed/implemented in version: 0.261.422** +**Related config.py version: `0.261.423`** ## 1. Overview and Purpose @@ -272,10 +273,38 @@ The **Document Search** action exposes a `get_document_accessibility_report` Sem |---|---| | `document_id` | The document to read (required). | | `doc_scope` | `all`, `personal`, `group`, or `public`. | -| `sections` | Comma-separated subset of `accessibility`, `style`, `issues`. Defaults to all three. | +| `sections` | Comma-separated subset of `accessibility`, `style`, `issues`. Defaults to all three. Since 0.261.422, opt-in `comments` reads a workspace Word original. | | `page_number` | Optional 1-based page, slide, or Excel indexed evidence segment. Also returns that segment's stored chunk text. Excel segments are not physical print pages. | -Access follows the existing Document Search rules: the current user's workspace, group, and public access and the action's allowed scopes. Chat-scope uploads are refused, and documents without stored metadata return guidance to re-upload. There is deliberately no REST endpoint for this. +Access follows the existing Document Search rules: the current user's workspace, group, and public access and the action's allowed scopes. Chat-scope uploads are refused. Stored-report requests without metadata return extraction guidance; Word comment inspection can use an available original even when stored style metadata is absent. There is deliberately no separate REST endpoint for this function. + +### Opt-in native Word comments (0.261.422) + +Fixed/Implemented in version: **0.261.422** +(`application/single_app/config.py`). + +Structured Word comments are parsed during ingestion and incorporated into +searchable comment chunks; ordinary stored Word accessibility/style metadata is +not that structured inventory. `sections="comments"` reuses +`functions_office_media.extract_docx_comments` over the authorized active original, +with strict inspection that distinguishes an inspected empty package from +unavailable, malformed, or over-limit parts. + +The `word_inspection` result contains parsed root comments and supported linked +replies, available author/date/resolution, bounded anchored text, counts, and +limitations. Anchors are at most 300 characters; clipped or unclosed ranges are +not marked complete. Missing resolution or threading metadata is not invented. +Default stored-metadata calls and default ingestion parsing keep their previous +behavior. A comment-only request can proceed without stored metadata; a combined +request still requires the stored sections rather than returning empty style or +issue data for an uninspected section. + +Orchestration dispatches this work by actual source format and retains the parsed +records before composition. Native citations reopen the exact saved result with +source-access, version/revision, and integrity checks; they do not choose a fresh +search hit. Both chat citation interfaces distinguish Word evidence from workbook +cells. Existing Search, Analyze, accessibility, and OCR/vision paths for other +formats are unchanged. ### Building Custom Checks @@ -304,3 +333,7 @@ Rules that need data SimpleChat does not extract (for example full tint/shade co - Validates `get_document_accessibility_report` section filtering, page chunks, chat-scope and missing-metadata errors, and the plugin and citation wiring. - **Functional Test**: `functional_tests/test_excel_document_accessibility_extraction.py` (implemented in **0.261.399**) - Exercises real OOXML and synthetic legacy BIFF workbooks, native properties, comments/replies, alt text/decorative flags, controls, formatting, print metadata, unchanged source bytes, bounded/failing inspection, both tabular ingestion modes, workspace scope propagation, schema retries, authorized report/citations and native-versus-data question routing. +- **Functional Test**: `functional_tests/test_orchestration_attachment_native_evidence.py` (Word coverage added in **0.261.422**) + - Exercises DOCX/DOCM/DOTX parsing, authorized original reads, exact retained comments/replies and citations, empty/unavailable/malformed/partial inventories, anchor limits, checker-label repair, unsupported findings, and content-free logging. +- **UI Test**: `ui_tests/test_v2_stream_citation_delivery.py` (Word coverage added in **0.261.422**) + - Checks Word anchors/replies/resolution and inert reviewer text in classic and V2 citation views at mobile and desktop widths, alongside existing Excel evidence. diff --git a/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md index 0d04d9718..3a26c5d8b 100644 --- a/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md +++ b/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md @@ -3,6 +3,17 @@ Fixed/Implemented in version: **0.261.413**, recorded in `application/single_app/config.py`. +Current behavior updated in version: **0.261.422**. +The [pipeline reliability follow-up](ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md) +adds conservative single-source wiring and two shared model corrections. +The [planner contract recovery follow-up](ORCHESTRATION_PLANNER_CONTRACT_RECOVERY_FIX.md) +aligns defaulted Analyze output variants, gives native document actions an enforced +source binding, and carries their provenance/citations through retention and +composition. It also classifies additional model-draft contract failures within +the same shared correction allowance and preserves typed failure information on +the saved question. These changes do not identify the exact historical model +JSON behind a generic rejection. + ## Issue and root cause Document-backed turns could finish upload/source preparation but stop with @@ -58,7 +69,9 @@ omissions separately. `functions_orchestration_planner.py` validates selected requirements inside the existing correction loop. Targeted feedback preserves every selection and asks for compatible, explicit source bindings. All repairable categories share -`PLAN_REPAIR_ATTEMPTS = 1`: a second different error does not start another budget. +`PLAN_REPAIR_ATTEMPTS = 2` since **0.261.422** (originally 1): a different error +does not start another budget. The initial proposal and at most two corrections +share that limit. An unrenderable clarification stays in that same loop rather than recursively restarting the planner and resetting its allowance. The same planner/model is used, and uncorrectable proposals return stable, specific @@ -96,7 +109,7 @@ for event names, properties, and a bounded query. request/context/planner/validator/logger boundaries with external I/O isolated. It covers PNG, PDF, DOCX, CSV and XLSX selections, manual/automatic approval, Auto routing, correctable and terminal option/source failures, mixed errors -sharing one correction, implicit scoped search, multiple/disabled sources, source-free +sharing one allowance, implicit scoped search, multiple/disabled sources, source-free controls, existing-plan narrowing, legacy semantics, explicit requirement bounds, provider failures, and debug privacy/gating/serialization. diff --git a/docs/explanation/fixes/ORCHESTRATION_PLANNER_CONTRACT_RECOVERY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_PLANNER_CONTRACT_RECOVERY_FIX.md new file mode 100644 index 000000000..50035fc23 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_PLANNER_CONTRACT_RECOVERY_FIX.md @@ -0,0 +1,285 @@ +# Orchestration planner contract and source-binding recovery (v0.261.419) + +Fixed/Implemented in version: **0.261.419**, tracked in +`application/single_app/config.py`. + +Current integrated version: **0.261.420**. The updated React V2 base adds +saved-findings publication identifiers and status codes to the logger; those +properties and this fix's consumer-run/schema diagnostics are preserved together. +Post-integration validation passed 179 logging/native/saved-findings functional +tests and 75 browser source-inspector checks. The merged V2 build, documentation +checks and reviewed Cosmos inventory also passed. + +Current planning behavior updated in **0.261.422**: +[pipeline reliability follow-up](ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md). +It adds conservative single-source defaults and two shared model corrections +without changing the source, output, or approval contracts described here. + +## Issue and root cause + +Similar document requests could either run normally or stop with "The request +could not be planned." Read-only Application Insights and App Service console +investigation distinguished several planning failures on deployment version +0.261.418: + +- A native inspection action did not count as selected-document work because its + contract had no structured document binding. +- Some replacement plans failed output declarations or producer/consumer kind + validation. Those failures could stop an initial draft without using the + available correction. +- Generic `plan_invalid` events did not identify their precise rejecting rule, + and the browser/saved-question boundary lost the typed planning reason. + +The failing and successful turns used the same recorded planner binding and Auto +routing. A later Word review completed. These observations identify validation +failures, not an upload/extraction failure or evidence of intermittent Azure +transport failure. + +The exact rejected production JSON was not retained. The investigation therefore +does not attribute a particular historical generic rejection to an invented field +or claim that every failure had one cause. + +A separate real-module reproduction established an effective-contract defect: +the planner was offered Analyze's optional `records` output, but omitted options +became v2 `source_observations`, which does not expose that projection. A draft +following the advertised output list could then be rejected by server defaults. +This defect requires contract alignment, not merely another retry. + +## Technical changes + +### One effective capability contract + +`functions_orchestration_registry.py` projects Analyze's actual default arguments +and output variants through the existing output resolver. The default projection +offers `findings` and `coverage`, plus optional `report`, not optional `records`. +Explicit legacy semantics and v2 `reconciled_records` retain their record output. +The planner is instructed to declare the appropriate semantics for required +records, rather than silently discard a requested projection. + +Fixed producers can omit output declarations and use the server's canonical +defaults. Explicit declarations remain strictly validated. + +### Explicit native document action sources + +`functions_action_manifest.py` identifies registered Document Search types, not +configured action names or task wording. The planner marks those resolved actions +as supporting document binding. Their `action_invoke` arguments can carry one to +100 exact, unique workspace `document_ids`; other integrations cannot claim this +source coverage. + +The compiler, executor, adapter and action runner preserve that binding. The +adapter resolves authorized source snapshots and rechecks identity, scope and +revision before each tool call. The actual loaded plugin must be Document Search. +Its native metadata, chunk and summary reads reject other IDs; search calls with +no model-supplied filter receive the bound IDs before source I/O. + +Existing action workspace restrictions and current source-authority checks still +apply. The existing task-local selected-source mechanism pins workspace identity +and isolates concurrent invocations. A document-read error is an explicit action +failure, not successful findings. Native properties that are absent or +inaccessible within an available report remain limitations, not invented facts. + +The existing plan UI displays these documents through its standard source chips +and narrowing controls. No extra Search or Analyze step is required solely to +declare that a native action reads the selected file. + +### Retention, citations and final composition + +`functions_orchestration_result_runtime.py` admits workspace provenance alongside +the current external action reference only for a matching, explicitly bound +Document Search action. Missing or mismatched sources, another integration, or an +undeclared binding remain rejected. + +Both kinds of lineage survive result storage and restart. Native tool citations +are projected from the exact retained result, checked against its workspace +lineage, and passed to composition. A citation cannot name an unbound document. +This prevents a valid plan from failing later at retention or losing its native +source references before the final answer. + +The executor also preserves the explicit binding at its post-action source +handoff. Its former external-Gather path cleared workspace sources before calling +retention, even though the action had read them. The authenticated Auto-run +regression exercises this handoff rather than substituting a prepared action +result at the adapter boundary. + +Retained results keep their existing conversation/run access semantics; recorded +native sources are provenance, not permission to reread an original. External +integration access continues to be checked. The change does not replace those +policies or promote partial findings into complete datasets. + +### Shared targeted corrections, not blanket retries + +`functions_orchestration_schema.py` and +`functions_orchestration_result_contracts.py` identify owned step-shape, +argument, output, named-input and graph failures. The same planner receives +contract-aware feedback, and the complete replacement plan passes every normal +validator. + +All eligible failures share **two correction attempts total** since **0.261.422** +(originally one). A different error does not reset that allowance. Unknown model argument names can be +corrected without logging those names. Permission failures, unavailable +capabilities, untrusted retained aliases, held sources, provider refusals and +unclassified exceptions do not become blanket-retryable. Workflow-owned argument +validation keeps its existing correction/drop policy. + +No tool executes during planning or correction. No selection, required operation, +approval or exact-data validation is disabled to make a plan succeed. + +### Failure meaning survives reload + +`functions_orchestration_events.py`, `route_backend_orchestration.py` and +`functions_orchestration_planning.py` carry known failure classifications through +SSE and the saved question: + +| Stream field | Saved planning field | Meaning | +| --- | --- | --- | +| `code` | `error_code` | Stable owned failure category | +| `error_stage` | `error_stage` | Rejecting planning phase | +| `error_rule` | `error_rule` | Specific rule, when known | +| `error_attempt` | `error_attempt` | Proposal attempt, when known | + +The stream can still use HTTP 200; an error frame and failed planning state are +not a completed request. Explicit retry keeps the original question/selections +and clears obsolete failure metadata when the new plan is ready. + +Debug diagnostics add only known schema keywords, registered argument-field +names or `unknown`, result-kind codes, counts and hashed step identity. Raw model +JSON, prompts, arbitrary property names, document contents and provider exceptions +are not added to logging. + +### Debug checkpoints from binding to final preparation + +With the existing debug switch enabled, correlated checkpoints now cover binding, +snapshot rechecks, native function attempts/outcomes, the executor source handoff, +retention, citation projection and final content preparation. They contain hashes, +counts, known operations, shape/presence flags and safe error types. Cross-run +consumption uses a validated consumer-run hash. + +These events explicitly gate the ordinary structured logger, reaching both console +and Application Insights. Existing console-only debug calls are not globally +exported. Diagnostics use already available data and do not reread a source, +invoke another model, or serialize its private content to build a log message. +See the [lifecycle checkpoint reference](../../reference/logging-tags.md#native-action-and-retained-result-lifecycle-checkpoints) +for event names and a bounded workspace KQL query. + +## Validation and impact + +The new functional suites cover effective Analyze contracts, source-free and +document-backed correction, mixed errors sharing one budget, protected sources, +real Semantic Kernel native calls, authority/revision changes, scope isolation, +SDK-default retention, integration reauthorization, citation confinement and +native findings reaching final composition with completeness checking enabled. +The browser regression verifies bound source visibility and narrowing using the +existing components, without production traffic. + +`test_orchestration_document_action_end_to_end.py` submits a real authenticated +HTTP request with automatic approval and requires a persisted cited answer. It +uses the real plugin loader, native report projection, execution leases, SDK-shaped +stores, retention, composition, completeness and publication. PDF and Word cases +run with debug on/off and with successful/failed native reads. Known native +metadata, its limits and its citation must reach composition unchanged. Failure +must stop before retention/composition instead of producing successful findings. + +The same tests check saved turn state, one original question, model/source call +counts, absent debug checkpoints when disabled, structured log privacy when +enabled, and the absence of private error text from the stream. The shared SDK +double now accepts the bounded read transport options used by real external +identity checks while retaining partition routing, ETags, failures and conditional +writes. Its behavior has a dedicated regression. + +Additional suites: + +- `test_orchestration_planner_contract_recovery.py`: effective defaults, owned + draft errors, single-budget repair, source constraints and non-document work. +- `test_orchestration_document_action_binding.py`: real native tool filters, + scope isolation, snapshot checks and debug-sink/call-count parity. +- `test_orchestration_document_action_lineage.py`: mixed lineage, restart/current + integration access, citation confinement and final composition, with completeness + and debug independently enabled/disabled. +- `test_orchestration_planning_persistence.py`: typed terminal reasons, reload, + explicit retry, claim fencing, cancellation and Auto handoff. +- `ui_tests/test_v2_orchestration_actions.py`: existing source-chip review and + narrowing controls for bound actions. + +| Before | After | +| --- | --- | +| A valid advertised Analyze projection could conflict with later defaults. | Default and conditional output projections agree with the shared resolver. | +| A native action could not declare selected-file coverage. | A supported action binds its exact sources without redundant reading work. | +| Passing planning could still fail at mixed native/action retention. | Both authorized lineage kinds and source citations reach retained readers and composition. | +| Draft shape/type mistakes could terminate before correction. | Classified model-draft errors receive the same single targeted correction. | +| Reload showed only a generic failure category. | Known phase, rule and attempt remain available in saved planning state. | + +This is branch-local implementation and offline/browser verification, not a +live-model success-rate claim. It does not change Azure configuration, deploy the +application, reprocess files, alter the user's conversations, or guarantee that +every model-generated draft will be valid. + +### Executed checks + +The combined 18-file shared-contract, native-action, source-selection, external +authorization, retained-citation, observation, available-information and saved-turn +selection passed **507 tests and 184 subtests**. The full HTTP regression by +itself passed **9 tests**: + +```powershell +$env:PYTHONPATH = "$PWD\application\single_app;$PWD\functional_tests" +python -m pytest -q -p no:cacheprovider --disable-warnings --tb=short --show-capture=no .\functional_tests\test_orchestration_document_action_end_to_end.py +``` + +The source-review browser regression and planning-retry suite passed **16 tests**: + +```powershell +python -m pytest -q -p no:cacheprovider --disable-warnings --tb=short .\ui_tests\test_v2_orchestration_actions.py::test_native_action_bound_sources_are_visible_and_can_be_narrowed .\ui_tests\test_v2_orchestration_planning_retry.py +``` + +The production V2 build, including TypeScript checking, passed with output isolated +under UI-test artifacts. Route Blueprint inventory, unauthenticated policy, +route-test coverage and logger initialization passed **31 tests** together. +Documentation surface coverage passed **7/7**, and site quality **6/6**. +Regenerating the application surface did not add a setting, route, plugin or UI +control. The reviewed Cosmos inventory has **1,094 owners, 425 doubles and zero +review gaps**. + +The changed SDK owners and shared double also passed **251 tests** with +`test_support.cosmos_network_guard` loaded before collection. This selection covers +the normalizer, SDK orchestration responses, saved planning, conversation/source +authorization, shared/regenerated turns, retained citations, attachments and the +new authenticated native Auto-run path. Inventory updates are limited to +22 reviewed planning/route owners and the SDK-default conditional container; +there are no added or removed owner/double entries. + +The full reviewed storage-contract gate also passed **2,015 tests and 431 +subtests**, with network access blocked before collection: + +```powershell +python .\scripts\check_cosmos_response_boundaries.py --run-contracts +``` + +### Broader baseline limitations + +These are not presented as a fully green repository-wide test run. Five unrelated +existing checks were also executed against unchanged `HEAD` production modules +and reproduced the same failures: + +- The tabular composition-profile assertion expects two profiles although + `document_transform_v1` is already registered. +- An older hand-off check expects that capability to be last although conversation + capabilities already follow it. +- The hand-off-off golden context expects a 3,600-second interval although the + current definition already permits one second. +- A workflow-proposal result-alias fixture is rejected as + `result_reference_untrusted`. +- The legacy AST-only research harness lacks `STORAGE_FAILURE_CODE` in its + extracted namespace. + +An existing debug-settings integration check also needs cloud configuration +outside the isolated harness; it is not counted as passing. A broader browser +selection reported 49 passed, two failed restored-state selector checks and two +teardown errors from an unexpected resource request. Application frontend source +was unchanged, and those failures are not suppressed or counted as passing. +The targeted browser selection above is independently green. + +Related explanations: +[selected-document planning recovery](ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md), +[action access](../features/CHAT_ORCHESTRATION_ACTIONS.md), and +[Document Search](../../reference/actions/document-search.md). diff --git a/docs/explanation/fixes/ORCHESTRATION_PLUGIN_DISCOVERY_IDENTITY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_PLUGIN_DISCOVERY_IDENTITY_FIX.md new file mode 100644 index 000000000..8b573b5ee --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_PLUGIN_DISCOVERY_IDENTITY_FIX.md @@ -0,0 +1,213 @@ +# Orchestration plugin discovery identity fix (v0.261.421) + +Fixed/Implemented in version: **0.261.421**, tracked in +`application/single_app/config.py`. + +## Issue and root cause + +Selected-document orchestration could validate its plan and permissions, load the +Document Search action, and then fail before its first document-reading function. +Later report-writing steps were skipped because their required evidence had never +been collected. Similar failures across different files were not evidence that +each file was corrupt. + +In plain language, the application made two copies of the same tool. A safety +check expected one copy, but the loader supplied the other. Python treated them +as different classes even though they shared a name and source file. + +`semantic_kernel_plugins/plugin_loader.py` executed a fresh module object for each +discovery call without reusing Python's registered module. The source-binding +guard added in 0.261.419 imported the canonical `DocumentSearchPlugin` and used +`isinstance` to verify the actual reader. That correct guard exposed the older +duplicate-class loading behavior. + +The previous authenticated end-to-end fixture replaced discovery with a direct +reference to the canonical class. Its successful replay therefore did not test +the loader/guard combination used in production. + +Read-only Application Insights, Log Analytics, and App Service console correlation +on version 0.261.420 showed the same boundary in the six reported runs: verified +source identity/access, successful plugin registration, immediate action failure, +and no native read checkpoint. An isolated replay using real discovery reproduced +the exact reader rejection with zero action-model calls and zero report reads. +Hosting metrics did not indicate CPU or memory exhaustion as their cause. + +## Changes + +`plugin_loader.py` now uses `importlib.import_module` for installed plugins. +Repeated and concurrent discovery reuse canonical class objects while logged +loading continues to construct separate instances with their own configurations. +No additional registry, reload mechanism, source widening, or class-name-only +validation was introduced. Failed module imports emit a tagged warning with the +module name and exception type, not the exception message. + +`functions_orchestration_actions.py` attaches an application-owned failure code +only at known initialization, reader identity, document-result error, and missing +function-result boundaries. Unclassified action errors still preserve their +existing provider causes. The document-reader type guard, source rechecks, +function-call budget, and cancellation behavior remain in place. + +`functions_orchestration_schema.py` admits those codes through the existing safe +failure contract. `functions_orchestration_adapters.py` logs the safe code/stage +and hashed run/step/conversation identity. Saved step details and the final +failure explanation retain the original action failure instead of explaining +only the dependent step's skip. + +| Code | Meaning and response | +| --- | --- | +| `action_initialization_failed` | Check action/model configuration; the tools or function-calling capabilities could not initialize. | +| `document_reader_identity_mismatch` | Repair or update the deployed application before retrying. Re-uploading or changing timeouts does not repair duplicate class identity. | +| `document_action_read_failed` | Check the selected source's availability and requested inspection; a failed read was not accepted as findings. | +| `action_results_missing` | No usable function results were returned; review the action/model tool support if it repeats. | + +These codes are not automatic transient retries. Permitted manual failed-step +recovery remains available after the underlying problem is repaired. No unknown +exception code or private exception message is accepted as public failure text. + +### Fresh-container template dependency + +The container startup probe exposed a separate installation problem: the +configured feed installed Semantic Kernel's Handlebars adapter without its +required `pymeta` module. This was not the cause of the six historical failures, +which had already reached planning and action loading. + +`application/single_app/requirements.txt` now explicitly pins the existing +template-parser dependency, `PyMeta3==0.5.1`, matching the working test environment. +Fresh normal/optimized import tests and the Linux image probe exercise the real +Semantic Kernel import rather than treating a completed package install as proof +that the application starts. + +## Validation + +`functional_tests/test_orchestration_document_action_end_to_end.py` now runs +production discovery. Its authenticated HTTP-to-saved-answer cases verify native +PDF/Word facts, limits, citations, failed reads, and debug on/off. + +`functional_tests/test_plugin_discovery_identity.py` verifies every discovered +class against its registered module, repeated/concurrent loading, separate +request-local users and configuration, safe import-error logging, and real +bootstrap/web/headless imports in fresh normal and optimized processes. + +`functional_tests/test_orchestration_document_loader_reliability.py` derives its +format matrix from `config.py` rather than listing only the reported extensions. +It exercises actual source checks, SDK-shaped document records, ordered chunk +retrieval, real action loading, retention, and saved cited answers using indexed +text. This matrix does not claim that every upload format has native PDF/Office +properties or that a filename change proves extraction support. + +Fault-injected initialization, wrong reader identity, document-read, and missing +result cases verify the exact safe code in the saved step API, its user-facing +explanation, hashed telemetry, attempt flags, privacy, and absence of composition +after failed acquisition. Provider timeout/connection causes remain transient; +the new code faults do not. + +Existing real-package PDF, Word, PowerPoint, and Excel accessibility/native +inspection fixtures separately cover extraction and its stated limitations. +The CI storage-contract workflow explicitly executes the real discovery and +document-action regressions with network access blocked before collection. + +| Before | After | +| --- | --- | +| Discovery made another class object that the native-reader guard rejected. | Discovery and the guard use the same registered class. | +| A shortcut fixture hid production loading behavior. | The authenticated regression uses the production loader. | +| Distinct action failures collapsed into a generic operation message. | Safe initialization, identity, read, and missing-result meanings survive saved details and the failure explanation. | +| Debug-off errors lacked usable run/step correlation at the action boundary. | The first action error retains safe stage/code and hashed correlation with debug on or off. | + +### Executed checks + +The shared discovery/native/action-runtime selection passed **81 tests** with the +eight fresh-process import-order cases excluded from that combined selection. +Those eight normal/optimized startup cases passed in the preceding discovery +selection. Real PDF/Word/PowerPoint/Excel extraction and Excel native-inspection +fixtures passed **63 tests** separately. + +The complete config-derived stored-text format matrix passed independently, +requiring a saved cited answer for every allowed extension. The **15** remaining +failure/contract cases passed independently, including the eight debug-on/off +saved-step failures. These selections are not presented as an aggregate test count. + +```powershell +$env:PYTHONPATH = "$PWD\application\single_app;$PWD\functional_tests" +python -m pytest -q -p no:cacheprovider -p test_support.cosmos_network_guard --disable-warnings --tb=short --show-capture=no .\functional_tests\test_plugin_discovery_identity.py .\functional_tests\test_orchestration_document_action_end_to_end.py .\functional_tests\test_orchestration_action_runtime.py .\functional_tests\test_orchestration_document_action_binding.py -k 'not real_cold_import_orders' +``` + +```powershell +python -m pytest -q -p no:cacheprovider -p test_support.cosmos_network_guard --disable-warnings --tb=short --show-capture=no .\functional_tests\test_document_accessibility_extraction.py .\functional_tests\test_excel_document_accessibility_extraction.py .\functional_tests\test_excel_native_inspection_chat_evidence.py +``` + +The reviewed SDK boundary inventory remains at **1,094 owners, 425 doubles and +zero review gaps**; regeneration introduced no inventory changes. + +After the final version update, the exact new CI regression selection passed +**37 tests**, including the complete **81-extension** retrieval matrix and all +fresh-process cases. The route-policy/manual-recovery selection passed **77 +tests**. The full reviewed offline SDK gate passed **2,026 tests and 431 subtests**: + +```powershell +python .\scripts\check_cosmos_response_boundaries.py --run-contracts +``` + +Documentation surface coverage and site quality passed **7/7** and **6/6** +checks. Changed Python compilation, CI YAML/network-guard validation, the changed +browser-sink scan and whitespace checks passed. + +The fresh local production Docker build passed the V2/native build stages but +stopped during dependency installation with an HTTPS handshake failure to +`files.pythonhosted.org`. Normal certificate-verified host and container probes +reproduced that network failure. TLS verification was not disabled, and this is +not attributed to the document-reader defect. + +Using the machine's existing verified-HTTPS package feed in a temporary build +context resolved that transport problem. The feed's missing FFmpeg metadata +sidecar was avoided by selecting the exact verified `ffmpeg-binaries-compat` +1.0.1 wheel with its SHA-256 hash in that temporary context; the repository's +FFmpeg pin and pip configuration were not changed. + +After the explicit PyMeta3 declaration, the full CI-equivalent regression gate +passed **38 tests**. The production Linux image built successfully and its real +application/WSGI authentication, canonical reader loading, compiled V2 assets, +five implementation-file hashes, Gunicorn configuration, and installed-package +consistency checks passed with external networking disabled. The default +external healthcheck switch remains disabled in the offline fixture; its 400 +response was not reported as a healthy endpoint or silently enabled in Azure. + +Release pages were regenerated and preserve all **313** source releases. + +### Packaged-runtime replay + +The actual Linux production image also passed **101 existing regression cases**: +the **38** discovery/native-action/runtime cases, including the full +**81-extension** indexed-text matrix, plus **63** real PDF/Word/PowerPoint/Excel +extraction and evidence cases. There were zero errors, failures, or skipped +cases. + +All **221** original production package versions remained unchanged. Only the +four missing, pinned pytest tooling packages were installed into the disposable +container's owned scratch user site. The shipped image, production dependencies, +and application source were not replaced to make the tests pass. Docker +networking was disabled, fixtures were mounted read-only, and the existing +pre-collection network guard remained active. + +All five raw implementation-file hashes also matched the worktree. The earlier +smoke fingerprints explicitly correspond to UTF-8 text with CRLF converted to +LF; a raw-byte comparison against those normalized fingerprints is not a valid +stale-build test. + +## Scope and rollout + +No new upload type, extractor, model provider, administrator toggle, route, or +browser dependency is added. Existing workspace permissions, selected-source +bindings, native-inspection limitations, and historical run records are preserved. +A missing native property in an available report is still different from a report +that cannot be read. + +Offline regressions are not a guarantee that every live model response will be +valid. Deploy the verified version/build, then prove actual saved cited answers +with approved live canaries. HTTP 200 and a healthy web endpoint do not establish +successful orchestration. A separately observed partial document-search run was +not attributed to this loader defect and must be diagnosed independently if it +persists. + +Related references: [Document Search](../../reference/actions/document-search.md), +[action access](../features/CHAT_ORCHESTRATION_ACTIONS.md), and +[plugin/action failure logging](../../reference/logging-tags.md#plugin-discovery-and-action-failures). diff --git a/docs/explanation/fixes/ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md b/docs/explanation/fixes/ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md new file mode 100644 index 000000000..1b745f442 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_RELIABILITY_PIPELINE_FIX.md @@ -0,0 +1,236 @@ +# Orchestration pipeline reliability (v0.261.423) + +Fixed/Implemented in version: **0.261.422**. Development integration: +**0.261.423**, tracked in `application/single_app/config.py`. + +The original [PR #1907](https://github.com/microsoft/simplechat/pull/1907) +merged into `paullizer-react-v2-ui`, not the intended `Development` branch. +The corrected integration preserves `Development` changes and includes the +plan-contract and canonical plugin-loader prerequisites missing there. It does +not include the separate saved-findings Sources inspector changes or replace +`Development` release notes. The original merge is not reverted, and published +branch history is not rewritten. + +## Issue and root cause + +Straightforward document reviews could work in ordinary chat but fail in +Orchestrate. Read-only Application Insights and App Service console investigation +identified separate boundaries, not one general inability to read the files. + +| Boundary | Why work stopped | What the failure does not prove | +| --- | --- | --- | +| Planning | A proposed or corrected read step could still omit source wiring or contain another argument/output contract error. The previous shared allowance ended after one correction. | A missing file binding is not evidence that upload extraction or OCR failed. | +| Native Word comments | Valid Word comment requests reached an Excel-only native collector. Ordinary Word style/accessibility metadata was not the structured comment inventory. | A workbook-reader rejection does not mean the Word comments are unreadable. | +| Prepared-answer checking | A checker's evidence reference could fail its transport contract even when useful findings had already been retained. Repair feedback did not provide usable reference labels. | An invalid assessment reference is not itself proof that the prepared answer is factually incorrect. | + +Historical generic rejections did not retain every rejected field or raw model +response. This fix does not invent those values or attribute every historical +failure to the same cause. + +## Technical details + +### Source wiring and bounded plan correction + +`functions_orchestration_planner.py` now supplies request-specific source defaults +and fills an absent document binding only when exactly one selection has verified +authorization, available source metadata, and a supported source kind. It acts on +an existing enabled independent reader, including the verified Document Search +action. It does not create a missing step, overwrite explicit IDs, replace dynamic +inputs, narrow multiple selections, or restore disabled work. Editing keeps +explicit source choices. + +All normal validators still run on the repaired proposal. The same planner can +then receive at most **two model corrections**, shared across recognized errors: +three total proposals, not two new attempts per error. Repeated unrenderable +clarification uses the remaining original allowance rather than restarting it. +Permission, unavailable-source/capability, provider, and unknown failures remain +terminal. + +`functions_orchestration_schema.py` adds owned argument/source rules and bounded +feedback for up to eight safe argument-schema problems in a failing step. +Registered fields or `unknown` are reported instead of arbitrary supplied names +or values. Known incompatible native-comment formats are rejected before +execution. + +### Shared Word reader and retained citations + +`functions_search_service.py` exposes opt-in `sections="comments"` through the +existing Document Search metadata function. For `.docx`, `.docm`, and `.dotx`, +it uses the existing authorized original-byte service and +`functions_office_media.extract_docx_comments`; no second extraction engine or +new route is introduced. Default stored-metadata calls and default ingestion +parsing preserve their previous behavior. +Comment-only inspection can succeed without stored style metadata; a combined +request still requires its stored sections. A successful comment read cannot +certify absent style or issue metadata as empty. + +The strict native path preserves parsed reviewer text and supported replies, +available author/date/resolution, and bounded anchors. A missing resolution flag +stays unknown. Anchors are limited to 300 characters and require an unclipped +closed range to be marked complete. An inspected package without a comments part +is verified empty; missing originals, malformed parts, identity/thread failures, +and resource limits are not successful empty inventories. + +`functions_orchestration_excel.py` adds a Word collector alongside the existing +Excel paging/fragment collector. Whole thread records are retained within the +existing budget; over-budget coverage is explicitly partial. The adapter chooses +the collector by source format and rechecks source identity, scope, version, and +revision before retention. + +`functions_orchestration_result_runtime.py` supports Word/mixed native evidence +while retaining Excel content-scope compatibility. Native citation resolution +reads the exact committed record with integrity and current source checks, not a +fresh Search fallback. `chat-citations.js`, V2 `CitationChip.tsx`, and its API types +show Word anchors/replies/resolution without worksheet fields or an invalid Excel +preview. Reviewer text remains inert browser text. + +### Checker report repair is not answer repair + +`functions_task_completeness.py` closes new assessment reference schemas over +the evidence labels actually supplied to that check. Empty evidence permits no +references. Existing compatible exact paths/identifiers still normalize; arbitrary +prose and nonexistent references do not. + +One malformed assessment can receive one report repair with allowed labels and +an owned rejection rule. That does not rewrite the candidate or spend an answer +correction merely because the checker named its evidence incorrectly. Genuine +unsupported findings still trigger answer correction and required verification. +Repeated invalid reports, unavailable required checking, and unresolved substantive +deficiencies still block delivery. + +`functions_orchestration_composition.py` records content-free checker diagnostics +at both prepared-content and retained-final-answer review boundaries. Existing +checkpoint and answer-correction limits are preserved; they are separate from the +planner's two corrections. + +### Privacy and operating diagnostics + +The existing planner, adapter, and executor tags record stage, owned rule/code, +actual correction counts, known source format, counts, and applicable hashed +workflow identifiers. Native-reader and checker checkpoints work with debug +logging disabled. Detailed planner snapshots retain their existing debug gate. + +No raw failed reference, prompt, source text, filename, ID, or provider exception +message is added to those diagnostics. See the +[logging reference](../../reference/logging-tags.md#native-comments-and-checker-report-repair) +for `Properties.sc_message` queries and the distinction between a reader +checkpoint and a confirmed terminal execution outcome. + +## Validation and impact + +Regression coverage uses real planner/compiler, Word parser, shared metadata +service, collector, retained result, checker, citation HTTP, and browser rendering +boundaries with external storage/model transport isolated. + +| Regression area | Coverage | +| --- | --- | +| Planning | Unambiguous local wiring; immutable explicit/dynamic/disabled work; ambiguous and unavailable selections; mixed errors sharing two corrections; valid third proposals and exhaustion. | +| Native reading | DOCX/DOCM/DOTX comments/replies; absent metadata; empty, missing, malformed, partial, and unsupported inventories; clipped/unclosed anchors; source revocation/change. | +| Answer checking | Legacy and canonical assessment schemas; one invalid-reference repair without candidate rewriting; real serialized diagnostics; repeated unsupported findings still blocked. | +| Citation delivery | Exact retained Word records and replies; HTTP lookup and source checks; desktop/mobile classic and V2 text safety; unchanged Excel preview and saved identities. | +| Storage and imports | Renewed affected Cosmos owner-runtime reviews, real SDK contracts with network blocked, and canonical plugin identity. | + +Key suites are `test_orchestration_attachment_native_evidence.py`, +`test_orchestration_planner_contract_recovery.py`, +`test_orchestration_document_planning_recovery.py`, +`test_orchestration_source_kind_planning.py`, `test_task_completeness_engine.py`, +`test_task_completeness_runtime.py`, and `test_orchestration_failure_telemetry.py`. +Browser coverage is in `ui_tests/test_v2_stream_citation_delivery.py` and the +existing Excel/inline-citation suites. + +### Verified Development integration results (v0.261.423) + +The correction was integrated with `Development` commit +`4dd6d80230fe60e3cbfe8544d1d52308a14a7387`. The existing saved-analysis UI, +publication implementation, and release notes match that target unchanged. +The required source-binding, plan-contract, and canonical-loader prerequisites +remain included. + +| Check | Result | +| --- | --- | +| Combined planner, Word-native, checker, source-scope, persistence, action-loader, and retained-citation regressions | 515 passed. | +| Full network-blocked Cosmos contract gate | 2,015 passed; 431 subtests passed; 1,094 owners and 425 doubles reviewed with 0 review gaps. | +| Local Classic/V2 citation, Excel inspection, filename fallback, and action browser regressions | 49 passed. | +| Blueprint policy, unauthenticated policy, and route-coverage regressions | 30 passed. | +| V2 TypeScript/Vite build and Classic citation JavaScript syntax | Passed. | +| Target-relative access-control and XSS guardrails | Passed for 22 Python files and 25 supported rendering files, respectively. | +| Documentation inventory coverage and site quality | 7/7 and 6/6 passed, respectively. | + +The browser fixture now supplies the existing pending-question endpoint's empty +response for its synthetic conversation, matching the other local fixtures and +the real controller contract. All other unexpected requests still fail; no +production behavior or browser-error assertion was disabled. + +The five unchanged execution-module Cosmos reviews were restored from +`Development` only after checking exact source and owner/module fingerprints. +Other changed storage owners retain their concrete runtime evidence. + +### Verified original v0.261.422 local results + +These runs overlap; their counts are not an aggregate end-to-end success rate. + +| Check | Result | +| --- | --- | +| Combined planner, checker, native-reader, action, and citation regressions | 343 passed. | +| Selected-source, conversation-scope, and durable planning-failure regressions | 93 passed. | +| Coupled action, composition, source-integrity, import, extraction, and registry regressions | 262 passed; 3 subtests passed; 1 confirmed baseline case deselected. | +| Classic/V2 Word, Excel, and reopened-citation browser regressions | 30 passed. | +| Affected Search Service owner test using real SDK document shapes with network blocked | 1 passed. | +| Full network-blocked Cosmos contract gate | 2,026 passed; 431 subtests passed; 1,094 owners and 425 doubles reviewed with 0 review gaps. | +| V2 TypeScript/Vite build | Passed. | +| Impeccable detector on the two changed citation clients | Passed; no findings. | +| Documentation inventory coverage and site quality | 7/7 and 6/6 passed, respectively. | + +The coupled regression command is: + +```powershell +$env:PYTHONPATH = "$PWD\application\single_app;$PWD\functional_tests" +python -m pytest -q -p no:cacheprovider --disable-warnings --tb=short --show-capture=no ` + .\functional_tests\test_orchestration_selected_source_planning.py ` + .\functional_tests\test_orchestration_plan_revision_planner.py ` + .\functional_tests\test_orchestration_document_action_binding.py ` + .\functional_tests\test_orchestration_document_action_end_to_end.py ` + .\functional_tests\test_orchestration_composition_metadata_recovery.py ` + .\functional_tests\test_orchestration_compose_output_retry.py ` + .\functional_tests\test_orchestration_retained_analysis_publication.py ` + .\functional_tests\test_orchestration_search_result_integrity.py ` + .\functional_tests\test_orchestration_result_imports.py ` + .\functional_tests\test_plugin_discovery_identity.py ` + .\functional_tests\test_document_accessibility_extraction.py ` + .\functional_tests\test_excel_document_accessibility_extraction.py ` + .\functional_tests\test_orchestration_registry_contract.py ` + -k "not test_an_endpoint_refusing_json_mode_is_asked_again_without_it" +``` + +The excluded test has an unchanged exact-log expectation that omits the +`requested_format` and `next_format` fields already emitted by the endpoint +client. A separately attempted legacy planner-diagnostics suite also has ten +setup errors because its AST-only harness omits the imported +`STORAGE_FAILURE_CODE`. Both limitations were reproduced against unmodified +HEAD; neither was changed or treated as a product fix. These scoped exclusions +do not waive the full network-blocked storage contract gate. + +The full storage gate passed with: + +```powershell +$env:PYTHONPATH = "$PWD\application\single_app;$PWD\functional_tests" +python .\scripts\check_cosmos_response_boundaries.py --run-contracts +``` + +The remaining reproducible build/documentation checks are: + +```powershell +npm --prefix .\application\v2_ui run build +python .\scripts\build_docs_inventory.py +python .\functional_tests\test_docs_app_surface_coverage.py +python .\functional_tests\test_docs_site_quality.py +``` + +The documentation quality check verifies source templates and search indexing +contracts; a full Jekyll site build was not run. + +This is an offline implementation and contract-verification result, **not a 100% +deployed-model success-rate claim**. Deployment and live replay require separate +approval. The change does not automatically replay historical failures, repair +corrupt input files, certify accessibility compliance, widen source access, or +replace existing Analyze and OCR/vision paths. diff --git a/docs/guides/review-and-edit-orchestration-plans.md b/docs/guides/review-and-edit-orchestration-plans.md index a6a466602..b035ecf13 100644 --- a/docs/guides/review-and-edit-orchestration-plans.md +++ b/docs/guides/review-and-edit-orchestration-plans.md @@ -4,7 +4,7 @@ title: "Review and edit orchestration plans" description: "Refine proposed work with the planner before running it." section: "Guides" audience: user -version: "0.261.418" +version: "0.261.422" --- ## Decide what should run @@ -109,6 +109,24 @@ planner may Analyze or Compare those inputs without adding a Search step merely because they were pinned. Explicitly selected Search still constrains the plan, and the validator still rejects a plan that omits selected documents. +### Recover ordinary planning mistakes without widening the work + +Fixed/Implemented in version: **0.261.422** +(`application/single_app/config.py`). + +A missing file ID should not require you to rebuild an otherwise useful plan. +For a single verified available selection, SimpleChat can fill an absent binding +on an existing independent reader. The visible plan then contains the source it +will actually read. Explicit IDs, dependent/dynamic inputs, disabled work, +unavailable sources, and ambiguous multi-file selections are not overwritten. + +Other recognized draft mistakes receive specific feedback to the same planner. +The initial proposal and at most two corrections share one allowance across +different validation errors. No source reader or external action runs during +those corrections, and your selected approval mode still applies afterward. +Permissions, unavailable capabilities, provider refusals, and exhausted correction +attempts remain real failures rather than permission to run an invalid plan. + ### Open the evidence behind document answers Since **0.261.356**, Analyze and Compare supply source references through the same @@ -146,6 +164,18 @@ For workspace-backed Excel workbooks, annotation requests use native Notes and threaded-comment inspection, preserving available metadata and saved cell values. Reported pagination and resource limits still bound what was inspected. +Since **0.261.422**, workspace Word `.docx`, `.docm`, and `.dotx` review comments +use the Word reader instead. Ask to include reviewer text and replies, and open a +native citation to inspect the retained comment, its available anchored text, +author/date, and resolution metadata. A bounded or unavailable anchor is not a +complete quotation, and a missing resolution flag is not proof that a comment is +unresolved. Word evidence does not offer an Excel worksheet preview. + +The reader needs accessible original bytes; ordinary stored style metadata is not +the structured Word comment inventory. Unavailable or malformed originals remain +read failures. An inspected package with no comments is distinct from a failed +inspection. Historical failed runs are not automatically replayed by this change. + Fixed/Implemented in version: **0.261.341** (`application/single_app/config.py`). When you ask to rewrite, audit, summarize, or otherwise transform material diff --git a/docs/reference/actions/document-search.md b/docs/reference/actions/document-search.md index fef603be6..4354b0ead 100644 --- a/docs/reference/actions/document-search.md +++ b/docs/reference/actions/document-search.md @@ -4,6 +4,7 @@ title: "Document Search" description: "Reference for the Document Search SimpleChat action." section: "Reference" audience: user +version: "0.261.422" --- @@ -50,13 +51,76 @@ The action includes a `get_document_accessibility_report` function for agents th From **0.261.399**, workspace Excel files also expose native workbook properties, cell notes/threaded comments, object alternative text/decorative flags, formatting/layout and header/footer expressions. Modern `.xlsx`/`.xlsm` inspection has explicit saved-state coverage limits; legacy `.xls` is best effort. Use the report for the bounded workbook-wide inventory and document search for sheet/cell-addressed `Excel native inspection` evidence. Enhanced-citation workbooks still index a schema preview plus native evidence, not the complete worksheet data. -- Choose which parts to return with `sections` (`accessibility`, `style`, `issues`); all are returned by default. +- Choose which stored parts to return with `sections` (`accessibility`, `style`, `issues`); all three are returned by default. Opt-in Word `comments` inspection is separate. - Pass `page_number` to also get that page, slide or Excel indexed evidence segment's stored text. Excel segment numbers are not physical print-page numbers. - The stored workspace report does not grant access to chat uploads. For an existing workspace Excel file, retained active source bytes can be inspected on demand without re-uploading; unavailable originals need the normal re-upload/reprocessing workflow. - The issues list is heuristic. It is not a formal WCAG, Section 508, or PDF/UA conformance check. To apply your own rules, write them into the agent's instructions and have it compare them with the returned values. See [Document accessibility, structure, and comment extraction](https://github.com/microsoft/simplechat/blob/13db876626a043e2c7b30f054354f07c9a1633b9/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md) for what is extracted for each file type. +### Native Word review comments + +Fixed/Implemented in version: **0.261.422** +(`application/single_app/config.py`). + +For workspace `.docx`, `.docm`, or `.dotx` files, call +`get_document_accessibility_report(document_id=..., sections="comments")`. +This uses the existing authorized original-file reader and Word parser, returning +`word_inspection` with parsed comments, supported linked replies, inventory counts, +and limitations. Normal accessibility/style/issue calls do not add this original +file read. + +This is useful for checking what a reviewer actually wrote, independently of +document prose or a search summary. Anchors are limited to 300 characters; +`anchor_complete` distinguishes a complete parsed range from bounded or +unavailable text. Resolution is known only when `resolution_available` is true. +Relationships come from supported Word metadata, not guesses about author names +or reply wording. + +Excel cursors, worksheet/cell filters, and non-comment native sections do not apply +to Word. A comment-only request for an unsupported format, missing original, or +malformed package fails explicitly; it does not certify that there are no comments. +Stored Word style metadata can be absent while original comment inspection is +available. A combined comments/style/accessibility/issue request still requires +the stored sections; a successful comment read does not certify missing style or +issue data as empty. This does not grant native-report access to chat-only uploads. + +### Selected sources in Orchestrate + +Implemented in version: **0.261.419** (`application/single_app/config.py`). +When an orchestration plan uses this action for selected workspace files, it can +declare their exact IDs as `arguments.document_ids`. Inspect the Run view's source +chips before approval; its existing narrowing controls can remove individual +files from that step. + +This declaration is enforced on the server, not just added to a model instruction. +Native metadata, chunk, summary and search reads stay within the bound files and +the action's configured workspace restrictions. Omitting an ID filter on a tool +search does not broaden it to the whole workspace. + +No separate Search or Analyze step is required solely to make native inspection +count as selected-source work. Supported native facts and citations travel through +the retained action result to the answer, including explicit inspection limits. +A denied or unavailable report is not a successful inspection; a missing property +inside an available report must still be distinguished from an inaccessible one. +This does not add native-report access for chat-only attachments. + +### Loading failures and recovery + +From **0.261.421** (`application/single_app/config.py`), shared plugin discovery +reuses the registered Python module instead of creating another copy of the +document reader. This fixes selected-document actions being rejected before their +first read even when the plan and source permissions were valid. + +Run details distinguish a tool initialization or reader identity failure from a +failed document read and an action that returned no function results. For a reader +identity failure, ask an administrator to repair/update the application before +retrying; changing the model, increasing timeouts, or re-uploading the same source +does not fix that loading defect. + +The source checks and native-inspection limits are unchanged. A missing report or +unavailable native property does not become evidence of absence or compliance. + ### Paged Excel inspection Implemented in version: **0.261.408** (`application/single_app/config.py`). diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index d4b3310a7..4ec6952c6 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -4,27 +4,31 @@ title: "Chat interface controls" description: "Reference for every documented control in the SimpleChat chat interface." section: "Reference" audience: user -version: "0.261.418" +version: "0.261.423" --- ## How to use this reference Use this page when you can see a control in Chat but are not sure what it does or when to use it. The generated inventory lists 47 real controls plus 3 child sub-elements. The child elements `document-comparison-edit-btn-label`, `fork-conversation-button-label`, and `fork-conversation-button-spinner` are mentioned with their parent controls instead of receiving separate rows. -### Native workbook evidence +### Native Word and workbook evidence -Fixed/Implemented in version: **0.261.417** +Fixed/Implemented in version: **0.261.417**, extended in **0.261.422** (`application/single_app/config.py`). | Control | Purpose | Availability | | --- | --- | --- | | Native note/comment citation | Opens exact retained reviewer text and its worksheet/cell, author, reply metadata, and saved cell value. | A citation with an unambiguous retained native-evidence reference | -| Open workbook preview (separate from annotation evidence) | Opens worksheet data independently; the preview is not presented as the cited reviewer text. | Successfully authorized native-evidence view | +| Native Word comment citation | Shows the retained comment and linked replies, available author/date/resolution, and anchored text with its completeness limit. | Authorized native Word evidence from `.docx`, `.docm`, or `.dotx` | +| Open workbook preview (separate from annotation evidence) | Opens worksheet data independently; the preview is not presented as the cited reviewer text. | Successfully authorized workbook evidence, not Word comments | -Source access and the inspected workbook version are rechecked when evidence +Source access and the inspected document version are rechecked when evidence opens. An unavailable historical reference stays unavailable rather than choosing a newer comment at the same cell. See [Inspect Excel data, comments, and structure]({{ '/guides/inspect-excel-workbooks/' | relative_url }}). +Word comment evidence uses document anchors rather than invented page/cell +locations. An absent resolution flag remains unavailable, not automatically +unresolved. See [Review and edit orchestration plans]({{ '/guides/review-and-edit-orchestration-plans/' | relative_url }}). ## Conversation list diff --git a/docs/reference/logging-tags.md b/docs/reference/logging-tags.md index 2905bac41..9369804ce 100644 --- a/docs/reference/logging-tags.md +++ b/docs/reference/logging-tags.md @@ -42,6 +42,52 @@ and capability codes. A cancelled stream is informational, not an error. Orchestration records an outcome only after its saved state is confirmed; replaying a cached response does not record another execution. +## Plugin discovery and action failures + +Implemented in version **0.261.421**, tracked in +`application/single_app/config.py`. + +`[PLUGIN_LOADER] Plugin module could not be loaded.` is a Warning with the +installed module's `sc_plugin_name` and safe `sc_error_type`. Discovery continues +for other modules; the exception's message is not logged. + +`[ORCHESTRATION_ADAPTERS] Direct action execution failed.` is an Error that +records the first failed action's `sc_failure_code`, `sc_stage`, +`sc_capability_id`, `sc_error_type`, and hashed conversation/run/step identifiers. +These records are available with debug logging disabled. They distinguish the +action failure from later steps that did not run because their dependency failed. + +| Failure code | Stage | Meaning | +| --- | --- | --- | +| `action_initialization_failed` | `action_initialization` | Tools could not load, expose functions, or use the selected model's function-calling capabilities. | +| `document_reader_identity_mismatch` | `action_initialization` | The loaded plugin was not the application's verified document-reader class; no document-reading function ran. | +| `document_action_read_failed` | `document_action_read` | A bound document function returned a failed read, not successful findings. | +| `action_results_missing` | `action_results` | The action returned no usable function results, so none were retained as findings. | + +`sc_function_calls_attempted` is false for the two initialization classifications +and true for a failed document-function result. It is omitted where that fact is +unknown, including provider errors and missing-result cases. Other failures keep +their existing provider/cancellation/authority classifications. The new codes do +not enable automatic transient retries or disable permitted manual recovery. + +To inspect a specific failed run without logging its raw identifier: + +```kusto +let runHash = hash_sha256(""); +AppTraces +| where TimeGenerated > ago(2h) +| where tostring(Properties.sc_run_id_hash) == runHash +| where tostring(Properties.sc_message) == "[ORCHESTRATION_ADAPTERS] Direct action execution failed." +| project TimeGenerated, + code=tostring(Properties.sc_failure_code), + stage=tostring(Properties.sc_stage), + stepHash=tostring(Properties.sc_step_id_hash), + functionsAttempted=tostring(Properties.sc_function_calls_attempted), + errorType=tostring(Properties.sc_error_type) +| order by TimeGenerated asc +| take 50 +``` + ## SQL connection diagnostics Since **0.261.351**, `[SQL_CONNECTION]` records connection-check and runtime @@ -404,6 +450,7 @@ for availability states and live-UI validation limits. - `[PLUGIN_FUNCTION_LOGGER]` - `[PLUGIN_HEALTH]` - `[PLUGIN_INVOCATION]` +- `[PLUGIN_LOADER]` - `[PLUGIN_LOGGING_API]` - `[PLUGIN_RECOVERY]` - `[PLUGIN_REPAIR]` @@ -554,7 +601,7 @@ Insights `customDimensions` or Log Analytics `AppTraces.Properties`. | `sc_validation_code` | Validation category, including `deliverables_invalid`, `source_kind_invalid`, `source_binding_required`, `analysis_options_invalid`, or `selected_requirements_missing`. | | `sc_validation_rule` | Specific application-owned rejection, such as `invalid_quantity`, `non_file_format`, `answer_producer_mismatch`, `missing_final_response`, `file_format_mismatch`, `narrative_source_required`, or `document_sources_required`. | | `sc_response_failure` | A missing, incomplete, or refused planner completion, when applicable. | -| `sc_attempt` | Planner proposal number: 1 for the initial proposal and 2 for its single shared correction. Since 0.261.413, also present on Analyze-option and selected-requirement correction/failure events. | +| `sc_attempt` | Planner proposal number: 1 initially, then 2 or 3 for up to two shared model corrections since 0.261.422. Also present on Analyze-option and selected-requirement correction/failure events. | | `sc_execution_code`, `sc_output_code` | Safe execution or output-store failure category, where available. | | `sc_failure_code` | On step and adapter events, a step failure category, such as `step_timeout`, `run_timeout`, or `context_unavailable`. On `[CONTENT_SCREENING]` events, the source-authority code, such as `source_authority_unverified`, `source_authority_unavailable`, or `screening_revision_conflict`. | | `sc_authority_reason` | Which check failed for a source. With `source_authority_unverified`, the check that could not verify the source's current authority: `document_record_invalid`, `screening_marker_invalid`, `screening_state_unknown`, `screening_marker_incomplete`, `document_scope_invalid`, `workspace_record_invalid`, `scan_record_invalid`, `provenance_mismatch`, `document_id_invalid`, `manifest_context_invalid`, `manifest_scope_missing`, `manifest_revision_invalid`, or `manifest_batch_invalid`. With `screening_revision_conflict`, what saved evidence no longer matches in the current document: `source_revision_changed` (the document was re-versioned), `screening_generation_changed` (it was re-screened or remediated), `source_scope_changed` (its id or workspace differs), `provenance_shape_invalid` (the saved provenance has a different set of fields), `screened_record_changed` (a saved copy of the screened record differs), `source_version_changed` (a legacy reference without provenance), or `cached_evidence_unproven` (cached evidence from before screening enrollment). On `[ORCHESTRATION_EXECUTOR]` step, saved-wait, finalization and content-preparation failure events (since 0.261.209), the stable code of a refused external source, such as `external_identity_session_unavailable` (the step ran in a background continuation with no signed-in session), `external_identity_role_required`, `external_identity_access_restricted`, `external_identity_conversation_unavailable`, `external_identity_settings_unavailable`, or `external_configuration_changed`. On `[ORCHESTRATION_EXTERNAL_SOURCES]` refusal events (since 0.261.291), the same refusal code, such as `result_external_source_unavailable` or `result_external_capability_unavailable`, with the check that failed in `sc_reason`. | @@ -628,6 +675,7 @@ remain available with debug logging off. | `[ORCHESTRATION_PLANNER] Step binding diagnostic.` | One bounded step entry: hashed step ID, known capability, enabled state, binding-field presence, and Analyze option shape/known-value flags. | Stages are `planner_input`, `plan_proposal` (before defaults), `analysis_defaults`, +`selected_source_defaults` (after an eligible local wiring repair), `plan_normalized`, and `plan_validated`. A rejected attempt also uses its actual boundary, such as `plan_normalization` or `selected_requirements`. @@ -642,6 +690,7 @@ boundary, such as `plan_normalization` or `selected_requirements`. | `sc_required_capability_count`, `sc_missing_capability_count` | Positive-operation coverage, separate from document coverage. | | `sc_normalization_lost_document_count` | References present in the raw proposal but absent after normalization. Zero missing references at proposal time followed by loss here differs from a model omission. | | `sc_remaining_corrections` | Remaining shared proposal-correction allowance; never a new per-error budget. | +| `sc_source_binding_repair_count` | Eligible absent bindings filled locally for one verified available selected source since 0.261.422. This is not another model correction or permission to overwrite explicit work. | | `sc_model_routing_auto`, `sc_planner_binding_provided` | Auto-routing selection and whether planning uses a resolved binding rather than the legacy client resolver. | | `sc_step_count`, `sc_omitted_step_count`, `sc_omitted_document_count` | Total steps and explicit diagnostic omissions. At most 32 source and 32 step detail events are emitted per snapshot; summaries still count all references. | | `sc_options_version_present`, `sc_options_version_v1`, `sc_options_version_v2`, `sc_options_mode_present`, `sc_options_mode_observations`, `sc_options_mode_records`, `sc_options_mode_legacy` | Presence/known-value flags, not arbitrary supplied option strings. | @@ -655,6 +704,29 @@ Analyze rules include `options_object_required`, `unsupported_option_fields`, `calculation_rules_conflict`, `calculation_rules_invalid`, `observation_requirements_conflict`, and `analysis_options_schema_invalid`. +Since **0.261.419**, model-draft contract diagnostics also distinguish +`plan_structure_invalid`, `plan_arguments_invalid`, `plan_outputs_invalid`, and +`plan_inputs_invalid`, alongside specific graph-binding codes. Owned rules include +`step_shape_invalid`, `argument_schema_invalid`, `unknown_argument`, +`fixed_output_not_supported`, `fixed_output_missing`, `input_binding_invalid`, +and `input_kind_mismatch`. `sc_schema_keyword` and `sc_argument_field` contain only +a schema keyword and registered argument name (or `unknown`), never an arbitrary +model-supplied property name. `sc_result_kind` uses a known underscore-form code; +`sc_accepts_*` flags describe compatible kinds. + +Since **0.261.422**, additional owned rules identify `model_binding_object_required`, +`argument_string_empty`, `document_id_invalid`, `document_selection_duplicate`, +`document_limit_exceeded`, `document_authorization_required`, +`source_binding_conflict`, and `native_comments_format_unsupported`. Argument +correction feedback can describe up to eight safe schema problems in a failing +step; it never includes arbitrary supplied field names or values. Recognized +draft mistakes share two corrections. Source authorization and unavailable +capabilities remain terminal rather than being repaired by widening access. + +`[ORCHESTRATION_PLANNER] Selected source wiring repaired.` is an ordinary +content-free event, available with debug logging off. Detailed before/after +snapshots remain debug-gated. + A picked document missing from both `plan_proposal` and `plan_normalized` indicates an omitted recognized binding in the proposal, not a normalization loss. These structural diagnostics do not establish extraction fidelity or prove that every @@ -682,6 +754,135 @@ traces | take 500 ``` +### Native comments and checker report repair + +Implemented in version **0.261.422**, tracked in +`application/single_app/config.py`. These checkpoints use the normal application +logger and remain available with **Enable Debug Logging** off. + +| Event message | Boundary | +| --- | --- | +| `[ORCHESTRATION_ADAPTERS] Native comment read checkpoint.` | `native_comment_read`: started/completed/failed for the actual Word or Excel collector; failed source recheck or retention can follow a completed read. | +| `[ORCHESTRATION_EXECUTOR] Task completeness verification unavailable.` | `task_completeness_assessment`: an assessment failed during prepared-content checking, before possible report repair. | +| `[ORCHESTRATION_EXECUTOR] Retained final-answer verification unavailable.` | The same assessment boundary for an already retained final answer. | +| `[ORCHESTRATION_EXECUTOR] Retained final-answer verification could not complete.` | The outer review boundary failed before a usable completed check. | + +The assessment warning is not itself a terminal answer failure. A single invalid +report may be repaired without regenerating the candidate; unresolved substantive +findings or unavailable required checking still block delivery. + +| Property | Meaning | +| --- | --- | +| `sc_application_version` | Application build from `config.py`; a mutable container image tag does not identify the deployed build. | +| `sc_source_format` | Known native format: `docx`, `docm`, `dotx`, `xlsx`, `xlsm`, `xls`, or `unsupported`; no filename. | +| `sc_selected_document_count` | Selection size at read start, not a count to sum across repeated per-source events. | +| `sc_returned_record_count`, `sc_total_record_count` | Returned collection count and total inventory count reported at the completed read; these do not confirm later result publication. | +| `sc_returned_reply_count` | Replies included in returned Word records when that metadata is available; separate from root-record count. | +| `sc_partial` | The collection hit its retention allowance; complete inventory discovery is not the same as retaining every record. | +| `sc_validation_rule`, `sc_error_type` | Owned refusal boundary and exception class, never the raw exception message. | +| `sc_validation_code` | Checker failure category, such as `completeness_evidence_invalid`; not a finding that the candidate is factually wrong. | +| `sc_report_repair_count` | Repairs already spent when the assessment failed: zero initially, one on a failed repaired report. Separate from plan and answer corrections. | +| `sc_evidence_key_count` | Number of top-level evidence labels supplied to the checker, without the labels or source bodies. | +| `sc_evidence_ref_count`, `sc_evidence_refs_list` | Size/type of the failing reference list when validation can report it safely, without bad references. | + +Applicable conversation/run/step/turn hashes correlate these events. No failed +reference string, source prose, raw workflow identifier, filename, or provider +exception message is added. + +```kusto +let conversationHash = hash_sha256(""); +AppTraces +| where TimeGenerated > ago(2h) +| extend p=todynamic(Properties) +| where tostring(p.sc_conversation_id_hash) == conversationHash +| where tostring(p.sc_stage) in ("selected_source_defaults", "native_comment_read", "task_completeness_assessment") +| project TimeGenerated, + event=tostring(p.sc_message), + applicationVersion=tostring(p.sc_application_version), + runHash=tostring(p.sc_run_id_hash), + stepHash=tostring(p.sc_step_id_hash), + stage=tostring(p.sc_stage), + outcome=tostring(p.sc_outcome), + format=tostring(p.sc_source_format), + returnedRecords=toint(p.sc_returned_record_count), + totalRecords=toint(p.sc_total_record_count), + returnedReplies=toint(p.sc_returned_reply_count), + partial=tobool(p.sc_partial), + rule=tostring(p.sc_validation_rule), + checkerCode=tostring(p.sc_validation_code), + reportRepairs=toint(p.sc_report_repair_count) +| order by TimeGenerated asc +| take 500 +``` + +Read `Properties.sc_message`, not `AppTraces.Message`: the envelope message is +`[SIMPLE_CHAT_LOG_EVENT]`. Application Insights' `traces` table uses +`customDimensions.sc_message`. App Service console records mirror the same +static tags when console logging is configured. + +To measure reliability, use the existing confirmed terminal `[CHAT_EXECUTION]` +semantic outcomes and actual application version. Exclude pending/started events, +and distinguish complete, partial, failed, and uncertain outcomes. A validated +plan, successful native read, absent unhandled exception, or HTTP 200 is not an +end-to-end delivered-answer success. + +### Native action and retained-result lifecycle checkpoints + +Implemented in version **0.261.419**, tracked in +`application/single_app/config.py`. These content-free checkpoints use the same +**Enable Debug Logging** switch as planning snapshots. They explicitly gate the +normal structured `log_event` path so both console and Application Insights receive +them. The existing console-only behavior of `debug_only=True` is not widened. + +| Event message | Stage and useful fields | +| --- | --- | +| `[ORCHESTRATION_ADAPTERS] Document action binding started.` / `Document action binding verified.` | `document_action_binding`: selected/resolved counts and verified authorization. | +| `[ORCHESTRATION_ADAPTERS] Document action binding rechecked.` | `document_action_recheck`: expected/actual counts, identity/revision match flags, authorization flag, and outcome. | +| `[ORCHESTRATION_ADAPTERS] Document action read checkpoint.` | `document_action_read`: started/completed/failed, known operation, call index, bound count, object/metadata/citation presence, safe error type, and an authorized document hash when known. | +| `[ORCHESTRATION_EXECUTOR] Gather source handoff verified.` | `gather_source_handoff`: planned/resolved source counts and whether an explicit document-action binding is in use. | +| `[ORCHESTRATION_EXECUTOR] Gather retention started.` / `Gather result retained.` | `gather_retention` / `gather_retained`: workspace/integration lineage counts, output/upstream counts, and partial status. | +| `[ORCHESTRATION_EXECUTOR] Retained document citation projection verified.` | `retained_citation_projection`: total and located citation counts for the selected producer; `sc_consumer_run_id_hash` links a later consuming run. | +| `[ORCHESTRATION_EXECUTOR] Composition input contract verified.` / `Prepared content retained.` | `compose_inputs` / `compose_retained`: input/citation/output counts, narrative policy, partial status, and whether completeness checking ran. | + +Correlation uses hashed conversation/run/step identity and the turn when available. +`sc_consumer_run_id_hash`, like other identity hashes, is retained only as a valid +64-character lowercase SHA-256 value. No new checkpoint logs document text, +filenames, prompts, model replies, source versions, provider URLs, credentials or +exception messages. Counts and presence flags do not claim whole-document +coverage or formal conformance. + +Start and verification markers distinguish a read attempt from verified findings. +For example, a failed native read must not be followed by a retained result for that +step. A bound action must not lose its workspace-source count between the source +handoff and retention. A result can retain provenance yet have no located citation; +the citation count makes that different boundary visible. Existing step-failure +and `CHAT_EXECUTION` outcome events remain available without debug logging. + +For workspace-based Log Analytics, correlate a single conversation without +recording its raw identifier: + +```kusto +let conversationHash = hash_sha256(""); +AppTraces +| where TimeGenerated > ago(2h) +| where tostring(Properties.sc_conversation_id_hash) == conversationHash +| extend event=tostring(Properties.sc_message) +| where event startswith "[ORCHESTRATION_" or event startswith "[CHAT_EXECUTION]" +| project TimeGenerated, event, + stage=tostring(Properties.sc_stage), + runHash=tostring(Properties.sc_run_id_hash), + consumerRunHash=tostring(Properties.sc_consumer_run_id_hash), + stepHash=tostring(Properties.sc_step_id_hash), + outcome=tostring(Properties.sc_outcome), + sourceCount=coalesce(toint(Properties.sc_workspace_source_count), toint(Properties.sc_source_count)), + externalCount=toint(Properties.sc_external_source_count), + citationCount=toint(Properties.sc_citation_count), + locatedCitations=toint(Properties.sc_located_citation_count), + rule=tostring(Properties.sc_validation_rule) +| order by TimeGenerated asc +| take 500 +``` + ### Step, render and content-preparation events Since **0.261.141**, these events explain a slow or failed step without logging diff --git a/functional_tests/cosmos_response_boundaries.json b/functional_tests/cosmos_response_boundaries.json index 24f701112..a16fa43fe 100644 --- a/functional_tests/cosmos_response_boundaries.json +++ b/functional_tests/cosmos_response_boundaries.json @@ -5742,7 +5742,7 @@ }, "functional_tests/test_support/orchestration_revisions.py::AtomicMemoryContainer": { "boundary": { - "sha256": "45972adfa5bac7f069ae2a00047424aa5a208a8d14b6cc61afd1ab560397f5ee", + "sha256": "075f89bd674ec476bf4ba79797b83ef2fffd644534b8874e8ef8774a31de4562", "methods": [ "create_item", "delete_item", @@ -5756,9 +5756,11 @@ "imports_sha256": "a5271c5eba30cd26978bd66b16ae2c89bba4de3c753911b4d9298133304773cc" }, "classification": "sdk-default", - "contract": "Migrated Cosmos container: actual pinned SDK point wrappers by default, explicit sdk_responses=False compatibility, plain query JSON; existing identity/ETag/failure behavior is preserved.", + "contract": "Migrated Cosmos container: actual pinned SDK point wrappers by default, explicit sdk_responses=False compatibility, plain query JSON; existing identity/ETag/failure behavior is preserved. Bounded connection/read timeouts and retry_total are accepted as transport-only options; unknown options fail explicitly. SDK point wrappers, partition routing, conditional ETags, failure injection and transaction behavior remain unchanged.", "tests": [ - "functional_tests/test_cosmos_document_boundaries.py" + "functional_tests/test_cosmos_document_boundaries.py", + "functional_tests/test_orchestration_document_action_end_to_end.py::test_sdk_transport_options_preserve_partition_etag_and_failure_semantics", + "functional_tests/test_orchestration_document_action_end_to_end.py" ] }, "functional_tests/test_support/public_file_source_harness.py::FileSyncContainer": { @@ -20326,12 +20328,13 @@ ], "queries": [], "batches": [], - "module_sha256": "ae2373ab845b8f0c1837ca03e591a9056fc7c256af3cdfac79c298518f1d56c5" + "module_sha256": "2c071839f337fc0a554e5fd4255ac3c6b4e073f9522f541095aa442c639349d5" }, "classification": "normalized-strict", - "contract": "The SDK point mapping is normalized before domain checks or copying. The private attempt remains strict JSON; question identity, masks, deletion and conditional ETags are preserved.", + "contract": "The SDK point mapping is normalized before domain checks or copying. The private attempt remains strict JSON; question identity, masks, deletion and conditional ETags are preserved. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ - "functional_tests/test_orchestration_planning_persistence.py" + "functional_tests/test_orchestration_planning_persistence.py", + "functional_tests/test_orchestration_planning_persistence.py::test_typed_contract_failure_survives_reload_and_clears_on_explicit_retry" ], "evidence_scope": "owner-runtime" }, @@ -20347,13 +20350,14 @@ ], "queries": [], "batches": [], - "module_sha256": "ae2373ab845b8f0c1837ca03e591a9056fc7c256af3cdfac79c298518f1d56c5" + "module_sha256": "2c071839f337fc0a554e5fd4255ac3c6b4e073f9522f541095aa442c639349d5" }, "classification": "normalized-strict", - "contract": "The caller supplies a normalized question with its ETag. Conditional replacement preserves unrelated metadata; its SDK write document is normalized before the owner reads the saved state.", + "contract": "The caller supplies a normalized question with its ETag. Conditional replacement preserves unrelated metadata; its SDK write document is normalized before the owner reads the saved state. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_planning_persistence.py::test_a_racing_stop_cannot_be_overwritten_by_a_heartbeat", - "functional_tests/test_orchestration_planning_persistence.py::test_sdk_claim_fencing_stale_worker_and_public_history" + "functional_tests/test_orchestration_planning_persistence.py::test_sdk_claim_fencing_stale_worker_and_public_history", + "functional_tests/test_orchestration_planning_persistence.py::test_typed_contract_failure_survives_reload_and_clears_on_explicit_retry" ], "evidence_scope": "owner-runtime" }, @@ -20369,10 +20373,10 @@ ], "queries": [], "batches": [], - "module_sha256": "ae2373ab845b8f0c1837ca03e591a9056fc7c256af3cdfac79c298518f1d56c5" + "module_sha256": "2c071839f337fc0a554e5fd4255ac3c6b4e073f9522f541095aa442c639349d5" }, "classification": "normalized-strict", - "contract": "A cancellation point response is normalized before exact scoped-field validation. Only a real 404 means absence; malformed or unavailable storage cannot grant planning ownership.", + "contract": "A cancellation point response is normalized before exact scoped-field validation. Only a real 404 means absence; malformed or unavailable storage cannot grant planning ownership. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_planning_persistence.py::test_stop_before_acknowledgement_is_durable", "functional_tests/test_orchestration_planning_persistence.py::test_early_stop_is_not_erased_by_an_unrelated_conversation_write" @@ -20391,10 +20395,10 @@ ], "queries": [], "batches": [], - "module_sha256": "ae2373ab845b8f0c1837ca03e591a9056fc7c256af3cdfac79c298518f1d56c5" + "module_sha256": "2c071839f337fc0a554e5fd4255ac3c6b4e073f9522f541095aa442c639349d5" }, "classification": "response-unused", - "contract": "The immutable, submission-scoped cancellation create response is discarded. A 409 rereads the exact owned marker through planning_was_cancelled instead of overwriting another record.", + "contract": "The immutable, submission-scoped cancellation create response is discarded. A 409 rereads the exact owned marker through planning_was_cancelled instead of overwriting another record. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_planning_persistence.py::test_stop_before_acknowledgement_is_durable" ], @@ -23443,10 +23447,10 @@ ], "queries": [], "batches": [], - "module_sha256": "0afaa631fb35d52b9022fa8c7e777695aa84a450fc6fcbff2e7b6f1b04636a77" + "module_sha256": "8835a019046bdee0d13641133cc22180068071f27d8ec5f0258baec00c3e8859" }, "classification": "sdk-compatible", - "contract": "_authorize_chat_upload_conversation: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", + "contract": "_authorize_chat_upload_conversation consumes the pinned SDK dictionary subclass through isinstance/get and verifies the exact conversation owner; SDK representation does not bypass strict source authority. Review renewed for 0.261.422: this owner AST, point call and imported authority bindings are unchanged. Module-only Word original-comment parsing and the comment-only metadata guard do not change this chat-upload consumer. The representative real-owner SDK test verifies both allowed and denied users with network blocked.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workflow_authoring_sdk_boundaries.py::test_search_upload_owners_consume_real_sdk_point_documents" @@ -23470,10 +23474,10 @@ } ], "batches": [], - "module_sha256": "0afaa631fb35d52b9022fa8c7e777695aa84a450fc6fcbff2e7b6f1b04636a77" + "module_sha256": "8835a019046bdee0d13641133cc22180068071f27d8ec5f0258baec00c3e8859" }, "classification": "sdk-compatible", - "contract": "_resolve_chat_upload_context: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", + "contract": "_resolve_chat_upload_context reads pinned SDK point mappings and plain query rows via isinstance/get, applies conversation/source authority checks, and constructs the authorized upload projection from nested JSON fields. Review renewed for 0.261.422: this owner AST, point/query calls and authority imports are unchanged; module-only Word parsing and opt-in metadata behavior do not alter the chat-upload consumer. The existing real-owner SDK test verifies authorized text projection with network blocked.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workflow_authoring_sdk_boundaries.py::test_search_upload_owners_consume_real_sdk_point_documents" @@ -31400,10 +31404,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "The conversation point mapping proves owner/nondeleted state through field access and now rechecks shared-source membership before dependent reads; missing or unauthorized records retain ConversationContextError.", + "contract": "The conversation point mapping proves owner/nondeleted state through field access and now rechecks shared-source membership before dependent reads; missing or unauthorized records retain ConversationContextError. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py", "functional_tests/test_orchestration_conversation_sources.py" @@ -31422,10 +31426,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Artifact point mappings are JSON-encoded by the existing checkpoint fingerprint helper; SDK dictionary subclasses are supported and nested artifact validation is unchanged.", + "contract": "Artifact point mappings are JSON-encoded by the existing checkpoint fingerprint helper; SDK dictionary subclasses are supported and nested artifact validation is unchanged. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31443,10 +31447,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "The saved turn point mapping is normalized by normalize_history_message before role/conversation/fingerprint validation; history-disabled snapshots fail before reuse and source authorization is rechecked.", + "contract": "The saved turn point mapping is normalized by normalize_history_message before role/conversation/fingerprint validation; history-disabled snapshots fail before reuse and source authorization is rechecked. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py", "functional_tests/test_orchestration_conversation_sources.py" @@ -31475,10 +31479,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Requested conversation point mappings supply owner/scope/deletion fields; constructed create/upsert bodies are plain and write responses are discarded. Shared ownership is established separately, not inferred from wrapper type.", + "contract": "Requested conversation point mappings supply owner/scope/deletion fields; constructed create/upsert bodies are plain and write responses are discarded. Shared ownership is established separately, not inferred from wrapper type. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31496,10 +31500,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Point answer mappings supply conversation/visibility and nested thread fields; metadata patching retains its existing owner and stale or missing answers are not silently restored.", + "contract": "Point answer mappings supply conversation/visibility and nested thread fields; metadata patching retains its existing owner and stale or missing answers are not silently restored. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31523,10 +31527,10 @@ } ], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "A cutoff point mapping supplies authorized conversation/role/timestamp; plain bounded query rows pass through screening and saved-analysis sanitization before plain snapshot construction.", + "contract": "A cutoff point mapping supplies authorized conversation/role/timestamp; plain bounded query rows pass through screening and saved-analysis sanitization before plain snapshot construction. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31544,10 +31548,10 @@ } ], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Ledger source revalidation consumes plain projected query rows to check source visibility; failed reads omit the ledger rather than authorizing point documents or widening scope.", + "contract": "Ledger source revalidation consumes plain projected query rows to check source visibility; failed reads omit the ledger rather than authorizing point documents or widening scope. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31565,10 +31569,10 @@ } ], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Historical context discovery consumes plain partitioned query rows, applies retry-prefix/screening rules and builds a bounded plain conversation snapshot.", + "contract": "Historical context discovery consumes plain partitioned query rows, applies retry-prefix/screening rules and builds a bounded plain conversation snapshot. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31591,10 +31595,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Conversation SDK mappings feed existing mapping-compatible shared authorization. The saved question is normalized by public_planning_message before projection/mirroring; private claims are not shared, and required admission mirror failures are explicit.", + "contract": "Conversation SDK mappings feed existing mapping-compatible shared authorization. The saved question is normalized by public_planning_message before projection/mirroring; private claims are not shared, and required admission mirror failures are explicit. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_shared_conversations.py" ], @@ -31612,10 +31616,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "The review-required question point mapping uses nested metadata mapping updates and the existing conditional metadata writer. Planning ownership is checked before run publication; no raw SDK question reaches an exact-dictionary JSON contract.", + "contract": "The review-required question point mapping uses nested metadata mapping updates and the existing conditional metadata writer. Planning ownership is checked before run publication; no raw SDK question reaches an exact-dictionary JSON contract. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_planning_persistence.py::test_explicit_retry_uses_saved_inputs_and_one_question", "functional_tests/test_orchestration_message_regeneration.py" @@ -31634,10 +31638,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "The assistant point mapping supplies role/deletion/mask metadata to reply_is_retracted; absence is treated as removed, never as an empty valid response.", + "contract": "The assistant point mapping supplies role/deletion/mask metadata to reply_is_retracted; absence is treated as removed, never as an empty valid response. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31655,10 +31659,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "response-unused", - "contract": "The message upsert response is discarded; screening and source-availability checks occur before publication, and persistence errors retain the existing failure path.", + "contract": "The message upsert response is discarded; screening and source-availability checks occur before publication, and persistence errors retain the existing failure path. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [], "evidence_scope": "owner-runtime" }, @@ -31679,10 +31683,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Question SDK point mappings are consumed by normalize_history_message and metadata mapping operations. The projected history dictionary, not the SDK wrapper, is fingerprinted. Create-only admission and conditional lifecycle writes preserve the original question.", + "contract": "Question SDK point mappings are consumed by normalize_history_message and metadata mapping operations. The projected history dictionary, not the SDK wrapper, is fingerprinted. Create-only admission and conditional lifecycle writes preserve the original question. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_planning_persistence.py" ], @@ -31705,10 +31709,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "The question create response is discarded. A racing 409 rereads an SDK point and validates the existing question instead of upserting over it; mismatched content fails without a save acknowledgement.", + "contract": "The question create response is discarded. A racing 409 rereads an SDK point and validates the existing question instead of upserting over it; mismatched content fails without a save acknowledgement. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_planning_persistence.py::test_concurrent_question_creation_never_upserts_over_the_winner" ], @@ -31731,10 +31735,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "The conversation point mapping supplies owner/deletion and mutable index fields; replace response is discarded and the Mapping-compatible cache invalidator consumes the authorized input.", + "contract": "The conversation point mapping supplies owner/deletion and mutable index fields; replace response is discarded and the Mapping-compatible cache invalidator consumes the authorized input. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31752,10 +31756,10 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Artifact point mappings are checked by field access for conversation/user/visibility and nested content metadata; no top-level exact-dictionary result contract consumes them.", + "contract": "Artifact point mappings are checked by field access for conversation/user/visibility and nested content metadata; no top-level exact-dictionary result contract consumes them. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py" ], @@ -31773,10 +31777,10 @@ } ], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Source revalidation consumes plain bounded query rows through screening and strict saved-snapshot checks; no point wrapper is repaired into a domain context. Selected-history projection uses plain SDK query rows, existing visibility/fingerprint checks and admitted typed result references; new full-text projection and assistant-only result selection are exercised through real planning, execution and editor routes with SDK-default containers.", + "contract": "Source revalidation consumes plain bounded query rows through screening and strict saved-snapshot checks; no point wrapper is repaired into a domain context. Selected-history projection uses plain SDK query rows, existing visibility/fingerprint checks and admitted typed result references; new full-text projection and assistant-only result selection are exercised through real planning, execution and editor routes with SDK-default containers. Re-reviewed for module-only planning failure projection changes: this owner's AST and storage calls are unchanged; normalization, partitioning and ETag consumers remain intact.", "tests": [ "functional_tests/test_orchestration_conversation_context_routes.py", "functional_tests/test_orchestration_followup_citation_provenance.py" @@ -31785,7 +31789,7 @@ }, "application/single_app/route_backend_orchestration.py::register_route_backend_orchestration.orchestration_plan": { "boundary": { - "sha256": "d624632ffa277aa6b9996c012eea1db246319dac2d33087d1bddf0a32ad9423a", + "sha256": "29030245af72419dd617c1b504f90ad6fdc0a163ffb27c875e7445265c95569a", "points": [ { "method": "read_item", @@ -31795,12 +31799,14 @@ ], "queries": [], "batches": [], - "module_sha256": "12d7db58d0681d207f45c56bf3178f98eba71e20f6fc0331ac18fb3f15a36a55" + "module_sha256": "a41fab05d0e151ff73b17a3a4fb4e4c9b813cb14cac5d62d8cfe526bfc28d010" }, "classification": "sdk-compatible", - "contract": "Clarification admission reads the SDK conversation through Mapping-compatible field access and current ownership checks. Authorized attachment DTOs are selected before planning; raw SDK responses never enter strict planner source contracts. Durable planning remains delegated to the existing normalized planning owner.", + "contract": "Clarification admission reads the SDK conversation through Mapping-compatible field access and current ownership checks. Authorized attachment DTOs are selected before planning; raw SDK responses never enter strict planner source contracts. Durable planning remains delegated to the existing normalized planning owner. Typed planner failure propagation changes only SSE/saved error classification; the existing point-read ownership and Mapping consumers are unchanged. SDK-default authenticated native Auto runs and failure/reload/retry exercise the complete path.", "tests": [ - "functional_tests/test_orchestration_attachment_native_evidence.py::test_http_existing_image_repairs_redundant_choice_then_reads_authorized_text" + "functional_tests/test_orchestration_attachment_native_evidence.py::test_http_existing_image_repairs_redundant_choice_then_reads_authorized_text", + "functional_tests/test_orchestration_planning_persistence.py::test_typed_contract_failure_survives_reload_and_clears_on_explicit_retry", + "functional_tests/test_orchestration_document_action_end_to_end.py" ], "evidence_scope": "owner-runtime" }, diff --git a/functional_tests/test_orchestration_attachment_native_evidence.py b/functional_tests/test_orchestration_attachment_native_evidence.py index f6842b02c..e51f3f40f 100644 --- a/functional_tests/test_orchestration_attachment_native_evidence.py +++ b/functional_tests/test_orchestration_attachment_native_evidence.py @@ -1,16 +1,21 @@ # test_orchestration_attachment_native_evidence.py """Real planning, inventory paging, retained evidence and composition regressions. -Version: 0.261.417 +Version: 0.261.423 Implemented in: 0.261.416 +Word-native comment routing and evidence delivery: 0.261.422 Only external storage/model I/O is replaced; no provider calls are made. """ import importlib import json import base64 +import hashlib +import logging +import zipfile from io import BytesIO from copy import deepcopy +from pathlib import Path import pytest from PIL import Image @@ -264,6 +269,299 @@ def small_pages(*args, **kwargs): return step, calls, document +def word_comment_package(*, comments=True, extended=True, malformed=False, anchor="15000", closed_anchor=True): + namespace = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + range_end = '' if closed_anchor else '' + document = ( + f'' + 'Budget cap ' + f'{anchor}{range_end}' + '' + ) + comment_xml = ( + f'' + '' + 'Raise cap to 19500.' + '' + 'Keep cap at 15000.' + '' + ) + stream = BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("word/document.xml", document) + if comments: + archive.writestr("word/comments.xml", "' + '' + '' + '' + )) + return stream.getvalue() + + +def configure_word_inventory(runtime, monkeypatch, content, *, extension="docx"): + document = { + "id": "document-1", "user_id": "owner", "version": 1, + "file_name": f"review.{extension}", "accessibility_metadata": {"format": extension}, + } + service = importlib.import_module("functions_search_service") + monkeypatch.setattr(service, "resolve_document_context", lambda **kwargs: { + "document": deepcopy(document), "scope": "personal", + }) + monkeypatch.setattr(service, "assert_document_available", lambda value, **kwargs: value) + monkeypatch.setattr(service, "assert_evidence_available", lambda *args, **kwargs: None) + calls = [] + + def read_bytes(value, **kwargs): + calls.append(deepcopy(kwargs)) + if isinstance(content, Exception): + raise content + return deepcopy(value), content + + monkeypatch.setattr(service.screening_access, "read_available_document_bytes", read_bytes) + step = { + "step_id": "comments", "capability_id": "document_search", + "arguments": {"query": "Read every Word review comment and reply.", + "document_ids": ["document-1"], "native_section": "comments"}, + } + return step, calls, document + + +def test_word_metadata_reader_uses_original_comments_not_stored_style_metadata(runtime, monkeypatch): + _, calls, _ = configure_word_inventory(runtime, monkeypatch, word_comment_package()) + service = importlib.import_module("functions_search_service") + report = service.get_document_accessibility_payload("document-1", "owner", sections="comments") + assert report["word_inspection"]["records"][0]["text"] == "Raise cap to 19500." + assert len(calls) == 1 + + +@pytest.mark.parametrize("metadata", [None, {}]) +def test_word_comment_only_inspection_does_not_need_stored_metadata(runtime, monkeypatch, metadata): + _, calls, document = configure_word_inventory(runtime, monkeypatch, word_comment_package()) + document["accessibility_metadata"] = metadata + service = importlib.import_module("functions_search_service") + report = service.get_document_accessibility_payload("document-1", "owner", sections="comments") + assert report["word_inspection"]["records"][0]["text"] == "Raise cap to 19500." + assert report["format"] == "docx" and len(calls) == 1 + assert not set(report).intersection({"accessibility_metadata", "style_inventory", "issues"}) + + +@pytest.mark.parametrize("metadata", [None, {}]) +@pytest.mark.parametrize("section", ["accessibility", "style", "issues"]) +def test_word_comments_do_not_certify_missing_stored_sections(runtime, monkeypatch, metadata, section): + _, _, document = configure_word_inventory(runtime, monkeypatch, word_comment_package()) + document["accessibility_metadata"] = metadata + service = importlib.import_module("functions_search_service") + with pytest.raises(LookupError, match="No accessibility metadata is stored"): + service.get_document_accessibility_payload("document-1", "owner", sections=f"comments,{section}") + + +@pytest.mark.parametrize("metadata", [None, {}]) +def test_word_default_metadata_requests_do_not_read_the_original(runtime, monkeypatch, metadata): + _, calls, document = configure_word_inventory(runtime, monkeypatch, word_comment_package()) + document["accessibility_metadata"] = metadata + service = importlib.import_module("functions_search_service") + with pytest.raises(LookupError, match="No accessibility metadata is stored"): + service.get_document_accessibility_payload("document-1", "owner") + assert calls == [] + + +@pytest.mark.parametrize("extension", ["docx", "docm", "dotx"]) +@pytest.mark.parametrize("repair_checker", [False, True]) +@pytest.mark.parametrize("prepared_text", [False, True]) +def test_word_comments_use_the_real_word_reader_and_reach_retained_answers( + runtime, monkeypatch, caplog, extension, repair_checker, prepared_text, +): + content = word_comment_package() + step, calls, document = configure_word_inventory(runtime, monkeypatch, content, extension=extension) + candidate = "The reviewer requests 19500; the author replies to keep 15000." + completeness = importlib.import_module("functions_task_completeness") + assessment_item = { + "status": "satisfied", "reason": "", "evidence_refs": ["comments"], "answer_quote": candidate, + } + assessment = {"requirements": { + identifier: deepcopy(assessment_item) for identifier, _ in completeness.NARRATIVE_OBLIGATIONS + } if prepared_text else [ + {"id": "review", "requirement": "Explain the reviewer comment and reply.", **assessment_item}, + ]} + invalid_assessment = deepcopy(assessment) + for item in ( + invalid_assessment["requirements"].values() if prepared_text else invalid_assessment["requirements"] + ): + item["evidence_refs"] = ["PRIVATE_INVALID_SOURCE_LABEL"] + replies = [candidate] + if repair_checker: + replies.extend([json.dumps(invalid_assessment), json.dumps(assessment)]) + answer_step = compose("answer", inputs={"comments": source_input("comments", "prepared")}) + if prepared_text: + answer_step["arguments"]["response_policy"] = "available_information" + case = runtime.make([ + step, answer_step, + ], replies, final_response=binding("answer"), + settings={"enable_chat_orchestration_completeness_check": repair_checker}) + case.fixture.sources["document-1"] = {**source(), "file_name": document["file_name"]} + spec = importlib.util.spec_from_file_location( + "native_comment_telemetry", + Path(__file__).resolve().parents[1] / "application" / "single_app" / "functions_appinsights.py", + ) + telemetry = importlib.util.module_from_spec(spec) + spec.loader.exec_module(telemetry) + adapters = importlib.import_module("functions_orchestration_adapters") + logger = logging.getLogger("test.orchestration.native_comments") + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: {"enable_debug_logging": False}) + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + monkeypatch.setattr(adapters, "log_event", telemetry.log_event) + monkeypatch.setattr(adapters, "workflow_log_context", telemetry.workflow_log_context) + monkeypatch.setattr(runtime.composition, "log_event", telemetry.log_event) + monkeypatch.setattr(runtime.composition, "workflow_log_context", telemetry.workflow_log_context) + with caplog.at_level(logging.INFO, logger=logger.name): + result = execute(runtime, case) + assert result["status"] == "completed", { + row["step_id"]: row.get("failure") for row in result["steps"] + } + assert len(calls) == 1 and calls[0]["user_id"] == "owner" and calls[0]["purpose"] == "accessibility" + task = case.context.task_results["comments"] + prepared = case.fixture.restart().open_result(task.output("prepared")).read_value() + assert prepared["content_scope"] == "native_document_annotations" + record = json.loads(prepared["search_results"][1]["chunk_text"])["word_annotation"] + assert record["id"] == "0" and record["author"] == "Reviewer" + assert record["text"] == "Raise cap to 19500." and record["anchor_text"] == "15000" + assert record["date"] == "2026-01-01T10:00:00Z" + assert record["resolution_available"] and record["done"] is False + assert record["anchor_complete"] and record["position_known"] + assert record["replies"][0]["text"] == "Keep cap at 15000." + assert record["replies"][0]["author"] == "Author" and record["replies"][0]["done"] is True + coverage = prepared["coverage"]["native_inspection"][0] + assert coverage["total_records"] == coverage["returned_records"] == 1 + assert coverage["total_replies"] == 1 and not coverage["partial"] + assert coverage["source_sha256"] == hashlib.sha256(content).hexdigest() + payload = json.loads(case.model.calls[0][0][-1]["content"]) + assert "Raise cap to 19500." in json.dumps(payload) and "Keep cap at 15000." in json.dumps(payload) + reader = importlib.import_module("functions_orchestration_result_runtime") + citations = reader.read_result_document_citations(case.context, task.output("prepared")) + citation = next(item for item in citations if item.get("evidence_kind") == "native_word_annotation") + contracts = importlib.import_module("functions_orchestration_result_contracts") + cited = reader.read_native_citation_evidence( + case.fixture.restart(), contracts.ResultRef.from_dict(citation["native_evidence"]), + citation_id=citation["citation_id"], document_id="document-1", + ) + assert cited["native_annotation"] == record and cited["evidence_kind"] == "native_word_annotation" + checkpoints = [entry for entry in caplog.records if getattr(entry, "sc_stage", None) == "native_comment_read"] + assert [entry.sc_outcome for entry in checkpoints] == ["started", "completed"] + assert all(entry.sc_source_format == extension and entry.sc_run_id_hash and entry.sc_step_id_hash for entry in checkpoints) + serialized = json.dumps([{key: value for key, value in entry.__dict__.items() if key.startswith("sc_")} for entry in checkpoints]) + assert "Raise cap" not in serialized and document["file_name"] not in serialized + assert len(case.model.calls) == (3 if repair_checker else 1) + if repair_checker: + feedback = json.loads(case.model.calls[2][0][-1]["content"])["assessment_problem"] + assert feedback["allowed_evidence_refs"] == ["comments", "user_provided_content"] + diagnostics = [entry for entry in caplog.records if getattr(entry, "sc_stage", None) == "task_completeness_assessment"] + assert len(diagnostics) == 1 + assert diagnostics[0].sc_validation_code == "completeness_evidence_invalid" + assert diagnostics[0].sc_run_id_hash and diagnostics[0].sc_step_id_hash + assert diagnostics[0].sc_report_repair_count == 0 and diagnostics[0].sc_evidence_key_count == 2 + assert diagnostics[0].sc_evidence_ref_count == 1 and diagnostics[0].sc_evidence_refs_list is True + safe_fields = {key: value for key, value in diagnostics[0].__dict__.items() if key.startswith("sc_")} + assert "PRIVATE_INVALID_SOURCE_LABEL" not in json.dumps(safe_fields) + answer = case.fixture.restart().open_result(case.context.task_results["answer"].output("answer")).read_text() + assert answer == candidate + + +def test_word_evidence_does_not_bypass_repeated_unsupported_answer_findings(runtime, monkeypatch): + step, _, _ = configure_word_inventory(runtime, monkeypatch, word_comment_package()) + candidate = "The author accepted the higher cap of 19500." + assessment = json.dumps({"requirements": [{ + "id": "grounding", "requirement": "Describe what the author actually replied.", + "status": "missing", "reason": "The source says to keep 15000, not accept 19500.", + "evidence_refs": ["comments"], "answer_quote": "", + }]}) + case = runtime.make([ + step, compose("answer", inputs={"comments": source_input("comments", "prepared")}), + ], [ + candidate, assessment, "The higher cap of 19500 was approved.", assessment, + "The final approved policy cap is 19500.", assessment, + ], final_response=binding("answer"), + settings={"enable_chat_orchestration_completeness_check": True}) + case.fixture.sources["document-1"] = {**source(), "file_name": "review.docx"} + result = execute(runtime, case) + assert result["status"] == "failed" + answer = next(row for row in result["steps"] if row["step_id"] == "answer") + assert answer["failure"]["code"] == "completeness_incomplete" + assert len(case.model.calls) == 6 + assert "comments" in case.context.task_results and "answer" not in case.context.task_results + + +@pytest.mark.parametrize("anchor,closed", [(" " + "x" * 400, True), ("15000", False)]) +def test_word_anchor_completeness_requires_an_unclipped_closed_range(runtime, monkeypatch, anchor, closed): + configure_word_inventory(runtime, monkeypatch, word_comment_package(anchor=anchor, closed_anchor=closed)) + service = importlib.import_module("functions_search_service") + payload = service.get_document_accessibility_payload("document-1", "owner", sections="comments") + annotation = payload["word_inspection"]["records"][0] + assert annotation["anchor_text"] and not annotation["anchor_complete"] + + +def test_comment_only_metadata_request_rejects_an_unsupported_format(runtime, monkeypatch): + _, calls, _ = configure_word_inventory(runtime, monkeypatch, word_comment_package(), extension="pdf") + service = importlib.import_module("functions_search_service") + with pytest.raises(LookupError, match="native comment inventory is unavailable"): + service.get_document_accessibility_payload("document-1", "owner", sections="comments") + assert calls == [] + + +@pytest.mark.parametrize("problem", ["empty", "missing", "malformed", "revoked", "changed", "limit"]) +def test_word_native_unavailable_sources_never_become_empty_success(runtime, monkeypatch, problem): + content = ( + LookupError("PRIVATE_ORIGINAL_UNAVAILABLE") if problem == "missing" + else word_comment_package(comments=problem != "empty", malformed=problem == "malformed") + ) + step, _, _ = configure_word_inventory(runtime, monkeypatch, content) + case = runtime.make([step]) + case.fixture.sources["document-1"] = {**source(), "file_name": "review.docx"} + service = importlib.import_module("functions_search_service") + original = service.screening_access.read_available_document_bytes + + def read_then_change(*args, **kwargs): + value = original(*args, **kwargs) + if problem == "revoked": + case.fixture.denied.add("document-1") + elif problem == "changed": + case.fixture.sources["document-1"]["source_revision"] = "new-revision" + return value + + monkeypatch.setattr(service.screening_access, "read_available_document_bytes", read_then_change) + if problem == "limit": + collector = importlib.import_module("functions_orchestration_excel") + monkeypatch.setattr(collector, "MAX_VALUE_BYTES", 4) + result = execute(runtime, case) + if problem in ("empty", "limit"): + assert "comments" in case.context.task_results, result["steps"] + task = case.context.task_results["comments"] + prepared = case.fixture.restart().open_result(task.output("prepared"), allow_partial=True).read_value() + coverage = prepared["coverage"]["native_inspection"][0] + assert coverage["returned_records"] == 0 and coverage["partial"] is (problem == "limit") + assert task.output("prepared").completeness.status == ("partial" if problem == "limit" else "complete") + else: + assert result["status"] != "completed" and "comments" not in case.context.task_results + assert "PRIVATE_ORIGINAL" not in json.dumps(result) + + +def test_word_resolution_and_bounded_anchor_remain_explicitly_unavailable(runtime, monkeypatch): + step, _, _ = configure_word_inventory( + runtime, monkeypatch, word_comment_package(extended=False, anchor="a" * 400), + ) + case = runtime.make([step]) + case.fixture.sources["document-1"] = {**source(), "file_name": "review.docx"} + result = execute(runtime, case) + assert result["status"] == "completed", result["steps"] + prepared = case.fixture.restart().open_result(case.context.task_results["comments"].output("prepared")).read_value() + record = json.loads(prepared["search_results"][1]["chunk_text"])["word_annotation"] + assert record["done"] is None and record["resolution_available"] is False + assert len(record["anchor_text"]) == 300 and record["anchor_complete"] is False + assert prepared["coverage"]["native_inspection"][0]["total_records"] == 2 + + def test_native_workbook_comments_reach_retained_evidence_and_composition( runtime, monkeypatch, inventory_store, review_workbook, ): @@ -353,12 +651,16 @@ def test_native_citation_lookup_rejects_stale_or_untrusted_evidence( ) +@pytest.mark.parametrize("source_format", ["xlsx", "docx"]) def test_native_citation_http_lookup_uses_retained_result_not_search( - runtime, real_http_harness, monkeypatch, inventory_store, review_workbook, + runtime, real_http_harness, monkeypatch, inventory_store, review_workbook, source_format, ): - step, _, _ = configure_inventory(runtime, monkeypatch, inventory_store, review_workbook) + if source_format == "docx": + step, _, _ = configure_word_inventory(runtime, monkeypatch, word_comment_package()) + else: + step, _, _ = configure_inventory(runtime, monkeypatch, inventory_store, review_workbook) case = runtime.make([step]) - case.fixture.sources["document-1"] = {**source(), "file_name": "review.xlsx"} + case.fixture.sources["document-1"] = {**source(), "file_name": f"review.{source_format}"} result = execute(runtime, case) assert result["status"] == "completed" reader = importlib.import_module("functions_orchestration_result_runtime") diff --git a/functional_tests/test_orchestration_conversation_sources.py b/functional_tests/test_orchestration_conversation_sources.py index fdc5241bf..b52415c87 100644 --- a/functional_tests/test_orchestration_conversation_sources.py +++ b/functional_tests/test_orchestration_conversation_sources.py @@ -1,8 +1,9 @@ # test_orchestration_conversation_sources.py """ Functional regressions for exact conversation-source transformations. -Version: 0.261.349 +Version: 0.261.423 Implemented in: 0.261.341 +Shared two-correction allowance updated in: 0.261.422 Exercises real discovery, resolver, Flask planning and initialized execution with offline storage/model boundaries. Discovery previews never substitute for full @@ -200,10 +201,12 @@ def test_repeated_inconsistent_plan_fails_instead_of_broadening_scope(real_http_ _replace_history(runtime.harness, long_sources()) runtime.harness.replies = [ transform_resolution(), _search_then_answer_plan(REQUEST), _search_then_answer_plan(REQUEST), + _search_then_answer_plan(REQUEST), ] events = _plan(runtime, REQUEST) assert any('not a workspace search' in event.get('error', '') for event in events), events assert not any(event.get('type') == 'orchestration_plan' for event in events) + assert len(runtime.harness.model_calls) == 4 def test_explicit_workspace_retrieval_overrides_no_retrieval_resolution(real_http_harness): diff --git a/functional_tests/test_orchestration_document_action_binding.py b/functional_tests/test_orchestration_document_action_binding.py new file mode 100644 index 000000000..56a15e1a2 --- /dev/null +++ b/functional_tests/test_orchestration_document_action_binding.py @@ -0,0 +1,375 @@ +# test_orchestration_document_action_binding.py +""" +Functional regressions for enforced native document action source bindings. +Version: 0.261.423 +Implemented in: 0.261.419 + +The real action runner, Semantic Kernel filters, Document Search plugin and +citation projection run with model and source-service I/O doubled. Existing +source-authority suites cover the underlying readers' current-access checks. +""" + +import asyncio +import importlib +import json +import logging +import sys +from contextlib import nullcontext +from copy import deepcopy +from types import ModuleType, SimpleNamespace + +import pytest +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.contents import AuthorRole, ChatMessageContent, FunctionCallContent + +from test_orchestration_harness_routes import modules + + +__all__ = ["modules"] +SOURCE = "selected-source" + + +@pytest.fixture +def native_action(modules, monkeypatch): + runner = importlib.import_module("functions_orchestration_actions") + contexts = importlib.import_module("agent_execution_context") + action = { + "action_ref": "personal:actor:native", "id": "native", "name": "document_search", + "display_name": "Native inspection", "type": "document_search", + "scope_type": "personal", "scope_id": "actor", + } + state = SimpleNamespace( + runtime=runner, contexts=contexts, loop=modules.execution_loop, + action=action, manifest={**action, "additionalFields": {}}, + settings={ + "enable_chat_orchestration": True, "enable_chat_orchestration_actions": True, + "enable_semantic_kernel": True, "max_auto_invoke_attempts": 5, + }, + loads=[], requests=[], replies=[], + ) + state.context = SimpleNamespace( + action_catalog=[action], active_group_ids=[], token_usage={}, notes=[], + user_id="actor", conversation_id="conversation", + agent_execution_identity=contexts.ExecutionIdentity( + "actor", "conversation", bridge=lambda reference: nullcontext(), + ), + delegation_budget=contexts.DelegationBudget(), + ) + plugin_module = importlib.import_module("semantic_kernel_plugins.document_search_plugin") + selection = importlib.import_module("functions_workflow_document_selection") + logger = importlib.import_module("semantic_kernel_plugins.plugin_invocation_logger") + monkeypatch.setattr(logger, "get_current_user_id", lambda: "actor") + state.native_reads, state.native_payloads, state.source_checks = [], [], 0 + state.allowed, state.scope, state.scope_id = True, "personal", "actor" + state.metadata = { + "format": "pdf", "schema_version": "1.0", "is_tagged": False, + "inspection_limits": ["No structure tree is stored for this source."], + } + + def authorize(): + state.source_checks += 1 + if not state.allowed: + raise PermissionError("The source is no longer available.") + + def payload(operation, **kwargs): + state.native_reads.append((operation, deepcopy(kwargs))) + identifier = kwargs.get("document_id", SOURCE) + if not selection.selected_source_matches("actor", identifier, state.scope, state.scope_id): + raise LookupError("The source is outside the bound workspace.") + document = { + "id": SOURCE, "file_name": f"selected.{state.metadata['format']}", + "user_id": "actor", "scope": state.scope, "scope_id": state.scope_id, "version": 1, + } + result = { + "document": document, "scope": state.scope, "format": state.metadata["format"], + "accessibility_metadata": deepcopy(state.metadata), + } + if operation == "search": + result["results"] = [{ + "document_id": SOURCE, "file_name": document["file_name"], + "id": f"{SOURCE}_1", "page_number": 1, "chunk_text": "Stored native evidence.", + }] + elif operation == "chunks": + result["chunks"] = [{ + "document_id": SOURCE, "file_name": document["file_name"], + "id": f"{SOURCE}_1", "page_number": 1, "chunk_text": "Stored native evidence.", + }] + elif operation == "summary": + result["summary"] = "Source-grounded summary." + state.native_payloads.append(result) + return result + + monkeypatch.setattr(plugin_module, "get_current_user_id", lambda: "actor") + monkeypatch.setattr(plugin_module, "get_document_accessibility_payload", lambda **kwargs: payload("native", **kwargs)) + monkeypatch.setattr(plugin_module, "get_document_chunks_payload", lambda **kwargs: payload("chunks", **kwargs)) + monkeypatch.setattr(plugin_module, "summarize_document_content", lambda **kwargs: payload("summary", **kwargs)) + monkeypatch.setattr(plugin_module, "run_document_search", lambda **kwargs: payload("search", **kwargs)) + + class Loader: + def __init__(self, kernel): + self.kernel, self.plugin_instances = kernel, [] + + def load_plugin_from_manifest(self, manifest, user_id): + plugin = plugin_module.DocumentSearchPlugin(manifest) + self.plugin_instances.append(plugin) + state.loads.append((manifest["id"], user_id)) + self.kernel.add_plugin(plugin, plugin_name="document_search") + return True + + loader_module = ModuleType("semantic_kernel_plugins.logged_plugin_loader") + loader_module.create_logged_plugin_loader = Loader + monkeypatch.setitem(sys.modules, "semantic_kernel_plugins.logged_plugin_loader", loader_module) + preparation = ModuleType("semantic_kernel_loader") + preparation.get_max_auto_invoke_attempts = lambda settings: settings["max_auto_invoke_attempts"] + preparation.prepare_action_plugin_manifest = lambda manifest, settings: deepcopy(manifest) + monkeypatch.setitem(sys.modules, "semantic_kernel_loader", preparation) + monkeypatch.setattr( + importlib.import_module("functions_settings"), "get_settings", lambda: deepcopy(state.settings), + ) + monkeypatch.setattr(runner, "resolve_action_manifest", lambda *args, **kwargs: deepcopy(state.manifest)) + state.service = AzureChatCompletion( + service_id="orchestration-action", deployment_name="offline-model", + endpoint="https://offline.invalid", api_key="offline-fixture-not-a-credential", api_version="2024-10-21", + ) + monkeypatch.setattr(runner, "_build_action_model", lambda *args, **kwargs: state.service) + state.check_sources = authorize + + async def reply(self, history, settings): + state.requests.append(str(history)) + if state.replies: + return [state.replies.pop(0)] + native = state.native_payloads[-1] + return [ChatMessageContent( + role=AuthorRole.ASSISTANT, + content=json.dumps({ + "native_evidence": native["accessibility_metadata"], + "citation": native.get("citation"), + }), + )] + + monkeypatch.setattr(AzureChatCompletion, "_inner_get_chat_message_contents", reply) + yield state + if not state.service.client.is_closed(): + state.loop.run_until_complete(state.service.client.close()) + + +def tool_call(function, arguments): + return ChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[FunctionCallContent( + id="native-call", name=f"document_search-{function}", arguments=json.dumps(arguments), + )], + ) + + +async def execute(state): + return await state.runtime.invoke_action( + state.action["action_ref"], "Inspect only the selected source.", state.context, + settings=state.settings, user_id="actor", cancel_requested=lambda: False, + document_sources=[{"document_id": SOURCE, "scope": state.scope, "scope_id": state.scope_id}], + document_source_check=state.check_sources, + m365_origin={"step_id": "inspect", "run_id": getattr(state.context, "run_id", None)}, + ) + + +@pytest.mark.parametrize("scope,scope_id", [ + ("personal", "actor"), ("group", "group-1"), ("public", "workspace-1"), +]) +def test_native_metadata_and_citations_survive_the_real_action_loop(native_action, scope, scope_id): + state = native_action + state.scope, state.scope_id = scope, scope_id + state.manifest["additionalFields"] = { + "allowed_scopes": [scope], "allowed_group_ids": ["group-1"], + "allowed_public_workspace_ids": ["workspace-1"], + } + state.replies = [tool_call("get_document_accessibility_report", {"document_id": SOURCE})] + result = state.loop.run_until_complete(execute(state)) + evidence = json.loads(result["findings"]) + assert result["calls"] == 1 and state.source_checks == 2 + assert len(state.native_reads) == 1 and state.native_reads[0][1]["doc_scope"] == scope + assert evidence["native_evidence"]["is_tagged"] is False + assert evidence["native_evidence"]["inspection_limits"] == state.metadata["inspection_limits"] + assert evidence["citation"] and SOURCE in evidence["citation"] + assert state.contexts.current_agent_execution() is None + selection = importlib.import_module("functions_workflow_document_selection") + outside_matches = selection.selected_source_matches("actor", "outside", scope, scope_id) + assert outside_matches is True + + +def test_native_word_comments_are_not_replaced_with_body_text(native_action): + state = native_action + state.metadata = { + "format": "docx", "schema_version": "1.0", + "comments": [ + {"id": "comment-1", "text": "Review this date.", "author": "Reviewer", "anchor": "15 December"}, + {"id": "reply-1", "parent_id": "comment-1", "text": "Use the approved date."}, + ], + "inspection_limits": ["Resolution state is not exposed by this stored report."], + } + state.replies = [tool_call("get_document_accessibility_report", {"document_id": SOURCE})] + result = state.loop.run_until_complete(execute(state)) + evidence = json.loads(result["findings"]) + assert evidence["native_evidence"]["comments"] == state.metadata["comments"] + assert evidence["native_evidence"]["inspection_limits"] == state.metadata["inspection_limits"] + assert evidence["citation"] and SOURCE in evidence["citation"] + + +def test_search_without_a_model_filter_is_restricted_before_source_io(native_action): + state = native_action + state.replies = [tool_call("search_documents", {"query": "native evidence"})] + result = state.loop.run_until_complete(execute(state)) + assert result["calls"] == 1 and len(state.native_reads) == 1 + assert state.native_reads[0][1]["document_ids"] == SOURCE + + +@pytest.mark.parametrize("function,arguments", [ + ("get_document_accessibility_report", {"document_id": "outside"}), + ("retrieve_document_chunks", {"document_id": "outside"}), + ("summarize_document", {"document_id": "outside"}), + ("search_documents", {"query": "evidence", "document_ids": "outside"}), +]) +def test_unselected_documents_are_rejected_before_source_io(native_action, function, arguments): + state = native_action + state.replies = [tool_call(function, arguments)] + with pytest.raises(state.runtime.ActionExecutionError): + state.loop.run_until_complete(execute(state)) + assert state.native_reads == [] + + +def test_action_workspace_restrictions_remain_in_force(native_action): + state = native_action + state.manifest["additionalFields"] = {"allowed_scopes": ["group"]} + state.replies = [tool_call( + "get_document_accessibility_report", {"document_id": SOURCE, "doc_scope": "personal"}, + )] + with pytest.raises(state.runtime.ActionExecutionError): + state.loop.run_until_complete(execute(state)) + assert state.native_reads == [] + + +def test_revoked_authority_stops_the_tool_before_source_io(native_action): + state = native_action + checks = [] + + def authorize(): + checks.append(True) + if len(checks) > 1: + raise PermissionError("The source is no longer available.") + + state.check_sources = authorize + state.replies = [tool_call("get_document_accessibility_report", {"document_id": SOURCE})] + with pytest.raises(state.runtime.ActionExecutionError): + state.loop.run_until_complete(execute(state)) + assert len(checks) == 2 and state.native_reads == [] + + +def test_other_integrations_cannot_receive_a_document_source_binding(native_action): + state = native_action + state.action["type"] = state.manifest["type"] = "openapi" + with pytest.raises(state.runtime.ActionExecutionError): + state.loop.run_until_complete(execute(state)) + assert state.loads == state.requests == state.native_reads == [] + + +def test_source_bindings_are_isolated_between_concurrent_tasks(modules): + selection = importlib.import_module("functions_workflow_document_selection") + + async def inspect(document_id): + entries = [{"document": {"document_id": document_id, "scope_type": "personal", "scope_id": "actor"}}] + with selection.selected_document_sources(entries, "actor", restrict_ids=True): + await asyncio.sleep(0) + own = selection.selected_source_matches("actor", document_id, "personal", "actor") + other = selection.selected_source_matches("actor", f"other-{document_id}", "personal", "actor") + return own, other + + async def run(): + return await asyncio.gather(inspect("first"), inspect("second")) + + results = modules.execution_loop.run_until_complete(run()) + assert results == [(True, False), (True, False)] + + +@pytest.mark.parametrize("debug", [False, True]) +def test_read_checkpoints_are_debug_gated_correlated_and_content_free(native_action, monkeypatch, caplog, debug): + state = native_action + telemetry = importlib.import_module("functions_appinsights") + logger = logging.getLogger("test.orchestration.native.read") + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: {"enable_debug_logging": debug}) + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + state.context.run_id, state.context.turn_id = "native-run", "native-turn" + state.metadata["title"] = "PRIVATE_NATIVE_REPORT_CONTENT" + state.replies = [tool_call("get_document_accessibility_report", {"document_id": SOURCE})] + with caplog.at_level(logging.INFO, logger=logger.name): + result = state.loop.run_until_complete(execute(state)) + records = [ + record for record in caplog.records + if getattr(record, "sc_message", None) + == "[ORCHESTRATION_ADAPTERS] Document action read checkpoint." + ] + assert result["calls"] == 1 and len(state.native_reads) == 1 and state.source_checks == 2 + assert len(state.requests) == 2 + assert len(records) == (2 if debug else 0) + if debug: + assert [record.sc_outcome for record in records] == ["started", "completed"] + assert records[-1].sc_native_metadata_present is True + assert records[-1].sc_citation_present is True + for record in records: + assert len(record.sc_conversation_id_hash) == len(record.sc_run_id_hash) == 64 + assert len(record.sc_turn_id_hash) == len(record.sc_step_id_hash) == 64 + assert len(record.sc_document_id_hash) == 64 + serialized = json.dumps([ + {name: value for name, value in record.__dict__.items() if name.startswith("sc_")} + for record in records + ]) + assert "PRIVATE_NATIVE_REPORT_CONTENT" not in serialized + assert SOURCE not in serialized + + +@pytest.mark.parametrize("changed", [False, True]) +def test_adapter_passes_exact_authorized_sources_and_rechecks_the_snapshot(native_action, monkeypatch, changed): + state = native_action + adapters = importlib.import_module("functions_orchestration_adapters") + contracts = importlib.import_module("functions_orchestration_result_contracts") + registry = importlib.import_module("functions_orchestration_registry") + context = state.context + context.run_id, context.attempt_index, context.turn_id, context.seeds = "run-1", 1, "turn-1", {} + producer = contracts.ProducerIdentity( + "actor", "conversation", "run-1", 1, "inspect", "action_invoke", + registry.get_capability("action_invoke")["result_contract_version"], + ) + context.result_producer = lambda step: producer + context.external_source_preflight = lambda **kwargs: None + context.capture_external_source_configuration = lambda *args, **kwargs: True + current = {"version": 1} + reads = [] + + def resolve(identifiers): + reads.append(list(identifiers)) + return [{ + "document_id": identifier, "scope": "personal", "scope_id": "actor", + "source_version": current["version"], "source_revision": str(current["version"]), + "authorization_status": "authorized", + } for identifier in identifiers] + + context.resolve_source_manifest = resolve + + async def invoke(*args, document_sources, document_source_check, **kwargs): + assert [source["document_id"] for source in document_sources] == [SOURCE] + if changed: + current["version"] = 2 + document_source_check() + return { + "root_id": context.delegation_budget.root_id, "invocations": [], "calls": 1, + "findings": "Native evidence from the selected source.", "artifacts": [], + } + + monkeypatch.setattr(state.runtime, "invoke_action", invoke) + # Windows' offline guard keeps the existing pre-created loop, not a new loopback socket pair. + monkeypatch.setattr(asyncio, "run", state.loop.run_until_complete) + result = adapters.run_action_invoke({ + "step_id": "inspect", "capability_id": "action_invoke", + "arguments": {"action_ref": state.action["action_ref"], "task": "Inspect native evidence.", + "document_ids": [SOURCE]}, + }, context, settings=state.settings, user_id="actor", emit=None, cancel_requested=lambda: False) + assert reads == [[SOURCE], [SOURCE]] + assert result["status"] == ("failed" if changed else "completed") diff --git a/functional_tests/test_orchestration_document_action_end_to_end.py b/functional_tests/test_orchestration_document_action_end_to_end.py new file mode 100644 index 000000000..021248186 --- /dev/null +++ b/functional_tests/test_orchestration_document_action_end_to_end.py @@ -0,0 +1,295 @@ +# test_orchestration_document_action_end_to_end.py +""" +Authenticated native-document orchestration from HTTP submission to saved answer. +Version: 0.261.423 +Implemented in: 0.261.419 +Production plugin discovery coverage added in: 0.261.421 + +Real routes, plugin discovery, approval/leases, native action filters, +report/citation projection, retention, composition, completeness and publication +run with external storage, configuration and model I/O isolated. No production +service or data is touched. +""" + +import asyncio +import importlib +import json +import logging +import traceback +from copy import deepcopy +from types import SimpleNamespace + +import pytest +from azure.core import MatchConditions +from azure.core.exceptions import AzureError +from azure.cosmos import exceptions +from azure.cosmos._cosmos_responses import CosmosDict +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.contents import AuthorRole, ChatMessageContent, FunctionCallContent + +from functions_task_completeness import NARRATIVE_OBLIGATIONS +from test_orchestration_harness_routes import modules, real_http_harness +from test_orchestration_planner_contract_recovery import ACTION, action_proposal +from test_orchestration_source_kind_planning import LEFT, source_runtime +from test_support.orchestration_revisions import AtomicMemoryContainer + + +__all__ = ["modules", "real_http_harness", "source_runtime"] +TURN = "native-http-turn" +MESSAGE = "Inspect native metadata in only the selected source and disclose its limits." +PRIVATE = "PRIVATE_NATIVE_METADATA_SENTINEL" +TRACE_MESSAGES = { + "[ORCHESTRATION_ADAPTERS] Document action binding verified.", + "[ORCHESTRATION_ADAPTERS] Document action binding rechecked.", + "[ORCHESTRATION_ADAPTERS] Document action read checkpoint.", + "[ORCHESTRATION_EXECUTOR] Gather retention started.", + "[ORCHESTRATION_EXECUTOR] Gather source handoff verified.", + "[ORCHESTRATION_EXECUTOR] Gather result retained.", + "[ORCHESTRATION_EXECUTOR] Retained document citation projection verified.", + "[ORCHESTRATION_EXECUTOR] Composition input contract verified.", + "[ORCHESTRATION_EXECUTOR] Prepared content retained.", +} + + +@pytest.fixture +def native_http(source_runtime, modules, monkeypatch): + runtime = source_runtime + harness = runtime.harness + catalog = importlib.import_module("functions_action_catalog") + settings_owner = importlib.import_module("functions_settings") + governance = importlib.import_module("functions_governance") + runner = importlib.import_module("functions_orchestration_actions") + search = importlib.import_module("functions_search_service") + telemetry = importlib.import_module("functions_appinsights") + logger = logging.getLogger("test.orchestration.native.http") + harness.settings.update({ + "enable_semantic_kernel": True, "enable_chat_orchestration_actions": True, + "enable_user_workspace": True, "allow_user_plugins": True, + "per_user_semantic_kernel": True, "allow_group_plugins": False, + "max_auto_invoke_attempts": 4, + "enable_chat_orchestration_completeness_check": True, + }) + action = { + **ACTION, "id": "native-action", "user_id": "owner", "name": "document_search", + "action_ref": catalog._action_ref("personal", "owner", "native-action"), "is_enabled": True, + } + actions = AtomicMemoryContainer("user_id") + actions.create_item({key: value for key, value in action.items() if key != "action_ref"}) + users = AtomicMemoryContainer("id") + users.create_item({"id": "owner", "settings": {"enable_agents": True}}) + monkeypatch.setattr(harness.config, "cosmos_personal_actions_container", actions) + monkeypatch.setattr(harness.config, "cosmos_user_settings_container", users) + monkeypatch.setattr(settings_owner, "get_settings", lambda: deepcopy(harness.settings)) + monkeypatch.setattr(governance, "get_settings", lambda: deepcopy(harness.settings)) + monkeypatch.setattr( + importlib.import_module("functions_group"), "get_user_groups", lambda user_id: [], + ) + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: { + "enable_debug_logging": harness.settings.get("enable_debug_logging", False), + }) + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + state = SimpleNamespace( + runtime=runtime, modules=modules, action=action, actions=actions, users=users, + logger=logger, native_calls=[], report_reads=[], fail_report=False, failures=[], + ) + log_failure = harness.execution._log_failure + + def capture_failure(message, record, error, **kwargs): + state.failures.append("".join(traceback.format_exception(error))) + return log_failure(message, record, error, **kwargs) + + monkeypatch.setattr(harness.execution, "_log_failure", capture_failure) + executor = importlib.import_module("functions_orchestration_executor") + step_failure = executor._raise_dependency_service_failure + + def capture_step_failure(step, error): + state.failures.append("".join(traceback.format_exception(error))) + return step_failure(step, error) + + monkeypatch.setattr(executor, "_raise_dependency_service_failure", capture_step_failure) + + def read_personal(document_id, user_id): + state.report_reads.append((document_id, user_id)) + if state.fail_report: + raise PermissionError(PRIVATE) + document = harness.config.cosmos_user_documents_container.read_item(document_id, document_id) + if document.get("user_id") != user_id: + return None + return {"document": document, "scope": "personal", "group_id": None, "public_workspace_id": None} + + monkeypatch.setattr(search, "_resolve_personal_document_context", read_personal) + service = AzureChatCompletion( + service_id="orchestration-action", deployment_name="gpt-4o", + endpoint="https://offline.invalid", api_key="offline-fixture-not-a-credential", api_version="2024-10-21", + ) + configuration = { + "provider": "aoai", "protocol": "azure_openai", "endpoint": "https://offline.invalid", + "api_version": "2024-10-21", "deployment": "gpt-4o", "parameters": {}, + } + monkeypatch.setattr(runner, "_build_action_model", lambda *args, capture_configuration=False: ( + (service, deepcopy(configuration)) if capture_configuration else service + )) + + async def native_reply(self, history, settings): + state.native_calls.append(deepcopy([message.to_dict() for message in history.messages])) + if len(state.native_calls) == 1: + return [ChatMessageContent(role=AuthorRole.ASSISTANT, items=[FunctionCallContent( + id="read-native", name="document_search-get_document_accessibility_report", + arguments=json.dumps({"document_id": LEFT, "doc_scope": "personal"}), + )])] + results = [ + item.result for message in history.messages for item in message.items + if hasattr(item, "result") + ] + value = results[-1] + value = json.loads(value) if isinstance(value, str) else value + return [ChatMessageContent(role=AuthorRole.ASSISTANT, content=json.dumps({ + "native_metadata": value["accessibility_metadata"], "citation": value["citation"], + }))] + + monkeypatch.setattr(AzureChatCompletion, "_inner_get_chat_message_contents", native_reply) + # The offline Windows loop is created before network blocking; never permit a new socket pair. + monkeypatch.setattr(asyncio, "run", modules.execution_loop.run_until_complete) + yield state + if not service.client.is_closed(): + modules.execution_loop.run_until_complete(service.client.close()) + + +def frames(response): + return [ + json.loads(frame.partition("data:")[2]) + for frame in response.get_data(as_text=True).split("\n\n") if frame.startswith("data:") + ] + + +def test_sdk_transport_options_preserve_partition_etag_and_failure_semantics(): + container = AtomicMemoryContainer("scope") + created = container.create_item({"id": "record", "scope": "owner", "value": 1}) + record = container.read_item( + "record", "owner", connection_timeout=10.0, read_timeout=10.0, retry_total=0, + ) + assert type(record) is CosmosDict and record["_etag"] == created["_etag"] + with pytest.raises(exceptions.CosmosResourceNotFoundError): + container.read_item("record", "other", read_timeout=10.0) + updated = container.replace_item( + "record", {**record, "value": 2}, etag=record["_etag"], match_condition=MatchConditions.IfNotModified, + ) + assert type(updated) is CosmosDict and updated["_etag"] != record["_etag"] + with pytest.raises(exceptions.CosmosAccessConditionFailedError): + container.replace_item( + "record", {**record, "value": 3}, etag=record["_etag"], match_condition=MatchConditions.IfNotModified, + ) + container.fail_reads = True + with pytest.raises(AzureError): + container.read_item("record", "owner", connection_timeout=10.0, read_timeout=10.0, retry_total=0) + + +@pytest.mark.parametrize("extension", ["pdf", "docx"]) +@pytest.mark.parametrize("debug", [False, True]) +@pytest.mark.parametrize("report_fails", [False, True]) +def test_native_auto_turn_reaches_a_saved_cited_answer(native_http, caplog, extension, debug, report_fails): + state = native_http + runtime, harness = state.runtime, state.runtime.harness + harness.settings["enable_debug_logging"] = debug + state.fail_report = report_fails + documents = harness.config.cosmos_user_documents_container + document = documents.read_item(LEFT, LEFT) + document["file_name"] = f"selected.{extension}" + document["accessibility_metadata"] = { + "format": extension, "schema_version": "1.0", "title": PRIVATE, + "inspection_limits": ["Formal conformance is not determined."], + **({"is_tagged": False} if extension == "pdf" else { + "comments": [{"id": "comment-1", "text": "Review this date."}], + }), + } + documents.upsert_item(document) + citations = importlib.import_module("functions_agent_document_citations") + marker = citations.annotate_document_search_payload({ + "document": {"id": LEFT, "file_name": document["file_name"], "version": document["version"]}, + "scope": "personal", + }, "get_document_accessibility_report")["citation"] + finding = "The stored PDF tag flag is false." if extension == "pdf" else "Native comment: Review this date." + answer = f"{finding} Formal conformance is not determined. {marker}" + proposal = action_proposal((LEFT,), action=state.action) + proposal["steps"][0]["arguments"]["task"] = MESSAGE + proposal["steps"][1]["arguments"]["instruction"] = ( + "Report the native facts, source citations, and explicit inspection limits." + ) + + harness.replies = [ + json.dumps({ + "relationship": "new_topic", "resolved_message": MESSAGE, "message_ids": [], + "requires_retrieval": True, "clarification": "", + }), + json.dumps(proposal), + answer, + json.dumps({"requirements": [{ + "id": identifier, "requirement": requirement, "status": "satisfied", + "reason": "", "evidence_refs": [], "answer_quote": answer, + } for identifier, requirement in NARRATIVE_OBLIGATIONS]}), + ] + with caplog.at_level(logging.INFO, logger=state.logger.name): + response = runtime.client.post("/api/v2/orchestration/plan", json={ + "conversation_id": "conversation-1", "turn_id": TURN, "message": MESSAGE, + "planning_submission_id": "native-submit", "approval_mode": "auto", + "selected_document_ids": [LEFT], + }, buffered=True) + events = frames(response) + plan = next(event["plan"] for event in events if event.get("type") == "orchestration_plan") + saved = harness.runs.read_item(plan["run_id"], "conversation-1") + turn_response = runtime.client.get( + f"/api/v2/orchestration/turns/{TURN}", query_string={"conversation_id": "conversation-1"}, + ) + turn = turn_response.get_json()["turn"] + assert turn_response.status_code == 200 and turn["status"] == saved["status"] + assert turn["run_id"] == plan["run_id"] + questions = [ + message for message in harness.messages.items.values() + if (message.get("metadata") or {}).get("orchestration_turn_id") == TURN and message.get("role") == "user" + ] + assert len(questions) == 1 and questions[0]["content"] == MESSAGE + records = [ + record for record in caplog.records if getattr(record, "sc_message", None) in TRACE_MESSAGES + ] + assert bool(records) is debug + if debug: + serialized = json.dumps([ + {name: value for name, value in record.__dict__.items() if name.startswith("sc_")} + for record in records + ]) + assert PRIVATE not in serialized and LEFT not in serialized + assert document["file_name"] not in serialized + if report_fails: + assert response.status_code == 200 and saved["status"] == "failed" + assert saved.get("task_results") == {} + assert len(state.native_calls) == 1 and state.report_reads == [(LEFT, "owner")] + assert len(harness.model_calls) == 2 and len(harness.replies) == 2 + assert PRIVATE not in response.get_data(as_text=True) + if debug: + reads = [record for record in records if record.sc_stage == "document_action_read"] + assert [record.sc_outcome for record in reads] == ["started", "failed"] + assert not any(record.sc_stage in ("gather_retained", "compose_retained") for record in records) + return + assert response.status_code == 200 and saved["status"] == "completed", "\n".join(state.failures) + assert [step["capability_id"] for step in plan["steps"]] == ["action_invoke", "compose"] + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + answers = harness.assistant_messages() + assert len(answers) == 1 and marker in answers[0]["content"] + assert finding in answers[0]["content"] + assert "Formal conformance is not determined." in answers[0]["content"] + assert len(state.native_calls) == 2 and state.report_reads == [(LEFT, "owner")] + assert len(harness.model_calls) == 4 and harness.replies == [] + compose_call = next( + call for call in harness.model_calls if call["messages"][0]["content"].startswith("Prepare only") + ) + prepared = json.loads(compose_call["messages"][-1]["content"])["inputs"]["native_findings"] + native = json.loads(prepared["value"]["notes"][0].partition("\n")[2]) + assert native["native_metadata"] == document["accessibility_metadata"] + assert native["citation"] == prepared["document_citations"][0]["citation"] == marker + if debug: + stages = {record.sc_stage for record in records} + assert { + "document_action_binding", "document_action_recheck", "document_action_read", + "gather_retention", "gather_retained", "retained_citation_projection", + "gather_source_handoff", "compose_inputs", "compose_retained", + } <= stages diff --git a/functional_tests/test_orchestration_document_action_lineage.py b/functional_tests/test_orchestration_document_action_lineage.py new file mode 100644 index 000000000..feae22668 --- /dev/null +++ b/functional_tests/test_orchestration_document_action_lineage.py @@ -0,0 +1,229 @@ +# test_orchestration_document_action_lineage.py +""" +Functional regressions for native document action retention and composition. +Version: 0.261.423 +Implemented in: 0.261.419 + +Real external admission, SDK-default result storage, retained readers, compiler, +executor and composition run with external configuration/model I/O isolated. +Workspace provenance and current integration authority must both survive. +""" + +import importlib +import json +import logging +from copy import deepcopy +from types import SimpleNamespace + +import pytest + +from functions_task_completeness import NARRATIVE_OBLIGATIONS +from test_orchestration_harness_routes import modules +from test_support.orchestration_harness_execution import compose_step, input_binding + + +__all__ = ["modules"] +ANSWER = "The stored PDF tag flag is false. This does not establish formal conformance." + + +@pytest.fixture +def native_result(modules): + from test_orchestration_external_sources import ExternalSourceWorld + + with ExternalSourceWorld("action_invoke") as world: + action = world.services.records[("actions", "group")][("group-a", "action-1")] + action["type"] = "document_search" + step = world.run["plan"]["steps"][0] + step["arguments"]["document_ids"] = ["document-1"] + provider = world.provider() + service = world.service(provider) + context = SimpleNamespace( + result_service=service, action_catalog=world.catalog("owner", settings=world.settings), + result_producer=lambda current: world.fixture.producer, + result_guard_token_for_step=lambda step_id: "server-attempt-token", + result_input_fingerprint_for_step=lambda step_id: None, + external_source_admission=lambda **kwargs: provider.admit_gather_result( + **kwargs, selector=world.action_selector, + ), + ) + citations = importlib.import_module("functions_agent_document_citations") + payload = citations.annotate_document_search_payload({ + "document": {"id": "document-1", "file_name": "selected.pdf", "version": 1}, + "scope": "personal", + "accessibility_metadata": { + "format": "pdf", "is_tagged": False, + "inspection_limits": ["Formal conformance is not determined by this metadata."], + }, + }, "get_document_accessibility_report") + result = { + "status": "completed", "notes": [json.dumps(payload)], "citations": [{ + "plugin_name": "document_search", "function_name": "get_document_accessibility_report", + "function_result": payload, "success": True, + }], + } + yield SimpleNamespace( + world=world, provider=provider, service=service, context=context, + step=step, result=result, payload=payload, + sources=[deepcopy(world.fixture.sources["document-1"])], + ) + + +def retain(case): + runtime = importlib.import_module("functions_orchestration_result_runtime") + return runtime.retain_gather_result( + case.step, case.context, case.result, source_manifest=case.sources, + ) + + +def test_native_action_keeps_workspace_and_external_lineage_after_restart(native_result): + case = native_result + task = retain(case) + reader = case.world.service(case.provider).open_result(task.output("prepared")) + value = reader.read_value() + metadata = reader.metadata() + sources = reader.lineage_sources() + assert value["notes"] == case.result["notes"] + assert metadata["source_count"] == metadata["external_source_count"] == 1 + assert metadata["origin"] == "grounded" + assert sources == case.sources + assert case.payload["citation"] in value["notes"][0] + case.world.services.roles.clear() + with pytest.raises(PermissionError): + case.world.service(case.provider).open_result(task.output("prepared")) + + +@pytest.mark.parametrize("kind", ["wrong_action", "missing_binding", "missing_source", "wrong_source"]) +def test_other_integrations_and_mismatched_lineage_are_rejected(native_result, kind): + case = native_result + if kind == "wrong_action": + case.context.action_catalog[0]["type"] = "openapi" + elif kind == "missing_binding": + case.step["arguments"].pop("document_ids") + elif kind == "missing_source": + case.sources = [] + else: + case.step["arguments"]["document_ids"] = ["other-document"] + contracts = importlib.import_module("functions_orchestration_result_contracts") + with pytest.raises(contracts.ResultContractError) as failure: + retain(case) + assert failure.value.code == "result_external_lineage_unsupported" + assert not case.world.fixture.container.items + + +@pytest.mark.parametrize("checks_enabled", [False, True]) +@pytest.mark.parametrize("debug", [False, True]) +def test_bound_native_findings_reach_the_final_composed_answer( + native_result, checks_enabled, debug, monkeypatch, caplog, +): + case = native_result + telemetry = importlib.import_module("functions_appinsights") + logger = logging.getLogger("test.orchestration.native.lineage") + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: {"enable_debug_logging": debug}) + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + caplog.set_level(logging.INFO, logger=logger.name) + task = retain(case) + registry = importlib.import_module("functions_orchestration_registry") + schema = importlib.import_module("functions_orchestration_schema") + executor = importlib.import_module("functions_orchestration_executor") + capabilities = importlib.import_module("functions_model_capabilities") + output = task.output("prepared") + answer = compose_step("answer", inputs={ + "native_findings": { + "binding": { + "version": "orchestration-input-binding-v1", "step_id": None, + "output_name": None, "existing_result": "native_report", + }, + "allow_partial": False, + }, + }) + answer["arguments"].update( + knowledge_basis="sources", instruction="Report the native flag and its limitations.", + response_policy="available_information", + ) + plan = schema.normalize_plan({ + "run_id": "answer-run", "plan_id": "answer-plan", "steps": [answer], + "final_response": input_binding("answer"), + }, "conversation-1", "owner", available_capability_ids=["compose"], + existing_results={"native_report": output}, contract_version=2) + contracts = importlib.import_module("functions_orchestration_result_contracts") + producer = contracts.ProducerIdentity( + "owner", "conversation-1", "answer-run", 1, "answer", "compose", + registry.get_capability("compose")["result_contract_version"], + ) + case.world.fixture.add_producer(producer) + case.world.fixture.runs["answer-run"]["plan"] = plan + + class Model: + model_metadata = capabilities.ModelTokenBudget( + model_id="offline-composition", provider="openai", context_window=32768, + input_limit=24576, output_limit=2048, output_accounting="total_generation", + ) + provider = "openai" + output_tokens = 512 + + def __init__(self): + self.calls = [] + + def __call__(self, messages, **kwargs): + self.calls.append(deepcopy(messages)) + if messages[0]["content"].startswith("Prepare only"): + return f"{ANSWER} {case.payload['citation']}" + return json.dumps({"requirements": [{ + "id": identifier, "requirement": requirement, "status": "satisfied", + "reason": "", "evidence_refs": [], "answer_quote": ANSWER, + } for identifier, requirement in NARRATIVE_OBLIGATIONS]}) + + model = Model() + context = executor.RunContext( + run_id="answer-run", plan_id="answer-plan", conversation_id="conversation-1", user_id="owner", + user_message="Inspect the native PDF tag flag.", plan_contract_version=2, + result_service=case.world.service(case.provider), result_aliases={"native_report": output}, + result_guard_token_for_step=lambda step_id: "server-attempt-token", invoke_prompt=model, + ) + result = executor.execute_plan( + plan, context, settings={"enable_chat_orchestration_completeness_check": checks_enabled}, + user_id="owner", + ) + assert result["status"] == "completed", result + assert ANSWER in result["message"] and case.payload["citation"] in result["message"] + assert len(model.calls) == (2 if checks_enabled else 1) + sent = json.loads(model.calls[0][-1]["content"]) + retained = sent["inputs"]["native_findings"]["value"] + citations = sent["inputs"]["native_findings"]["document_citations"] + native = json.loads(retained["notes"][0]) + assert native["accessibility_metadata"]["is_tagged"] is False + assert native["accessibility_metadata"]["inspection_limits"] == ( + case.payload["accessibility_metadata"]["inspection_limits"] + ) + assert native["citation"] == case.payload["citation"] + assert len(citations) == 1 + assert citations[0]["document_id"] == "document-1" + assert citations[0]["citation"] == case.payload["citation"] + stages = {"gather_retention", "gather_retained", "retained_citation_projection", "compose_inputs", "compose_retained"} + records = [record for record in caplog.records if getattr(record, "sc_stage", None) in stages] + assert {record.sc_stage for record in records} == (stages if debug else set()) + assert case.world.fixture.source_reads == [] + if debug: + retained_record = next(record for record in records if record.sc_stage == "gather_retained") + input_record = next(record for record in records if record.sc_stage == "compose_inputs") + assert retained_record.sc_workspace_source_count == retained_record.sc_external_source_count == 1 + assert input_record.sc_document_citation_count == 1 + linked = [record for record in records if getattr(record, "sc_consumer_run_id_hash", None)] + assert linked and all(len(record.sc_consumer_run_id_hash) == 64 for record in linked) + serialized = json.dumps([ + {name: value for name, value in record.__dict__.items() if name.startswith("sc_")} + for record in records + ]) + assert ANSWER not in serialized and "document-1" not in serialized + assert "selected.pdf" not in serialized and case.payload["citation"] not in serialized + + +def test_native_tool_citations_cannot_name_another_source(native_result): + case = native_result + case.result["citations"][0]["function_result"]["document"]["id"] = "unselected-document" + task = retain(case) + runtime = importlib.import_module("functions_orchestration_result_runtime") + contracts = importlib.import_module("functions_orchestration_result_contracts") + with pytest.raises(contracts.ResultContractError) as failure: + runtime.read_result_document_citations(case.context, task.output("prepared")) + assert failure.value.code == "result_citation_source_invalid" diff --git a/functional_tests/test_orchestration_document_loader_reliability.py b/functional_tests/test_orchestration_document_loader_reliability.py new file mode 100644 index 000000000..7ac85bf33 --- /dev/null +++ b/functional_tests/test_orchestration_document_loader_reliability.py @@ -0,0 +1,290 @@ +# test_orchestration_document_loader_reliability.py +""" +Functional tests for shared document-action loading and safe failure outcomes. +Version: 0.261.423 +Implemented in: 0.261.421 + +Real authenticated routes, discovery, source checks, SDK-shaped document stores, +chunk readers, action filters, retention, and final publication run offline. +The all-format matrix reads already indexed text; it does not claim that every +format exposes native Office/PDF metadata or that extraction is tested here. +""" + +import hashlib +import importlib +import importlib.util +import json +import logging +from copy import deepcopy +from types import SimpleNamespace + +import pytest +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.contents import AuthorRole, ChatMessageContent, FunctionCallContent + +from functions_task_completeness import NARRATIVE_OBLIGATIONS +from test_orchestration_document_action_end_to_end import ( + LEFT, PRIVATE, frames, modules, native_http, real_http_harness, source_runtime, +) +from test_orchestration_planner_contract_recovery import action_proposal + + +__all__ = ["modules", "native_http", "real_http_harness", "source_runtime"] +FAILURE_CASES = ( + ("loading", "action_initialization_failed", "action_initialization", 0, 0), + ("identity", "document_reader_identity_mismatch", "action_initialization", 0, 0), + ("read", "document_action_read_failed", "document_action_read", 1, 1), + ("results", "action_results_missing", "action_results", 1, 0), +) + + +def submit_action_turn(state, message, answer, turn_id): + proposal = action_proposal((LEFT,), action=state.action) + proposal["steps"][0]["arguments"]["task"] = message + proposal["steps"][1]["arguments"]["instruction"] = ( + "Use only the returned source evidence and copy its citation." + ) + state.runtime.harness.replies = [ + json.dumps({ + "relationship": "new_topic", "resolved_message": message, "message_ids": [], + "requires_retrieval": True, "clarification": "", + }), + json.dumps(proposal), + answer, + json.dumps({"requirements": [{ + "id": identifier, "requirement": requirement, "status": "satisfied", + "reason": "", "evidence_refs": [], "answer_quote": answer, + } for identifier, requirement in NARRATIVE_OBLIGATIONS]}), + ] + response = state.runtime.client.post("/api/v2/orchestration/plan", json={ + "conversation_id": "conversation-1", "turn_id": turn_id, "message": message, + "planning_submission_id": f"{turn_id}-submit", "approval_mode": "auto", + "selected_document_ids": [LEFT], + }, buffered=True) + events = frames(response) + plans = [event["plan"] for event in events if event.get("type") == "orchestration_plan"] + if len(plans) != 1: + raise AssertionError(f"The request did not produce one approved plan: {response.get_data(as_text=True)}") + plan = plans[0] + saved = state.runtime.harness.runs.read_item(plan["run_id"], "conversation-1") + return response, events, plan, saved + + +def test_every_supported_format_reaches_a_saved_cited_stored_chunk_answer(native_http, monkeypatch): + state = native_http + harness = state.runtime.harness + documents = harness.config.cosmos_user_documents_container + owner = importlib.import_module("functions_documents") + citations = importlib.import_module("functions_agent_document_citations") + extensions = sorted(harness.config.get_allowed_extensions(enable_video=True, enable_audio=True)) + categories = harness.config.get_allowed_extension_categories( + enable_video=True, enable_audio=True, enable_xsd=True, + ) + covered = {extension for category in categories for extension in category["extensions"]} + assert covered == set(extensions) + search_calls = [] + native_attempts = [] + monkeypatch.setattr(owner, "cosmos_user_documents_container", documents) + plugin = importlib.import_module("semantic_kernel_plugins.document_search_plugin") + + def reject_native_report(**kwargs): + native_attempts.append(deepcopy(kwargs)) + raise AssertionError("Stored-text retrieval must not request or infer a native report.") + + monkeypatch.setattr(plugin, "get_document_accessibility_payload", reject_native_report) + + def stored_chunk(document): + return { + "id": f"{LEFT}_1", "document_id": LEFT, "user_id": "owner", + "file_name": document["file_name"], "version": document["version"], + "chunk_id": "1", "chunk_sequence": 1, "page_number": 1, + "chunk_text": "The stored source states that the review is complete.", + } + + def search(**kwargs): + search_calls.append(deepcopy(kwargs)) + if kwargs.get("filter") != f"document_id eq '{LEFT}'": + raise AssertionError("The stored-chunk reader did not use the bound document filter.") + document = documents.read_item(LEFT, LEFT) + return [stored_chunk(document)] + + def search_client(*, group_id=None, public_workspace_id=None): + if group_id is not None or public_workspace_id is not None: + raise AssertionError("A personal document was routed to another search index.") + return SimpleNamespace(search=search) + + monkeypatch.setattr(owner, "_get_search_client", search_client) + + async def stored_reply(self, history, settings): + state.native_calls.append(deepcopy([message.to_dict() for message in history.messages])) + results = [ + item.result for message in history.messages for item in message.items + if hasattr(item, "result") + ] + if not results: + return [ChatMessageContent(role=AuthorRole.ASSISTANT, items=[FunctionCallContent( + id="read-stored", name="document_search-retrieve_document_chunks", + arguments=json.dumps({"document_id": LEFT, "doc_scope": "personal"}), + )])] + value = json.loads(results[-1]) if isinstance(results[-1], str) else results[-1] + return [ChatMessageContent(role=AuthorRole.ASSISTANT, content=json.dumps({ + "stored_text": value["chunks"][0]["chunk_text"], "citation": value["chunks"][0]["citation"], + }))] + + monkeypatch.setattr(AzureChatCompletion, "_inner_get_chat_message_contents", stored_reply) + message = "Quote the selected file's already stored extracted text and cite it. Do not infer native properties." + for index, extension in enumerate(extensions): + document = documents.read_item(LEFT, LEFT) + document.update({ + "file_name": f"selected.{extension}", "num_chunks": 1, + "indexed_chunk_count_version": document["version"], + }) + for field in ("archived_blob_path", "accessibility_metadata"): + document.pop(field, None) + documents.upsert_item(document) + marker = citations.annotate_document_search_payload({ + "document": {"id": LEFT, "file_name": document["file_name"], "version": document["version"]}, + "scope": "personal", "chunks": [stored_chunk(document)], + }, "retrieve_document_chunks")["chunks"][0]["citation"] + answer = f"The stored source states that the review is complete. {marker}" + before_calls = len(state.native_calls) + before_search = len(search_calls) + before_context_reads = len(state.report_reads) + response, events, plan, saved = submit_action_turn( + state, message, answer, f"format-{index}-{extension}", + ) + assert response.status_code == 200 and saved["status"] == "completed", ( + extension, saved.get("failure"), state.failures, + ) + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert len(state.native_calls) - before_calls == 2, extension + assert len(search_calls) - before_search == 1, extension + assert len(state.report_reads) - before_context_reads == 1, extension + assert native_attempts == [], extension + assert len(harness.replies) == 0, extension + saved_answer = harness.assistant_messages()[-1] + assert answer == saved_answer["content"], extension + assert not any(event.get("type") == "error" for event in events), extension + assert len(search_calls) == len(extensions) + assert len(harness.assistant_messages()) == len(extensions) + + +@pytest.mark.parametrize("debug", [False, True]) +@pytest.mark.parametrize("mode,code,stage,model_calls,report_reads", FAILURE_CASES) +def test_owned_action_failure_survives_saved_steps_and_private_logging( + native_http, monkeypatch, caplog, debug, mode, code, stage, model_calls, report_reads, +): + state = native_http + harness = state.runtime.harness + harness.settings["enable_debug_logging"] = debug + document = harness.config.cosmos_user_documents_container.read_item(LEFT, LEFT) + document.update({ + "file_name": "selected.pdf", + "accessibility_metadata": { + "format": "pdf", "schema_version": "1.0", "title": PRIVATE, "is_tagged": False, + }, + }) + harness.config.cosmos_user_documents_container.upsert_item(document) + loader = importlib.import_module("semantic_kernel_plugins.logged_plugin_loader") + if mode == "loading": + monkeypatch.setattr(loader.LoggedPluginLoader, "load_plugin_from_manifest", lambda *args, **kwargs: False) + elif mode == "identity": + reader = importlib.import_module("semantic_kernel_plugins.document_search_plugin") + spec = importlib.util.spec_from_file_location(reader.__name__, reader.__file__) + duplicate = importlib.util.module_from_spec(spec) + spec.loader.exec_module(duplicate) + monkeypatch.setattr( + loader.LoggedPluginLoader, "_create_plugin_instance", + lambda self, manifest: duplicate.DocumentSearchPlugin(deepcopy(manifest)), + ) + elif mode == "read": + state.fail_report = True + else: + async def no_results(self, history, settings): + state.native_calls.append(deepcopy([message.to_dict() for message in history.messages])) + return [ChatMessageContent(role=AuthorRole.ASSISTANT, content=PRIVATE)] + + monkeypatch.setattr(AzureChatCompletion, "_inner_get_chat_message_contents", no_results) + with caplog.at_level(logging.INFO, logger=state.logger.name): + response, events, plan, saved = submit_action_turn( + state, "Inspect only the selected document's native properties.", + "This draft must never be composed after a failed acquisition.", f"failure-{mode}", + ) + assert response.status_code == 200 and saved["status"] == "failed" + assert saved.get("task_results") == {} + assert len(state.native_calls) == model_calls + assert len(state.report_reads) == report_reads + assert len(harness.model_calls) == 2 and len(harness.replies) == 2 + steps_response = state.runtime.client.get( + f"/api/v2/orchestration/runs/{plan['run_id']}/steps", + query_string={"conversation_id": "conversation-1"}, + ) + steps_payload = steps_response.get_json() + assert steps_response.status_code == 200 + failed_step = next( + step for step in steps_payload["steps"] if step["step_id"] == plan["steps"][0]["step_id"] + ) + assert failed_step["status"] == "failed" + assert failed_step["failure"]["code"] == code + schema = importlib.import_module("functions_orchestration_schema") + expected = schema.build_failure(code) + assert failed_step["failure"]["message"] == expected["message"] + answers = harness.assistant_messages() + assert len(answers) == 1 and expected["message"] in answers[0]["content"] + assert PRIVATE not in response.get_data(as_text=True) + json.dumps(steps_payload) + json.dumps(answers) + records = [ + record for record in caplog.records + if getattr(record, "sc_message", None) == "[ORCHESTRATION_ADAPTERS] Direct action execution failed." + ] + assert len(records) == 1 + record = records[0] + assert record.sc_failure_code == code and record.sc_stage == stage + assert record.sc_run_id_hash == hashlib.sha256(plan["run_id"].encode()).hexdigest() + assert record.sc_step_id_hash == hashlib.sha256(plan["steps"][0]["step_id"].encode()).hexdigest() + assert record.sc_conversation_id_hash == hashlib.sha256(b"conversation-1").hexdigest() + if stage in ("action_initialization", "document_action_read"): + assert record.sc_function_calls_attempted is (stage == "document_action_read") + else: + assert not hasattr(record, "sc_function_calls_attempted") + serialized = json.dumps({ + name: value for name, value in record.__dict__.items() if name.startswith("sc_") + }) + assert PRIVATE not in serialized and LEFT not in serialized + assert document["file_name"] not in serialized + + +@pytest.mark.parametrize("code", [case[1] for case in FAILURE_CASES]) +def test_owned_action_codes_are_safe_nontransient_and_allow_manual_recovery(modules, code): + actions = importlib.import_module("functions_orchestration_actions") + schema = importlib.import_module("functions_orchestration_schema") + failure = schema.failure_from_exception(actions.ActionExecutionError(PRIVATE, failure_code=code)) + transient = schema.failure_is_transient(failure) + repeats = schema.failure_repeats_on_retry(failure) + assert failure == schema.build_failure(code) + assert not transient and not repeats + assert PRIVATE not in json.dumps(failure) + + +@pytest.mark.parametrize("cause,code", [(TimeoutError, "provider_timeout"), (ConnectionError, "connection_failed")]) +def test_unclassified_action_errors_preserve_transient_provider_causes(modules, cause, code): + actions = importlib.import_module("functions_orchestration_actions") + schema = importlib.import_module("functions_orchestration_schema") + error = actions.ActionExecutionError(PRIVATE) + error.__cause__ = cause(PRIVATE) + failure = schema.failure_from_exception(error) + transient = schema.failure_is_transient(failure) + assert failure["code"] == code and transient + assert PRIVATE not in json.dumps(failure) + + +def test_arbitrary_exception_failure_codes_are_not_admitted(modules): + actions = importlib.import_module("functions_orchestration_actions") + schema = importlib.import_module("functions_orchestration_schema") + failure = schema.failure_from_exception(actions.ActionExecutionError(PRIVATE, failure_code=PRIVATE)) + expected = schema.build_failure("step_failed") + assert failure == expected + assert PRIVATE not in json.dumps(failure) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/functional_tests/test_orchestration_document_planning_recovery.py b/functional_tests/test_orchestration_document_planning_recovery.py index c23f7ed7c..974b7135d 100644 --- a/functional_tests/test_orchestration_document_planning_recovery.py +++ b/functional_tests/test_orchestration_document_planning_recovery.py @@ -2,9 +2,11 @@ #!/usr/bin/env python3 """ Functional regressions for picked-document planning and debug diagnostics. -Version: 0.261.415 +Version: 0.261.423 Implemented in: 0.261.413 Server-owned Auto handoff integration: 0.261.415 +Shared draft-contract recovery integration: 0.261.419 +Two shared corrective proposals and local source wiring: 0.261.422 Real authenticated routes, source metadata, planner, validation and logger serialization run with external model/storage I/O isolated. Diagnostics must @@ -90,12 +92,22 @@ def proposal(document_ids=(), *, capability="document_search", options=None): def run_plan( runtime, replies, *, identifiers=(LEFT,), context_updates=None, seed_updates=None, - message=PRIVATE, **options, + message=PRIVATE, prepare_sources=False, **options, ): harness = runtime.harness selected, seeds = candidates(runtime, identifiers) seeds.update(seed_updates or {}) - offered = enrich(runtime, selected, seeds) + if prepare_sources: + from functions_orchestration_source_context import build_selected_source_context + + offered = runtime.context.enrich_planner_candidates( + selected, "owner", conversation_id="conversation-1", seeds=seeds, + selected_source_context_reader=lambda sources: build_selected_source_context( + sources, read_bytes=lambda *args, **kwargs: pytest.fail("Narrative planning must not read file bytes."), + ), + ) + else: + offered = enrich(runtime, selected, seeds) context = runtime.context.build_planner_context(message, candidates=offered, seeds=seeds) context.update(context_updates or {}) harness.replies = [json.dumps(reply) if isinstance(reply, dict) else reply for reply in replies] @@ -128,12 +140,12 @@ def test_debug_diagnostics_identify_model_omission_in_both_sinks(planning, caplo planning.debug_flags["enable_debug_logging"] = debug with caplog.at_level(logging.INFO, logger=planning.logger.name): with pytest.raises(planning.harness.planner.PlannerError): - run_plan(planning, [proposal(), proposal()]) + run_plan(planning, [proposal(), proposal(), proposal()]) records = diagnostic_records(caplog) assert bool(records) is debug if debug: initial = diagnostic_records(caplog, stage="planner_input", message="[ORCHESTRATION_PLANNER] Planning diagnostic.")[0] - assert initial.sc_remaining_corrections == 1 + assert initial.sc_remaining_corrections == 2 summaries = [ record for record in diagnostic_records(caplog, stage="plan_normalized") if record.sc_message == "[ORCHESTRATION_PLANNER] Planning diagnostic." @@ -164,7 +176,7 @@ def test_debug_options_show_exact_owner_rule_before_defaulting(planning, caplog, invalid = proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}) with caplog.at_level(logging.INFO, logger=planning.logger.name): with pytest.raises(planning.harness.planner.PlannerError): - run_plan(planning, [invalid, invalid]) + run_plan(planning, [invalid, invalid, invalid]) steps = diagnostic_records(caplog, stage="plan_proposal", message="[ORCHESTRATION_PLANNER] Step binding diagnostic.") analysis = next(record for record in steps if record.sc_capability_id == "document_analyze") assert analysis.sc_options_mode_observations is True @@ -177,7 +189,7 @@ def test_debug_options_show_exact_owner_rule_before_defaulting(planning, caplog, def test_diagnostic_hashes_survive_serialization_without_expanding_text_allowlists(planning): fingerprint = hashlib.sha256(PRIVATE.encode()).hexdigest() - for key in ("document_id_hash", "planner_binding_hash", "document_set_hash"): + for key in ("document_id_hash", "planner_binding_hash", "document_set_hash", "consumer_run_id_hash"): valid = planning.telemetry._build_logger_extra("Diagnostic", {key: fingerprint}) invalid = planning.telemetry._build_logger_extra("Diagnostic", {key: PRIVATE}) assert valid[f"sc_{key}"] == fingerprint @@ -193,7 +205,7 @@ def test_debug_snapshot_bounds_are_explicit(planning, caplog): identifiers = tuple(f"document-{index}" for index in range(101)) with caplog.at_level(logging.INFO, logger=planning.logger.name): with pytest.raises(planning.harness.planner.PlannerError): - run_plan(planning, [proposal(), proposal()], identifiers=identifiers) + run_plan(planning, [proposal(), proposal(), proposal()], identifiers=identifiers) summary = diagnostic_records(caplog, stage="plan_proposal", message="[ORCHESTRATION_PLANNER] Planning diagnostic.")[0] assert summary.sc_selected_document_count == summary.sc_missing_document_count == 101 assert summary.sc_omitted_document_count == 101 - planning.harness.planner.PLANNER_DEBUG_MAX_ITEMS @@ -351,29 +363,26 @@ def test_http_picked_document_recovery_preserves_scope_and_does_not_execute( ("selection", "selection", "selected_requirements"), ("options", "options", "plan_normalization"), ]) -def test_every_failure_path_shares_exactly_one_correction(planning, caplog, first, second, stage): +def test_every_failure_path_shares_two_corrections(planning, caplog, first, second, stage): bad = { "options": proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}), "selection": proposal(), } with caplog.at_level(logging.INFO, logger=planning.logger.name): - with pytest.raises(planning.harness.planner.PlannerError) as failure: - run_plan(planning, [bad[first], bad[second], proposal([LEFT])]) - assert len(planning.harness.model_calls) == 2 - assert len(planning.harness.replies) == 1 - assert "PRIVATE_" not in str(failure.value) - expected_message = "selected" if second == "selection" else "document-analysis" - assert expected_message in str(failure.value) - terminal = next( - record for record in reversed(caplog.records) - if getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] The request could not be planned." - ) - assert terminal.sc_attempt == 2 and terminal.sc_stage == stage + kind, plan = run_plan(planning, [bad[first], bad[second], proposal([LEFT])]) + assert kind == "plan" and plan["validation"]["ok"] + assert len(planning.harness.model_calls) == 3 and not planning.harness.replies + corrections = [ + record for record in caplog.records + if getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] Asking the planner to correct a rejected plan." + ] + assert len(corrections) == 2 + assert corrections[-1].sc_attempt == 2 and corrections[-1].sc_stage == stage -def test_unrelated_malformed_arguments_are_not_made_repairable(planning): +def test_unavailable_capabilities_are_not_made_repairable(planning): invalid = proposal([LEFT]) - invalid["steps"][0]["arguments"][PRIVATE] = PRIVATE + invalid["steps"][0]["capability_id"] = "unavailable_capability" with pytest.raises(planning.harness.planner.PlannerError): run_plan(planning, [invalid, proposal([LEFT])]) assert len(planning.harness.model_calls) == 1 @@ -431,7 +440,7 @@ def test_edit_failure_preserves_previous_plan_and_legacy_options(planning): invalid = proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}) invalid["revised_request"] = "Describe the selected source." with pytest.raises(planning.harness.planner.PlannerError) as failure: - run_plan(planning, [invalid, invalid], edit_context={"current_plan": previous}) + run_plan(planning, [invalid, invalid, invalid], edit_context={"current_plan": previous}) assert "previous plan is unchanged" in str(failure.value) assert previous == before omitted = proposal([LEFT], capability="document_analyze") @@ -584,8 +593,8 @@ def test_unrenderable_question_cannot_reset_the_shared_correction_budget(plannin "question": question, }[first] with pytest.raises(planning.harness.planner.PlannerError): - run_plan(planning, [initial, question, proposal([LEFT])]) - assert len(planning.harness.model_calls) == 2 + run_plan(planning, [initial, question, question, proposal([LEFT])]) + assert len(planning.harness.model_calls) == 3 assert len(planning.harness.replies) == 1 diff --git a/functional_tests/test_orchestration_failure_telemetry.py b/functional_tests/test_orchestration_failure_telemetry.py index a0b05de42..cf7eeb249 100644 --- a/functional_tests/test_orchestration_failure_telemetry.py +++ b/functional_tests/test_orchestration_failure_telemetry.py @@ -1,12 +1,13 @@ # test_orchestration_failure_telemetry.py """Specific orchestration failure rules reach telemetry without private plan text. -Version: 0.261.342 +Version: 0.261.423 Implemented in: 0.261.140 Step, capability, format, and authority-reason codes covered in: 0.261.141 Partial-input producer capability and step hash fields covered in: 0.261.191 Semantic chat outcomes and persisted-state correlation implemented in: 0.261.341 Bounded, non-regex application-version validation implemented in: 0.261.342 +Two shared planning corrections covered in: 0.261.422 Uses real planner/validation/logging modules with offline provider replies. The synthetic invalid proposals test diagnostics, not the unknown historical @@ -76,7 +77,7 @@ def test_repair_and_terminal_failure_retain_safe_specific_rule( monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: {"enable_debug_logging": debug}) proposal = invalid_proposal(rule) - harness.replies = [json.dumps(proposal), json.dumps(proposal)] + harness.replies = [json.dumps(proposal)] * 3 with caplog.at_level(logging.INFO, logger=logger.name): with pytest.raises(harness.planner.PlannerError) as failure: harness.planner.plan_request( @@ -92,8 +93,8 @@ def test_repair_and_terminal_failure_retain_safe_specific_rule( "[ORCHESTRATION_PLANNER] The request could not be planned.", } ] - assert len(records) == len(harness.model_calls) == 2 - assert [record.sc_attempt for record in records] == [1, 2] + assert len(records) == len(harness.model_calls) == 3 + assert [record.sc_attempt for record in records] == [1, 2, 3] for record in records: assert record.sc_validation_rule == rule assert record.sc_validation_code == "deliverables_invalid" diff --git a/functional_tests/test_orchestration_planner_contract_recovery.py b/functional_tests/test_orchestration_planner_contract_recovery.py new file mode 100644 index 000000000..4c09015d1 --- /dev/null +++ b/functional_tests/test_orchestration_planner_contract_recovery.py @@ -0,0 +1,376 @@ +# test_orchestration_planner_contract_recovery.py +""" +Functional regressions for effective planner contracts and document action binding. +Version: 0.261.423 +Implemented in: 0.261.419; 0.261.422 + +Real registry, planner, validation, source preparation and logger owners run with +external model/storage I/O isolated. A corrected draft must retain the original +scope, model, deliverables and shared correction budget without executing tools. +""" + +import importlib +import json +import logging +from copy import deepcopy + +import pytest + +from test_orchestration_document_planning_recovery import planning, proposal, run_plan +from test_orchestration_harness_routes import modules, real_http_harness +from test_orchestration_source_kind_planning import LEFT, RIGHT, candidates, enrich, source_runtime +from test_support.orchestration_harness_execution import compose_step, input_binding + + +__all__ = ["planning", "modules", "real_http_harness", "source_runtime"] +ACTION = { + "action_ref": "personal:owner:native", "id": "native", "name": "Native inspection", + "display_name": "Native inspection", "type": "document_search", + "scope_type": "personal", "scope_id": "owner", "scope_label": "Personal", +} +PRIVATE = "PRIVATE_PROMPT_OR_DOCUMENT_CONTENT" + + +def action_proposal(document_ids=(LEFT,), *, action=ACTION): + reader = { + "step_id": "inspect", "capability_id": "action_invoke", + "arguments": {"action_ref": action["action_ref"], "task": PRIVATE}, + } + if document_ids is not None: + reader["arguments"]["document_ids"] = list(document_ids) + answer = compose_step("answer", inputs={ + "native_findings": { + "binding": input_binding("inspect", "prepared"), "allow_partial": False, + }, + }) + answer["arguments"]["knowledge_basis"] = "sources" + answer["delivers"] = ["answer"] + return { + "kind": "plan", + "deliverables": [{ + "id": "answer", "kind": "answer", "requested": "explicit", + "description": "Answer using the selected sources.", "status": "planned", + }], + "steps": [reader, answer], "final_response": input_binding("answer"), + } + + +def run_action_plan(runtime, replies, *, identifiers=(LEFT,), action=ACTION, prepare_sources=False): + harness = runtime.harness + harness.settings.update(enable_semantic_kernel=True, enable_chat_orchestration_actions=True) + selected, seeds = candidates(runtime, identifiers) + if prepare_sources: + from functions_orchestration_source_context import build_selected_source_context + + offered = runtime.context.enrich_planner_candidates( + selected, "owner", conversation_id="conversation-1", seeds=seeds, + selected_source_context_reader=lambda sources: build_selected_source_context( + sources, read_bytes=lambda *args, **kwargs: pytest.fail("Narrative planning must not read file bytes."), + ), + ) + else: + offered = enrich(runtime, selected, seeds) + context = runtime.context.build_planner_context(PRIVATE, candidates=offered, seeds=seeds, actions=[action]) + caller = {**harness.services().capability_request_bindings(), "action_catalog": [action]} + harness.replies = [json.dumps(reply) for reply in replies] + return harness.planner.plan_request( + PRIVATE, context, "conversation-1", "owner", + settings=harness.settings, seeds=seeds, authorized_document_ids=list(identifiers), + turn_id="contract-turn", request_context=caller, + ) + + +def correction_records(caplog): + return [ + record for record in caplog.records + if getattr(record, "sc_message", None) + == "[ORCHESTRATION_PLANNER] Asking the planner to correct a rejected plan." + ] + + +@pytest.mark.parametrize("options,records", [ + (None, False), + ({}, True), + ({"version": "analysis-options-v1"}, True), + ({"version": "analysis-options-v2", "mode": "source_observations"}, False), + ({"version": "analysis-options-v2", "mode": "reconciled_records"}, True), + ({"version": "analysis-options-v1", "mode": "narrative_findings"}, True), +]) +def test_advertised_analyze_variants_match_effective_validation(planning, options, records): + registry = importlib.import_module("functions_orchestration_registry") + capability = registry.get_capability("document_analyze") + advertised = registry.build_planner_capability_projection([capability])[0] + assert "records" not in advertised["optional_result_outputs"] + assert advertised["default_arguments"]["analysis_options"]["mode"] == "source_observations" + analysis = importlib.import_module("functions_document_analysis_results") + effective = ( + advertised["default_arguments"]["analysis_options"] if options is None + else analysis.normalize_analysis_options(options) + ) + variant = next( + item for item in advertised["result_output_variants"] + if all(effective.get(key) == value for key, value in item["analysis_options"].items()) + ) + assert ("records" in variant["optional_result_outputs"]) is records + raw = proposal([LEFT], capability="document_analyze", options=options) + raw["steps"][0]["outputs"] = [ + {"name": name, "kind": kind} for name, kind in variant["result_outputs"].items() + ] + if records: + raw["steps"][0]["outputs"].append({"name": "records", "kind": "records-v1"}) + kind, plan = run_plan(planning, [raw]) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert ("records" in {item["name"] for item in plan["steps"][0]["outputs"]}) is records + + +def test_requested_records_are_corrected_by_explicit_semantics_not_dropped(planning, caplog): + raw = proposal([LEFT], capability="document_analyze") + raw["steps"][0]["outputs"] = [ + {"name": "findings", "kind": "records-v1"}, + {"name": "coverage", "kind": "structured-v1"}, + {"name": "records", "kind": "records-v1"}, + ] + corrected = deepcopy(raw) + corrected["steps"][0]["arguments"]["analysis_options"] = { + "version": "analysis-options-v2", "mode": "reconciled_records", "required_fields": ["amount"], + } + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan(planning, [raw, corrected]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert "records" in {item["name"] for item in plan["steps"][0]["outputs"]} + assert plan["steps"][0]["arguments"]["analysis_options"]["required_fields"] == ["amount"] + assert correction_records(caplog)[0].sc_validation_rule == "fixed_output_not_supported" + assert {call["model"] for call in planning.harness.model_calls} == {"gpt-4o"} + + +@pytest.mark.parametrize("invalid", ["kind", "missing", "shape", "input"]) +def test_owned_contract_errors_receive_one_precise_correction(planning, caplog, invalid): + raw = proposal([LEFT]) + if invalid == "kind": + raw["steps"][0]["outputs"] = [{"name": "prepared", "kind": "markdown-v1"}] + elif invalid == "missing": + raw["steps"][0]["outputs"] = [{"name": "prepared", "kind": "structured-v1"}] + elif invalid == "shape": + raw["steps"][0]["outputs"] = "not-a-list" + else: + raw["steps"][-1]["inputs"] = {"evidence": {}} + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan(planning, [raw, proposal([LEFT])]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + record = correction_records(caplog)[0] + assert record.sc_validation_code in ("plan_outputs_invalid", "plan_inputs_invalid") + assert record.sc_validation_rule + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert planning.harness.assistant_messages() == [] + + +@pytest.mark.parametrize("extension", ["pdf", "docx", "png", "csv", "xlsx"]) +def test_native_action_binds_a_selected_document_without_a_redundant_reader(planning, extension): + documents = planning.harness.config.cosmos_user_documents_container + source = documents.read_item(LEFT, LEFT) + source["file_name"] = f"selected.{extension}" + documents.upsert_item(source) + kind, plan = run_action_plan(planning, [action_proposal()]) + assert kind == "plan" + assert [step["capability_id"] for step in plan["steps"]] == ["action_invoke", "compose"] + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert planning.harness.schema.effective_plan_document_ids(plan) == [LEFT] + assert len(planning.harness.model_calls) == 1 + context = json.loads(planning.harness.model_calls[0]["messages"][1]["content"]) + assert context["actions"][0]["document_binding_supported"] is True + assert planning.harness.assistant_messages() == [] and planning.harness.blobs.file_uploads == 0 + + +def test_missing_action_source_declaration_is_corrected_without_extra_work(planning, caplog): + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_action_plan( + planning, [action_proposal(None), action_proposal((LEFT, RIGHT))], identifiers=(LEFT, RIGHT), + ) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert [step["capability_id"] for step in plan["steps"]] == ["action_invoke", "compose"] + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT, RIGHT] + assert correction_records(caplog)[0].sc_validation_rule == "selected_documents_missing" + + +def test_single_selected_action_source_is_connected_locally(planning, caplog): + raw = action_proposal(None) + original = deepcopy(raw) + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_action_plan(planning, [raw], prepare_sources=True) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert raw == original + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert [step["capability_id"] for step in plan["steps"]] == ["action_invoke", "compose"] + assert plan["validation"]["ok"] and plan["validation"]["repairs"] + assert not correction_records(caplog) + context = json.loads(planning.harness.model_calls[0]["messages"][1]["content"]) + assert context["actions"][0]["default_arguments"]["document_ids"] == [LEFT] + assert planning.harness.assistant_messages() == [] and planning.harness.blobs.file_uploads == 0 + + +def test_single_selected_analyze_source_is_connected_before_validation(planning): + raw = proposal([LEFT], capability="document_analyze") + del raw["steps"][0]["arguments"]["document_ids"] + kind, plan = run_plan(planning, [raw], prepare_sources=True) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert plan["steps"][0]["arguments"]["analysis_options"]["mode"] == "source_observations" + assert plan["steps"][-1]["inputs"]["source"]["binding"]["step_id"] == "read" + assert plan["validation"]["repairs"] + context = json.loads(planning.harness.model_calls[0]["messages"][1]["content"]) + analyze = next(item for item in context["capabilities"] if item["id"] == "document_analyze") + assert analyze["default_arguments"]["document_ids"] == [LEFT] + + +@pytest.mark.parametrize("change", ["explicit", "disabled", "dynamic", "ambiguous", "unavailable"]) +def test_local_source_wiring_never_replaces_explicit_or_ambiguous_work(planning, change): + planner = planning.harness.planner + raw = action_proposal(None) + selected_sources = [{ + "document_id": LEFT, "source_kind": "narrative", "availability": "available", + }] + if change == "explicit": + raw["steps"][0]["arguments"]["document_ids"] = [] + elif change == "disabled": + raw["steps"][0]["enabled"] = False + elif change == "dynamic": + raw["steps"][0]["inputs"] = {"sources": {"binding": input_binding("discover", "sources")}} + elif change == "ambiguous": + selected_sources.append({"document_id": RIGHT, "source_kind": "tabular", "availability": "available"}) + else: + selected_sources[0]["availability"] = "unavailable" + original = deepcopy(raw) + result, repairs = planner._with_selected_source_bindings( + raw, selected_sources=selected_sources, + selected_document_ids=[item["document_id"] for item in selected_sources], + authorized_document_ids=[LEFT, RIGHT], actions=[ACTION], + ) + assert result == original and raw == original and repairs == [] + + +def test_non_document_action_cannot_claim_sources_using_a_forged_catalog_flag(planning): + action = {**ACTION, "type": "openapi", "document_binding_supported": True} + with pytest.raises(planning.harness.planner.PlannerError): + run_action_plan(planning, [action_proposal(action=action)], action=action) + assert len(planning.harness.model_calls) == 1 + + +def test_source_set_cannot_be_passed_as_prepared_action_findings(planning, caplog): + action = {**ACTION, "type": "openapi"} + raw = action_proposal(None, action=action) + raw["steps"].insert(0, deepcopy(proposal([LEFT])["steps"][0])) + raw["steps"][1]["inputs"] = { + "evidence": {"binding": input_binding("read", "sources"), "allow_partial": False}, + } + corrected = deepcopy(raw) + corrected["steps"][1]["inputs"]["evidence"]["binding"] = input_binding("read", "prepared") + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_action_plan(planning, [raw, corrected], action=action) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert correction_records(caplog)[0].sc_validation_rule == "input_kind_mismatch" + assert plan["steps"][1]["inputs"]["evidence"]["binding"]["output_name"] == "prepared" + + +def test_different_contract_errors_share_two_corrections_and_keep_private_values_out(planning, caplog): + first = proposal([LEFT]) + first["steps"][0]["outputs"] = [] + second = proposal([LEFT]) + second["steps"][-1]["inputs"] = {"private": {}} + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan(planning, [first, second, proposal([LEFT])]) + assert kind == "plan" and plan["validation"]["ok"] + assert len(planning.harness.model_calls) == 3 and not planning.harness.replies + assert len(correction_records(caplog)) == 2 + records = [ + {name: value for name, value in record.__dict__.items() if name.startswith("sc_")} + for record in caplog.records + ] + assert PRIVATE not in json.dumps(records) + + +def test_third_invalid_proposal_stops_without_a_new_error_budget(planning, caplog): + first = proposal([LEFT]) + first["steps"][0]["outputs"] = [] + second = proposal([LEFT], capability="document_analyze", options={"version": "invalid"}) + third = proposal([LEFT]) + third["steps"][-1]["inputs"] = {"evidence": {}} + with caplog.at_level(logging.INFO, logger=planning.logger.name): + with pytest.raises(planning.harness.planner.PlannerError) as failure: + run_plan(planning, [first, second, third, proposal([LEFT])]) + assert len(planning.harness.model_calls) == 3 and len(planning.harness.replies) == 1 + assert len(correction_records(caplog)) == 2 + assert failure.value.code == "plan_inputs_invalid" + assert failure.value.rule == "input_binding_required" + assert planning.harness.assistant_messages() == [] + + +def test_source_free_draft_uses_the_same_contract_correction_without_adding_retrieval(planning): + invalid = proposal() + invalid["steps"][0]["outputs"] = [] + kind, plan = run_plan(planning, [invalid, proposal()], identifiers=()) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert [step["capability_id"] for step in plan["steps"]] == ["compose"] + assert planning.harness.schema.effective_plan_document_ids(plan) == [] + + +def test_argument_type_feedback_names_only_the_registered_field(planning, caplog): + invalid = proposal([LEFT]) + invalid["steps"][0]["arguments"]["query"] = 42 + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan(planning, [invalid, proposal([LEFT])]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + diagnostic = next( + record for record in caplog.records + if getattr(record, "sc_schema_keyword", None) == "type" + ) + assert diagnostic.sc_argument_field == "query" + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + + +def test_correction_cannot_authorize_an_unselected_source(planning): + invalid = proposal([LEFT]) + invalid["steps"][0]["outputs"] = [] + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [invalid, proposal([RIGHT]), proposal([LEFT])]) + assert len(planning.harness.model_calls) == 2 and len(planning.harness.replies) == 1 + assert planning.harness.assistant_messages() == [] + + +def test_unknown_argument_names_are_corrected_but_never_logged(planning, caplog): + invalid = proposal([LEFT]) + invalid["steps"][0]["arguments"][PRIVATE] = PRIVATE + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan(planning, [invalid, proposal([LEFT])]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert correction_records(caplog)[0].sc_validation_rule == "unknown_argument" + records = [ + {name: value for name, value in record.__dict__.items() if name.startswith("sc_")} + for record in caplog.records + ] + assert PRIVATE not in json.dumps(records) + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + + +@pytest.mark.parametrize("shape", ["field", "id", "dependencies", "flags", "arguments", "cycle", "duplicate"]) +def test_model_owned_step_and_graph_declarations_are_corrected_once(planning, shape): + invalid = proposal([LEFT]) + first = invalid["steps"][0] + if shape == "field": + first[PRIVATE] = PRIVATE + elif shape == "id": + first["step_id"] = "invalid step identifier" + elif shape == "dependencies": + first["depends_on"] = "answer" + elif shape == "flags": + first["enabled"] = "yes" + elif shape == "arguments": + first["arguments"] = [] + elif shape == "cycle": + first["depends_on"] = ["answer"] + else: + invalid["steps"][-1]["step_id"] = first["step_id"] + kind, plan = run_plan(planning, [invalid, proposal([LEFT])]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert [step["capability_id"] for step in plan["steps"]] == ["document_search", "compose"] diff --git a/functional_tests/test_orchestration_planning_persistence.py b/functional_tests/test_orchestration_planning_persistence.py index d9144ce49..4966bcc17 100644 --- a/functional_tests/test_orchestration_planning_persistence.py +++ b/functional_tests/test_orchestration_planning_persistence.py @@ -1,8 +1,10 @@ # test_orchestration_planning_persistence.py """ Regression coverage for durable initial orchestration turns. -Version: 0.261.415 +Version: 0.261.423 Implemented in: 0.261.415 +Typed planner failure persistence added in: 0.261.419 +Shared two-correction allowance updated in: 0.261.422 Real authenticated routes and SDK-default offline storage exercise admission, clarification, failure, disconnect, Auto handoff, cancellation, and explicit retry. @@ -85,7 +87,7 @@ def test_initial_question_and_outcome_are_saved(empty_history_runtime, outcome): if outcome == "clarification": assert current["elicitation"]["message"] == "Choose a response style." if outcome == "failed": - assert current["error"] and current["error_code"] == "planning_failed" + assert current["error"] and current["error_code"] == "invalid_plan_work" if outcome == "success": record = runtime.harness.runs.read_item(current["run_id"], CONVERSATION) assert record["conversation_context"]["messages"] == [] @@ -283,6 +285,39 @@ def test_explicit_retry_uses_saved_inputs_and_one_question(empty_history_runtime assert current["status"] == "awaiting_review" +def test_typed_contract_failure_survives_reload_and_clears_on_explicit_retry(empty_history_runtime): + runtime = empty_history_runtime + invalid = plan_doc() + invalid["steps"][0]["outputs"] = [] + runtime.harness.replies.extend([json.dumps(invalid)] * 3) + response = runtime.client.post(PLAN_PATH, json=body(), buffered=True) + failure = next(frame for frame in sse_frames(response) if frame.get("error")) + expected = { + "error_stage": "plan_normalization", + "error_rule": "outputs_list_required", + "error_attempt": 3, + } + assert failure["code"] == "plan_outputs_invalid" + assert all(failure.get(key) == value for key, value in expected.items()) + calls = len(runtime.harness.model_calls) + saved = status(runtime, include_request=True) + reloaded = status(runtime) + assert calls == len(runtime.harness.model_calls) == 3 + assert saved["status"] == reloaded["status"] == "failed" + assert saved["error_code"] == reloaded["error_code"] == failure["code"] + assert all(saved.get(key) == reloaded.get(key) == value for key, value in expected.items()) + assert not runtime.harness.runs.items and not runtime.harness.assistant_messages() + runtime.harness.replies.append(json.dumps(plan_doc())) + retried = runtime.client.post(PLAN_PATH, json={ + **saved["request"], "planning_submission_id": "contract-retry", + "requires_fresh_review": True, "approval_mode": "manual", + }, buffered=True) + assert any(frame.get("type") == "orchestration_plan" for frame in sse_frames(retried)) + current = status(runtime) + assert current["status"] == "awaiting_review" and len(questions(runtime)) == 1 + assert not any(key in current for key in ("error", "error_code", *expected)) + + def test_stop_before_acknowledgement_is_durable(empty_history_runtime): runtime = empty_history_runtime stopped = runtime.client.post( diff --git a/functional_tests/test_orchestration_selected_source_planning.py b/functional_tests/test_orchestration_selected_source_planning.py index e7040fc5b..1de2e9ca4 100644 --- a/functional_tests/test_orchestration_selected_source_planning.py +++ b/functional_tests/test_orchestration_selected_source_planning.py @@ -1,8 +1,9 @@ # test_orchestration_selected_source_planning.py """Selected attachments remain usable during planning and clarification. -Version: 0.261.415 +Version: 0.261.423 Implemented in: 0.261.355 +Shared two-correction allowance updated in: 0.261.422 Uses real source projection, parsers, resolution and HTTP planning with external storage/model I/O replaced. Previews must never become complete export records. @@ -181,7 +182,7 @@ def test_full_planner_repairs_source_question_instead_of_publishing_it(source_ru "type": "object", "properties": {"rows": {"type": "string"}}, "required": ["rows"], }, } - harness.replies = [json.dumps(question), json.dumps(question)] + harness.replies = [json.dumps(question)] * 3 with pytest.raises(harness.planner.PlannerError) as failure: harness.planner.plan_request( "What columns does my file have?", context, "conversation-1", "owner", @@ -190,7 +191,7 @@ def test_full_planner_repairs_source_question_instead_of_publishing_it(source_ru request_context=harness.services().capability_request_bindings(), ) assert failure.value.reason == "selected_source_already_available" - assert len(harness.model_calls) == 2 + assert len(harness.model_calls) == 3 assert harness.blobs.file_uploads == 0 @@ -584,7 +585,7 @@ def test_source_diagnostics_survive_logging_without_source_text(source_runtime, "kind": "elicitation", "message": "PRIVATE_MODEL_QUESTION", "clarification_context": {"reason": "source_content", "document_ids": ["tabular-doc"]}, } - harness.replies = [json.dumps(question), json.dumps(question)] + harness.replies = [json.dumps(question)] * 3 with pytest.raises(harness.planner.PlannerError): harness.planner.plan_request( request, context, "conversation-1", "owner", turn_id="private-turn", @@ -595,7 +596,9 @@ def test_source_diagnostics_survive_logging_without_source_text(source_runtime, logging._build_logger_extra(message, kwargs["extra"]) for message, kwargs in recorded if (kwargs.get("extra") or {}).get("stage") == "plan_clarification" ] - assert len(diagnostics) >= 2 + assert len(diagnostics) >= 3 + assert {entry["sc_attempt"] for entry in diagnostics} == {1, 2, 3} + assert len(harness.model_calls) == 3 assert diagnostics[0]["sc_stage"] == "plan_clarification" assert diagnostics[0]["sc_reason"] == "selected_source_already_available" assert diagnostics[0]["sc_selected_source_count"] == 1 diff --git a/functional_tests/test_orchestration_source_kind_planning.py b/functional_tests/test_orchestration_source_kind_planning.py index 4091b3849..0dc3a290b 100644 --- a/functional_tests/test_orchestration_source_kind_planning.py +++ b/functional_tests/test_orchestration_source_kind_planning.py @@ -1,7 +1,7 @@ # test_orchestration_source_kind_planning.py """Plan mixed document types from authorized metadata, not user-facing labels. -Version: 0.261.355 +Version: 0.261.423 Implemented in: 0.261.140 The real manifest classifier, context projection, planner, compiler and HTTP @@ -278,14 +278,14 @@ def test_source_kind_repair_stays_bounded_and_never_drops_selected_sources(sourc offered = enrich(runtime, selected, seeds) payload = runtime.context.build_planner_context("Compare the documents.", candidates=offered, seeds=seeds) rejected = proposal(narrative_comparison=True) - harness.replies = [json.dumps(rejected), json.dumps(rejected)] + harness.replies = [json.dumps(rejected)] * 3 with pytest.raises(harness.planner.PlannerError) as failure: harness.planner.plan_request( "Compare the documents.", payload, "conversation-1", "owner", settings=harness.settings, seeds=seeds, authorized_document_ids=[LEFT, RIGHT], request_context=harness.services().capability_request_bindings(), ) - assert len(harness.model_calls) == 2 + assert len(harness.model_calls) == 3 assert failure.value.message == harness.planner.SOURCE_KIND_FAILURE_MESSAGE assert harness.blobs.file_uploads == 0 and harness.assistant_messages() == [] diff --git a/functional_tests/test_plugin_discovery_identity.py b/functional_tests/test_plugin_discovery_identity.py new file mode 100644 index 000000000..209b83c2c --- /dev/null +++ b/functional_tests/test_plugin_discovery_identity.py @@ -0,0 +1,243 @@ +# test_plugin_discovery_identity.py +""" +Functional tests for canonical plugin discovery and instance isolation. +Version: 0.261.423 +Implemented in: 0.261.421 + +Production discovery, logged registration, classic loading, and real bootstrap +imports run with external I/O isolated. Fresh normal and optimized processes +check import order without substituting config or plugin modules. +""" + +import importlib +import json +import logging +import os +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from importlib.metadata import version +from pathlib import Path + +import pytest +from flask import Flask, session +from semantic_kernel import Kernel + +from test_orchestration_harness_routes import modules + + +__all__ = ["modules"] +ROOT = Path(__file__).resolve().parents[1] +APP = ROOT / "application" / "single_app" +TESTS = ROOT / "functional_tests" +IMPORT_ORDERS = ( + ("document_search_plugin", "plugin_loader", "logged_plugin_loader", "classic"), + ("plugin_loader", "logged_plugin_loader", "classic", "document_search_plugin"), + ("logged_plugin_loader", "classic", "plugin_loader", "document_search_plugin"), + ("classic", "plugin_loader", "logged_plugin_loader", "document_search_plugin"), +) +COLD_IMPORT_PROBE = """ +import importlib +import json +import socket +import sys +from pathlib import Path +from unittest.mock import patch +from test_support.offline_bootstrap import offline_app_imports + +dns_attempts = [] +def no_dns(*args, **kwargs): + dns_attempts.append(True) + raise AssertionError("Cold plugin import attempted DNS access.") + +with offline_app_imports() as environment, patch.object(socket, "getaddrinfo", no_dns): + for name in json.loads(sys.argv[1]): + module_name = "semantic_kernel_loader" if name == "classic" else "semantic_kernel_plugins." + name + importlib.import_module(module_name) + if name == "plugin_loader": + discovered_early = importlib.import_module(module_name).discover_plugins() + reader_early = importlib.import_module("semantic_kernel_plugins.document_search_plugin") + if discovered_early.get("DocumentSearchPlugin") is not reader_early.DocumentSearchPlugin: + raise AssertionError("Discovery-first loading created a different reader class.") + discovery = importlib.import_module("semantic_kernel_plugins.plugin_loader") + first = discovery.discover_plugins() + second = discovery.discover_plugins() + if "DocumentSearchPlugin" not in first or first.keys() != second.keys(): + raise AssertionError("Installed plugin discovery changed between calls.") + for name, plugin_class in first.items(): + owner = importlib.import_module(plugin_class.__module__) + canonical = vars(owner).get(plugin_class.__name__) + if canonical is not plugin_class or second[name] is not plugin_class: + raise AssertionError("Installed plugin class identity was not preserved.") + importlib.import_module("functions_orchestration_bootstrap") + importlib.import_module("app") + importlib.import_module("functions_orchestration_execution") + config = importlib.import_module("config") + if Path(config.__file__).resolve() != Path(sys.argv[2]).resolve(): + raise AssertionError("The real config module was not imported.") + if environment.network_attempts or dns_attempts: + raise AssertionError("A blocked network attempt was swallowed during startup.") + print("PLUGIN_IDENTITY_RESULT:" + json.dumps({"plugins": len(first), "network_attempts": 0})) +""" + + +@pytest.fixture +def discovery(modules): + return importlib.import_module("semantic_kernel_plugins.plugin_loader") + + +@pytest.fixture +def request_app(): + app = Flask(__name__) + app.secret_key = "plugin-identity-test" + return app + + +def test_repeated_discovery_reuses_every_canonical_plugin_class(discovery): + first = discovery.discover_plugins() + second = discovery.discover_plugins() + assert "DocumentSearchPlugin" in first + assert first.keys() == second.keys() + for name, plugin_class in first.items(): + owner = importlib.import_module(plugin_class.__module__) + canonical = vars(owner).get(plugin_class.__name__) + assert canonical is plugin_class, name + assert second[name] is plugin_class, name + + +def test_required_template_parser_is_explicitly_pinned_and_importable(modules): + requirements = (APP / "requirements.txt").read_text(encoding="utf-8").splitlines() + pins = [line.partition("==")[2] for line in requirements if line.lower().startswith("pymeta3==")] + parser = importlib.import_module("pymeta.grammar") + installed = version("PyMeta3") + assert len(pins) == 1 and installed == pins[0] + assert callable(parser.OMeta) + + +def test_logged_reader_instances_keep_separate_configuration(discovery, request_app): + loader_module = importlib.import_module("semantic_kernel_plugins.logged_plugin_loader") + reader_module = importlib.import_module("semantic_kernel_plugins.document_search_plugin") + manifests = [ + { + "name": name, "type": "document_search", + "additionalFields": {"allowed_scopes": [scope]}, + } + for name, scope in (("personal_reader", "personal"), ("public_reader", "public")) + ] + originals = deepcopy(manifests) + loaders = [loader_module.LoggedPluginLoader(Kernel()) for _ in manifests] + loaded = [ + loader.load_plugin_from_manifest(manifest, user_id="owner") + for loader, manifest in zip(loaders, manifests) + ] + assert loaded == [True, True] + readers = [loader.plugin_instances[0] for loader in loaders] + assert all(type(reader) is reader_module.DocumentSearchPlugin for reader in readers) + assert readers[0] is not readers[1] + with request_app.test_request_context(): + session["user"] = {"oid": "owner"} + initial_scopes = [reader._resolve_doc_scope_requests("all") for reader in readers] + readers[0].manifest["additionalFields"]["allowed_scopes"].append("group") + changed_scopes = [reader._resolve_doc_scope_requests("all") for reader in readers] + assert initial_scopes == [["personal"], ["public"]] + assert changed_scopes == [["personal", "group"], ["public"]] + assert manifests == originals + functions = [loader.kernel.get_list_of_function_metadata({}) for loader in loaders] + assert all(len(items) == 4 for items in functions) + + +def test_concurrent_discovery_and_loading_keep_reader_identity_and_scope(discovery, request_app): + loader_module = importlib.import_module("semantic_kernel_plugins.logged_plugin_loader") + reader_module = importlib.import_module("semantic_kernel_plugins.document_search_plugin") + + def load_reader(index): + scope = ("personal", "group", "public")[index % 3] + user_id = f"owner-{index}" + with request_app.test_request_context(): + session["user"] = {"oid": user_id} + classes = discovery.discover_plugins() + loader = loader_module.LoggedPluginLoader(Kernel()) + loaded = loader.load_plugin_from_manifest({ + "name": f"reader_{index}", "type": "document_search", + "additionalFields": {"allowed_scopes": [scope]}, + }, user_id=user_id) + reader = loader.plugin_instances[0] if loaded else None + scopes = reader._resolve_doc_scope_requests("all") if reader is not None else None + observed_user = reader._get_user_id() if reader is not None else None + return loaded, classes.get("DocumentSearchPlugin"), reader, scopes, scope, observed_user, user_id + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(load_reader, range(12))) + readers = [] + for loaded, plugin_class, reader, scopes, expected_scope, observed_user, expected_user in results: + assert loaded + assert plugin_class is reader_module.DocumentSearchPlugin + assert type(reader) is reader_module.DocumentSearchPlugin + assert scopes == [expected_scope] + assert observed_user == expected_user + readers.append(reader) + assert len({id(reader) for reader in readers}) == len(readers) + + +def test_module_import_failure_is_logged_without_private_exception_text( + discovery, monkeypatch, caplog, +): + telemetry = importlib.import_module("functions_appinsights") + logger = logging.getLogger("test.plugin.discovery") + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: {}) + original_import = importlib.import_module + + def fail_optional_module(name, package=None): + if name == "semantic_kernel_plugins.unavailable_plugin": + raise ImportError("PRIVATE_PLUGIN_IMPORT_SENTINEL") + return original_import(name, package) + + monkeypatch.setattr(discovery.os, "listdir", lambda path: [ + "document_search_plugin.py", "unavailable_plugin.py", "base_plugin.py", "__init__.py", + ]) + monkeypatch.setattr(discovery.importlib, "import_module", fail_optional_module) + with caplog.at_level(logging.WARNING, logger=logger.name): + found = discovery.discover_plugins() + assert "DocumentSearchPlugin" in found + records = [ + record for record in caplog.records + if getattr(record, "sc_message", None) == "[PLUGIN_LOADER] Plugin module could not be loaded." + ] + assert len(records) == 1 + assert records[0].sc_error_type == "ImportError" + properties = [ + {key: value for key, value in record.__dict__.items() if key.startswith("sc_")} + for record in records + ] + assert "PRIVATE_PLUGIN_IMPORT_SENTINEL" not in json.dumps(properties) + + +@pytest.mark.parametrize("order", IMPORT_ORDERS, ids=("reader-first", "discovery-first", "logged-first", "classic-first")) +@pytest.mark.parametrize("optimized", [False, True], ids=("normal", "optimized")) +def test_real_cold_import_orders_preserve_identity_and_network_free_startup(order, optimized): + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join((str(APP), str(TESTS))) + environment["PYTHONDONTWRITEBYTECODE"] = "1" + command = [ + sys.executable, *(["-O"] if optimized else []), "-c", COLD_IMPORT_PROBE, + json.dumps(order), str(APP / "config.py"), + ] + result = subprocess.run( + command, cwd=ROOT, env=environment, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=180, + ) + assert result.returncode == 0, result.stdout + result.stderr + records = [ + line.removeprefix("PLUGIN_IDENTITY_RESULT:") + for line in result.stdout.splitlines() if line.startswith("PLUGIN_IDENTITY_RESULT:") + ] + assert len(records) == 1 + report = json.loads(records[0]) + assert report["plugins"] > 1 + assert report["network_attempts"] == 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/functional_tests/test_support/orchestration_revisions.py b/functional_tests/test_support/orchestration_revisions.py index e2112b84d..0246e83cd 100644 --- a/functional_tests/test_support/orchestration_revisions.py +++ b/functional_tests/test_support/orchestration_revisions.py @@ -2,10 +2,11 @@ """ In-memory Cosmos boundary for conditional orchestration revision tests. -Version: 0.261.324 +Version: 0.261.420 Implemented in: 0.261.102 SDK-shaped point response coverage added in: 0.261.140 SDK responses default in: 0.261.324; sdk_responses=False is a compatibility probe. +Bounded SDK read transport options supported in: 0.261.419 Batch operations are formatted by the installed SDK and applied to a copy. A failed operation never makes preceding operations visible. Hooks permit deterministic races. @@ -66,7 +67,9 @@ def create_item(self, body): raise exceptions.CosmosResourceExistsError(status_code=409, message='Test record exists') return self._save(body) - def read_item(self, item, partition_key): + def read_item(self, item, partition_key, **kwargs): + if set(kwargs) - {'connection_timeout', 'read_timeout', 'retry_total'}: + raise TypeError('Unsupported test-container read options.') with self._lock: if self.fail_reads: raise AzureError('Private test read failure') diff --git a/functional_tests/test_task_completeness_engine.py b/functional_tests/test_task_completeness_engine.py index 3ccf25b9f..efba78667 100644 --- a/functional_tests/test_task_completeness_engine.py +++ b/functional_tests/test_task_completeness_engine.py @@ -1,8 +1,8 @@ # test_task_completeness_engine.py """Grounded, bounded task checking and recovery. -Version: 0.261.418 -Implemented in: 0.261.382; 0.261.384; 0.261.416; 0.261.418 +Version: 0.261.423 +Implemented in: 0.261.382; 0.261.384; 0.261.416; 0.261.418; 0.261.422 """ import json @@ -61,6 +61,52 @@ def test_complete_original_has_no_correction(): assert state["corrections"] == 0 +@pytest.mark.parametrize("evidence", [{}, {"findings": {"private": "source text"}, "coverage": {}}]) +def test_assessment_transport_only_accepts_actual_evidence_keys(evidence): + response_format = assessment_response_format(evidence=evidence) + schema = response_format["json_schema"]["schema"] + validator = Draft202012Validator(schema) + report = json.loads(assessment(refs=list(evidence))) + valid_errors = list(validator.iter_errors(report)) + assert not valid_errors + report["requirements"][0]["evidence_refs"] = ["invented_reference"] + invalid_errors = list(validator.iter_errors(report)) + assert invalid_errors + assert "source text" not in json.dumps(schema) + + +def test_evidence_reference_repair_gets_valid_labels_without_rewriting_the_answer(): + calls, diagnostics = [], [] + replies = iter([assessment(refs=["PRIVATE_WRONG_REFERENCE"]), assessment()]) + + def invoke(messages, stage, *, response_format=None): + calls.append((deepcopy(messages), stage, deepcopy(response_format))) + return next(replies) + + candidate = "Evaluation" + result, state = check_and_correct( + candidate, instruction="Include the status.", contract={"kind": "text-v1"}, + evidence={"record": {"status": "Evaluation"}}, + policy={"enabled": True, "max_corrections": 2}, + invoke=invoke, validate=lambda value: [], on_diagnostic=diagnostics.append, + ) + assert result == candidate and state["status"] == "complete" + assert state["corrections"] == 0 and state["report_repairs"] == 1 + assert [call[1] for call in calls] == ["completeness_check", "completeness_check"] + repair_payload = json.loads(calls[1][0][-1]["content"]) + problem = repair_payload["assessment_problem"] + assert problem["code"] == "completeness_evidence_invalid" + assert problem["allowed_evidence_refs"] == ["record"] + assert problem["evidence_ref_count"] == 1 + assert "PRIVATE_WRONG_REFERENCE" not in json.dumps(problem) + refs = calls[0][2]["json_schema"]["schema"]["properties"]["requirements"]["items"]["properties"]["evidence_refs"] + assert refs["items"]["enum"] == ["record"] + assert diagnostics[0]["validation_code"] == "completeness_evidence_invalid" + assert diagnostics[0]["validation_rule"] == "evidence_reference_unknown" + assert diagnostics[0]["report_repair_count"] == 0 and diagnostics[0]["evidence_ref_count"] == 1 + assert "PRIVATE_WRONG_REFERENCE" not in json.dumps(diagnostics) + + def test_actual_missing_tool_facts_are_repaired_and_rechecked(): corrected = '{"status":"Evaluation","proposal_due":"2027-02-15","award_target":"2027-04-01","response_days":15}' result, state, calls, _ = run([assessment("missing"), corrected, assessment()]) diff --git a/ui_tests/test_v2_orchestration_actions.py b/ui_tests/test_v2_orchestration_actions.py index 3da1a406d..c586b87dc 100644 --- a/ui_tests/test_v2_orchestration_actions.py +++ b/ui_tests/test_v2_orchestration_actions.py @@ -1,9 +1,10 @@ # test_v2_orchestration_actions.py """ Browser coverage for action identity and tool citations in orchestration. -Version: 0.261.321 +Version: 0.261.423 Implemented in: 0.261.098 Operation intent added in: 0.261.321 +Document action source binding coverage added in: 0.261.419 Use the existing orchestration harness and real stores/components. Serve its local bundle through request interception so local and Azure Playwright browsers @@ -15,7 +16,7 @@ import re import sys from pathlib import Path -from urllib.parse import urlsplit +from urllib.parse import parse_qs, urlsplit import pytest from playwright.sync_api import expect @@ -60,6 +61,13 @@ def serve(route): route.fulfill(path=str(hb.BUNDLE), content_type="application/javascript") elif f"{parsed.scheme}://{parsed.netloc}" == ORIGIN and parsed.path == "/favicon.ico": route.fulfill(status=204) + elif ( + f"{parsed.scheme}://{parsed.netloc}" == ORIGIN + and route.request.method == "GET" + and parsed.path == "/api/v2/orchestration/questions" + and parse_qs(parsed.query).get("conversation_id") == [CONVERSATION] + ): + route.fulfill(json={"questions": [], "csrf_token": "fixture-token"}) elif f"{parsed.scheme}://{parsed.netloc}" == ORIGIN and is_pending_actions_list( route.request.method, parsed.path ): @@ -72,7 +80,7 @@ def serve(route): page.goto(ORIGIN, wait_until="domcontentloaded") page.wait_for_function("() => Boolean(window.OrchHarness)") yield page - assert not errors, errors + assert not errors, {"errors": errors, "unexpected_requests": unexpected_requests} assert not unexpected_requests, unexpected_requests @@ -283,6 +291,29 @@ def test_action_step_keeps_existing_narrowing_controls(action_page): expect(row.get_by_text("Run Look up the ticket", exact=True)).to_be_visible() +def test_native_action_bound_sources_are_visible_and_can_be_narrowed(action_page): + plan = _plan() + plan["intent"]["summary"] = "Inspect native document evidence" + plan["inputs"]["documents"] = [ + {"document_id": "native-pdf", "display_name": "Selected PDF.pdf", "selected_by_user": True}, + {"document_id": "native-word", "display_name": "Selected Word.docx", "selected_by_user": True}, + ] + step = plan["steps"][0] + step["arguments"]["document_ids"] = ["native-pdf", "native-word"] + step["outputs"] = [{"name": "prepared", "kind": "structured-v1"}] + _seed(action_page, plan) + row = _action_row(action_page) + expect(row.get_by_text("Selected PDF.pdf", exact=True)).to_be_visible() + expect(row.get_by_text("Selected Word.docx", exact=True)).to_be_visible() + row.get_by_role("button", name="Remove document native-pdf from this step", exact=True).click() + edits = action_page.evaluate("""({conversation, turn}) => { + const api = window.OrchHarness.stores.orchestration; + return api.selectEdits(api.useOrchestrationStore.getState(), conversation, turn); + }""", {"conversation": CONVERSATION, "turn": TURN}) + assert edits["removed_document_ids"] == {"lookup": ["native-pdf"]} + expect(row.get_by_text("Selected Word.docx", exact=True)).to_be_visible() + + @pytest.mark.parametrize("status,summary", [ ("running", "Calling the selected action."), ("completed", "Found the ticket status."), diff --git a/ui_tests/test_v2_stream_citation_delivery.py b/ui_tests/test_v2_stream_citation_delivery.py index 4fa37b466..1a6cf8de3 100644 --- a/ui_tests/test_v2_stream_citation_delivery.py +++ b/ui_tests/test_v2_stream_citation_delivery.py @@ -1,14 +1,16 @@ # test_v2_stream_citation_delivery.py """ UI regressions for live tool/web evidence and recorded tool outcomes. -Version: 0.261.417 +Version: 0.261.423 Implemented in: 0.261.365 +Word comment inspection fixed in: 0.261.422 Drives the real send/SSE/store/message inspector locally with synthetic HTTP boundaries. The existing Azure Playwright-compatible fixture also supports local browsers. No live agents, actions, conversations, model requests, or connector requests are created. """ +import json from pathlib import Path import re @@ -17,6 +19,7 @@ from ui_tests.fixtures.playwright_connection import connect_options # noqa: F401 from ui_tests.fixtures.orchestration import harness_build +from ui_tests.test_inline_citation_filename_fallback import CitationPage ROOT = Path(__file__).resolve().parents[1] @@ -220,6 +223,120 @@ def test_native_annotation_first_preserves_streamed_and_saved_identity(citation_ expect(dialog.get_by_role("button", name="Open workbook preview (separate from annotation evidence)", exact=True)).to_have_count(0) +def _word_comment_evidence(): + reference = { + "producer": {"user_id": "owner", "conversation_id": "evidence-chat", "run_id": "native-run", + "step_id": "comments", "attempt_index": 1, "capability_id": "document_search"}, + "output_name": "prepared", "kind": "structured-v1", "manifest_sha256": "a" * 64, + } + citation = { + "citation_id": "word:native-comments:0", "document_id": "word", "file_name": "review.docx", + "location_label": "Location", "location_value": "Word comment 0", "native_evidence": reference, + "source_version": 1, "annotation_id": "0", + } + payload = { + "file_name": "review.docx", "evidence_kind": "native_word_annotation", + "native_annotation": { + "id": "0", "author": "Reviewer A", "date": "2026-01-01T10:00:00Z", + "anchor_text": "Original policy paragraph", "anchor_complete": False, + "done": None, "resolution_available": False, + "replies": [{ + "id": "1", "author": "Reviewer B", "date": "2026-01-02T10:00:00Z", + "text": 'Reply ', "done": True, + "resolution_available": True, + }], + }, + "cited_text": "Check the policy. ", + } + content = "Review (Source: review.docx, Location: Word comment 0) [#word:native-comments:0]" + return citation, payload, content + + +@pytest.mark.parametrize("width", [390, 1440]) +def test_word_comment_inspector_retains_anchor_replies_and_saved_evidence(citation_page, width): + page = citation_page + page.set_viewport_size({"width": width, "height": 900}) + citation, payload, content = _word_comment_evidence() + message = _send(page, {"full_content": content, "hybrid_citations": [citation], "augmented": True}) + page.evaluate("""(payload) => { + window.wordRequests = []; + window.wordDenied = false; + const originalFetch = window.fetch; + window.fetch = async (url, init = {}) => { + if (new URL(String(url), location.origin).pathname !== '/api/get_citation') { + return originalFetch(url, init); + } + window.wordRequests.push(JSON.parse(init.body)); + return new Response(JSON.stringify(window.wordDenied + ? {error: 'The cited document is no longer available.'} : payload), + {status: window.wordDenied ? 403 : 200, headers: {'Content-Type': 'application/json'}}); + }; + }""", payload) + chip = message.get_by_role("button", name="review.docx Word comment 0", exact=True) + chip.focus() + chip.press("Enter") + dialog = page.get_by_role("dialog", name="Citation", exact=True) + comment = dialog.get_by_role("region", name="Native Word comment") + expect(comment).to_contain_text("Anchored text: Original policy paragraph") + expect(comment).to_contain_text("Anchor coverage: Bounded or unavailable") + expect(comment).to_contain_text("Resolved: Unavailable") + replies = dialog.get_by_role("region", name="Word comment replies") + expect(replies).to_contain_text("Author: Reviewer B") + expect(replies).to_contain_text("Resolved: Yes") + expect(replies).to_contain_text('Reply ') + expect(dialog).to_contain_text(payload["cited_text"]) + expect(dialog.get_by_text(re.compile("Worksheet / cell:|Saved cell value:"))).to_have_count(0) + expect(dialog.get_by_role("button", name=re.compile("workbook preview"))).to_have_count(0) + assert dialog.locator("script, img").count() == 0 + assert dialog.evaluate("element => element.scrollWidth <= element.clientWidth") + dialog.get_by_role("button", name="Close citation", exact=True).click() + page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().reloadMessages()") + chip.click() + expect(dialog).to_contain_text("Author: Reviewer B") + requests = page.evaluate("() => window.wordRequests") + assert len(requests) == 2 and all(request == requests[0] for request in requests) + assert requests[0]["native_evidence"] == citation["native_evidence"] + assert requests[0]["document_id"] == "word" + dialog.get_by_role("button", name="Close citation", exact=True).click() + page.evaluate("() => window.wordDenied = true") + chip.click() + expect(dialog).to_contain_text("The cited document is no longer available.") + expect(dialog.get_by_role("region", name="Native Word comment")).to_have_count(0) + + +@pytest.mark.parametrize("width", [390, 1440]) +def test_classic_word_comment_inspector_renders_exact_native_evidence(page, width): + page.set_viewport_size({"width": width, "height": 900}) + fixture = CitationPage(page, "classic") + citation, payload, content = _word_comment_evidence() + requests = [] + + def get_citation(route): + requests.append(route.request.post_data_json) + route.fulfill(content_type="application/json", body=json.dumps(payload)) + + page.route("**/api/get_citation", get_citation) + fixture.open() + fixture.load([{ + "id": "word-reply", "conversation_id": "evidence-chat", "role": "assistant", + "content": content, "hybrid_citations": [citation], "augmented": True, "metadata": {}, + }]) + fixture.row("word-reply").locator(".message-text a.citation-link").click() + dialog = page.get_by_role("dialog") + expect(dialog).to_contain_text("Native Word evidence: review.docx") + expect(dialog).to_contain_text("Anchored text: Original policy paragraph") + expect(dialog).to_contain_text("Anchor coverage: Bounded or unavailable") + expect(dialog).to_contain_text("Resolved: Unavailable") + expect(dialog).to_contain_text("Author: Reviewer B") + expect(dialog).to_contain_text("Resolved: Yes") + expect(dialog).to_contain_text(payload["cited_text"]) + expect(dialog).to_contain_text('Reply ') + expect(dialog.get_by_role("button", name=re.compile("workbook preview"))).to_have_count(0) + assert dialog.locator("script, img").count() == 0 + assert requests[0]["native_evidence"] == citation["native_evidence"] + assert fixture.errors == [] + + @pytest.mark.parametrize("result", [{"error": "HTTP 502", "status_code": 502}, None]) def test_live_failed_tool_retains_error_arguments_and_available_result(citation_page, result): failure = {**TOOL, "success": False, "error_message": "Synthetic upstream unavailable.", "function_result": result}