From dbceb227168ba7351678f8daaf99ba4b1aaf558e Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 11 Nov 2026 10:00:09 -0500 Subject: [PATCH] Add admin diagnostic support ZIP exports Build scoped, privacy-reviewed support bundles and owner-approved personal conversation snapshots in the classic and V2 admin interfaces. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 11 + application/single_app/admin_settings_nav.py | 1 + application/single_app/app.py | 21 +- application/single_app/background_tasks.py | 16 + application/single_app/config.py | 9 +- .../single_app/content_screening/model.py | 28 +- .../single_app/functions_appinsights.py | 47 +- .../single_app/functions_support_exports.py | 189 +++++ .../route_backend_support_exports.py | 202 +++++ .../single_app/static/js/support-exports.js | 424 ++++++++++ .../single_app/support_export_contracts.py | 196 +++++ .../single_app/support_export_privacy.py | 330 ++++++++ .../single_app/support_export_service.py | 751 ++++++++++++++++++ .../single_app/support_export_sources.py | 413 ++++++++++ .../single_app/support_export_store.py | 212 +++++ .../templates/admin/_panes/logging.html | 2 + .../templates/admin/_support_log_export.html | 87 ++ .../single_app/templates/admin_settings.html | 1 + .../single_app/templates/approvals.html | 14 + .../components/admin/SupportExportPanel.tsx | 345 ++++++++ .../approvals/SupportExportConsentPanel.tsx | 120 +++ application/v2_ui/src/lib/supportExports.ts | 180 +++++ .../v2_ui/src/pages/AdminSettingsPage.tsx | 3 + application/v2_ui/src/pages/ApprovalsPage.tsx | 11 +- docs/_data/app_surface.yml | 6 +- docs/_data/features.yml | 19 +- docs/_features/admin-support-exports.md | 4 + docs/admin/operations.md | 31 + .../features/ADMIN_SUPPORT_LOG_EXPORT.md | 213 +++++ docs/guides/export-support-logs.md | 117 +++ docs/guides/review-approval-requests.md | 26 +- docs/reference/logging-tags.md | 16 + .../cosmos_response_boundaries.json | 217 ++++- .../test_route_blueprint_policy_inventory.py | 4 +- .../test_admin_support_export_api.py | 77 ++ .../test_admin_support_export_filters.py | 93 +++ .../test_admin_support_export_jobs.py | 205 +++++ .../test_admin_support_export_logging.py | 73 ++ .../test_admin_support_export_model_budget.py | 105 +++ .../test_admin_support_export_privacy.py | 132 +++ .../test_admin_support_export_sources.py | 171 ++++ .../test_admin_support_export_startup.py | 79 ++ .../test_support/support_exports.py | 223 ++++++ .../test_unified_logging_entrypoint.py | 13 +- .../test_v2_admin_support_export_logic.mjs | 31 + ui_tests/fixtures/support_export_backend.py | 98 +++ ui_tests/test_admin_support_export.py | 67 ++ ui_tests/test_support_export_consent.py | 58 ++ ui_tests/test_v2_admin_support_export.py | 91 +++ 49 files changed, 5753 insertions(+), 29 deletions(-) create mode 100644 application/single_app/functions_support_exports.py create mode 100644 application/single_app/route_backend_support_exports.py create mode 100644 application/single_app/static/js/support-exports.js create mode 100644 application/single_app/support_export_contracts.py create mode 100644 application/single_app/support_export_privacy.py create mode 100644 application/single_app/support_export_service.py create mode 100644 application/single_app/support_export_sources.py create mode 100644 application/single_app/support_export_store.py create mode 100644 application/single_app/templates/admin/_support_log_export.html create mode 100644 application/v2_ui/src/components/admin/SupportExportPanel.tsx create mode 100644 application/v2_ui/src/components/approvals/SupportExportConsentPanel.tsx create mode 100644 application/v2_ui/src/lib/supportExports.ts create mode 100644 docs/_features/admin-support-exports.md create mode 100644 docs/explanation/features/ADMIN_SUPPORT_LOG_EXPORT.md create mode 100644 docs/guides/export-support-logs.md create mode 100644 functional_tests/test_admin_support_export_api.py create mode 100644 functional_tests/test_admin_support_export_filters.py create mode 100644 functional_tests/test_admin_support_export_jobs.py create mode 100644 functional_tests/test_admin_support_export_logging.py create mode 100644 functional_tests/test_admin_support_export_model_budget.py create mode 100644 functional_tests/test_admin_support_export_privacy.py create mode 100644 functional_tests/test_admin_support_export_sources.py create mode 100644 functional_tests/test_admin_support_export_startup.py create mode 100644 functional_tests/test_support/support_exports.py create mode 100644 functional_tests/test_v2_admin_support_export_logic.mjs create mode 100644 ui_tests/fixtures/support_export_backend.py create mode 100644 ui_tests/test_admin_support_export.py create mode 100644 ui_tests/test_support_export_consent.py create mode 100644 ui_tests/test_v2_admin_support_export.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 23162d646..845de90df 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -8508,6 +8508,17 @@ "default": False, }, ], + "support-log-export-section": [ + { + "type": "component", + "component": "support-log-export", + "label": "Support Log Export", + "help": ( + "Collect retained diagnostics by user, conversation and time into a private, " + "redacted support ZIP. Whole personal-conversation snapshots need their owner's approval." + ), + }, + ], "debug-logging-section": [ { "key": "enable_debug_logging", diff --git a/application/single_app/admin_settings_nav.py b/application/single_app/admin_settings_nav.py index 57f25fdca..c78f44b9a 100644 --- a/application/single_app/admin_settings_nav.py +++ b/application/single_app/admin_settings_nav.py @@ -577,6 +577,7 @@ "sections": [ {"id": "application-insights-section", "label": "Application Insights", "icon": "bi-graph-up"}, {"id": "debug-logging-section", "label": "Debug Logging", "icon": "bi-bug"}, + {"id": "support-log-export-section", "label": "Support Log Export", "icon": "bi-file-earmark-zip"}, {"id": "file-processing-logs-section", "label": "File Process Logging", "icon": "bi-file-earmark-text"}, {"id": "health-check-section", "label": "Health Check", "icon": "bi-heart-pulse"}, {"id": "swagger-section", "label": "API Documentation", "icon": "bi-file-earmark-code"}, diff --git a/application/single_app/app.py b/application/single_app/app.py index 9a8bb3119..a36280dac 100644 --- a/application/single_app/app.py +++ b/application/single_app/app.py @@ -44,7 +44,7 @@ import threading import time from datetime import datetime -from flask import Blueprint, g, make_response +from flask import Blueprint, g, has_request_context, make_response from urllib.parse import urlencode, urlparse from route_frontend_authentication import * @@ -69,6 +69,10 @@ from route_backend_chats import * from route_backend_analysis_results import register_route_backend_analysis_results +from route_backend_support_exports import ( + register_route_backend_support_exports, + register_route_backend_support_export_consents, +) from route_backend_v2 import ( register_route_backend_v2, register_route_backend_v2_admin, @@ -391,6 +395,19 @@ def ensure_application_initialized(): initialize_application() +def _support_request_diagnostics(): + if not has_request_context(): + return {} + authorized = getattr(g, 'authorized_chat_context', None) + return { + 'user_id': get_current_user_id_or_none(), + 'conversation_id': authorized.get('conversation_id') if isinstance(authorized, dict) else None, + } + + +configure_diagnostic_context_provider(_support_request_diagnostics) + + def get_idle_timeout_settings(settings=None): """ Resolve and normalize idle timeout settings used for warning and logout enforcement. @@ -1581,6 +1598,8 @@ def finalize_m365_json_request(response): # ------------------- API Control Center Routes --------- register_route_blueprint('backend_control_center', register_route_backend_control_center, login_required_blueprint) +register_route_blueprint('backend_support_exports', register_route_backend_support_exports, admin_required_blueprint) +register_route_blueprint('backend_support_export_consents', register_route_backend_support_export_consents, user_required_blueprint) # ------------------- API Notifications Routes ---------- register_route_blueprint('backend_notifications', register_route_backend_notifications, user_required_blueprint) diff --git a/application/single_app/background_tasks.py b/application/single_app/background_tasks.py index dcf3ed3b0..13966697b 100644 --- a/application/single_app/background_tasks.py +++ b/application/single_app/background_tasks.py @@ -44,6 +44,7 @@ check_due_key_vault_secret_reminders_once, ) from functions_tabular_generated_exports import check_due_tabular_generated_output_runs_once +from functions_support_exports import check_due_support_exports_once from functions_personal_workflows import ( compute_next_run_at, get_due_personal_workflows, @@ -973,6 +974,20 @@ def run_data_management_scheduler_loop(app=None): time.sleep(60) +def run_support_export_scheduler_loop(app=None): + """Run durable, fenced exports in web workers or the dedicated scheduler.""" + while True: + try: + with app.app_context() if app is not None else nullcontext(): + check_due_support_exports_once() + except Exception as exc: + log_event( + "[SUPPORT_EXPORT] Scheduler slice failed.", + extra={"error_type": type(exc).__name__}, level=logging.ERROR, + ) + time.sleep(15) + + def run_document_search_metadata_sync_loop(): """Retry search chunk metadata syncs that failed or never ran in their original process.""" while True: @@ -1108,6 +1123,7 @@ def start_background_task_threads(app=None): ('Orchestration scheduler background task started.', lambda: run_orchestration_scheduler_loop(app=app)), ('Workflow chat delivery background task started.', lambda: run_workflow_chat_delivery_loop(app=app)), ('Data Management scheduler background task started.', lambda: run_data_management_scheduler_loop(app=app)), + ('Support export scheduler background task started.', lambda: run_support_export_scheduler_loop(app=app)), ('Document search metadata sync background task started.', run_document_search_metadata_sync_loop), ('Content screening scheduler background task started.', lambda: run_content_screening_scheduler_loop(app=app)), ('App maintenance background task started.', run_app_maintenance_loop), diff --git a/application/single_app/config.py b/application/single_app/config.py index 1960c5b23..e2cbaa229 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.425" +VERSION = "0.261.426" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform @@ -699,6 +699,13 @@ def _create_container_if_not_exists_with_conflict_recovery(*args, **kwargs): partition_key=PartitionKey(path="/user_id") ) +cosmos_support_exports_container_name = "support_exports" +cosmos_support_exports_container = cosmos_database.create_container_if_not_exists( + id=cosmos_support_exports_container_name, + partition_key=PartitionKey(path="/partition_key"), + default_ttl=-1, +) + cosmos_data_management_jobs_container_name = "data_management_jobs" DATA_MANAGEMENT_HISTORY_INDEXING_POLICY = { "indexingMode": "consistent", diff --git a/application/single_app/content_screening/model.py b/application/single_app/content_screening/model.py index 1f9b3d2df..2e10f550a 100644 --- a/application/single_app/content_screening/model.py +++ b/application/single_app/content_screening/model.py @@ -1060,13 +1060,18 @@ def _checkpoint_record(window, check, fingerprint, model_fingerprint, findings, def evaluate_model_units( units: list[ContentUnit], check: dict, *, settings=None, client=None, on_progress=None, - checkpoint_store=None, + checkpoint_store=None, token_ceiling=None, input_ceiling=None, binding_validator=None, ) -> DetectorResult: """Inspect every required window or return a safe error/incomplete result. Shared policy limits bound source size, windows, wall time, and findings. Output tokens, total tokens, response size, and per-request time have internal hard ceilings, not additional editable policy keys. + Trusted callers may lower ``token_ceiling`` and ``input_ceiling`` for an + operation. Conservative complete-window reservations are checked before + inference; defaults preserve the document scanner's existing budgets. + A trusted ``binding_validator`` may additionally require the freshly resolved + binding to match the host's disclosure before constructing an inference client. Missing/placeholder provider usage is charged a conservative request/output reservation, not reported as measured token usage. Progress never includes evidence or IDs. ``usage.coverage`` binds the canonical source/rule and lists only fully @@ -1117,6 +1122,13 @@ def evaluate_model_units( if checkpoint_store is not None: result.usage["checkpoint_windows"] = 0 try: + total_token_limit = _MODEL_MAX_TOTAL_TOKENS if token_ceiling is None else token_ceiling + if ( + type(total_token_limit) is not int or not 1 <= total_token_limit <= _MODEL_MAX_TOTAL_TOKENS + or (input_ceiling is not None and (type(input_ceiling) is not int or input_ceiling < 1)) + or (binding_validator is not None and not callable(binding_validator)) + ): + raise ScreeningValidationError() if isinstance(units, (list, tuple)): result.required_units = len(units) units = normalize_units(units) @@ -1155,10 +1167,18 @@ def evaluate_model_units( ) ) binding = _resolve_binding(settings, check["model_selection"]) + if binding_validator is not None and binding_validator(binding) is not True: + raise _ModelFailure("model_configuration_unavailable") client, deployment, protocol, behavior = _inference_configuration( binding, settings, client, deadline, result, on_progress, initialize_client=checkpoint_store is None, ) + if token_ceiling is not None or input_ceiling is not None: + reservations = [_parameters(window, check, binding, deployment, protocol, behavior)[1] for window in windows] + if sum(reservations) > total_token_limit: + raise _ModelFailure("model_token_limit", status="incomplete") + if input_ceiling is not None and any(reservation > input_ceiling for reservation in reservations): + raise _ModelFailure("model_input_limit", status="incomplete") checkpoint_context, model_fingerprint = ( _checkpoint_context( checkpoint_store, binding, check, check_metadata, planner.source_fingerprint, protocol, @@ -1184,13 +1204,13 @@ def evaluate_model_units( ) if cached is not None: findings, window_usage = cached - if result.usage["budgeted_tokens"] + window_usage["budgeted_tokens"] > _MODEL_MAX_TOTAL_TOKENS: + if result.usage["budgeted_tokens"] + window_usage["budgeted_tokens"] > total_token_limit: raise _ModelFailure("model_token_limit", status="incomplete") for name in _MODEL_USAGE_FIELDS: result.usage[name] += window_usage[name] result.usage["checkpoint_windows"] += 1 else: - if result.usage["budgeted_tokens"] + reservation > _MODEL_MAX_TOTAL_TOKENS: + if result.usage["budgeted_tokens"] + reservation > total_token_limit: raise _ModelFailure("model_token_limit", status="incomplete") _emit_progress(result, on_progress) if client is None: @@ -1209,7 +1229,7 @@ def evaluate_model_units( reported = _record_usage(response, result) if reported is not None: result.usage["budgeted_tokens"] += reported - reservation - if result.usage["budgeted_tokens"] > _MODEL_MAX_TOTAL_TOKENS: + if result.usage["budgeted_tokens"] > total_token_limit: raise _ModelFailure("model_token_limit", status="incomplete") content = _response_text(response, _MODEL_MAX_RESPONSE_CHARACTERS) result.usage["output_characters"] += len(content) diff --git a/application/single_app/functions_appinsights.py b/application/single_app/functions_appinsights.py index 1c023410a..08c5afab9 100644 --- a/application/single_app/functions_appinsights.py +++ b/application/single_app/functions_appinsights.py @@ -6,6 +6,8 @@ import re import sys import threading +from contextlib import contextmanager +from contextvars import ContextVar from datetime import date, datetime from typing import Any, Dict, Optional, Tuple @@ -20,6 +22,8 @@ _appinsights_logger = None _azure_monitor_configured = False _logging_settings_load_state = threading.local() +_diagnostic_context = ContextVar("simplechat_diagnostic_context", default=None) +_diagnostic_context_provider = None CONSOLE_ERROR_HANDLER_ATTRIBUTE = "_simplechat_console_error_handler" CONSOLE_ERROR_LOG_FORMAT = "[%(asctime)s] %(levelname)s in %(name)s: %(message)s" REDACTED_LOG_VALUE = "***REDACTED***" @@ -133,7 +137,7 @@ "taskname", }) LOGGER_WORKFLOW_HASH_KEYS = frozenset({ - "conversationidhash", "turnidhash", "runidhash", "stepidhash", "producerstepidhash", + "useridhash", "conversationidhash", "turnidhash", "runidhash", "stepidhash", "producerstepidhash", "documentidhash", "plannerbindinghash", "documentsethash", "consumerrunidhash", }) @@ -146,6 +150,41 @@ }) +def configure_diagnostic_context_provider(provider): + """The web owner supplies request context without importing Flask/bootstrap here.""" + global _diagnostic_context_provider + if not callable(provider): + raise TypeError("A diagnostic context provider must be callable.") + _diagnostic_context_provider = provider + + +@contextmanager +def diagnostic_log_context(*, user_id=None, conversation_id=None): + """Explicit, isolated correlation for non-request and background work.""" + context = dict(_diagnostic_context.get() or {}) + context.update({key: value for key, value in ( + ("user_id", user_id), ("conversation_id", conversation_id), + ) if value}) + token = _diagnostic_context.set(context) + try: + yield + finally: + _diagnostic_context.reset(token) + + +def _diagnostic_hashes(extra): + context = dict(_diagnostic_context.get() or {}) + if _diagnostic_context_provider is not None: + context.update(_diagnostic_context_provider()) + fields = {} + extra = extra if isinstance(extra, dict) else {} + for key in ("user_id", "conversation_id"): + value = context.get(key) or (extra or {}).get(key) + if isinstance(value, str) and 0 < len(value) <= 128: + fields[f"sc_{key}_hash"] = hashlib.sha256(value.encode("utf-8", errors="replace")).hexdigest() + return fields + + def workflow_log_context(*, conversation_id=None, turn_id=None, run_id=None, step_id=None): """Correlate workflow requests without recording their raw identifiers.""" return { @@ -360,6 +399,7 @@ def _build_logger_extra( else: logger_extra[normalized_key] = _logger_safe_scalar(value) + logger_extra.update(_diagnostic_hashes(extra)) return logger_extra @@ -581,7 +621,8 @@ def log_event( includeStack (bool, optional): If True, includes the current stack trace in the log. stacklevel (int, optional): How many levels up the stack to report as the source. exceptionTraceback (Any, optional): If set to True, includes exception traceback. - debug_only (bool, optional): If True, emit only debug-gated console output. + debug_only (bool, optional): If True, emit debug-gated console output and a + tagged Application Insights debug trace when the exporter is configured. category (str, optional): Category label used for debug-only console output. flush (bool, optional): Flush console output immediately for debug-only output. message_args (tuple, optional): Optional printf-style formatting arguments. @@ -593,6 +634,8 @@ def log_event( if debug_only: _emit_debug_message(cache, formatted_message, category, flush, safe_extra) + if cache.get('enable_debug_logging', False): + _emit_appinsights_debug_trace(formatted_message, category, safe_extra) return try: diff --git a/application/single_app/functions_support_exports.py b/application/single_app/functions_support_exports.py new file mode 100644 index 000000000..dcf894776 --- /dev/null +++ b/application/single_app/functions_support_exports.py @@ -0,0 +1,189 @@ +# functions_support_exports.py +"""Operation-time wiring for support exports; no resource initialization on import.""" + +import logging +import uuid + +from azure.identity import AzureAuthorityHosts, DefaultAzureCredential +from azure.core.exceptions import ResourceNotFoundError + +import config +from content_screening.access import public_history_messages +from content_screening.contracts import ScreeningConfigurationError +from content_screening.model import evaluate_model_units, validate_scanner_model_configuration +from cosmos_document_utils import normalize_cosmos_document +from functions_ai_connections import AIConnectionError, CHAT_CAPABILITY, resolve_capability_binding +from functions_appinsights import log_event +from functions_collaboration import assert_user_can_view_collaboration_conversation +from functions_model_capabilities import ModelTokenBudgetError, resolve_model_token_budget +from functions_notifications import create_notification +from functions_orchestration_attempts import exclude_superseded_orchestration_attempts +from functions_saved_analysis import sanitize_saved_analysis_messages +from functions_settings import get_settings +from support_export_contracts import SupportExportError, digest, fail, iso, normalize_configuration, utc_now +from support_export_service import SupportExportService +from support_export_sources import SupportExportSources, build_support_monitor_client +from support_export_store import SupportExportStore + + +def audit_support_export(operation, **details): + log_event( + "[SUPPORT_EXPORT] Support export state changed.", + extra={"operation": operation, **details}, + level=logging.ERROR if "error_type" in details else logging.INFO, + ) + actor = details.pop("actor", None) or "system" + config.cosmos_activity_logs_container.create_item(body={ + "id": uuid.uuid4().hex, "user_id": actor, "activity_type": "support_export", + "timestamp": iso(utc_now()), "operation": operation, "additional_context": details, + }) + + +def _binding_fingerprint(binding): + return digest({ + "selection": binding.selection, "provider": binding.selection["provider"], + "connection": binding.endpoint.get("connection", {}), "model": binding.model, + "operation_settings": binding.operation_settings, + }) + + +def _model_info(): + try: + response = config.cosmos_settings_container.read_item(item="app_settings", partition_key="app_settings") + settings = normalize_cosmos_document(response) + binding = resolve_capability_binding(settings, CHAT_CAPABILITY) + validate_scanner_model_configuration(settings, binding.selection) + output_limit = min(binding.model.get("responseLength") or 8192, 8192) + budget = resolve_model_token_budget( + binding.model, binding.endpoint, provider=binding.selection["provider"], request_output_limit=output_limit, + ) + allowance = budget.remaining_input() + if allowance < 1024: + fail("model_context_small", "The default text model has too little configured input context.", 503) + return { + "available": True, "label": binding.model.get("displayName") or binding.model.get("deploymentName") or binding.selection["model_id"], + "provider": binding.selection["provider"], "selection": dict(binding.selection), + "fingerprint": _binding_fingerprint(binding), "input_ceiling": allowance + output_limit, + "message": "Uses the configured default text model; only already-scrubbed data is sent.", + } + except (AIConnectionError, ModelTokenBudgetError, SupportExportError, ScreeningConfigurationError, ResourceNotFoundError) as exc: + log_event("[SUPPORT_EXPORT] Default privacy model is unavailable.", extra={"error_type": type(exc).__name__}, level=logging.WARNING) + return { + "available": False, + "message": "Configure a supported default text model in AI Connections, including its context and total-generation token limits.", + } + + +def _inspect_ai(units, selection, progress, token_budget, *, expected_fingerprint): + info = _model_info() + if not info.get("available") or info["selection"] != selection or info["fingerprint"] != expected_fingerprint: + fail("model_changed", "The default privacy model changed. Create a new export.", 409) + + def on_progress(_result): + if progress: + progress() + current = _model_info() + return current.get("fingerprint") == info["fingerprint"] + + return evaluate_model_units( + units, { + "id": "support-privacy", "enabled": True, + "model_selection": {key: selection[key] for key in ("endpoint_id", "model_id")}, + "instructions": ( + "Identify sensitive personal or confidential information remaining in these diagnostic fields, " + "including personal names, street addresses, credentials, and private business content. " + "Do not infer facts or alter diagnostic codes. Return grounded exact spans only." + ), + "severity": "high", "category": "sensitive_information", + "window_unit": "chunks", "window_size": 1, "max_characters": 12000, "overlap_characters": 500, + "limits": { + "max_units": 100000, "max_total_characters": 20000000, "max_windows": 100000, + "max_findings": 1000, "max_runtime_seconds": 300, + }, + }, + on_progress=on_progress, token_ceiling=token_budget, input_ceiling=info["input_ceiling"], + binding_validator=lambda binding: _binding_fingerprint(binding) == expected_fingerprint, + ) + + +def _notify(owner_id, job_id): + create_notification( + user_id=owner_id, notification_type="system_announcement", + title="A support export needs your approval", + message="An administrator requested whole personal conversation snapshots. Review the exact scope before approving.", + link_url="/approvals?support_export=" + job_id, + idempotency_key="support-export:" + job_id, strict=True, + ) + + +def _visible_messages(messages, owner_id): + active = [] + for message in messages: + metadata = message.get("metadata") or {} + if metadata.get("is_deleted") is True or (metadata.get("thread_info") or {}).get("active_thread") is False: + continue + active.append(message) + active = sanitize_saved_analysis_messages(active, owner_id) + active = public_history_messages(active, owner_id) + return exclude_superseded_orchestration_attempts(active) + + +def get_support_export_service(): + settings = get_settings() + blobs = None + blob_service = None + if settings.get("office_docs_storage_account_url") or settings.get("office_docs_storage_account_blob_endpoint"): + blob_service = config.build_enhanced_citations_blob_service_client(settings) + blobs = blob_service.get_container_client("support-exports") + store = SupportExportStore(config.cosmos_support_exports_container, blobs) + configuration = normalize_configuration(store.configuration() or { + "cloud": "public", "app_resource_id": "", "site_resource_id": "", + }) + monitor = None + credential = None + if configuration["app_resource_id"] or configuration["site_resource_id"]: + government = configuration.get("cloud") == "usgovernment" + authority = AzureAuthorityHosts.AZURE_GOVERNMENT if government else AzureAuthorityHosts.AZURE_PUBLIC_CLOUD + credential = DefaultAzureCredential(authority=authority) + monitor = build_support_monitor_client(configuration, credential) + sources = SupportExportSources({ + "users": config.cosmos_user_settings_container, + "conversations": config.cosmos_conversations_container, "messages": config.cosmos_messages_container, + "group_conversations": config.cosmos_group_conversations_container, "group_messages": config.cosmos_group_messages_container, + "collaboration": config.cosmos_collaboration_conversations_container, + "collaboration_messages": config.cosmos_collaboration_messages_container, + "collaboration_states": config.cosmos_collaboration_user_state_container, + "activity": config.cosmos_activity_logs_container, "file_processing": config.cosmos_file_processing_container, + }, monitor=monitor, authorize_shared=lambda user_id, document: assert_user_can_view_collaboration_conversation( + user_id, document, read_only=True, + ), filter_messages=_visible_messages) + def close_resources(): + try: + if monitor is not None: + monitor.close() + finally: + try: + if credential is not None: + credential.close() + finally: + if blob_service is not None: + blob_service.close() + + return SupportExportService( + store, sources, version=config.VERSION, model_info=_model_info, inspect_ai=_inspect_ai, + notify=_notify, audit=audit_support_export, + storage_binding=digest({"container": blobs.url if blobs is not None else ""}), + close_resources=close_resources, + ) + + +def check_due_support_exports_once(): + metadata = SupportExportStore(config.cosmos_support_exports_container, None) + jobs = metadata.query("SELECT TOP 1 c.id FROM c WHERE c.record_kind = 'export'", [], limit=1) + if not jobs and metadata.read("blob-cleanup", "configuration") is None: + return + service = get_support_export_service() + try: + service.tick() + finally: + service.close() diff --git a/application/single_app/route_backend_support_exports.py b/application/single_app/route_backend_support_exports.py new file mode 100644 index 000000000..31d8e295b --- /dev/null +++ b/application/single_app/route_backend_support_exports.py @@ -0,0 +1,202 @@ +# route_backend_support_exports.py +"""Separate administrator collection and subject-owner consent API surfaces.""" + +import io +import logging + +from flask import jsonify, request, send_file, session + +from functions_appinsights import log_event +from functions_authentication import admin_required, get_current_user_id, login_required, user_required +from support_export_contracts import SupportExportError, fail +from support_export_service import require_admin +from swagger_wrapper import get_auth_security, swagger_route + + +def _service(factory): + if factory is not None: + return factory() + # Storage/settings wiring belongs to the operation, not route-module import. + from functions_support_exports import get_support_export_service + + return get_support_export_service() + + +def _roles(): + user = session.get("user") + roles = user.get("roles") if isinstance(user, dict) else None + return roles if isinstance(roles, list) else [] + + +def _body(): + body = request.get_json(silent=True) + if not isinstance(body, dict): + fail("invalid_request", "A JSON request object is required.") + return body + + +def _respond(factory, operation): + try: + if request.path.startswith("/api/admin/support-exports"): + require_admin(get_current_user_id(), _roles()) + service = _service(factory) + try: + result = operation(service) + finally: + service.close() + response = result if hasattr(result, "headers") else jsonify(result) + except SupportExportError as exc: + response = jsonify({"error": exc.public_message, "code": exc.code}) + response.status_code = exc.status + except Exception as exc: + log_event("[SUPPORT_EXPORT] Support export request failed.", extra={"error_type": type(exc).__name__}, level=logging.ERROR) + response = jsonify({"error": "Support export could not complete. Check configuration and application logs.", "code": "support_export_failed"}) + response.status_code = 503 + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + return response + + +def register_route_backend_support_exports(bp, service_factory=None): + @bp.route("/api/admin/support-exports/readiness", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_readiness(): + return _respond(service_factory, lambda service: service.readiness()) + + @bp.route("/api/admin/support-exports/configuration", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_configuration(): + return _respond(service_factory, lambda service: service.configure(get_current_user_id(), _roles(), _body())) + + @bp.route("/api/admin/support-exports/initialize-storage", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_initialize(): + def initialize(service): + if _body() != {"confirmed": True}: + fail("confirmation_required", "Confirm initialization of the private support-export container.") + return service.initialize_storage(get_current_user_id(), _roles()) + return _respond(service_factory, initialize) + + @bp.route("/api/admin/support-exports/test-sources", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_test_sources(): + return _respond(service_factory, lambda service: service.test_sources(get_current_user_id(), _roles())) + + @bp.route("/api/admin/support-exports/users", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_users(): + return _respond(service_factory, lambda service: service.sources.users(request.args.get("q", ""), request.args.get("after", ""))) + + @bp.route("/api/admin/support-exports/conversations", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_conversations(): + return _respond(service_factory, lambda service: service.sources.conversations( + request.args.get("user_id"), after=request.args.get("after", ""), term=request.args.get("q", ""), + )) + + @bp.route("/api/admin/support-exports", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_list(): + return _respond(service_factory, lambda service: service.list_jobs(get_current_user_id(), _roles(), request.args.get("after", ""))) + + @bp.route("/api/admin/support-exports/review", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_review(): + return _respond(service_factory, lambda service: service.prepare(get_current_user_id(), _roles(), _body())) + + @bp.route("/api/admin/support-exports", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_create(): + def create(service): + body = _body() + if set(body) != {"request", "retry_key", "review_id"}: + fail("invalid_request", "An export request, current scope review, and retry key are required.") + if not isinstance(body["review_id"], str) or not body["review_id"]: + fail("review_required", "Review the export scope before generating.") + return service.create(get_current_user_id(), _roles(), body["request"], body["retry_key"], body["review_id"]) + return _respond(service_factory, create) + + @bp.route("/api/admin/support-exports/", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_status(job_id): + return _respond(service_factory, lambda service: service.status(get_current_user_id(), _roles(), job_id)) + + @bp.route("/api/admin/support-exports//cancel", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_cancel(job_id): + return _respond(service_factory, lambda service: service.cancel(get_current_user_id(), _roles(), job_id)) + + @bp.route("/api/admin/support-exports//preview", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_preview(job_id): + return _respond(service_factory, lambda service: service.preview(get_current_user_id(), _roles(), job_id)) + + @bp.route("/api/admin/support-exports//download", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def support_export_download(job_id): + def download(service): + data = service.download(get_current_user_id(), _roles(), job_id) + return send_file(io.BytesIO(data), mimetype="application/zip", as_attachment=True, + download_name=f"simplechat-support-{job_id}.zip", max_age=0) + return _respond(service_factory, download) + + +def register_route_backend_support_export_consents(bp, service_factory=None): + @bp.route("/api/support-export-consents", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def support_export_consent_list(): + return _respond(service_factory, lambda service: service.list_consents(get_current_user_id(), request.args.get("after", ""))) + + @bp.route("/api/support-export-consents/", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def support_export_consent_details(consent_id): + return _respond(service_factory, lambda service: service.consent_details(get_current_user_id(), consent_id)) + + @bp.route("/api/support-export-consents//preview", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def support_export_consent_preview(consent_id): + return _respond(service_factory, lambda service: service.consent_preview(get_current_user_id(), consent_id)) + + @bp.route("/api/support-export-consents//decision", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def support_export_consent_decision(consent_id): + def decide(service): + body = _body() + if set(body) - {"action", "fingerprint"}: + fail("invalid_request", "Unsupported consent decision.") + return service.decide(get_current_user_id(), consent_id, body.get("action"), body.get("fingerprint")) + return _respond(service_factory, decide) diff --git a/application/single_app/static/js/support-exports.js b/application/single_app/static/js/support-exports.js new file mode 100644 index 000000000..615fe6d06 --- /dev/null +++ b/application/single_app/static/js/support-exports.js @@ -0,0 +1,424 @@ +// support-exports.js + +const exportRoot = '/api/admin/support-exports'; +const consentRoot = '/api/support-export-consents'; +const sourceLabels = { + telemetry: 'Application Insights telemetry', + activity: 'Activity and audit logs', + file_processing: 'File-processing diagnostics', + console: 'App Service console logs' +}; +const activeStates = new Set(['waiting_for_consent', 'queued', 'collecting', 'redacting', 'ai_review']); + +function node(tag, text = '', className = '') { + const element = document.createElement(tag); + element.textContent = text; + element.className = className; + return element; +} + +function button(text, action, className = 'btn btn-outline-secondary') { + const element = node('button', text, className); + element.type = 'button'; + element.addEventListener('click', action); + return element; +} + +async function api(path, body) { + const response = await fetch(path, { + method: body === undefined ? 'GET' : 'POST', + credentials: 'same-origin', cache: 'no-store', + headers: { Accept: 'application/json', ...(body === undefined ? {} : { 'Content-Type': 'application/json' }) }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + if (!response.headers.get('content-type')?.includes('application/json')) { + throw new Error('Sign in again and refresh the support export page.'); + } + const result = await response.json(); + if (!response.ok) throw new Error(typeof result.error === 'string' ? result.error : 'The support export operation failed.'); + return result; +} + +function localTime(date) { + const pad = (value) => String(value).padStart(2, '0'); + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}`; +} + +function utc(value) { + const date = new Date(value); + if (!value || Number.isNaN(date.getTime()) || localTime(date) !== value.slice(0, 16)) { + throw new Error('Choose a valid local date and time; daylight-saving gaps are not valid times.'); + } + for (const shift of [-86400000, 86400000]) { + const offset = new Date(date.getTime() + shift).getTimezoneOffset() - date.getTimezoneOffset(); + if (offset && localTime(new Date(date.getTime() + offset * 60000)) === value.slice(0, 16)) { + throw new Error('This local time repeats during a clock change. Choose unambiguous endpoints spanning the change.'); + } + } + return date.toISOString(); +} + +async function downloadZip(id) { + const response = await fetch(`${exportRoot}/${encodeURIComponent(id)}/download`, { credentials: 'same-origin', cache: 'no-store' }); + if (!response.ok || !response.headers.get('content-type')?.includes('application/zip')) { + const problem = response.headers.get('content-type')?.includes('application/json') ? await response.json() : null; + throw new Error(typeof problem?.error === 'string' ? problem.error : 'The ZIP could not be downloaded. Refresh its status.'); + } + const url = URL.createObjectURL(await response.blob()); + const link = document.createElement('a'); + link.href = url; + link.download = 'simplechat-support.zip'; + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 1000); +} + +function initializeAdmin(root) { + const get = (name) => root.querySelector(`#support-export-${name}`); + const state = { + ready: null, busy: false, people: [], peopleNext: null, conversations: [], conversationNext: null, + selected: new Map(), jobs: [], jobsNext: null, job: null, review: null, preview: null, poll: true + }; + let conversationController = null; + const show = (name, visible) => get(name).classList.toggle('d-none', !visible); + const error = (failure) => { + get('error').textContent = failure instanceof Error ? failure.message : 'Support export failed.'; + show('error', true); + }; + const updateControls = () => { + get('fields').disabled = state.busy || state.review !== null; + get('review').disabled = state.busy || !state.ready?.storage.available; + get('generate').disabled = state.busy || !get('review-ack').checked; + get('job').disabled = state.busy; + get('download').disabled = state.busy || !get('download-ack').checked || !state.job?.can_download; + get('transcripts').disabled = get('all').checked || state.selected.size === 0 + || [...state.selected.values()].some((item) => !item.transcript_eligible); + const conversationMode = get('mode').value === 'conversations'; + show('person-fields', !get('all').checked); + show('margins', conversationMode); + show('time-fields', !conversationMode || get('narrow').checked); + show('conversation-fields', !get('all').checked && !!get('user').value); + show('ai-notice', get('privacy').value === 'ai'); + }; + const act = async (operation) => { + if (state.busy) return; + state.busy = true; show('error', false); root.setAttribute('aria-busy', 'true'); + root.querySelectorAll('button').forEach((item) => { item.disabled = true; }); + updateControls(); + try { await operation(); } catch (failure) { error(failure); } finally { + state.busy = false; root.setAttribute('aria-busy', 'false'); + root.querySelectorAll('button').forEach((item) => { item.disabled = false; }); + updateControls(); + } + }; + const renderPeople = () => { + const selected = get('user').value; + get('user').replaceChildren(new Option('Select a user', '')); + for (const person of state.people) get('user').appendChild(new Option(`${person.name}${person.email ? ` (${person.email})` : ''}`, person.id)); + get('user').value = state.people.some((item) => item.id === selected) ? selected : ''; + show('more-users', !!state.peopleNext); + }; + const loadPeople = async (more = false) => { + const query = new URLSearchParams({ q: get('user-search').value, ...(more ? { after: state.peopleNext } : {}) }); + const page = await api(`${exportRoot}/users?${query}`); + state.people = more ? [...state.people, ...page.items.filter((item) => !state.people.some((other) => other.id === item.id))] : page.items; + state.peopleNext = page.next; renderPeople(); + }; + const renderConversations = () => { + const list = get('conversations'); + list.replaceChildren(); + for (const item of state.conversations) { + const label = node('label', '', 'd-flex align-items-start gap-2 small mb-2 text-break'); + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; checkbox.className = 'form-check-input flex-shrink-0'; + checkbox.checked = state.selected.has(item.id); + checkbox.disabled = state.selected.size >= 20 && !checkbox.checked; + checkbox.addEventListener('change', () => { + if (checkbox.checked) state.selected.set(item.id, item); else state.selected.delete(item.id); + if ([...state.selected.values()].some((other) => !other.transcript_eligible)) get('transcripts').checked = false; + renderConversations(); updateControls(); + }); + label.append(checkbox, node('span', `${item.id} (${item.kind}; last activity ${item.last_activity || 'unknown'})`)); + list.appendChild(label); + } + if (!state.conversations.length) list.appendChild(node('p', 'No conversations match this user and ID search.', 'small text-muted')); + get('selected').replaceChildren(node('p', `Selected: ${state.selected.size}/20`)); + for (const item of state.selected.values()) { + get('selected').appendChild(button(`Remove ${item.id}`, () => { + state.selected.delete(item.id); renderConversations(); updateControls(); + }, 'btn btn-sm btn-outline-secondary me-1 mb-1 text-break')); + } + show('more-conversations', !!state.conversationNext); + }; + const loadConversations = async (more = false) => { + conversationController?.abort(); + conversationController = new AbortController(); + const user = get('user').value; + if (!user || get('all').checked) return; + const query = new URLSearchParams({ + user_id: user, q: get('conversation-search').value, + ...(more ? { after: state.conversationNext } : {}) + }); + const response = await fetch(`${exportRoot}/conversations?${query}`, { + credentials: 'same-origin', cache: 'no-store', signal: conversationController.signal, + headers: { Accept: 'application/json' } + }); + if (!response.headers.get('content-type')?.includes('application/json')) throw new Error('Sign in again before loading conversations.'); + const page = await response.json(); + if (!response.ok) throw new Error(page.error || 'Conversations could not be loaded.'); + if (get('user').value !== user || get('all').checked) return; + state.conversations = more ? [...state.conversations, ...page.items.filter((item) => !state.conversations.some((other) => other.id === item.id))] : page.items; + state.conversationNext = page.next; renderConversations(); + }; + const resetUser = () => { + conversationController?.abort(); state.selected.clear(); state.conversations = []; state.conversationNext = null; + get('transcripts').checked = false; get('conversation-search').value = ''; + if (get('all').checked) { get('user').value = ''; get('mode').value = 'time'; } + get('mode').querySelector('option[value="conversations"]').disabled = get('all').checked; + renderConversations(); updateControls(); + }; + const renderReady = (ready) => { + state.ready = ready; + get('cloud').value = ready.configuration.cloud; + get('app-resource').value = ready.configuration.app_resource_id; + get('site-resource').value = ready.configuration.site_resource_id; + get('storage-status').textContent = ready.storage.message; + get('model').textContent = `${ready.model.message || ''}${ready.model.label ? ` Default model: ${ready.model.label} (${ready.model.provider}).` : ''}`; + get('privacy').querySelector('option[value="ai"]').disabled = !ready.model.available; + for (const [source, status] of Object.entries(ready.sources)) { + root.querySelector(`[data-source-note="${source}"]`).textContent = `${status.message}${status.access ? ` Access: ${status.access}.` : ''}`; + } + updateControls(); + }; + const renderJob = () => { + const job = state.job; + get('job-status').replaceChildren(); + get('job-actions').replaceChildren(); + show('preview', state.preview !== null); + show('download-review', state.preview !== null); + show('download', state.preview !== null && !!job?.can_download); + if (!job) { get('job-status').textContent = 'No support exports requested by this administrator.'; return; } + get('job-status').dataset.state = job.status; + get('job-status').appendChild(node('p', `${job.phase} (${job.status}). Expires ${job.expires_at}. Only you can download this export.`, 'small text-break')); + if (job.error) get('job-status').appendChild(node('p', job.error.message, 'alert alert-danger')); + get('job-actions').appendChild(button('Refresh export status', () => act(async () => selectJob(await api(`${exportRoot}/${encodeURIComponent(job.id)}`))))); + if (activeStates.has(job.status) || job.status === 'ready') { + get('job-actions').appendChild(button('Cancel export', () => act(async () => selectJob(await api(`${exportRoot}/${encodeURIComponent(job.id)}/cancel`, {}))), 'btn btn-outline-danger')); + } + if (job.can_download) { + get('job-actions').appendChild(button('Review redacted preview', () => act(async () => { + state.preview = await api(`${exportRoot}/${encodeURIComponent(job.id)}/preview`); + get('preview').textContent = JSON.stringify(state.preview, null, 2); + get('download-ack').checked = false; renderJob(); + }))); + } + updateControls(); + }; + const renderJobs = () => { + get('job').replaceChildren(new Option('Select an export', '')); + for (const item of state.jobs) get('job').appendChild(new Option(`${item.created_at} - ${item.status} - ${item.id}`, item.id)); + get('job').value = state.job?.id || ''; show('more-jobs', !!state.jobsNext); + }; + const selectJob = (job) => { + state.job = job; state.preview = null; state.poll = true; get('download-ack').checked = false; + state.jobs = [job, ...state.jobs.filter((item) => item.id !== job.id)]; + renderJobs(); renderJob(); + }; + const loadJobs = async (more = false) => { + const page = await api(`${exportRoot}${more ? `?${new URLSearchParams({ after: state.jobsNext })}` : ''}`); + state.jobs = more ? [...state.jobs, ...page.items.filter((item) => !state.jobs.some((other) => other.id === item.id))] : page.items; + state.jobsNext = page.next; + if (!state.job && state.jobs.length) { + const ordered = [...state.jobs].sort((a, b) => b.created_at.localeCompare(a.created_at)); + state.job = ordered.find((item) => activeStates.has(item.status)) || ordered[0]; + } + renderJobs(); renderJob(); + }; + for (const [source, labelText] of Object.entries(sourceLabels)) { + const label = node('label', '', 'd-flex align-items-start gap-2 mb-2 small'); + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; checkbox.className = 'form-check-input flex-shrink-0'; + checkbox.dataset.source = source; checkbox.checked = ['telemetry', 'activity'].includes(source); + const text = node('span', labelText); + const note = node('span', 'Checking source readiness...', 'd-block text-muted'); + note.dataset.sourceNote = source; text.appendChild(note); label.append(checkbox, text); get('sources').appendChild(label); + } + const now = new Date(); + get('end').value = localTime(now); get('start').value = localTime(new Date(now.getTime() - 3600000)); + get('timezone').textContent = `Input timezone: ${Intl.DateTimeFormat().resolvedOptions().timeZone}. Review shows exact UTC intervals. Maximum effective span: 7 days.`; + get('all').addEventListener('change', resetUser); + get('user').addEventListener('change', () => { resetUser(); act(loadConversations); }); + get('user-search').addEventListener('input', () => { get('user').value = ''; resetUser(); }); + get('search-users').addEventListener('click', () => act(() => loadPeople())); + get('more-users').addEventListener('click', () => act(() => loadPeople(true))); + get('search-conversations').addEventListener('click', () => act(() => loadConversations())); + get('more-conversations').addEventListener('click', () => act(() => loadConversations(true))); + get('more-jobs').addEventListener('click', () => act(() => loadJobs(true))); + for (const name of ['mode', 'narrow', 'privacy', 'review-ack', 'download-ack']) get(name).addEventListener('change', updateControls); + get('save-config').addEventListener('click', () => act(async () => { + await api(`${exportRoot}/configuration`, { + cloud: get('cloud').value, + app_resource_id: get('app-resource').value.trim(), site_resource_id: get('site-resource').value.trim() + }); + state.review = null; show('review-panel', false); renderReady(await api(`${exportRoot}/readiness`)); + })); + get('initialize').addEventListener('click', () => act(async () => renderReady(await api(`${exportRoot}/initialize-storage`, { confirmed: true })))); + get('test').addEventListener('click', () => act(async () => renderReady(await api(`${exportRoot}/test-sources`, {})))); + get('review').addEventListener('click', () => act(async () => { + const request = { + all_users: get('all').checked, user_id: get('user').value, + conversation_ids: [...state.selected.keys()], mode: get('mode').value, + before_minutes: Number(get('before').value), after_minutes: Number(get('after').value), + sources: [...root.querySelectorAll('[data-source]:checked')].map((item) => item.dataset.source), + include_system_context: get('context').checked, include_transcripts: get('transcripts').checked, + privacy: get('privacy').value, purpose: get('purpose').value + }; + if (request.mode === 'time' || get('narrow').checked) { request.start = utc(get('start').value); request.end = utc(get('end').value); } + const scope = await api(`${exportRoot}/review`, request); + state.review = { scope, request }; + get('review-details').replaceChildren(); + for (const text of [ + scope.request.all_users ? 'All users' : `User: ${scope.request.user_id}`, + ...scope.intervals.map(([start, end]) => `UTC: ${start} through ${end} (end exclusive)`), + `Sources: ${scope.request.sources.map((source) => sourceLabels[source]).join(', ')}`, + scope.request.include_transcripts ? 'Whole personal-conversation snapshots will wait for owner consent.' : 'Logs only; no transcripts.', + `Privacy: ${scope.request.privacy}${scope.model_label ? ` (${scope.model_label})` : ''}. Review expires ${scope.expires_at}.` + ]) get('review-details').appendChild(node('p', text)); + get('review-ack').checked = false; show('review-panel', true); get('review-heading').focus(); + })); + get('edit').addEventListener('click', () => { state.review = null; show('review-panel', false); updateControls(); }); + get('generate').addEventListener('click', () => act(async () => { + const review = state.review; + if (!review || !get('review-ack').checked) throw new Error('Review and acknowledge the exact scope first.'); + const job = await api(exportRoot, { request: review.request, retry_key: review.scope.review_id, review_id: review.scope.review_id }); + state.review = null; show('review-panel', false); selectJob(job); + })); + get('job').addEventListener('change', () => { + const selected = state.jobs.find((item) => item.id === get('job').value); + if (selected) selectJob(selected); + else { state.job = null; state.preview = null; renderJob(); updateControls(); } + }); + get('download').addEventListener('click', () => act(async () => { + if (!get('download-ack').checked || !state.job?.can_download) throw new Error('Review the scrubbed preview before downloading.'); + await downloadZip(state.job.id); + })); + root.addEventListener('keydown', (event) => { + if (event.key === 'Enter' && event.target instanceof HTMLInputElement) { + event.preventDefault(); + if (event.target === get('user-search')) get('search-users').click(); + if (event.target === get('conversation-search')) get('search-conversations').click(); + } + }); + const timer = setInterval(async () => { + if (state.busy || !state.poll || !state.job || !activeStates.has(state.job.status)) return; + const id = state.job.id; + try { + const job = await api(`${exportRoot}/${encodeURIComponent(id)}`); + if (state.job?.id === id) { state.job = job; state.jobs = state.jobs.map((item) => item.id === id ? job : item); renderJobs(); renderJob(); } + } catch (failure) { state.poll = false; error(failure); } + }, 5000); + window.addEventListener('pagehide', () => { clearInterval(timer); conversationController?.abort(); }, { once: true }); + act(async () => { await Promise.all([api(`${exportRoot}/readiness`).then(renderReady), loadPeople(), loadJobs()]); }); +} + +function initializeConsents(root) { + const list = root.querySelector('[data-support-consent-list]'); + const detail = root.querySelector('[data-support-consent-detail]'); + const errorNode = root.querySelector('[data-support-consent-error]'); + const refresh = root.querySelector('[data-support-consent-refresh]'); + const more = root.querySelector('[data-support-consent-more]'); + let items = []; + let next = null; + let selected = null; + let fingerprint = ''; + let busy = false; + let approveButton = null; + let acknowledgementInput = null; + const act = async (operation) => { + if (busy) return; + busy = true; errorNode.classList.add('d-none'); + root.querySelectorAll('button').forEach((item) => { item.disabled = true; }); + try { await operation(); } catch (failure) { + errorNode.textContent = failure instanceof Error ? failure.message : 'Support-data consent failed.'; + errorNode.classList.remove('d-none'); + } finally { + busy = false; + root.querySelectorAll('button').forEach((item) => { item.disabled = false; }); + if (approveButton) approveButton.disabled = !fingerprint || !acknowledgementInput?.checked; + if (acknowledgementInput) acknowledgementInput.disabled = !fingerprint; + } + }; + const renderDetail = () => { + detail.replaceChildren(); fingerprint = ''; approveButton = null; acknowledgementInput = null; + if (!selected) { detail.appendChild(node('p', 'Choose a support-data request to review its scope.', 'text-muted')); return; } + const consent = selected; + detail.appendChild(node('h4', 'Personal conversation data release', 'h6')); + detail.appendChild(node('p', consent.purpose, 'text-break')); + detail.appendChild(node('p', consent.scope_notice, 'alert alert-warning')); + for (const text of [ + `Status: ${consent.status}. Requesting administrator: ${consent.requester_id}`, + `Whole snapshot cutoff: ${consent.source_cutoff}. Expires: ${consent.expires_at}`, + `Privacy: ${consent.privacy}${consent.model_label ? ` (${consent.model_label})` : ''}`, + ...consent.conversations.map((item) => `Personal conversation: ${item.id} (${item.message_count ?? 'unknown'} recorded messages)`) + ]) detail.appendChild(node('p', text, 'small text-break')); + if (consent.privacy === 'ai') detail.appendChild(node('p', 'Approving also allows already-scrubbed snapshots to be sent to the default model shown above. Privacy checking is not a guarantee of anonymity.', 'alert alert-warning')); + const decide = async (action) => { + const response = await api(`${consentRoot}/${encodeURIComponent(consent.id)}/decision`, { action, ...(action === 'approve' ? { fingerprint } : {}) }); + items = items.map((item) => item.id === response.id ? response : item); selected = response; render(); + }; + if (consent.status === 'pending' && !consent.expired) { + const preview = node('pre', '', 'bg-body-secondary border rounded p-3 overflow-auto text-break d-none'); + preview.style.maxHeight = '20rem'; preview.style.whiteSpace = 'pre-wrap'; + const acknowledgement = document.createElement('input'); + acknowledgement.type = 'checkbox'; acknowledgement.className = 'form-check-input me-2'; acknowledgement.disabled = true; + const approve = button('Approve whole-conversation snapshots', () => act(async () => { + if (!fingerprint || !acknowledgement.checked) throw new Error('Review the snapshot scope and explicitly approve it first.'); + await decide('approve'); + }), 'btn btn-primary'); + approveButton = approve; acknowledgementInput = acknowledgement; + approve.disabled = true; + const label = node('label', '', 'd-block small my-3'); + label.append(acknowledgement, document.createTextNode('I approve releasing the whole requested snapshots to this administrator, including selected privacy processing. Downloaded copies cannot be recalled.')); + acknowledgement.addEventListener('change', () => { approve.disabled = !acknowledgement.checked || !fingerprint; }); + detail.appendChild(button('Review my snapshot preview', () => act(async () => { + const result = await api(`${consentRoot}/${encodeURIComponent(consent.id)}/preview`); + fingerprint = result.fingerprint; preview.textContent = JSON.stringify(result.preview, null, 2); + preview.classList.remove('d-none'); acknowledgement.disabled = false; acknowledgement.checked = false; + approve.disabled = true; + }))); + detail.append(preview, node('p', 'Preview shows at most 10 messages per conversation; approval covers the whole snapshot through the cutoff, not just this preview.', 'small text-muted'), label); + const actions = node('div', '', 'd-flex flex-wrap gap-2'); + actions.append(approve, button('Decline transcript release', () => act(() => decide('decline')), 'btn btn-outline-danger')); + detail.appendChild(actions); + } + if (consent.status === 'approved' && !consent.expired) detail.appendChild(button('Revoke further transcript downloads', () => act(() => decide('revoke')), 'btn btn-outline-danger')); + }; + const render = () => { + list.replaceChildren(); + for (const item of items) list.appendChild(button(`${item.purpose} (${item.status})`, () => { + selected = item; renderDetail(); + }, 'btn btn-outline-secondary text-start text-break mb-2 w-100')); + if (!items.length) list.appendChild(node('p', 'No support-data consent requests.', 'text-muted')); + more.classList.toggle('d-none', !next); renderDetail(); + }; + const load = async (append = false) => { + const page = await api(`${consentRoot}${append ? `?${new URLSearchParams({ after: next })}` : ''}`); + items = append ? [...items, ...page.items.filter((item) => !items.some((other) => other.id === item.id))] : page.items; + next = page.next; + const linked = new URLSearchParams(window.location.search).get('support_export'); + if (linked && !selected) { + const id = 'consent-' + linked; + selected = items.find((item) => item.id === id) || await api(`${consentRoot}/${encodeURIComponent(id)}`); + if (!items.some((item) => item.id === selected.id)) items.unshift(selected); + } else if (selected) selected = items.find((item) => item.id === selected.id) || selected; + render(); + if (linked) root.scrollIntoView({ block: 'start' }); + }; + refresh.addEventListener('click', () => act(() => load())); + more.addEventListener('click', () => act(() => load(true))); + act(() => load()); +} + +document.querySelectorAll('[data-support-exports="admin"]').forEach(initializeAdmin); +document.querySelectorAll('[data-support-exports="consent"]').forEach(initializeConsents); diff --git a/application/single_app/support_export_contracts.py b/application/single_app/support_export_contracts.py new file mode 100644 index 000000000..665c0e9f1 --- /dev/null +++ b/application/single_app/support_export_contracts.py @@ -0,0 +1,196 @@ +# support_export_contracts.py +"""Bootstrap-free, bounded contracts for administrator support bundles.""" + +import hashlib +import json +import re +from datetime import datetime, timedelta, timezone + + +SCHEMA_VERSION = 1 +MAX_RECORDS = 100000 +MAX_BYTES = 50 * 1024 * 1024 +MAX_CONVERSATIONS = 20 +MAX_DAYS = 7 +AI_TOKEN_BUDGET = 200000 +EXPIRY_SECONDS = 24 * 60 * 60 +SOURCES = ("telemetry", "activity", "file_processing", "console") +ACTIVE_STATES = ("waiting_for_consent", "queued", "collecting", "redacting", "ai_review") +RUNNING_STATES = ("collecting", "redacting", "ai_review") +IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z") +PUBLIC_LIMITS = { + "max_days": MAX_DAYS, "max_conversations": MAX_CONVERSATIONS, + "max_records": MAX_RECORDS, "max_bytes": MAX_BYTES, + "ai_input_tokens": AI_TOKEN_BUDGET, "expiry_seconds": EXPIRY_SECONDS, +} + + +class SupportExportError(Exception): + """A stable error safe to show without including provider/source content.""" + + def __init__(self, code, message, status=400): + super().__init__(message) + self.code = code + self.public_message = message + self.status = status + + +def fail(code, message, status=400): + raise SupportExportError(code, message, status) + + +def utc_now(): + return datetime.now(timezone.utc) + + +def timestamp(value, *, stored=False): + if not isinstance(value, str) or len(value) > 64: + fail("invalid_time", "Use a valid timestamp with an explicit timezone.") + if ":" not in value: + fail("invalid_time", "A calendar date alone is not a reliable diagnostic timestamp.") + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + fail("invalid_time", "Use a valid timestamp with an explicit timezone.") + if result.tzinfo is None: + if not stored: + fail("invalid_time", "Use a valid timestamp with an explicit timezone.") + result = result.replace(tzinfo=timezone.utc) + return result.astimezone(timezone.utc) + + +def iso(value): + return value.astimezone(timezone.utc).isoformat() + + +def identifier(value): + if not isinstance(value, str) or not IDENTIFIER.fullmatch(value): + fail("invalid_identifier", "A valid user or conversation ID is required.") + return value + + +def digest(value): + encoded = json.dumps(value, sort_keys=True, ensure_ascii=True, allow_nan=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def identity_hash(value): + return hashlib.sha256(identifier(value).encode("utf-8")).hexdigest() + + +def normalize_configuration(value): + allowed = {"cloud", "app_resource_id", "site_resource_id"} + if not isinstance(value, dict) or set(value) - allowed: + fail("invalid_configuration", "Unsupported support-export configuration.") + result = {key: value.get(key, "") for key in allowed} + if result["cloud"] not in ("public", "usgovernment"): + fail("invalid_cloud", "Choose Azure Commercial or Azure US Government.") + if any(not isinstance(item, str) or len(item) > 512 for item in result.values()): + fail("invalid_configuration", "Invalid support-export configuration.") + prefix = r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]+/providers/" + for key, resource in ( + ("app_resource_id", r"Microsoft\.Insights/components/"), + ("site_resource_id", r"Microsoft\.Web/sites/"), + ): + result[key] = result[key].strip() + if result[key] and not re.fullmatch(prefix + resource + r"[A-Za-z0-9_.()-]+", result[key], re.I): + fail("invalid_resource", "Use a complete Application Insights or App Service resource ID.") + return result + + +def normalize_request(value, now=None): + allowed = { + "all_users", "user_id", "conversation_ids", "mode", "start", "end", + "before_minutes", "after_minutes", "sources", "include_system_context", + "include_transcripts", "privacy", "purpose", + } + if not isinstance(value, dict) or set(value) - allowed: + fail("invalid_request", "Unsupported support-export request.") + now = now or utc_now() + result = {} + for key in ("all_users", "include_system_context", "include_transcripts"): + item = value.get(key, False) + if type(item) is not bool: + fail("invalid_request", "Export switches must be boolean values.") + result[key] = item + result["user_id"] = "" if result["all_users"] else identifier(value.get("user_id")) + conversations = value.get("conversation_ids", []) + if not isinstance(conversations, list) or len(conversations) > MAX_CONVERSATIONS: + fail("conversation_limit", "Select at most 20 conversations.") + result["conversation_ids"] = sorted({identifier(item) for item in conversations}) + result["mode"] = value.get("mode", "time") + if result["mode"] not in ("time", "conversations"): + fail("invalid_mode", "Choose time-based or conversation-based collection.") + if result["all_users"] and (conversations or result["include_transcripts"]): + fail("invalid_scope", "Select one user before selecting conversations or transcripts.") + if (result["mode"] == "conversations" or result["include_transcripts"]) and not conversations: + fail("conversation_required", "Select at least one conversation.") + for key in ("before_minutes", "after_minutes"): + item = value.get(key, 15) + if type(item) is not int or not 0 <= item <= 1440: + fail("invalid_margin", "Before/after margins must be whole minutes from 0 to 1440.") + result[key] = item + result["start"] = result["end"] = None + if result["mode"] == "time" or value.get("start") or value.get("end"): + start = timestamp(value.get("start") or iso(now - timedelta(hours=1))) + end = timestamp(value.get("end") or iso(now)) + if end > now + timedelta(minutes=1) or start >= end or end - start > timedelta(days=MAX_DAYS): + fail("invalid_range", "Choose a past time range of at most 7 days, with start before end.") + result.update(start=iso(start), end=iso(min(end, now))) + selected_sources = value.get("sources", ["telemetry", "activity"]) + if ( + not isinstance(selected_sources, list) or not selected_sources + or any(item not in SOURCES for item in selected_sources) + ): + fail("invalid_sources", "Select at least one available log source.") + result["sources"] = sorted(set(selected_sources)) + if "console" in selected_sources and not (result["all_users"] or result["include_system_context"]): + fail("console_context_required", "Console logs require the nearby system context option for one-user exports.") + result["privacy"] = value.get("privacy", "deterministic") + if result["privacy"] not in ("deterministic", "ai"): + fail("invalid_privacy", "Choose deterministic or deterministic plus AI privacy checking.") + purpose = value.get("purpose", "Troubleshoot a SimpleChat issue.") + if not isinstance(purpose, str) or not purpose.strip() or len(purpose) > 500: + fail("invalid_purpose", "Provide a support purpose of at most 500 characters.") + result["purpose"] = purpose.strip() + return result + + +def resolve_intervals(plan, conversations, now=None): + cutoff = now or utc_now() + if plan["mode"] == "time": + return [[plan["start"], plan["end"]]] + windows = [] + for conversation in conversations: + first, last = conversation.get("first_activity"), conversation.get("last_activity") + if not first or not last: + fail("conversation_time_unknown", "A conversation has no reliable activity times. Use a time-based export.") + start = timestamp(first, stored=True) - timedelta(minutes=plan["before_minutes"]) + end = min(cutoff, timestamp(last, stored=True) + timedelta(minutes=plan["after_minutes"], seconds=1)) + if plan["start"]: + start = max(start, timestamp(plan["start"])) + end = min(end, timestamp(plan["end"])) + if start < end: + windows.append((start, end)) + if not windows: + fail("empty_intervals", "The selected conversations do not overlap the chosen time range.") + windows.sort() + if max(end for _, end in windows) - windows[0][0] > timedelta(days=MAX_DAYS): + fail("range_limit", "Conversation windows span more than 7 days. Narrow the explicit time range.") + merged = [] + for start, end in windows: + if merged and start <= merged[-1][1]: + merged[-1][1] = max(end, merged[-1][1]) + else: + merged.append([start, end]) + return [[iso(start), iso(end)] for start, end in merged] + + +def within_intervals(value, intervals): + instant = timestamp(value, stored=True) + return any(timestamp(start) <= instant < timestamp(end) for start, end in intervals) + + +def check_size(records, byte_count): + if records > MAX_RECORDS or byte_count > MAX_BYTES: + fail("bundle_limit", "The bundle exceeds 100,000 records or 50 MiB. Narrow its scope.", 413) diff --git a/application/single_app/support_export_privacy.py b/application/single_app/support_export_privacy.py new file mode 100644 index 000000000..1dea4a3e5 --- /dev/null +++ b/application/single_app/support_export_privacy.py @@ -0,0 +1,330 @@ +# support_export_privacy.py +"""Complete structured support-data redaction without settings or Azure bootstrap.""" + +import copy +import hashlib +import hmac +import ipaddress +import json +import re +import secrets +from collections import Counter +from urllib.parse import unquote + +import regex + +from content_screening.contracts import ContentUnit +from content_screening.deterministic import evaluate_deterministic_units +from content_screening.policies import STARTER_RULE_TEMPLATES +from functions_appinsights import AUTHORIZATION_VALUE_RE, SECRET_ASSIGNMENT_RE, _is_sensitive_log_key +from support_export_contracts import AI_TOKEN_BUDGET, check_size, fail + + +SECRET_TEXT = regex.compile( + r"""(?ix) + (?P["']?(?:api[-_]?key|accountkey|access[-_]?token|client[-_]?secret| + connection[-_]?string|password|passwd|pwd|secret|subscription[-_]?key|token| + authorization|cookie|sig|signature)["']?\s*[:=]\s*) + (?:"[^"]*"|'[^']*'|[^\s&,;]+) + """ +) +PRIVATE_KEY = regex.compile( + r"-----BEGIN (?:[A-Z ]{0,30})PRIVATE KEY-----.*?-----END (?:[A-Z ]{0,30})PRIVATE KEY-----", + regex.DOTALL, +) +TOKEN = regex.compile( + r"\b(?:gh[pousr]_[A-Za-z0-9]{30,255}|github_pat_[A-Za-z0-9_]{30,255}|" + r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b" +) +IP_CANDIDATE = regex.compile(r"(? 12000: + units.append(ContentUnit(str(len(units)), packed)) + locations.append(ranges) + packed, ranges = "", [] + start = len(packed) + len(header) + packed += header + text + ranges.append((start, len(packed), index)) + if packed: + units.append(ContentUnit(str(len(units)), packed)) + locations.append(ranges) + result = inspect(units, selection, progress, AI_TOKEN_BUDGET) + if result.status not in ("pass", "findings"): + fail("ai_incomplete", "AI privacy inspection did not complete. Narrow the export or create a deterministic-only export.", 422) + if result.completed_units != len(units) or result.completed_windows != result.required_windows: + fail("ai_incomplete", "AI privacy inspection did not cover the complete bundle.", 422) + by_unit = {} + for finding in result.findings: + try: + unit_index = int(finding.unit_id) + if not 0 <= unit_index < len(units): + raise IndexError() + text = units[unit_index].text + except (ValueError, IndexError): + fail("ai_invalid_span", "AI privacy inspection returned invalid references.", 422) + if ( + type(finding.start) is not int or type(finding.end) is not int + or not 0 <= finding.start < finding.end <= len(text) + or text[finding.start:finding.end] != finding.evidence + ): + fail("ai_invalid_span", "AI privacy inspection needs manual span review. The export was not released.", 422) + containing = [ + (start, end, index) for start, end, index in locations[unit_index] + if start <= finding.start < finding.end <= end + ] + if len(containing) != 1: + fail("ai_invalid_span", "AI privacy inspection crossed field boundaries. The export was not released.", 422) + start, _, index = containing[0] + by_unit.setdefault(index, []).append((finding.start - start, finding.end - start)) + replacements = {} + for index, spans in by_unit.items(): + merged = [] + for start, end in sorted(spans): + if merged and start <= merged[-1][1]: + merged[-1][1] = max(end, merged[-1][1]) + else: + merged.append([start, end]) + text = texts[index] + for start, end in reversed(merged): + self.counts["ai_sensitive"] += fields[texts[index]] + text = text[:start] + self.alias("ai_sensitive", texts[index][start:end]) + text[end:] + replacements[texts[index]] = text + redacted = map_text(value, lambda text: self.credentials(replacements.get(text, text))) + return redacted, { + "status": "complete", "fields_checked": sum(fields.values()), + "input_tokens": result.usage.get("input_tokens", 0), + "budgeted_tokens": result.usage.get("budgeted_tokens", 0), + "model_selection": dict(selection), + } + + def report(self, ai=None): + return { + "deterministic": "complete", "fields_checked": self.fields_checked, + "redaction_counts": dict(self.counts), "ai": ai or {"status": "not_requested"}, + "limitations": "Best-effort privacy checks are not a guarantee of anonymity. Review before sharing.", + } + + +def encoded_json(value): + data = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8") + check_size(0, len(data)) + return data diff --git a/application/single_app/support_export_service.py b/application/single_app/support_export_service.py new file mode 100644 index 000000000..b0b9dc3dd --- /dev/null +++ b/application/single_app/support_export_service.py @@ -0,0 +1,751 @@ +# support_export_service.py +"""Server-authoritative support exports, private snapshots, and owner-only consent.""" + +import io +import hashlib +import json +import secrets +import time +import uuid +import zipfile +from collections import Counter +from datetime import timedelta + +from functions_appinsights import diagnostic_log_context +from support_export_contracts import ( + ACTIVE_STATES, EXPIRY_SECONDS, PUBLIC_LIMITS, RUNNING_STATES, SCHEMA_VERSION, SOURCES, + SupportExportError, check_size, digest, fail, identifier, iso, normalize_configuration, + normalize_request, resolve_intervals, timestamp, utc_now, +) +from support_export_privacy import BundleRedactor, encoded_json, map_text, scrub_credentials +from support_export_store import LEASE_SECONDS + + +WARNINGS = [ + "Only retained, ingested telemetry can be exported. Sampling, ingestion delay, disabled logging, and expired retention leave gaps.", + "Historical events without user/conversation correlation may be absent from scoped exports.", + "Debug logs may contain request text. Privacy checks are best effort; review the bundle before sharing.", +] + + +def bounded_preview(value): + remaining = 32000 + + def project(item): + nonlocal remaining + remaining -= 1 + if remaining <= 0: + return "[Preview limit reached; complete reviewed content remains in the ZIP.]" + if isinstance(item, str): + allowance = min(2000, remaining) + remaining -= min(len(item), allowance) + return item if len(item) <= allowance else item[:allowance] + " [Preview truncated]" + if isinstance(item, list): + result = [] + for child in item: + result.append(project(child)) + if remaining <= 0: + break + return result + if isinstance(item, dict): + result = {} + for index, (key, child) in enumerate(item.items()): + safe_key = key if len(key) <= 256 else key[:256] + f" [Preview key {index} truncated]" + remaining -= len(safe_key) + result[safe_key] = project(child) + if remaining <= 0: + break + return result + return item + + result = project(value) + serialized = json.dumps(result, ensure_ascii=False) + if len(serialized.encode("utf-8")) > 256 * 1024: + return {"notice": "Preview size limited; complete reviewed content remains in the ZIP.", "sample": serialized[:64000]} + return result + + +def require_admin(actor, roles): + identifier(actor) + if not isinstance(roles, (list, tuple, set)) or "Admin" not in roles: + fail("admin_required", "Administrator access is required.", 403) + + +def _intent(job): + return { + key: job[key] for key in ( + "plan", "intervals", "source_configuration", "source_cutoff", "model_disclosure", + "privacy_salt", "storage_binding", "requested_by", + ) + } + + +def _check_integrity(job): + if job.get("fingerprint") != digest(_intent(job)): + fail("export_changed", "The immutable export request changed. Create a new export.", 409) + + +class SupportExportService: + def __init__(self, store, sources, *, version, model_info, inspect_ai, notify, audit, storage_binding="", close_resources=None): + self.store = store + self.sources = sources + self.version = version + self.model_info = model_info + self.inspect_ai = inspect_ai + self.notify = notify + self.audit = audit + self.storage_binding = storage_binding + self.last_heartbeat = 0.0 + self.close_resources = close_resources + + def close(self): + if self.close_resources is not None: + self.close_resources() + + def configuration(self): + stored = self.store.configuration() + return normalize_configuration(stored or { + "cloud": "public", "app_resource_id": "", "site_resource_id": "", + }) + + def readiness(self): + configuration = self.configuration() + availability = { + "activity": {"available": True, "message": "Retained Cosmos activity/audit records."}, + "file_processing": {"available": True, "message": "Retained Cosmos file-processing records; only present when logging collected them."}, + } + for key, resource in (("telemetry", "app_resource_id"), ("console", "site_resource_id")): + ready = bool(configuration[resource]) + availability[key] = { + "available": ready, "access": "not_tested", + "message": "Configured; read permissions are checked during collection." if ready + else "Configure this application's resource ID.", + } + storage = {"available": False, "message": "Initialize the private support-export container."} + try: + self.store.require_private_storage() + storage = {"available": True, "message": "Private export storage is accessible."} + except SupportExportError as exc: + storage["message"] = exc.public_message + except Exception as exc: + self.audit("storage_check_failed", error_type=type(exc).__name__) + storage["message"] = "Application Blob storage could not be verified. Check its configuration and data-plane permissions." + model = self.model_info() + return { + "configuration": configuration, "sources": availability, "storage": storage, + "model": {key: model.get(key) for key in ("available", "label", "provider", "message")}, + "limits": PUBLIC_LIMITS, "warnings": WARNINGS, + } + + def initialize_storage(self, actor, roles): + require_admin(actor, roles) + self.store.require_private_storage(create=True) + self.store.set_configuration(self.configuration()) + self.audit("storage_initialized", actor=actor) + return self.readiness() + + def test_sources(self, actor, roles): + require_admin(actor, roles) + result = self.readiness() + configuration = self.configuration() + for source in ("telemetry", "console"): + if not result["sources"][source]["available"]: + continue + try: + self.sources.probe_monitor(source, configuration) + result["sources"][source].update(access="verified", message="Read access verified; empty or missing retained data is still possible.") + except Exception as exc: + self.audit("source_check_failed", actor=actor, source=source, error_type=type(exc).__name__) + result["sources"][source].update( + available=False, access="failed", + message="Read access could not be verified. Check the resource IDs, cloud, collected tables, and resource-context Azure Monitor read permissions.", + ) + return result + + def configure(self, actor, roles, value): + require_admin(actor, roles) + normalized = normalize_configuration(value) + self.store.set_configuration(normalized) + self.audit("configuration_saved", actor=actor) + return {"configuration": normalized} + + def _own_job(self, actor, roles, job_id, *, allow_expired=False): + require_admin(actor, roles) + identifier(job_id) + job = self.store.read(job_id, actor) + if not job or job.get("record_kind") != "export" or job.get("requested_by") != actor: + fail("export_not_found", "Support export not found.", 404) + _check_integrity(job) + if not allow_expired and timestamp(job["expires_at"]) <= utc_now(): + fail("export_expired", "This support export has expired.", 410) + return job + + def public_job(self, job): + result = {key: job.get(key) for key in ( + "id", "status", "created_at", "updated_at", "expires_at", "plan", "intervals", + "error", "counts", "warnings", "phase", "privacy_report", "source_cutoff", + )} + result["expired"] = timestamp(job["expires_at"]) <= utc_now() + if result["expired"]: + result["status"] = "expired" + result["can_download"] = job["status"] == "ready" and not result["expired"] + return result + + def list_jobs(self, actor, roles, after=""): + require_admin(actor, roles) + if after: + identifier(after) + rows = self.store.query( + "SELECT TOP 51 * FROM c WHERE c.record_kind = 'export' AND c.requested_by = @actor " + "AND c.id > @after ORDER BY c.id", + [{"name": "@actor", "value": actor}, {"name": "@after", "value": after}], + partition=actor, limit=51, + ) + return {"items": [self.public_job(row) for row in rows[:50]], "next": rows[49]["id"] if len(rows) > 50 else None} + + def _reserve_admin(self, actor, job_id, expires_at): + lock = self.store.read("active-export", actor) + if lock and timestamp(lock["expires_at"]) > utc_now() and lock["job_id"] != job_id: + other = self.store.read(lock["job_id"], actor) + if other and other["status"] in ACTIVE_STATES: + fail("export_busy", "You already have an active export. Finish or cancel it first.", 409) + body = { + "id": "active-export", "partition_key": actor, "record_kind": "admin_lock", + "job_id": job_id, "expires_at": expires_at, + } + if lock: + body["_etag"] = lock["_etag"] + self.store.save(body) + else: + try: + self.store.create(body) + except Exception as exc: + if getattr(exc, "status_code", None) == 409: + fail("export_busy", "Another export request is being created. Retry after refreshing.", 409) + raise + + def _prepare_data(self, actor, roles, data): + require_admin(actor, roles) + now = utc_now() + plan = normalize_request(data, now) + if plan["user_id"]: + self.sources.require_registered_user(plan["user_id"]) + conversations = [ + self.sources.conversation(plan["user_id"], item, bounds=plan["mode"] == "conversations") + for item in plan["conversation_ids"] + ] + if plan["include_transcripts"] and any(not item["transcript_eligible"] for item in conversations): + fail("personal_only", "Shared and group conversations are logs-only. Select personal conversations for transcripts.", 403) + configuration = self.configuration() + for source in plan["sources"]: + if source in ("telemetry", "console") and not ( + configuration["site_resource_id" if source == "console" else "app_resource_id"] + ): + fail("source_unavailable", "A selected Azure source is not configured. Configure it or explicitly deselect it.", 503) + model = self.model_info() if plan["privacy"] == "ai" else {"available": False} + if plan["privacy"] == "ai" and not model.get("available"): + fail("model_unavailable", "The default text model cannot perform privacy inspection. Configure AI Connections or use deterministic-only checking.", 503) + self.store.require_private_storage() + plan["purpose"] = BundleRedactor().deterministic(plan["purpose"]) + return { + "plan": plan, "conversations": conversations, "source_configuration": configuration, + "source_cutoff": iso(now), "intervals": resolve_intervals(plan, conversations, now), + "model_disclosure": model if plan["privacy"] == "ai" else {}, + } + + def prepare(self, actor, roles, data): + prepared = self._prepare_data(actor, roles, data) + review_id = "review-" + uuid.uuid4().hex + expires_at = iso(utc_now() + timedelta(minutes=15)) + self.store.create({ + "id": review_id, "partition_key": actor, "record_kind": "review", + "input_fingerprint": digest(data), "prepared": prepared, "expires_at": expires_at, + "storage_binding": self.storage_binding, "ttl": 900, + }) + return { + "review_id": review_id, "request": prepared["plan"], "intervals": prepared["intervals"], + "conversations": prepared["conversations"], "expires_at": expires_at, + "model_label": prepared["model_disclosure"].get("label", ""), "warnings": WARNINGS, + } + + def create(self, actor, roles, data, retry_key, review_id=None): + require_admin(actor, roles) + identifier(retry_key) + input_fingerprint = digest(data) + job_id = digest({"actor": actor, "retry_key": retry_key})[:32] + existing = self.store.read(job_id, actor) + if existing: + if existing.get("input_fingerprint") != input_fingerprint: + fail("retry_changed", "An export retry cannot change its requested scope.", 409) + _check_integrity(existing) + return self.public_job(existing) + if review_id: + identifier(review_id) + review = self.store.read(review_id, actor) + if ( + not review or review.get("record_kind") != "review" + or review["input_fingerprint"] != input_fingerprint + or timestamp(review["expires_at"]) <= utc_now() + or review["storage_binding"] != self.storage_binding + ): + fail("review_expired", "Review this exact export scope again before generating.", 409) + prepared = review["prepared"] + if prepared["source_configuration"] != self.configuration(): + fail("configuration_changed", "Source configuration changed. Review the export again.", 409) + for item in prepared["conversations"]: + current = self.sources.conversation(prepared["plan"]["user_id"], item["id"]) + if prepared["plan"]["include_transcripts"] and not current["transcript_eligible"]: + fail("personal_only", "A conversation is no longer eligible for transcript release.", 403) + if prepared["plan"]["privacy"] == "ai" and ( + self.model_info().get("fingerprint") != prepared["model_disclosure"].get("fingerprint") + ): + fail("model_changed", "The default AI destination changed. Review the export again.", 409) + else: + prepared = self._prepare_data(actor, roles, data) + plan = prepared["plan"] + conversations = prepared["conversations"] + model = prepared["model_disclosure"] + now = utc_now() + job = { + "id": job_id, "partition_key": actor, "record_kind": "export", "requested_by": actor, + "input_fingerprint": input_fingerprint, "plan": plan, + "source_configuration": prepared["source_configuration"], "storage_binding": self.storage_binding, + "source_cutoff": prepared["source_cutoff"], "intervals": prepared["intervals"], + "model_disclosure": model, + "privacy_salt": secrets.token_hex(32), "created_at": iso(now), "updated_at": iso(now), + "expires_at": iso(now + timedelta(seconds=EXPIRY_SECONDS)), + "status": "waiting_for_consent" if plan["include_transcripts"] else "queued", + "phase": "Waiting for owner approval" if plan["include_transcripts"] else "Queued", + "warnings": WARNINGS, "ttl": 7 * 24 * 60 * 60, + } + job["fingerprint"] = digest(_intent(job)) + self._reserve_admin(actor, job_id, job["expires_at"]) + self.audit("requested", actor=actor, export_id=job_id, transcripts=plan["include_transcripts"]) + job = self.store.create(job) + if plan["include_transcripts"]: + consent = self.store.create({ + "id": "consent-" + job_id, "partition_key": plan["user_id"], "record_kind": "consent", + "owner_id": plan["user_id"], "requester_id": actor, "job_id": job_id, + "plan_fingerprint": job["fingerprint"], "status": "pending", "purpose": plan["purpose"], + "created_at": job["created_at"], "expires_at": job["expires_at"], + "source_cutoff": job["source_cutoff"], "conversations": conversations, + "privacy": plan["privacy"], "model_label": scrub_credentials(model.get("label", "")), + "ttl": 7 * 24 * 60 * 60, + }) + try: + self.notify(plan["user_id"], job_id) + except Exception as exc: + self.audit("consent_notification_failed", actor=actor, error_type=type(exc).__name__) + job.update(status="failed", phase="Consent notification failed", error={ + "code": "notification_failed", "message": "The consent notification could not be delivered. Create a new export.", + }) + self.store.save(job) + fail("notification_failed", "The consent notification could not be delivered. Create a new export.", 503) + return self.public_job(job) + + def status(self, actor, roles, job_id): + job = self._own_job(actor, roles, job_id, allow_expired=True) + if job["plan"]["include_transcripts"] and job["status"] == "ready" and timestamp(job["expires_at"]) > utc_now(): + self._grant(job) + return self.public_job(job) + + def cancel(self, actor, roles, job_id): + job = self._own_job(actor, roles, job_id) + if job["status"] != "cancelled": + job.update(status="cancelled", phase="Cancelled", updated_at=iso(utc_now()), lease_token=None) + job = self.store.save(job) + self.audit("cancelled", actor=actor, export_id=job_id) + if job["plan"]["include_transcripts"]: + consent = self._consent(job["plan"]["user_id"], "consent-" + job["id"]) + if consent["status"] in ("pending", "approved"): + consent["status"] = "withdrawn" + self.store.save(consent) + return self.public_job(job) + + def _consent(self, owner, consent_id): + identifier(owner) + identifier(consent_id) + consent = self.store.read(consent_id, owner) + if not consent or consent.get("record_kind") != "consent" or consent.get("owner_id") != owner: + fail("consent_not_found", "Support-data consent request not found.", 404) + return consent + + def list_consents(self, owner, after=""): + identifier(owner) + if after: + identifier(after) + rows = self.store.query( + "SELECT TOP 51 * FROM c WHERE c.record_kind = 'consent' AND c.owner_id = @owner " + "AND c.id > @after ORDER BY c.id", + [{"name": "@owner", "value": owner}, {"name": "@after", "value": after}], + partition=owner, limit=51, + ) + return {"items": [self.public_consent(row) for row in rows[:50]], "next": rows[49]["id"] if len(rows) > 50 else None} + + def consent_details(self, owner, consent_id): + return self.public_consent(self._consent(owner, consent_id)) + + def public_consent(self, consent): + result = {key: consent.get(key) for key in ( + "id", "job_id", "requester_id", "status", "purpose", "created_at", "expires_at", + "source_cutoff", "conversations", "privacy", "model_label", "approved_at", + )} + result["expired"] = timestamp(consent["expires_at"]) <= utc_now() + if result["expired"]: + result["status"] = "expired" + result["scope_notice"] = ( + "Approval releases whole personal conversations, not just the diagnostic time window. " + "Downloaded copies and AI inputs already submitted cannot be recalled." + ) + return result + + def consent_preview(self, owner, consent_id): + consent = self._consent(owner, consent_id) + if consent["status"] != "pending" or timestamp(consent["expires_at"]) <= utc_now(): + fail("consent_closed", "This consent request is no longer pending.", 409) + job = self._consent_job(consent) + snapshots = self.sources.snapshot(owner, job["plan"]["conversation_ids"], job["source_cutoff"]) + preview = [{"conversation_id": item["conversation_id"], "message_count": len(item["messages"]), + "messages": item["messages"][:10]} for item in snapshots] + redactor = BundleRedactor(job["privacy_salt"]) + preview = redactor.deterministic(preview, identifiers=[owner, *job["plan"]["conversation_ids"]]) + return {"consent": self.public_consent(consent), "fingerprint": digest(snapshots), "preview": bounded_preview(preview)} + + def _consent_job(self, consent): + job = self.store.read(consent["job_id"], consent["requester_id"]) + if ( + not job or job.get("record_kind") != "export" or not job["plan"]["include_transcripts"] + or job["plan"]["user_id"] != consent["owner_id"] + or consent["plan_fingerprint"] != job["fingerprint"] + or timestamp(job["expires_at"]) <= utc_now() or job["status"] in ("failed", "cancelled", "expired") + ): + fail("consent_closed", "The related export is no longer available.", 409) + _check_integrity(job) + return job + + def decide(self, owner, consent_id, action, fingerprint=None): + consent = self._consent(owner, consent_id) + if action not in ("approve", "decline", "revoke"): + fail("invalid_consent_action", "Choose approve, decline, or revoke.") + if timestamp(consent["expires_at"]) <= utc_now(): + fail("consent_expired", "This consent request has expired.", 410) + job = self._consent_job(consent) + if action == "approve": + if consent["status"] == "approved" and consent.get("snapshot_fingerprint") == fingerprint: + return self.public_consent(consent) + if consent["status"] != "pending" or job["status"] != "waiting_for_consent": + fail("consent_closed", "This consent request is no longer pending.", 409) + snapshots = self.sources.snapshot(owner, job["plan"]["conversation_ids"], job["source_cutoff"]) + if not isinstance(fingerprint, str) or fingerprint != digest(snapshots): + fail("snapshot_changed", "The conversation snapshot changed. Review the refreshed preview before approving.", 409) + if job["plan"]["privacy"] == "ai" and self.model_info().get("fingerprint") != job["model_disclosure"].get("fingerprint"): + fail("model_changed", "The default AI destination changed. Ask the administrator to create a new export.", 409) + redactor = BundleRedactor(job["privacy_salt"]) + cleaned = redactor.deterministic(snapshots, identifiers=[owner, *job["plan"]["conversation_ids"]]) + expires_at = iso(utc_now() + timedelta(seconds=EXPIRY_SECONDS)) + path = f"{job['id']}/{uuid.uuid4().hex}/snapshot.json" + self.store.put(path, encoded_json(cleaned), expires_at) + consent.update( + status="approved", approved_at=iso(utc_now()), expires_at=expires_at, + snapshot_blob=path, snapshot_fingerprint=fingerprint, privacy_report=redactor.report(), + approved_by=owner, snapshot_checksum=hashlib.sha256(encoded_json(cleaned)).hexdigest(), + ) + consent = self.store.save(consent) + job.update(status="queued", phase="Owner approved; queued", expires_at=expires_at, updated_at=iso(utc_now())) + else: + expected = "pending" if action == "decline" else "approved" + if consent["status"] != expected: + fail("consent_closed", "This consent decision is no longer available.", 409) + consent.update(status="denied" if action == "decline" else "revoked", decided_at=iso(utc_now())) + consent = self.store.save(consent) + job.update(status="cancelled", phase="Transcript consent " + consent["status"], lease_token=None, updated_at=iso(utc_now())) + self.store.save(job) + self.audit("consent_" + action, actor=owner, export_id=job["id"]) + return self.public_consent(consent) + + def _grant(self, job, *, check_snapshot=False): + if not job["plan"]["include_transcripts"]: + return None + consent = self._consent(job["plan"]["user_id"], "consent-" + job["id"]) + if ( + consent["status"] != "approved" or timestamp(consent["expires_at"]) <= utc_now() + or consent["plan_fingerprint"] != job["fingerprint"] or consent["requester_id"] != job["requested_by"] + or consent.get("approved_by") != job["plan"]["user_id"] + or not isinstance(consent.get("snapshot_blob"), str) + or not consent["snapshot_blob"].startswith(job["id"] + "/") + or not consent["snapshot_blob"].endswith("/snapshot.json") + ): + fail("consent_required", "A current owner approval is required for these transcripts.", 403) + for conversation_id in job["plan"]["conversation_ids"]: + metadata = self.sources.conversation(job["plan"]["user_id"], conversation_id) + if not metadata["transcript_eligible"]: + fail("personal_only", "A selected conversation is no longer a personal conversation.", 403) + if check_snapshot: + current = self.sources.snapshot(job["plan"]["user_id"], job["plan"]["conversation_ids"], job["source_cutoff"]) + if digest(current) != consent["snapshot_fingerprint"]: + fail("snapshot_changed", "Conversation revisions or availability changed after approval. Create a new export.", 409) + return consent + + def _downloadable(self, actor, roles, job_id): + job = self._own_job(actor, roles, job_id) + if job["status"] != "ready": + fail("export_not_ready", "The support export is not ready.", 409) + self._grant(job, check_snapshot=True) + if job["storage_binding"] != self.storage_binding: + fail("storage_changed", "Export storage changed. Restore its configuration or create a new export.", 409) + return job + + def _artifact(self, job, field, filename): + path = job.get(field) + if not isinstance(path, str) or not path.startswith(job["id"] + "/") or not path.endswith("/" + filename): + fail("invalid_artifact", "The support-export artifact is not bound to this request.", 503) + data = self.store.get(path) + if hashlib.sha256(data).hexdigest() != job.get(field + "_checksum"): + fail("artifact_changed", "The support-export artifact changed. Create a new export.", 503) + return data + + def preview(self, actor, roles, job_id): + job = self._downloadable(actor, roles, job_id) + data = self._artifact(job, "preview_blob", "preview.json") + self._downloadable(actor, roles, job_id) + return json.loads(data) + + def download(self, actor, roles, job_id): + job = self._downloadable(actor, roles, job_id) + data = self._artifact(job, "archive_blob", "bundle.zip") + self._downloadable(actor, roles, job_id) + self.audit("downloaded", actor=actor, export_id=job_id) + return data + + def _heartbeat(self, job, slot, phase=None, *, force=False): + if not phase and not force and time.monotonic() - self.last_heartbeat < 5: + return + current = self.store.read(job["id"], job["partition_key"]) + if ( + current is None or current["status"] not in RUNNING_STATES + or current.get("lease_token") != job["lease_token"] + or timestamp(current["lease_until"]) <= utc_now() or timestamp(current["expires_at"]) <= utc_now() + ): + fail("lease_lost", "The export was cancelled or its worker lease expired.", 409) + _check_integrity(current) + self._grant(current) + self.store.renew_slot(slot) + current.update(updated_at=iso(utc_now()), lease_until=iso(utc_now() + timedelta(seconds=LEASE_SECONDS))) + if phase: + current.update(status=phase, phase=phase.replace("_", " ").capitalize()) + saved = self.store.save(current) + job.clear() + job.update(saved) + self.last_heartbeat = time.monotonic() + + def _archive(self, job, data, report, expires_at): + logs, conversations = data["logs"], data["conversations"] + files = {} + source_files = {"telemetry": "telemetry", "activity": "activity", "file_processing": "file-processing", "console": "system-context"} + groups = {} + for row in logs: + name = "system-context" if row["context_scope"] == "system_context" else source_files[row["source"]] + groups.setdefault(name, []).append(row) + for name, rows in groups.items(): + files[f"logs/{name}.jsonl"] = b"\n".join(encoded_json(row) for row in rows) + b"\n" + files["logs/diagnostics.txt"] = "\n\n".join( + f"{row['timestamp']} [{row['source']}] [{row['severity']}] {row['message']}\n" + + json.dumps(row["properties"], ensure_ascii=False, indent=2) for row in logs + ).encode("utf-8") + for index, conversation in enumerate(conversations, 1): + files[f"conversations/conversation-{index}.json"] = encoded_json(conversation) + files[f"conversations/conversation-{index}.md"] = "\n\n".join( + f"## {message['role']} - {message['timestamp']}\n\n{message['content']}" + for message in conversation["messages"] + ).encode("utf-8") + report = map_text(report, scrub_credentials) + files["privacy-report.json"] = encoded_json(report) + files["README.md"] = ( + "# SimpleChat support bundle\n\n" + "See manifest.json for scope, UTC windows, source exclusions, and checksums. " + "logs/diagnostics.txt is a readable timeline; JSONL files preserve diagnostic structure.\n\n" + "Only approved personal conversation snapshots are included. Logs may still contain " + "sensitive information: privacy inspection is best effort, not an anonymity guarantee. " + "Review before manually sharing with support. This application did not send the bundle.\n\n" + + "\n".join("- " + warning for warning in WARNINGS) + ).encode("utf-8") + redactor = BundleRedactor(job["privacy_salt"]) + counts = dict(Counter(row["source"] for row in logs)) + manifest = { + "schema_version": SCHEMA_VERSION, "application_version": self.version, "export_id": job["id"], + "created_at": iso(utc_now()), "source_cutoff": job["source_cutoff"], "expires_at": expires_at, + "intervals_utc": job["intervals"], "all_users": job["plan"]["all_users"], + "subject": redactor.alias("identifier", job["plan"]["user_id"]) if job["plan"]["user_id"] else "all_users", + "conversations": [redactor.alias("identifier", item) for item in job["plan"]["conversation_ids"]], + "sources": { + source: {"status": "collected" if source in job["plan"]["sources"] else "excluded_by_admin", + "records": counts.get(source, 0)} for source in SOURCES + }, + "consent_reference": "consent-" + job["id"] if conversations else None, + "privacy": report, "warnings": WARNINGS, + "checksums_sha256": {name: hashlib.sha256(body).hexdigest() for name, body in files.items()}, + } + files["manifest.json"] = encoded_json(map_text(manifest, scrub_credentials)) + check_size(len(logs) + sum(len(item["messages"]) for item in conversations), sum(map(len, files.values()))) + output = io.BytesIO() + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + for name, body in files.items(): + archive.writestr(name, body) + return output.getvalue(), counts + + def run(self, job, slot): + with diagnostic_log_context(user_id=job["requested_by"]): + self._run(job, slot) + + def _run(self, job, slot): + current = self.store.read(job["id"], job["partition_key"]) + if ( + current is None or current["status"] not in ("queued", *RUNNING_STATES) + or timestamp(current["expires_at"]) <= utc_now() + or (current["status"] in RUNNING_STATES and timestamp(current["lease_until"]) > utc_now()) + ): + return + _check_integrity(current) + current.update( + status="collecting", phase="Collecting", lease_token=uuid.uuid4().hex, + lease_until=iso(utc_now() + timedelta(seconds=LEASE_SECONDS)), updated_at=iso(utc_now()), + ) + job = self.store.save(current) + worker_token = job["lease_token"] + try: + if job["storage_binding"] != self.storage_binding or job["source_configuration"] != self.configuration(): + fail("configuration_changed", "Collection configuration changed. Create a new export.", 409) + self.store.require_private_storage() + grant = self._grant(job) + for conversation_id in job["plan"]["conversation_ids"]: + self.sources.conversation(job["plan"]["user_id"], conversation_id) + progress = lambda: self._heartbeat(job, slot) + records = [] + seen = set() + byte_count = 0 + for source in job["plan"]["sources"]: + rows = ( + self.sources.collect_monitor(source, job["source_configuration"], job["plan"], job["intervals"], progress) + if source in ("telemetry", "console") + else self.sources.collect_cosmos(source, job["plan"], job["intervals"], progress) + ) + for row in rows: + row_digest = digest(row) + if row_digest in seen: + continue + seen.add(row_digest) + byte_count += len(encoded_json(row)) + check_size(len(records) + 1, byte_count) + records.append(row) + progress() + records.sort(key=lambda row: timestamp(row["timestamp"], stored=True)) + if grant: + snapshot_bytes = self.store.get(grant["snapshot_blob"]) + if hashlib.sha256(snapshot_bytes).hexdigest() != grant["snapshot_checksum"]: + fail("snapshot_changed", "The approved snapshot artifact changed. Create a new export.", 409) + snapshots = json.loads(snapshot_bytes) + else: + snapshots = [] + self._heartbeat(job, slot, "redacting") + redactor = BundleRedactor(job["privacy_salt"]) + cleaned = { + "logs": redactor.deterministic( + records, identifiers=[job["plan"]["user_id"], *job["plan"]["conversation_ids"]], progress=progress, + ), + # The checksum-bound owner snapshots already completed deterministic inspection. + "conversations": snapshots, + } + ai = None + if job["plan"]["privacy"] == "ai": + self._heartbeat(job, slot, "ai_review") + if self.model_info().get("fingerprint") != job["model_disclosure"].get("fingerprint"): + fail("model_changed", "The configured default AI destination changed. Create a new export.", 409) + cleaned, ai = redactor.inspect_ai( + cleaned, selection=job["model_disclosure"]["selection"], + inspect=lambda units, selection, callback, budget: self.inspect_ai( + units, selection, callback, budget, + expected_fingerprint=job["model_disclosure"]["fingerprint"], + ), progress=progress, + ) + report = redactor.report(ai) + if grant: + report["approved_snapshot_redactions"] = grant["privacy_report"] + report["fields_checked"] += grant["privacy_report"]["fields_checked"] + report["redaction_counts"] = dict( + Counter(report["redaction_counts"]) + Counter(grant["privacy_report"]["redaction_counts"]), + ) + self._heartbeat(job, slot, force=True) + expires_at = iso(utc_now() + timedelta(seconds=EXPIRY_SECONDS)) + if grant: + expires_at = min(expires_at, grant["expires_at"]) + archive, counts = self._archive(job, cleaned, report, expires_at) + attempt = f"{job['id']}/{job['lease_token']}" + archive_blob, preview_blob = attempt + "/bundle.zip", attempt + "/preview.json" + preview = bounded_preview({ + "logs": cleaned["logs"][:20], + "conversations": [{"messages": item["messages"][:10]} for item in cleaned["conversations"]], + "privacy_report": report, "notice": "Preview only; the ZIP contains all collected records.", + }) + self.store.put(archive_blob, archive, expires_at, content_type="application/zip") + self.store.put(preview_blob, encoded_json(preview), expires_at) + self._heartbeat(job, slot, force=True) + job.update( + status="ready", phase="Ready for review and download", expires_at=expires_at, + archive_blob=archive_blob, preview_blob=preview_blob, privacy_report=report, + archive_blob_checksum=hashlib.sha256(archive).hexdigest(), + preview_blob_checksum=hashlib.sha256(encoded_json(preview)).hexdigest(), + counts=counts, lease_token=None, updated_at=iso(utc_now()), + ) + self.store.save(job) + self.audit("completed", actor=job["requested_by"], export_id=job["id"], records=len(records)) + except Exception as exc: + self.audit("failed", actor=job["requested_by"], export_id=job["id"], error_type=type(exc).__name__) + latest = self.store.read(job["id"], job["partition_key"]) + if latest and latest.get("lease_token") == worker_token and latest["status"] in RUNNING_STATES: + latest.update(status="failed", phase="Export failed", lease_token=None, error={ + "code": exc.code if isinstance(exc, SupportExportError) else "collection_failed", + "message": exc.public_message if isinstance(exc, SupportExportError) + else "Diagnostic collection failed. Check source read permissions, storage, and application logs.", + }) + self.store.save(latest) + + def tick(self): + now = utc_now() + expired = self.store.query( + "SELECT TOP 100 * FROM c WHERE c.record_kind = 'export' AND c.expires_at <= @now " + "AND (c.status != 'expired' OR NOT IS_DEFINED(c.cleanup_complete) OR c.cleanup_complete != true)", + [{"name": "@now", "value": iso(now)}], + ) + for job in expired: + if job["status"] != "expired": + job.update(status="expired", phase="Expired", lease_token=None) + job = self.store.save(job) + try: + self.store.delete_artifacts(job["id"]) + job["cleanup_complete"] = True + self.store.save(job) + self.audit("expired", export_id=job["id"]) + except Exception as exc: + self.audit("cleanup_failed", export_id=job["id"], error_type=type(exc).__name__) + cleanup = self.store.read("blob-cleanup", "configuration") + if self.store.blobs is not None and cleanup and ( + not cleanup.get("next_scan_at") or timestamp(cleanup["next_scan_at"]) <= now + ): + try: + self.store.sweep_expired_blobs() + except Exception as exc: + self.audit("cleanup_failed", error_type=type(exc).__name__) + due = self.store.query( + "SELECT TOP 1 * FROM c WHERE c.record_kind = 'export' AND c.expires_at > @now " + "AND (c.status = 'queued' OR (ARRAY_CONTAINS(@running, c.status) AND c.lease_until <= @now))", + [{"name": "@now", "value": iso(now)}, {"name": "@running", "value": list(RUNNING_STATES)}], limit=1, + ) + if not due: + return + slot = self.store.acquire_slot() + if slot is None: + return + try: + self.run(due[0], slot) + finally: + self.store.release_slot(slot) diff --git a/application/single_app/support_export_sources.py b/application/single_app/support_export_sources.py new file mode 100644 index 000000000..2359debb0 --- /dev/null +++ b/application/single_app/support_export_sources.py @@ -0,0 +1,413 @@ +# support_export_sources.py +"""Bounded support collectors with explicit subject/resource relationships.""" + +import json +import re +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone + +from azure.core.exceptions import ResourceNotFoundError +from azure.monitor.query import LogsQueryClient, LogsQueryStatus + +from collaboration_models import is_shared_conversation_backing +from cosmos_document_utils import normalize_cosmos_document +from functions_control_center_activity import ACTIVITY_ACTOR_FIELDS, cosmos_property_path +from functions_azure_endpoint_validation import validate_azure_monitor_query_endpoint +from functions_appinsights import log_event +from support_export_contracts import ( + MAX_RECORDS, check_size, fail, identifier, identity_hash, iso, + normalize_configuration, timestamp, within_intervals, +) + + +CONVERSATION_FIELDS = ( + "c.id, c.user_id, c.created_at, c.created_date, c.last_updated, c.updated_at, " + "c.last_message_at, c.message_count, c.conversation_kind, c.collaboration_conversation_id, " + "c.converted_to_collaboration_at, c.group_id" +) +OMIT_PROPERTY_KEYS = { + "headers", "requestheaders", "responseheaders", "settings", "configuration", "requestbody", + "responsebody", "prompt", "systemprompt", "messages", "messagecontent", "documentcontent", + "requestcontent", "responsecontent", "rawcontent", "body", +} + + +def build_support_monitor_client(configuration, credential, *, transport=None): + configuration = normalize_configuration(configuration) + origin = validate_azure_monitor_query_endpoint( + "https://api.loganalytics.us" if configuration["cloud"] == "usgovernment" + else "https://api.loganalytics.io", + ) + # The pinned 1.4.1 SDK treats endpoint as the full /v1 base, not an origin. + return LogsQueryClient( + credential, endpoint=origin + "/v1", audience=origin, + **({"transport": transport} if transport is not None else {}), + ) + + +INLINE_DATA = re.compile(r"data:[A-Za-z0-9.+/-]{1,120};base64,[A-Za-z0-9+/=]+", re.I) + + +def json_value(value, *, omit_payload=True): + if isinstance(value, datetime): + return iso(value if value.tzinfo else value.replace(tzinfo=timezone.utc)) + if isinstance(value, Mapping): + return { + str(key): json_value(item, omit_payload=omit_payload) for key, item in value.items() + if not omit_payload or "".join(char for char in str(key).lower() if char.isalnum()) not in OMIT_PROPERTY_KEYS + } + if isinstance(value, (list, tuple)): + return [json_value(item, omit_payload=omit_payload) for item in value] + if value is None or type(value) in (str, int, float, bool): + return value + fail("invalid_source_value", "A log source returned unsupported diagnostic data.", 422) + + +def _bounded_rows(container, query, parameters, *, partition=None, maximum=MAX_RECORDS, omit_payload=True): + options = {"max_item_count": min(maximum + 1, 200)} + if partition is None: + options["enable_cross_partition_query"] = True + else: + options["partition_key"] = partition + result = [] + byte_count = 0 + for row in container.query_items(query=query, parameters=parameters, **options): + item = json_value(row, omit_payload=omit_payload) + byte_count += len(json.dumps(item, ensure_ascii=False).encode("utf-8")) + check_size(len(result) + 1, byte_count) + if len(result) >= maximum: + fail("source_limit", "A source has too many matching records. Narrow the export.", 413) + result.append(item) + return result + + +def _metadata(record, kind): + first = record.get("created_at") or record.get("created_date") + last = record.get("last_message_at") or record.get("last_updated") or record.get("updated_at") or first + personal = kind == "personal" and not ( + is_shared_conversation_backing(record) or record.get("collaboration_conversation_id") + or record.get("converted_to_collaboration_at") + or record.get("conversation_kind") in ("collaborative", "collaboration_source") or record.get("group_id") + ) + return { + "id": record["id"], "kind": "personal" if personal else ("shared" if kind == "personal" else kind), + "first_activity": first, "last_activity": last, "message_count": record.get("message_count"), + "transcript_eligible": personal, + } + + +class SupportExportSources: + def __init__(self, containers, *, monitor=None, authorize_shared=None, filter_messages=None): + self.containers = containers + self.monitor = monitor + self.authorize_shared = authorize_shared + self.filter_messages = filter_messages + + def require_registered_user(self, user_id): + user_id = identifier(user_id) + rows = _bounded_rows( + self.containers["users"], "SELECT VALUE c.id FROM c WHERE c.id = @user", + [{"name": "@user", "value": user_id}], partition=user_id, maximum=1, + ) + if not rows: + fail("subject_not_found", "The selected SimpleChat user is unavailable.", 404) + + def users(self, term="", after=""): + if not isinstance(term, str) or len(term) > 200: + fail("invalid_search", "Search text must be at most 200 characters.") + if after: + identifier(after) + query = ( + "SELECT TOP 51 c.id, c.display_name, c.email, c.settings.displayName AS displayName, " + "c.settings.display_name AS settings_name, c.settings.email AS settings_email " + "FROM c WHERE c.id > @after AND (" + "@term = '' OR CONTAINS(c.id, @term, true) OR CONTAINS(c.display_name, @term, true) " + "OR CONTAINS(c.email, @term, true) OR CONTAINS(c.settings.displayName, @term, true) " + "OR CONTAINS(c.settings.display_name, @term, true) OR CONTAINS(c.settings.email, @term, true)) ORDER BY c.id" + ) + rows = _bounded_rows( + self.containers["users"], query, + [{"name": "@after", "value": after}, {"name": "@term", "value": term.strip()}], maximum=51, + ) + return { + "items": [{ + "id": row["id"], "name": row.get("display_name") or row.get("displayName") + or row.get("settings_name") or row.get("email") or row["id"], + "email": row.get("email") or row.get("settings_email") or "", + } for row in rows[:50]], + "next": rows[49]["id"] if len(rows) > 50 else None, + } + + def _shared(self, user_id, conversation_id): + states = _bounded_rows( + self.containers["collaboration_states"], + "SELECT TOP 1 c.conversation_id FROM c WHERE c.user_id = @user " + "AND c.conversation_id = @id AND c.membership_status = 'accepted'", + [{"name": "@user", "value": user_id}, {"name": "@id", "value": conversation_id}], + partition=user_id, maximum=1, + ) + if not states or self.authorize_shared is None: + return None + try: + response = self.containers["collaboration"].read_item(item=conversation_id, partition_key=conversation_id) + except ResourceNotFoundError: + return None + document = normalize_cosmos_document(response) + try: + self.authorize_shared(user_id, document) + except (PermissionError, LookupError): + log_event("[SUPPORT_EXPORT] Shared metadata is no longer accessible.", extra={"status": "not_accessible"}, debug_only=True) + return None + return document + + def conversations(self, user_id, *, after="", term=""): + self.require_registered_user(user_id) + if after: + identifier(after) + if not isinstance(term, str) or len(term) > 128: + fail("invalid_search", "Search conversation IDs using at most 128 characters.") + rows = {} + frontiers = [] + parameters = [ + {"name": "@user", "value": user_id}, {"name": "@after", "value": after}, + {"name": "@term", "value": term}, + ] + for kind, container in (("personal", "conversations"), ("group", "group_conversations")): + records = _bounded_rows( + self.containers[container], + f"SELECT TOP 51 {CONVERSATION_FIELDS} FROM c WHERE c.user_id = @user " + "AND c.id > @after AND (@term = '' OR CONTAINS(c.id, @term, true)) ORDER BY c.id", + parameters, maximum=51, + ) + if len(records) == 51: + frontiers.append(records[-1]["id"]) + for record in records: + rows[record["id"]] = _metadata(record, kind) + states = _bounded_rows( + self.containers["collaboration_states"], + "SELECT TOP 51 c.conversation_id FROM c WHERE c.user_id = @user " + "AND c.membership_status = 'accepted' AND c.conversation_id > @after " + "AND (@term = '' OR CONTAINS(c.conversation_id, @term, true)) ORDER BY c.conversation_id", + parameters, partition=user_id, maximum=51, + ) + if len(states) == 51: + frontiers.append(states[-1]["conversation_id"]) + for state in states: + document = self._shared(user_id, state["conversation_id"]) + if document: + rows[document["id"]] = _metadata(document, "shared") + frontier = min(frontiers) if frontiers else None + ordered = sorted( + (row for row in rows.values() if frontier is None or row["id"] <= frontier), + key=lambda row: row["id"], + ) + return {"items": ordered[:50], "next": ordered[49]["id"] if len(ordered) > 50 else frontier} + + def conversation(self, user_id, conversation_id, *, bounds=False): + self.require_registered_user(user_id) + identifier(conversation_id) + parameters = [{"name": "@id", "value": conversation_id}, {"name": "@user", "value": user_id}] + matches = [] + for kind, container in (("personal", "conversations"), ("group", "group_conversations")): + records = _bounded_rows( + self.containers[container], + f"SELECT TOP 1 {CONVERSATION_FIELDS} FROM c WHERE c.id = @id AND c.user_id = @user", + parameters, partition=conversation_id, maximum=1, + ) + matches.extend((_metadata(record, kind), "messages" if kind == "personal" else "group_messages") for record in records) + shared = self._shared(user_id, conversation_id) + if shared: + matches = [(_metadata(shared, "shared"), "collaboration_messages")] + if len(matches) != 1: + fail("conversation_not_found", "A selected conversation is unavailable for this user.", 404) + metadata, message_container = matches[0] + if bounds: + rows = _bounded_rows( + self.containers[message_container], + "SELECT MIN(c.timestamp) AS first_time, MAX(c.timestamp) AS last_time, COUNT(1) AS total " + "FROM c WHERE c.conversation_id = @id AND IS_STRING(c.timestamp)", + [{"name": "@id", "value": conversation_id}], partition=conversation_id, maximum=1, + ) + if rows and rows[0].get("first_time"): + metadata.update( + first_activity=rows[0]["first_time"], last_activity=rows[0]["last_time"], + message_count=rows[0]["total"], + ) + return metadata + + def snapshot(self, user_id, conversation_ids, cutoff): + snapshots = [] + count = size = 0 + for conversation_id in conversation_ids: + metadata = self.conversation(user_id, conversation_id) + if not metadata["transcript_eligible"]: + fail("personal_only", "Only personal conversations can be released with owner consent.", 403) + messages = _bounded_rows( + self.containers["messages"], + "SELECT * FROM c WHERE c.conversation_id = @id AND c.role IN ('user', 'assistant') " + "ORDER BY c.timestamp", + [{"name": "@id", "value": conversation_id}], partition=conversation_id, + omit_payload=False, + ) + if self.filter_messages is None: + fail("transcript_filter_required", "Transcript availability checks are unavailable.", 503) + messages = self.filter_messages(messages, user_id) + visible = [] + for message in messages: + if message.get("role") not in ("user", "assistant") or not message.get("timestamp"): + continue + if timestamp(message["timestamp"], stored=True) > timestamp(cutoff): + continue + content = message.get("content") + metadata = message.get("metadata") or {} + if any(metadata.get(key) is True for key in ("is_hidden", "is_internal", "hidden")): + continue + if isinstance(content, list): + content = "\n\n".join( + part["text"] for part in content if isinstance(part, dict) + and part.get("type") in ("text", "input_text", "output_text") and isinstance(part.get("text"), str) + ) + if not isinstance(content, str) or content.startswith("data:") or message.get("message_type") == "image": + continue + content = INLINE_DATA.sub("[Binary data omitted]", content) + item = { + "id": message.get("id"), "role": message["role"], + "timestamp": message["timestamp"], "content": content, + } + count += 1 + size += len(json.dumps(item, ensure_ascii=False).encode("utf-8")) + check_size(count, size) + visible.append(item) + snapshots.append({"conversation_id": conversation_id, "messages": visible}) + return snapshots + + def collect_cosmos(self, source, plan, intervals, progress): + first = timestamp(intervals[0][0]).date() + last = timestamp(intervals[-1][1]).date() + timedelta(days=1) + clauses = ["c.timestamp >= @start", "c.timestamp < @end"] + parameters = [{"name": "@start", "value": first.isoformat()}, {"name": "@end", "value": last.isoformat()}] + if not plan["all_users"]: + actors = ACTIVITY_ACTOR_FIELDS if source == "activity" else ("user_id",) + clauses.append("(" + " OR ".join(f"{cosmos_property_path(key)} = @user" for key in actors) + ")") + parameters.append({"name": "@user", "value": plan["user_id"]}) + rows = _bounded_rows( + self.containers[source], "SELECT * FROM c WHERE " + " AND ".join(clauses), parameters, + ) + result = [] + for index, row in enumerate(rows): + if index % 100 == 0: + progress() + if not within_intervals(row["timestamp"], intervals): + continue + conversation_id = row.get("conversation_id") or (row.get("conversation") or {}).get("id") + if plan["conversation_ids"] and conversation_id and conversation_id not in plan["conversation_ids"]: + continue + properties = {key: item for key, item in row.items() if not key.startswith("_") and key not in ("id", "timestamp")} + result.append({ + "timestamp": row["timestamp"], "source": source, "severity": "info", + "message": row.get("log") or row.get("description") or row.get("activity_type") or "Activity", + "properties": properties, "context_scope": "correlated", + }) + return result + + def collect_monitor(self, source, configuration, plan, intervals, progress): + if self.monitor is None: + fail("monitor_not_configured", "Azure Monitor query access is not configured.", 503) + resource = configuration["site_resource_id" if source == "console" else "app_resource_id"] + table = ( + "AppServiceConsoleLogs" if source == "console" + else "union isfuzzy=true withsource=SourceTable AppTraces, AppExceptions, AppRequests, AppDependencies, AppEvents" + ) + windows = " or ".join( + f"(TimeGenerated >= datetime({start}) and TimeGenerated < datetime({end}))" for start, end in intervals + ) + base = ( + f"{table} | where tolower(_ResourceId) == tolower({json.dumps(resource)}) " + f"| where {windows} " + "| extend p=todynamic(column_ifexists('Properties', dynamic({}))) " + "| extend u=tostring(p.sc_user_id_hash), c=tostring(p.sc_conversation_id_hash), " + "lu=coalesce(tostring(p.sc_user_id), tostring(p.user_id)), " + "lc=coalesce(tostring(p.sc_conversation_id), tostring(p.conversation_id)), " + "op=tostring(column_ifexists('OperationId', ''))" + ) + if plan["all_users"]: + query = f"let b=({base}); b | extend ContextScope='correlated'" + else: + user = json.dumps(plan["user_id"]) + user_hash = json.dumps(identity_hash(plan["user_id"])) + ids = json.dumps(plan["conversation_ids"]) + hashes = json.dumps([identity_hash(item) for item in plan["conversation_ids"]]) + subject = f"(u == {user_hash} or lu == {user})" + selected = f"(c in (dynamic({hashes})) or lc in (dynamic({ids})))" + no_user = "(isempty(u) and isempty(lu))" + no_conversation = "(isempty(c) and isempty(lc))" + anchor = f"({subject} or {selected})" if plan["conversation_ids"] else subject + if plan["conversation_ids"]: + anchor += f" and ({no_conversation} or {selected})" + query = ( + f"let b=({base}); let a=(b | where {anchor}); " + "let ops=(a | where isnotempty(op) | distinct op); " + f"b | where ({anchor}) or (op in (ops) and ({no_user} or {subject}) " + f"and ({no_conversation} or {selected}))" + ) + if plan["include_system_context"]: + query += f" or ({no_user} and {no_conversation})" + query += f" | extend ContextScope=iff({anchor}, 'correlated', 'system_context')" + query += ( + " | project TimeGenerated, ContextScope, " + "Message=coalesce(tostring(p.sc_message), tostring(column_ifexists('Message', '')), " + "tostring(column_ifexists('OuterMessage', '')), tostring(column_ifexists('ResultDescription', '')), " + "tostring(column_ifexists('Name', ''))), " + "Severity=tostring(column_ifexists('SeverityLevel', '')), " + "Table=tostring(column_ifexists('SourceTable', 'AppServiceConsoleLogs')), Operation=op, Properties=p, " + "Details=column_ifexists('Details', dynamic(null)), " + "Success=column_ifexists('Success', bool(null)), Duration=column_ifexists('DurationMs', real(null)), " + "ResultCode=tostring(column_ifexists('ResultCode', '')), Target=tostring(column_ifexists('Target', '')), " + "ExceptionType=tostring(column_ifexists('ExceptionType', '')), " + "RequestId=tostring(column_ifexists('Id', '')), ParentId=tostring(column_ifexists('ParentId', '')), " + "Instance=tostring(column_ifexists('AppRoleInstance', '')) " + f"| take {MAX_RECORDS + 1}" + ) + progress() + response = self.monitor.query_resource( + resource, query, + timespan=(timestamp(intervals[0][0]), timestamp(intervals[-1][1])), server_timeout=120, + ) + if response.status != LogsQueryStatus.SUCCESS: + fail("monitor_partial", "Azure Monitor returned incomplete results. Narrow the export and retry.", 503) + result = [] + for table_result in response.tables: + columns = [column if isinstance(column, str) else column.name for column in table_result.columns] + for row in table_result.rows: + check_size(len(result) + 1, 0) + record = json_value(dict(zip(columns, row))) + result.append({ + "timestamp": record["TimeGenerated"], "source": source, "severity": record["Severity"], + "message": record["Message"], "context_scope": record["ContextScope"], + "properties": { + "table": record["Table"], "operation_id": record["Operation"], + "dimensions": record["Properties"], "exception": record["Details"], + "success": record["Success"], "duration_ms": record["Duration"], + "result_code": record["ResultCode"], "target": record["Target"], + "exception_type": record["ExceptionType"], "request_id": record["RequestId"], + "parent_id": record["ParentId"], "instance_id": record["Instance"], + }, + }) + progress() + return result + + def probe_monitor(self, source, configuration): + if self.monitor is None: + fail("monitor_not_configured", "Azure Monitor query access is not configured.", 503) + resource = configuration["site_resource_id" if source == "console" else "app_resource_id"] + table = ( + "AppServiceConsoleLogs" if source == "console" + else "union isfuzzy=true AppTraces, AppExceptions, AppRequests, AppDependencies, AppEvents" + ) + response = self.monitor.query_resource( + resource, + f"{table} | where tolower(_ResourceId) == tolower({json.dumps(resource)}) | project TimeGenerated | take 1", + timespan=timedelta(minutes=5), server_timeout=30, + ) + if response.status != LogsQueryStatus.SUCCESS: + fail("monitor_partial", "Azure Monitor access testing returned an incomplete result.", 503) diff --git a/application/single_app/support_export_store.py b/application/single_app/support_export_store.py new file mode 100644 index 000000000..df6e815d1 --- /dev/null +++ b/application/single_app/support_export_store.py @@ -0,0 +1,212 @@ +# support_export_store.py +"""Injected Cosmos/Blob persistence with conditional writes and private artifacts.""" + +import copy +import re +import uuid +from collections.abc import Mapping +from datetime import timedelta + +from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError +from azure.storage.blob import ContentSettings + +from cosmos_document_utils import normalize_cosmos_document +from support_export_contracts import SupportExportError, fail, iso, timestamp, utc_now + + +BLOB_PATH = re.compile(r"[0-9a-f]{32}/[0-9a-f]{32}/(?:snapshot\.json|bundle\.zip|preview\.json)\Z") +LEASE_SECONDS = 900 + + +class SupportExportStore: + def __init__(self, records, blobs): + self.records = records + self.blobs = blobs + + def read(self, record_id, partition): + try: + response = self.records.read_item(item=record_id, partition_key=partition) + except ResourceNotFoundError: + return None + return normalize_cosmos_document(response) + + def create(self, document): + response = self.records.create_item(body=document) + return normalize_cosmos_document(response) + + def save(self, document): + etag = document.get("_etag") + if not isinstance(etag, str) or not etag: + fail("storage_contract", "Support-export storage returned an invalid revision.", 503) + body = {key: copy.deepcopy(item) for key, item in document.items() if not key.startswith("_")} + try: + response = self.records.replace_item( + item=document["id"], body=body, etag=etag, + match_condition=MatchConditions.IfNotModified, + ) + except Exception as exc: + if getattr(exc, "status_code", None) in (409, 412): + fail("export_conflict", "This export changed. Refresh its status before trying again.", 409) + raise + return normalize_cosmos_document(response) + + def query(self, query, parameters, *, partition=None, limit=100): + options = {"max_item_count": min(limit, 100)} + if partition is None: + options["enable_cross_partition_query"] = True + else: + options["partition_key"] = partition + rows = self.records.query_items(query=query, parameters=parameters, **options) + result = [] + for row in rows: + if not isinstance(row, Mapping): + fail("storage_contract", "Support-export storage returned an invalid query row.", 503) + result.append(dict(row)) + if len(result) >= limit: + break + return result + + def configuration(self): + document = self.read("configuration", "configuration") + return document.get("configuration", {}) if document else {} + + def set_configuration(self, value): + document = self.read("configuration", "configuration") + if document: + document["configuration"] = value + self.save(document) + else: + self.create({ + "id": "configuration", "partition_key": "configuration", + "record_kind": "configuration", "configuration": value, + }) + + def require_private_storage(self, *, create=False): + if self.blobs is None: + fail("storage_required", "Configure application Blob storage before generating support exports.", 503) + try: + properties = self.blobs.get_container_properties() + except ResourceNotFoundError: + if not create: + fail("storage_container_missing", "Initialize the private support-export container before generating.", 503) + try: + self.blobs.create_container() + except ResourceExistsError: + pass + properties = self.blobs.get_container_properties() + public_access = properties.get("public_access") + if public_access: + fail("public_export_storage", "The support-export Blob container must have public access disabled.", 503) + + def put(self, path, data, expires_at, *, content_type="application/json"): + if not BLOB_PATH.fullmatch(path): + fail("invalid_artifact", "Invalid support-export artifact.", 503) + self.require_private_storage() + if self.read("blob-cleanup", "configuration") is None: + try: + self.create({ + "id": "blob-cleanup", "partition_key": "configuration", "record_kind": "cleanup", + "continuation": None, + }) + except Exception as exc: + if getattr(exc, "status_code", None) != 409: + raise + if self.read("blob-cleanup", "configuration") is None: + raise + self.blobs.get_blob_client(path).upload_blob( + data, overwrite=False, + metadata={"expires_at": str(int(timestamp(expires_at).timestamp()))}, + content_settings=ContentSettings(content_type=content_type), + ) + + def get(self, path): + if not isinstance(path, str) or not BLOB_PATH.fullmatch(path): + fail("invalid_artifact", "Invalid support-export artifact.", 503) + self.require_private_storage() + return self.blobs.get_blob_client(path).download_blob().readall() + + def delete_artifacts(self, job_id): + if not re.fullmatch(r"[0-9a-f]{32}", job_id): + fail("invalid_artifact", "Invalid support-export artifact.", 503) + if self.blobs is None: + fail("storage_required", "Support-export cleanup requires application Blob storage.", 503) + for blob in self.blobs.list_blobs(name_starts_with=job_id + "/"): + if not BLOB_PATH.fullmatch(blob.name): + fail("invalid_artifact", "An unexpected support-export artifact needs administrator review.", 503) + try: + self.blobs.delete_blob(blob.name) + except ResourceNotFoundError: + pass + + def sweep_expired_blobs(self): + """Bounded orphan recovery also works after a job document's TTL elapsed.""" + self.require_private_storage() + cursor = self.read("blob-cleanup", "configuration") + pages = self.blobs.list_blobs(include=["metadata"], results_per_page=100).by_page( + continuation_token=cursor.get("continuation") if cursor else None, + ) + page = next(pages, None) + if page is not None: + for blob in page: + if not BLOB_PATH.fullmatch(blob.name): + fail("invalid_artifact", "An unexpected support-export artifact needs administrator review.", 503) + try: + expires_at = int((blob.metadata or {})["expires_at"]) + except (KeyError, ValueError, TypeError): + fail("invalid_artifact", "A support-export artifact has invalid expiry metadata.", 503) + if expires_at <= int(utc_now().timestamp()): + try: + self.blobs.delete_blob(blob.name) + except ResourceNotFoundError: + pass + document = { + "id": "blob-cleanup", "partition_key": "configuration", "record_kind": "cleanup", + "continuation": pages.continuation_token, + "next_scan_at": iso(utc_now() + timedelta(minutes=1)), + } + if cursor: + document["_etag"] = cursor["_etag"] + self.save(document) + else: + self.create(document) + + def acquire_slot(self, now=None): + now = now or utc_now() + for index in range(2): + slot_id = f"worker-{index}" + slot = self.read(slot_id, "worker-slots") + token = uuid.uuid4().hex + if slot is None: + try: + return self.create({ + "id": slot_id, "partition_key": "worker-slots", "record_kind": "lease", + "lease_token": token, "lease_until": iso(now + timedelta(seconds=LEASE_SECONDS)), + }) + except Exception as exc: + if getattr(exc, "status_code", None) == 409: + continue + raise + if timestamp(slot["lease_until"]) > now: + continue + slot.update(lease_token=token, lease_until=iso(now + timedelta(seconds=LEASE_SECONDS))) + try: + return self.save(slot) + except SupportExportError as exc: + if exc.status == 409: + continue + raise + return None + + def renew_slot(self, slot): + current = self.read(slot["id"], "worker-slots") + if current is None or current["lease_token"] != slot["lease_token"] or timestamp(current["lease_until"]) <= utc_now(): + fail("lease_lost", "The export worker lease expired. Retry the export.", 409) + current["lease_until"] = iso(utc_now() + timedelta(seconds=LEASE_SECONDS)) + return self.save(current) + + def release_slot(self, slot): + current = self.read(slot["id"], "worker-slots") + if current and current["lease_token"] == slot["lease_token"]: + current["lease_until"] = iso(utc_now()) + self.save(current) diff --git a/application/single_app/templates/admin/_panes/logging.html b/application/single_app/templates/admin/_panes/logging.html index f4361b0ca..958464334 100644 --- a/application/single_app/templates/admin/_panes/logging.html +++ b/application/single_app/templates/admin/_panes/logging.html @@ -91,6 +91,8 @@
+ {% include "admin/_support_log_export.html" %} +
File Process Logging diff --git a/application/single_app/templates/admin/_support_log_export.html b/application/single_app/templates/admin/_support_log_export.html new file mode 100644 index 000000000..bf8537ec0 --- /dev/null +++ b/application/single_app/templates/admin/_support_log_export.html @@ -0,0 +1,87 @@ +
+
Support Log Export
+

Collect retained diagnostics for a support case. Nothing is sent to the development team automatically. These actions run separately from Save Settings.

+ +
+ Source and storage setup +
+
+
+
+
+
+

Queries use Azure Monitor resource context. Give the application identity read access to these resources; the connected workspace must allow resource permissions. An ingestion connection string does not grant query access.

+

Checking private export storage...

+
+ + + +
+
+
+ Choose diagnostic scope +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+
+ +
+

Metadata only: messages and sensitive titles are not available without transcript consent. Select at most 20 conversations.

+
+
+ +
+
+ Log sources +
+

Explicitly deselect unavailable sources. Historical console output is not recoverable unless Azure already collected it.

+
+
+
+ Privacy and optional transcripts +
+

Transcripts are optional, not restricted to the diagnostic time window, and unavailable for shared/group conversations.

+ + +

+
Only already-scrubbed data is sent to the configured default model. The complete inspection has a 200,000-input-token budget; oversized or incomplete checks block release.
+ + +
+

Limits: 7 days, 20 conversations, 100,000 records, 50 MiB uncompressed. Download expires after 24 hours. No raw export or automatic sharing.

+ +
+
+
Review the exact collection scope
+
+
+
+
+
+
Your support exports
+ + +
+
+ 

+        

+        
+    

+

diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html
index 991dad222..a3484234c 100644
--- a/application/single_app/templates/admin_settings.html
+++ b/application/single_app/templates/admin_settings.html
@@ -2399,6 +2399,7 @@ 

                 
             

 
+            

+                

+                    
+                    
+                

+                
Only requests for your own personal conversations appear here. An admin role cannot approve another person's data release.

+                
+                

+                    
Loading support-data requests...

+                    

+                

+            

+
             {% if settings.enable_content_screening %}
             
             

@@ -331,6 +344,7 @@ 
Paused Microsoft 365 requests

 {% endblock %}
 
 {% block scripts %}
+
 
 
 
diff --git a/application/v2_ui/src/components/admin/SupportExportPanel.tsx b/application/v2_ui/src/components/admin/SupportExportPanel.tsx
new file mode 100644
index 000000000..263347bcb
--- /dev/null
+++ b/application/v2_ui/src/components/admin/SupportExportPanel.tsx
@@ -0,0 +1,345 @@
+// SupportExportPanel.tsx
+
+import { useEffect, useRef, useState } from 'react';
+import { api, apiUrl, CREDENTIALS_MODE } from '../../lib/apiClient';
+import { saveBlob } from '../../lib/endpoints';
+import {
+    SUPPORT_EXPORT_ROOT, SUPPORT_SOURCE_LABELS, SUPPORT_SOURCES, buildSupportRequest,
+    supportDefaults, supportExportPath, supportJobActive,
+    type SupportConfiguration, type SupportConversation, type SupportForm, type SupportJob,
+    type SupportPage, type SupportPerson, type SupportReadiness, type SupportRequest, type SupportReview,
+} from '../../lib/supportExports';
+import { v2ApiFetch } from '../../lib/v2SourcePolicy';
+import { Notice, StatusBadge } from '../approvals/ApprovalParts';
+import { GlassButton } from '../ui/primitives';
+import { inputClass } from './fields';
+
+const configFields = [
+    ['app_resource_id', 'Application Insights resource ID'],
+    ['site_resource_id', 'App Service resource ID (optional console source)'],
+] as const;
+
+function message(error: unknown): string {
+    return error instanceof Error ? error.message : 'The support export operation failed.';
+}
+
+export function SupportExportPanel() {
+    const [readiness, setReadiness] = useState(null);
+    const [configuration, setConfiguration] = useState(null);
+    const [form, setForm] = useState(() => supportDefaults());
+    const [people, setPeople] = useState([]);
+    const [peopleNext, setPeopleNext] = useState(null);
+    const [search, setSearch] = useState('');
+    const [conversationSearch, setConversationSearch] = useState('');
+    const [conversations, setConversations] = useState([]);
+    const [conversationNext, setConversationNext] = useState(null);
+    const [selected, setSelected] = useState([]);
+    const [jobs, setJobs] = useState([]);
+    const [jobsNext, setJobsNext] = useState(null);
+    const [job, setJob] = useState(null);
+    const [review, setReview] = useState<{ scope: SupportReview; request: SupportRequest } | null>(null);
+    const [reviewAcknowledged, setReviewAcknowledged] = useState(false);
+    const [preview, setPreview] = useState(null);
+    const [downloadAcknowledged, setDownloadAcknowledged] = useState(false);
+    const [busy, setBusy] = useState(false);
+    const [loadingConversations, setLoadingConversations] = useState(false);
+    const [polling, setPolling] = useState(true);
+    const [error, setError] = useState('');
+    const reviewHeading = useRef(null);
+    const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+    const run = async (operation: () => Promise) => {
+        setBusy(true);
+        setError('');
+        try { await operation(); } catch (failure) { setError(message(failure)); } finally { setBusy(false); }
+    };
+    const rememberJob = (item: SupportJob) => {
+        setJob(item);
+        setJobs((current) => [item, ...current.filter((other) => other.id !== item.id)]);
+        setPreview(null);
+        setDownloadAcknowledged(false);
+        setPolling(true);
+    };
+
+    useEffect(() => {
+        const controller = new AbortController();
+        Promise.all([
+            api.get(`${SUPPORT_EXPORT_ROOT}/readiness`, controller.signal),
+            api.get>(SUPPORT_EXPORT_ROOT, controller.signal),
+        ]).then(([ready, page]) => {
+            setReadiness(ready); setConfiguration(ready.configuration); setJobs(page.items); setJobsNext(page.next);
+            const latest = [...page.items].sort((a, b) => b.created_at.localeCompare(a.created_at));
+            setJob(latest.find((item) => supportJobActive(item.status)) ?? latest[0] ?? null);
+        }).catch((failure) => { if (!controller.signal.aborted) setError(message(failure)); });
+        return () => controller.abort();
+    }, []);
+
+    useEffect(() => {
+        const controller = new AbortController();
+        const timer = setTimeout(() => {
+            const query = new URLSearchParams({ q: search });
+            api.get>(`${SUPPORT_EXPORT_ROOT}/users?${query}`, controller.signal)
+                .then((page) => { setPeople(page.items); setPeopleNext(page.next); })
+                .catch((failure) => { if (!controller.signal.aborted) setError(message(failure)); });
+        }, 200);
+        return () => { clearTimeout(timer); controller.abort(); };
+    }, [search]);
+
+    useEffect(() => {
+        if (!form.user_id || form.all_users) return;
+        const controller = new AbortController();
+        setLoadingConversations(true);
+        const query = new URLSearchParams({ user_id: form.user_id, q: conversationSearch });
+        api.get>(`${SUPPORT_EXPORT_ROOT}/conversations?${query}`, controller.signal)
+            .then((page) => { setConversations(page.items); setConversationNext(page.next); })
+            .catch((failure) => { if (!controller.signal.aborted) setError(message(failure)); })
+            .finally(() => { if (!controller.signal.aborted) setLoadingConversations(false); });
+        return () => controller.abort();
+    }, [form.user_id, form.all_users, conversationSearch]);
+
+    useEffect(() => {
+        if (!job || !supportJobActive(job.status) || !polling) return;
+        const controller = new AbortController();
+        const timer = setInterval(() => {
+            api.get(supportExportPath(job.id), controller.signal).then((updated) => {
+                setJob(updated); setJobs((current) => current.map((item) => item.id === updated.id ? updated : item));
+            }).catch((failure) => {
+                if (!controller.signal.aborted) { setError(message(failure)); setPolling(false); }
+            });
+        }, 5000);
+        return () => { clearInterval(timer); controller.abort(); };
+    }, [job?.id, job?.status, polling]);
+
+    const changeUser = (id: string, allUsers: boolean) => {
+        setForm((current) => ({
+            ...current, user_id: id, all_users: allUsers, conversation_ids: [], include_transcripts: false,
+            mode: allUsers ? 'time' : current.mode,
+        }));
+        setSelected([]); setConversations([]); setConversationNext(null); setConversationSearch('');
+    };
+    const chooseConversation = (item: SupportConversation, checked: boolean) => {
+        const next = checked ? [...selected, item] : selected.filter((other) => other.id !== item.id);
+        setSelected(next);
+        setForm((current) => ({
+            ...current, conversation_ids: next.map((other) => other.id),
+            include_transcripts: current.include_transcripts && next.length > 0 && next.every((other) => other.transcript_eligible),
+        }));
+    };
+    const refreshReadiness = async () => {
+        const ready = await api.get(`${SUPPORT_EXPORT_ROOT}/readiness`);
+        setReadiness(ready); setConfiguration(ready.configuration); setReview(null);
+    };
+
+    return (
+        

+            

+                Collect retained diagnostics for a support case. Nothing is sent to the development team automatically.
+                These actions run separately from Save settings.
+            

+            {error ? {error} : null}
+            

+                Source and storage setup
+                {configuration ? 

+                    
+                    {configFields.map(([key, label]) => )}
+                    
Queries use Azure Monitor resource context. Give the application identity read access to these resources; the connected workspace must allow resource permissions. The ingestion connection string does not grant query access.

+                    
{readiness?.storage.message}

+                    

+                         void run(async () => {
+                            await api.post(`${SUPPORT_EXPORT_ROOT}/configuration`, configuration); await refreshReadiness();
+                        })}>Save export configuration
+                         void run(async () => {
+                            const ready = await api.post(`${SUPPORT_EXPORT_ROOT}/initialize-storage`, { confirmed: true }); setReadiness(ready);
+                        })}>Initialize private export storage
+                         void run(async () => {
+                            setReadiness(await api.post(`${SUPPORT_EXPORT_ROOT}/test-sources`, {}));
+                        })}>Test source access
+                    

+                
 : 
Source setup could not be loaded.
}
+            

+
+            

+                Choose diagnostic scope
+                
+                {!form.all_users ? 

+                    
+                    
+                    {peopleNext ?  void run(async () => {
+                        const page = await api.get>(`${SUPPORT_EXPORT_ROOT}/users?${new URLSearchParams({ q: search, after: peopleNext })}`);
+                        setPeople((current) => [...current, ...page.items.filter((item) => !current.some((other) => other.id === item.id))]); setPeopleNext(page.next);
+                    })}>Load more users : null}
+                
 : null}
+
+                
+                {form.mode === 'conversations' ? 

+                    

+                        
+                        
+                    

+                    
+                
 : null}
+                {(form.mode === 'time' || form.narrow_range) ? 

+                    
+                    
+                
 : null}
+                
Input timezone: {timezone}. Review shows exact UTC intervals. Maximum effective span: 7 days.

+
+                {!form.all_users && form.user_id ? 

+                    
+                    
Metadata only: messages and sensitive titles are not available to admins without transcript consent. Selected: {selected.length}/20.

+                    {loadingConversations ? 
Loading conversations...
 : null}
+                    
    
+                        {conversations.map((item) => 
  • 
+                            
+                        
    • )}
+                    

+                    {!loadingConversations && !conversations.length ? 
No conversations match this user and ID search.
 : null}
+                    {selected.length ? 
Selected conversation IDs

+                        {selected.map((item) => 
{item.id}
+                             chooseConversation(item, false)}>Remove {item.id}
)}
+                    
 : null}
+                    {conversationNext ?  void run(async () => {
+                        const page = await api.get>(`${SUPPORT_EXPORT_ROOT}/conversations?${new URLSearchParams({ user_id: form.user_id, q: conversationSearch, after: conversationNext })}`);
+                        setConversations((current) => [...current, ...page.items.filter((item) => !current.some((other) => other.id === item.id))]); setConversationNext(page.next);
+                    })}>Load more conversations : null}
+                
 : null}
+
+                

+                    Log sources
+                    {SUPPORT_SOURCES.map((source) => )}
+                    
Explicitly deselect unavailable sources. Historical console output is not recoverable unless Azure already collected it.

+                    
+                

+
+                

+                    Privacy and optional transcripts
+                    
+                    
Transcripts are optional, not restricted to the diagnostic time window, and unavailable for shared/group conversations.

+                    
+                    
{readiness?.model.message} {readiness?.model.label ? `Default model: ${readiness.model.label} (${readiness.model.provider}).` : ''}

+                    {form.privacy === 'ai' ? Only already-scrubbed data is sent to the configured default model. The complete inspection has a 200,000-input-token budget; oversized or incomplete checks block release. : null}
+