From 6d9c04bee3e0361f58f52d38c8113ec1489ee3bb Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 18 Aug 2026 10:26:04 -0400 Subject: [PATCH 1/3] Use Content Understanding for Enhanced extraction Enhanced extraction for PDFs and images now uses Azure AI Content Understanding (prebuilt-documentSearch) instead of Document Intelligence Layout, because it also returns AI-generated descriptions of figures, charts, and diagrams. Standard extraction is unchanged and still uses Document Intelligence prebuilt-read. Enhanced never becomes a hard dependency. Content Understanding is not offered in Azure Government, so resolve_enhanced_extraction_engine falls back to Document Intelligence Layout when the cloud is unsupported, when Content Understanding is unconfigured, or when a request fails. The reason is recorded on the document and surfaced in workspace tooltips. A new enable_enhanced_extraction toggle gates the feature and defaults the mode selector to Auto. The admin UI is environment-aware: Azure Public shows the Content Understanding configuration with a test button and setup guide, while Government and custom clouds show that Enhanced uses Document Intelligence Layout and needs no configuration. Existing deployments already set to Enhanced or Auto are migrated on first settings read so they do not silently downgrade to Standard. Auto still samples first pages with Document Intelligence Layout as the cheaper detector, now also upgrading when it finds figures or images. Neither engine describes figures inside Office files, so embedded images are extracted from DOCX and PPTX packages and analyzed with whichever engine backs the selected mode, then indexed as their own citable chunks. Cost is bounded by a minimum size filter, content-hash de-duplication, and a per-document cap. Uploaded Office files are treated as untrusted: output names are generated rather than reused from archive entries, entry sizes are checked before decompression, and slide relationship parts are parsed with defusedxml. The Change Extraction action now covers images as well as PDFs. Existing setting and document field names are unchanged, so no Cosmos or search index migration is required. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- application/single_app/functions_content.py | 80 ++ .../functions_content_understanding.py | 599 +++++++++++++++ .../functions_document_access_index.py | 5 + application/single_app/functions_documents.py | 266 ++++++- .../single_app/functions_office_media.py | 190 +++++ application/single_app/functions_settings.py | 191 +++++ .../single_app/route_backend_documents.py | 2 + .../route_backend_group_documents.py | 2 + .../route_backend_public_documents.py | 2 + .../single_app/route_backend_settings.py | 47 +- .../route_frontend_admin_settings.py | 49 ++ .../single_app/route_frontend_chats.py | 7 +- .../smart_http_plugin.py | 11 +- .../static/js/admin/admin_settings.js | 104 ++- .../static/js/public/public_workspace.js | 62 +- .../js/workspace/workspace-documents.js | 59 +- .../single_app/templates/admin_settings.html | 371 ++++++++-- .../templates/group_workspaces.html | 52 +- ...NTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md | 251 +++++++ docs/explanation/release_notes.md | 38 + ...content_understanding_extraction_engine.py | 682 ++++++++++++++++++ ...nt_intelligence_auto_reprocess_contract.py | 8 +- ..._intelligence_pdf_image_extraction_mode.py | 9 +- ...t_document_intelligence_test_button_fix.py | 6 +- .../test_office_embedded_image_extraction.py | 427 +++++++++++ 26 files changed, 3379 insertions(+), 143 deletions(-) create mode 100644 application/single_app/functions_content_understanding.py create mode 100644 application/single_app/functions_office_media.py create mode 100644 docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md create mode 100644 functional_tests/test_content_understanding_extraction_engine.py create mode 100644 functional_tests/test_office_embedded_image_extraction.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 70f8cf21c..a0c5b5b0a 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.217" +VERSION = "0.250.218" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index 563544adf..50051cf16 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -10,6 +10,7 @@ from functions_debug import debug_print from config import * +from functions_office_media import extract_office_embedded_images import functions_settings from functions_settings import * from functions_logging import * @@ -607,6 +608,85 @@ def extract_content_with_azure_di(file_path, extraction_mode='read', pages=None) raise e +def resolve_extraction_engine_for_mode(extraction_mode, settings): + """Resolve which engine backs the given extraction mode, plus a human-readable reason.""" + if extraction_mode != 'layout': + return functions_settings.EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, '' + return functions_settings.resolve_enhanced_extraction_engine(settings) + + +def extract_content_with_extraction_engine( + file_path, + extraction_mode, + extraction_engine=None, + settings=None, + pages=None, +): + """Extract page content with the engine that backs the requested extraction mode. + + Standard extraction always uses Document Intelligence ``prebuilt-read``. Enhanced extraction + prefers Azure AI Content Understanding, but always falls back to Document Intelligence + ``prebuilt-layout`` so a Content Understanding outage never blocks ingestion. + + Returns ``(pages_data, engine_used, fallback_reason)``. + """ + resolved_settings = settings if settings is not None else get_settings() + if extraction_engine is None: + extraction_engine, _ = resolve_extraction_engine_for_mode(extraction_mode, resolved_settings) + + if ( + extraction_mode != 'layout' + or extraction_engine != functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + ): + return ( + extract_content_with_azure_di(file_path, extraction_mode=extraction_mode, pages=pages), + functions_settings.EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + '', + ) + + from functions_content_understanding import extract_content_with_content_understanding + + try: + content_understanding_pages = extract_content_with_content_understanding( + file_path, + pages=pages, + settings=resolved_settings, + ) + if content_understanding_pages: + return ( + content_understanding_pages, + functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, + '', + ) + fallback_reason = ( + 'Content Understanding returned no content, so Document Intelligence Layout was used' + ) + log_event( + f"[EXTRACTION_ENGINE] Content Understanding returned no content for " + f"{os.path.basename(file_path)}; falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + except Exception as content_understanding_error: + # The reason is persisted and rendered in workspace tooltips, so keep it short. + error_summary = " ".join(str(content_understanding_error).split())[:240] + fallback_reason = ( + f'Content Understanding failed ({error_summary}), ' + 'so Document Intelligence Layout was used' + ) + log_event( + f"[EXTRACTION_ENGINE] Content Understanding failed for " + f"{os.path.basename(file_path)}: {content_understanding_error}. " + "Falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + + return ( + extract_content_with_azure_di(file_path, extraction_mode='layout', pages=pages), + functions_settings.EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + fallback_reason, + ) + + def extract_table_file(file_path, file_ext): try: if file_ext == '.csv': diff --git a/application/single_app/functions_content_understanding.py b/application/single_app/functions_content_understanding.py new file mode 100644 index 000000000..4bbd6d56d --- /dev/null +++ b/application/single_app/functions_content_understanding.py @@ -0,0 +1,599 @@ +# functions_content_understanding.py + +"""Azure AI Content Understanding client used for Enhanced document extraction. + +Standard extraction always uses Azure Document Intelligence ``prebuilt-read``. Enhanced extraction +uses Azure AI Content Understanding when the app runs in an Azure commercial cloud and the service is +configured; otherwise callers fall back to Document Intelligence ``prebuilt-layout``. + +The public extraction helper intentionally mirrors the return contract of +``functions_content.extract_content_with_azure_di`` so the ingestion pipeline can swap engines +without reshaping downstream chunking. +""" + +import json +import logging +import os +import time + +import requests +from azure.identity import DefaultAzureCredential + +from config import AZURE_ENVIRONMENT, cognitive_services_scope +from functions_appinsights import log_event +from functions_debug import debug_print +import functions_settings + + +CONTENT_UNDERSTANDING_ANALYZE_TIMEOUT_SECONDS = 600 +CONTENT_UNDERSTANDING_POLL_INTERVAL_SECONDS = 3 +CONTENT_UNDERSTANDING_SUBMIT_TIMEOUT_SECONDS = 300 +CONTENT_UNDERSTANDING_POLL_TIMEOUT_SECONDS = 60 +CONTENT_UNDERSTANDING_TOKEN_REFRESH_BUFFER_SECONDS = 300 + +# Content Understanding refuses LLM-backed analyzers until the Foundry resource has default model +# deployments configured. That failure is common enough during setup to deserve its own guidance. +CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_HINTS = ( + 'model deployment', + 'modeldeployment', + 'no default', + 'default model', + 'deployment not found', + 'deploymentnotfound', +) + +CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_MESSAGE = ( + "Content Understanding rejected the request because the Foundry resource has no default model " + "deployments. Deploy a completion model and an embedding model, then set them as Content " + "Understanding defaults before using Enhanced extraction." +) + + +class ContentUnderstandingError(Exception): + """Raised when a Content Understanding request cannot be completed.""" + + +class ContentUnderstandingNotConfiguredError(ContentUnderstandingError): + """Raised when Content Understanding is unavailable or missing required configuration.""" + + +def _get_settings(): + return functions_settings.get_settings() + + +def _resolve_config(settings=None, config_override=None): + """Return normalized Content Understanding configuration from settings or an explicit override.""" + if config_override: + return { + 'endpoint': functions_settings.normalize_content_understanding_endpoint( + config_override.get('endpoint') + ), + 'key': str(config_override.get('key') or '').strip(), + 'authentication_type': functions_settings.normalize_content_understanding_authentication_type( + config_override.get('authentication_type') + ), + 'api_version': functions_settings.normalize_content_understanding_api_version( + config_override.get('api_version') + ), + 'analyzer_id': functions_settings.normalize_content_understanding_analyzer_id( + config_override.get('analyzer_id'), + functions_settings.CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + ), + 'image_analyzer_id': functions_settings.normalize_content_understanding_analyzer_id( + config_override.get('image_analyzer_id'), + functions_settings.CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + ), + } + + return functions_settings.get_content_understanding_config( + settings if settings is not None else _get_settings() + ) + + +_TOKEN_CACHE = {'token': None, 'expires_on': 0} + + +def _get_bearer_token(): + """Return a cached Entra ID access token for Content Understanding.""" + now = time.time() + cached_token = _TOKEN_CACHE.get('token') + cached_expiry = _TOKEN_CACHE.get('expires_on') or 0 + if cached_token and cached_expiry - CONTENT_UNDERSTANDING_TOKEN_REFRESH_BUFFER_SECONDS > now: + return cached_token + + credential = DefaultAzureCredential() + access_token = credential.get_token(cognitive_services_scope) + _TOKEN_CACHE['token'] = access_token.token + _TOKEN_CACHE['expires_on'] = getattr(access_token, 'expires_on', 0) or 0 + return access_token.token + + +def _build_auth_headers(config): + """Build authentication headers for either key or managed identity access.""" + if config.get('authentication_type') == 'managed_identity': + return {'Authorization': f"Bearer {_get_bearer_token()}"} + + key = config.get('key') + if not key: + raise ContentUnderstandingNotConfiguredError( + "Content Understanding key authentication is selected but no key is configured." + ) + return {'Ocp-Apim-Subscription-Key': key} + + +def _validate_config(config): + """Raise when the environment or configuration cannot support a Content Understanding call.""" + if not functions_settings.is_content_understanding_supported_environment(): + raise ContentUnderstandingNotConfiguredError( + f"Content Understanding is not available in the {AZURE_ENVIRONMENT} cloud." + ) + if not config.get('endpoint'): + raise ContentUnderstandingNotConfiguredError( + "Content Understanding endpoint is not configured." + ) + + +def _build_analyze_binary_url(config, analyzer_id, page_range=None): + url = ( + f"{config['endpoint']}/contentunderstanding/analyzers/{analyzer_id}:analyzeBinary" + f"?api-version={config['api_version']}" + ) + if page_range: + url = f"{url}&range={page_range}" + return url + + +def _describe_http_error(response): + """Return a readable message for a failed Content Understanding HTTP response.""" + detail = '' + try: + payload = response.json() + error_payload = payload.get('error') if isinstance(payload, dict) else None + if isinstance(error_payload, dict): + detail = str(error_payload.get('message') or error_payload.get('code') or '') + if not detail and isinstance(payload, dict): + detail = str(payload.get('message') or '') + if not detail: + detail = json.dumps(payload)[:500] + except (ValueError, TypeError): + detail = (response.text or '')[:500] + + if _looks_like_missing_model_deployment(detail): + return CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_MESSAGE + + return f"Content Understanding request failed with HTTP {response.status_code}: {detail}".strip() + + +def _looks_like_missing_model_deployment(message): + normalized_message = str(message or '').lower() + return any(hint in normalized_message for hint in CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_HINTS) + + +def analyze_file_with_content_understanding( + file_path, + analyzer_id=None, + page_range=None, + settings=None, + config_override=None, + max_wait_seconds=CONTENT_UNDERSTANDING_ANALYZE_TIMEOUT_SECONDS, +): + """Submit a local file to Content Understanding and return the completed ``result`` payload.""" + config = _resolve_config(settings=settings, config_override=config_override) + _validate_config(config) + + resolved_analyzer_id = functions_settings.normalize_content_understanding_analyzer_id( + analyzer_id, + config['analyzer_id'], + ) + + with open(file_path, 'rb') as file_handle: + file_bytes = file_handle.read() + + if not file_bytes: + raise ContentUnderstandingError(f"Cannot analyze empty file: {os.path.basename(file_path)}") + + headers = {'Content-Type': 'application/octet-stream'} + headers.update(_build_auth_headers(config)) + + submit_url = _build_analyze_binary_url(config, resolved_analyzer_id, page_range=page_range) + debug_print( + f"[CONTENT_UNDERSTANDING] Submitting {os.path.basename(file_path)} " + f"({len(file_bytes)} bytes) to analyzer {resolved_analyzer_id}" + + (f" range={page_range}" if page_range else "") + ) + + response = requests.post( + submit_url, + headers=headers, + data=file_bytes, + timeout=CONTENT_UNDERSTANDING_SUBMIT_TIMEOUT_SECONDS, + ) + + if response.status_code >= 400: + raise ContentUnderstandingError(_describe_http_error(response)) + + # Content extraction analyzers can answer synchronously for small inputs. + if response.status_code == 200: + payload = response.json() + if isinstance(payload, dict) and payload.get('result'): + return payload['result'] + if isinstance(payload, dict) and payload.get('contents'): + return payload + + operation_location = response.headers.get('Operation-Location') or response.headers.get('operation-location') + if not operation_location: + raise ContentUnderstandingError( + "Content Understanding did not return an Operation-Location header for the analysis request." + ) + + return _poll_analysis_result( + operation_location, + config, + max_wait_seconds=max_wait_seconds, + ) + + +def _poll_analysis_result(operation_location, config, max_wait_seconds): + """Poll a Content Understanding operation until it succeeds, fails, or times out.""" + poll_headers = _build_auth_headers(config) + start_time = time.time() + + while True: + elapsed_seconds = time.time() - start_time + if elapsed_seconds > max_wait_seconds: + raise TimeoutError( + f"Content Understanding analysis did not finish within {max_wait_seconds} seconds." + ) + + time.sleep(CONTENT_UNDERSTANDING_POLL_INTERVAL_SECONDS) + + poll_response = requests.get( + operation_location, + headers=poll_headers, + timeout=CONTENT_UNDERSTANDING_POLL_TIMEOUT_SECONDS, + ) + if poll_response.status_code >= 400: + raise ContentUnderstandingError(_describe_http_error(poll_response)) + + payload = poll_response.json() + status = str(payload.get('status') or '').strip().lower() + + if status == 'succeeded': + result = payload.get('result') + if not result: + raise ContentUnderstandingError( + "Content Understanding reported success but returned no result payload." + ) + return result + + if status == 'failed': + error_payload = payload.get('error') or {} + message = str(error_payload.get('message') or 'Content Understanding analysis failed.') + if _looks_like_missing_model_deployment(message): + raise ContentUnderstandingError(CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_MESSAGE) + raise ContentUnderstandingError(f"Content Understanding analysis failed: {message}") + + if status == 'canceled': + raise ContentUnderstandingError("Content Understanding analysis was canceled.") + + +def _iter_document_contents(result): + """Yield document content entries from a Content Understanding result payload.""" + contents = (result or {}).get('contents') + if not isinstance(contents, list): + return [] + return [content for content in contents if isinstance(content, dict)] + + +def _slice_markdown_by_spans(markdown_text, spans): + """Return the markdown covered by a page's spans.""" + if not markdown_text or not spans: + return '' + + slices = [] + for span in spans: + if not isinstance(span, dict): + continue + try: + offset = int(span.get('offset', 0)) + length = int(span.get('length', 0)) + except (TypeError, ValueError): + continue + if length <= 0: + continue + slices.append(markdown_text[offset:offset + length]) + + return ''.join(slices) + + +def _build_figure_summaries(content): + """Map page-agnostic figure offsets to readable description blocks. + + The description and any structured diagram source are kept separate so de-duplication can drop + a description that Content Understanding already inlined into the markdown without discarding + the machine-readable diagram alongside it. + """ + figures = content.get('figures') + if not isinstance(figures, list): + return [] + + summaries = [] + for figure in figures: + if not isinstance(figure, dict): + continue + + description = str(figure.get('description') or '').strip() + figure_content = figure.get('content') + figure_kind = str(figure.get('kind') or '').strip().lower() + + diagram_block = '' + if figure_kind == 'mermaid' and isinstance(figure_content, str) and figure_content.strip(): + diagram_block = f"```mermaid\n{figure_content.strip()}\n```" + + if not description and not diagram_block: + continue + + caption = figure.get('caption') + caption_text = '' + if isinstance(caption, dict): + caption_text = str(caption.get('content') or '').strip() + elif isinstance(caption, str): + caption_text = caption.strip() + + label = caption_text or str(figure.get('id') or '').strip() or 'Figure' + + span = figure.get('span') + offset = None + if isinstance(span, dict): + try: + offset = int(span.get('offset')) + except (TypeError, ValueError): + offset = None + + summaries.append({ + 'offset': offset, + 'label': label, + 'description': description, + 'diagram_block': diagram_block, + }) + + return summaries + + +def _render_figure_summary(summary, page_text=''): + """Render a figure summary block, omitting a description already present in the page markdown.""" + description = summary.get('description') or '' + diagram_block = summary.get('diagram_block') or '' + + lines = [] + # prebuilt-documentSearch already inlines most descriptions into the markdown. + if description and description not in page_text: + lines.append(description) + if diagram_block and diagram_block not in page_text: + lines.append(diagram_block) + + if not lines: + return '' + + return f"Figure ({summary.get('label') or 'Figure'}): " + "\n".join(lines) + + +def _assign_figures_to_pages(figure_summaries, page_ranges): + """Group figure summary blocks by the page whose span range contains them.""" + figures_by_page = {} + for summary in figure_summaries: + offset = summary.get('offset') + target_page = None + + if offset is not None: + for page_number, start, end in page_ranges: + if start <= offset < end: + target_page = page_number + break + + if target_page is None: + target_page = page_ranges[0][0] if page_ranges else 1 + + figures_by_page.setdefault(target_page, []).append(summary) + + return figures_by_page + + +def build_pages_from_content_understanding_result(result): + """Convert a Content Understanding result into ``[{page_number, content}]`` entries. + + Page text is reconstructed by slicing the content-level ``markdown`` with each page's spans, which + is the documented way to recover per-page content. Figure descriptions are appended to the page + that contains them when they are not already present in that page's markdown. + """ + pages_data = [] + + for content in _iter_document_contents(result): + markdown_text = str(content.get('markdown') or '') + pages = content.get('pages') + start_page_number = content.get('startPageNumber') + try: + start_page_number = int(start_page_number) + except (TypeError, ValueError): + start_page_number = 1 + + if not isinstance(pages, list) or not pages: + fallback_content = markdown_text.strip() + if fallback_content: + pages_data.append({ + 'page_number': start_page_number or 1, + 'content': fallback_content, + }) + continue + + page_ranges = [] + page_markdown = {} + for page in pages: + if not isinstance(page, dict): + continue + try: + page_number = int(page.get('pageNumber')) + except (TypeError, ValueError): + page_number = len(page_markdown) + start_page_number + + spans = page.get('spans') if isinstance(page.get('spans'), list) else [] + page_markdown[page_number] = _slice_markdown_by_spans(markdown_text, spans) + + span_starts = [] + span_ends = [] + for span in spans: + if not isinstance(span, dict): + continue + try: + offset = int(span.get('offset', 0)) + length = int(span.get('length', 0)) + except (TypeError, ValueError): + continue + span_starts.append(offset) + span_ends.append(offset + length) + + if span_starts: + page_ranges.append((page_number, min(span_starts), max(span_ends))) + + figures_by_page = _assign_figures_to_pages(_build_figure_summaries(content), page_ranges) + + for page_number in sorted(page_markdown.keys()): + page_text = page_markdown.get(page_number, '') + figure_blocks = [] + for summary in figures_by_page.get(page_number, []): + rendered_summary = _render_figure_summary(summary, page_text) + if rendered_summary: + figure_blocks.append(rendered_summary) + + if figure_blocks: + page_text = f"{page_text.rstrip()}\n\n" + "\n\n".join(figure_blocks) + + page_text = page_text.strip() + if not page_text: + continue + + pages_data.append({ + 'page_number': page_number, + 'content': page_text, + }) + + return pages_data + + +def extract_content_with_content_understanding(file_path, pages=None, settings=None): + """Extract page-by-page content with Content Understanding. + + Mirrors the return contract of ``functions_content.extract_content_with_azure_di`` so the + ingestion pipeline can switch engines without reshaping downstream chunking. + """ + result = analyze_file_with_content_understanding( + file_path, + page_range=str(pages) if pages else None, + settings=settings, + ) + + pages_data = build_pages_from_content_understanding_result(result) + debug_print( + f"[CONTENT_UNDERSTANDING] Extracted {len(pages_data)} page(s) from " + f"{os.path.basename(file_path)}" + ) + return pages_data + + +def analyze_image_with_content_understanding(image_path, settings=None): + """Analyze a standalone image and return its description text, or an empty string.""" + resolved_settings = settings if settings is not None else _get_settings() + config = _resolve_config(settings=resolved_settings) + + result = analyze_file_with_content_understanding( + image_path, + analyzer_id=config['image_analyzer_id'], + settings=resolved_settings, + ) + + text_blocks = [] + for content in _iter_document_contents(result): + markdown_text = str(content.get('markdown') or '').strip() + if markdown_text: + text_blocks.append(markdown_text) + + for summary in _build_figure_summaries(content): + rendered_summary = _render_figure_summary(summary, markdown_text) + if rendered_summary: + text_blocks.append(rendered_summary) + + return "\n\n".join(block for block in text_blocks if block).strip() + + +def test_content_understanding_connection(config_override, sample_file_path=None): + """Validate Content Understanding connectivity and return ``(ok, message)``.""" + if not functions_settings.is_content_understanding_supported_environment(): + return False, ( + f"Content Understanding is not available in the {AZURE_ENVIRONMENT} cloud. " + "Enhanced extraction uses Document Intelligence Layout here instead." + ) + + config = _resolve_config(config_override=config_override) + + if not config['endpoint']: + return False, "Content Understanding endpoint is required." + if config['authentication_type'] == 'key' and not config['key']: + return False, "Content Understanding key is required when key authentication is selected." + + try: + headers = _build_auth_headers(config) + except ContentUnderstandingError as auth_error: + return False, str(auth_error) + + analyzer_id = config['analyzer_id'] + analyzer_url = ( + f"{config['endpoint']}/contentunderstanding/analyzers/{analyzer_id}" + f"?api-version={config['api_version']}" + ) + + try: + response = requests.get( + analyzer_url, + headers=headers, + timeout=CONTENT_UNDERSTANDING_POLL_TIMEOUT_SECONDS, + ) + except requests.RequestException as request_error: + return False, f"Content Understanding connection error: {request_error}" + + if response.status_code == 404: + return False, ( + f"Content Understanding reached the endpoint but analyzer '{analyzer_id}' was not found. " + "Verify the analyzer id and API version." + ) + if response.status_code in (401, 403): + return False, ( + "Content Understanding rejected the credentials. For managed identity, assign the " + "Cognitive Services User role on the Foundry resource." + ) + if response.status_code >= 400: + return False, _describe_http_error(response) + + if not sample_file_path or not os.path.exists(sample_file_path): + return True, ( + f"Content Understanding connection successful. Analyzer '{analyzer_id}' is reachable." + ) + + try: + analyze_file_with_content_understanding( + sample_file_path, + analyzer_id=analyzer_id, + config_override=config, + max_wait_seconds=180, + ) + except (ContentUnderstandingError, TimeoutError) as analyze_error: + return False, str(analyze_error) + except Exception as analyze_error: # noqa: BLE001 - surface any client failure to the admin + log_event( + f"[CONTENT_UNDERSTANDING] Test analysis failed: {analyze_error}", + level=logging.WARNING, + ) + return False, f"Content Understanding test analysis failed: {analyze_error}" + + return True, ( + f"Content Understanding connection successful. Analyzer '{analyzer_id}' analyzed the test document." + ) diff --git a/application/single_app/functions_document_access_index.py b/application/single_app/functions_document_access_index.py index f7ba2d743..fa53e9c05 100644 --- a/application/single_app/functions_document_access_index.py +++ b/application/single_app/functions_document_access_index.py @@ -1739,6 +1739,8 @@ def _build_base_row(document_item, source_scope, scope_type, scope_id, access_ro 'publication_date': document_item.get('publication_date'), 'enhanced_citations': _has_persisted_blob_reference(document_item), 'document_intelligence_extraction_mode': document_item.get('document_intelligence_extraction_mode'), + 'extraction_engine': document_item.get('extraction_engine'), + 'extraction_engine_reason': document_item.get('extraction_engine_reason'), 'generated_artifact_promotion_status': document_item.get('generated_artifact_promotion_status'), 'generated_artifact_requested_by_user_id': document_item.get('generated_artifact_requested_by_user_id'), 'file_sync': document_item.get('file_sync'), @@ -2153,6 +2155,7 @@ def _query_candidate_projection_rows_for_scope(scope_key, source_scope): 'c.shared_user_ids, c.shared_group_ids, c.file_name, c.title, c.document_classification, c.tags, c.authors, c.keywords, ' 'c.abstract, c.status, c.percentage_complete, c.number_of_pages, c.publication_date, ' 'c.enhanced_citations, c.document_intelligence_extraction_mode, ' + 'c.extraction_engine, c.extraction_engine_reason, ' 'c.generated_artifact_promotion_status, c.generated_artifact_requested_by_user_id, ' 'c.file_sync, c.created_from_chat_upload, ' 'c.conversation_id, c.conversation_title_at_upload, c.upload_date, c.last_updated, ' @@ -2407,6 +2410,8 @@ def _projection_row_to_document(row, source_scope): 'publication_date': row.get('publication_date'), 'enhanced_citations': row.get('enhanced_citations'), 'document_intelligence_extraction_mode': row.get('document_intelligence_extraction_mode'), + 'extraction_engine': row.get('extraction_engine'), + 'extraction_engine_reason': row.get('extraction_engine_reason'), 'generated_artifact_promotion_status': row.get('generated_artifact_promotion_status'), 'generated_artifact_requested_by_user_id': row.get('generated_artifact_requested_by_user_id'), 'file_sync': row.get('file_sync'), diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index ecf6afa39..f0c735eda 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -26,6 +26,7 @@ ) from functions_visio import build_visio_page_markdown, parse_vsdx_pages from functions_content import * +from functions_content_understanding import analyze_image_with_content_understanding from functions_settings import * from functions_search import * from functions_logging import * @@ -252,6 +253,10 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): ) DI_MARKDOWN_TABLE_SEPARATOR_PATTERN = re.compile(r'(?m)^\s*\|(?:\s*:?-{3,}:?\s*\|)+\s*$') DI_MARKDOWN_TABLE_ROW_PATTERN = re.compile(r'(?m)^\s*\|.+\|\s*$') +# Document Intelligence Layout emits figures as markdown
blocks or image tags. Auto mode +# treats them as a signal that Enhanced extraction is worth the extra cost, because Content +# Understanding is the only engine that describes figures. +DI_MARKDOWN_FIGURE_PATTERN = re.compile(r'(|!\[[^\]]*\]\()', re.IGNORECASE) def is_pdf_file_name(file_name): @@ -288,12 +293,155 @@ def _get_document_intelligence_auto_layout_reason(sampled_pages): return 'selection marks or checkbox states detected in the sampled pages' if DI_MARKDOWN_TABLE_ROW_PATTERN.search(sampled_text) and DI_MARKDOWN_TABLE_SEPARATOR_PATTERN.search(sampled_text): return 'table structure detected in the sampled pages' + if DI_MARKDOWN_FIGURE_PATTERN.search(sampled_text): + return 'figures or images detected in the sampled pages' return '' +def _analyze_single_embedded_image(image_path, extraction_engine, image_extraction_mode, settings): + """Analyze one extracted Office image with the active engine and return its text.""" + if extraction_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING: + return analyze_image_with_content_understanding(image_path, settings=settings) + + image_pages = extract_content_with_azure_di(image_path, extraction_mode=image_extraction_mode) + return "\n\n".join( + str(page.get('content', '') or '').strip() + for page in image_pages or [] + if isinstance(page, dict) and str(page.get('content', '') or '').strip() + ).strip() + + +def _build_office_embedded_image_chunks( + file_path, + settings, + update_callback, + starting_page_number=1, +): + """Analyze images embedded in an Office file and return them as labelled content chunks. + + Neither engine describes figures inside Office files, so the images are pulled out of the OOXML + package and analyzed individually with whichever engine backs the admin's selected mode. + + Returns ``(chunks, analyzed_count, extraction_engine)``. + """ + if not settings.get('enable_office_embedded_image_analysis', True): + return [], 0, EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + + min_pixels = normalize_office_embedded_image_min_pixels( + settings.get('office_embedded_image_min_pixels') + ) + max_images = normalize_office_embedded_image_max_per_document( + settings.get('office_embedded_image_max_per_document') + ) + if max_images <= 0: + return [], 0, EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + + # Embedded images are images, so they follow the image side of the admin's configured mode. + admin_extraction_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) + image_extraction_mode = 'read' if admin_extraction_mode == 'read' else 'layout' + extraction_engine, _ = _resolve_extraction_engine_for_mode(image_extraction_mode, settings) + + chunks = [] + analyzed_count = 0 + temp_image_dir = None + + try: + temp_image_dir = tempfile.mkdtemp(prefix='office_images_') + embedded_images = extract_office_embedded_images( + file_path, + temp_image_dir, + min_pixels=min_pixels, + max_images=max_images, + ) + + if not embedded_images: + return [], 0, extraction_engine + + engine_label = ( + "Content Understanding" + if extraction_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + else "Document Intelligence" + ) + total_images = len(embedded_images) + update_callback( + status=f"Analyzing {total_images} embedded image(s) with {engine_label}..." + ) + + for image_index, embedded_image in enumerate(embedded_images, start=1): + try: + analysis_text = _analyze_single_embedded_image( + embedded_image['path'], + extraction_engine, + image_extraction_mode, + settings, + ) + except Exception as image_error: + log_event( + f"[OFFICE_EMBEDDED_IMAGES] Failed to analyze {embedded_image.get('name')}: {image_error}", + level=logging.WARNING, + ) + continue + + if not analysis_text: + continue + + location_label = '' + slide_number = embedded_image.get('slide_number') + if slide_number: + location_label = f" on slide {slide_number}" + + heading = ( + f"### Embedded image {image_index} of {total_images}: " + f"{embedded_image.get('name')}{location_label}" + ) + chunks.append({ + 'page_number': starting_page_number + len(chunks), + 'content': f"{heading}\n\n{analysis_text}", + }) + analyzed_count += 1 + + if analyzed_count: + update_callback( + status=f"Analyzed {analyzed_count} embedded image(s) with {engine_label}." + ) + except Exception as embedded_image_error: + log_event( + f"[OFFICE_EMBEDDED_IMAGES] Embedded image analysis failed for " + f"{os.path.basename(file_path)}: {embedded_image_error}", + level=logging.WARNING, + ) + finally: + if temp_image_dir and os.path.isdir(temp_image_dir): + shutil.rmtree(temp_image_dir, ignore_errors=True) + + return chunks, analyzed_count, extraction_engine + + +def _resolve_extraction_engine_for_mode(extraction_mode, settings): + """Resolve which engine backs the given extraction mode, plus a human-readable reason.""" + return resolve_extraction_engine_for_mode(extraction_mode, settings) + + +def _extract_pages_with_extraction_engine( + file_path, + extraction_mode, + extraction_engine, + settings=None, + pages=None, +): + """Extract page content with the resolved engine, falling back to Document Intelligence Layout.""" + return extract_content_with_extraction_engine( + file_path, + extraction_mode=extraction_mode, + extraction_engine=extraction_engine, + settings=settings, + pages=pages, + ) + + def _resolve_document_intelligence_auto_mode(temp_file_path, is_pdf, is_image, page_count, sample_pages, update_callback): if is_image: - return 'layout', 'image input benefits from Enhanced extraction for spatial structure and selection marks' + return 'layout', 'image input benefits from Enhanced extraction for figures, spatial structure, and selection marks' if not is_pdf: return 'read', 'Auto mode is only evaluated for PDFs and images' @@ -7004,18 +7152,27 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, print(f"Warning: Failed to extract initial metadata for {original_filename}: {e}") # Continue processing even if metadata fails - # --- DI Processing Logic --- + # --- Extraction Engine Resolution --- + # Standard extraction is always Document Intelligence prebuilt-read. Enhanced prefers Content + # Understanding, falling back to Document Intelligence prebuilt-layout when it is unavailable. settings = get_settings() # Assuming get_settings is accessible chunk_config = get_chunk_size_config(settings) + enhanced_extraction_enabled = is_enhanced_extraction_enabled(settings) document_intelligence_extraction_mode = 'read' document_intelligence_requested_mode = 'read' document_intelligence_auto_sample_pages = get_document_intelligence_auto_sample_pages(settings) document_intelligence_auto_reason = '' + extraction_engine = EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + extraction_engine_reason = '' if is_pdf or is_image: if extraction_mode_override: document_intelligence_requested_mode = normalize_document_intelligence_manual_extraction_mode(extraction_mode_override) else: - document_intelligence_requested_mode = get_document_intelligence_pdf_image_extraction_mode(settings) + document_intelligence_requested_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) + + if document_intelligence_requested_mode != 'read' and not enhanced_extraction_enabled: + document_intelligence_requested_mode = 'read' + extraction_engine_reason = 'Enhanced extraction is disabled, so Standard extraction was used' if document_intelligence_requested_mode == 'auto': document_intelligence_extraction_mode, document_intelligence_auto_reason = _resolve_document_intelligence_auto_mode( @@ -7030,11 +7187,20 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, document_intelligence_extraction_mode = document_intelligence_requested_mode document_intelligence_auto_reason = '' + resolved_engine, resolved_engine_reason = _resolve_extraction_engine_for_mode( + document_intelligence_extraction_mode, + settings, + ) + extraction_engine = resolved_engine + extraction_engine_reason = resolved_engine_reason or extraction_engine_reason + update_callback( document_intelligence_extraction_mode=document_intelligence_extraction_mode, document_intelligence_extraction_mode_requested=document_intelligence_requested_mode, document_intelligence_auto_sample_pages=document_intelligence_auto_sample_pages, document_intelligence_auto_reason=document_intelligence_auto_reason, + extraction_engine=extraction_engine, + extraction_engine_reason=extraction_engine_reason, ) di_limit_bytes = 500 * 1024 * 1024 @@ -7139,29 +7305,44 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, except Exception as e: raise Exception(f"Error extracting content from {chunk_effective_filename} with the legacy PowerPoint extractor: {str(e)}") else: - # Send chunk to Azure DI - update_callback(status=f"Sending {chunk_effective_filename} to Azure Document Intelligence...") + # Send chunk to the resolved extraction engine + engine_label = ( + "Azure AI Content Understanding" + if extraction_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + else "Azure Document Intelligence" + ) + update_callback(status=f"Sending {chunk_effective_filename} to {engine_label}...") try: - di_extracted_pages = extract_content_with_azure_di( + di_extracted_pages, engine_used, engine_fallback_reason = _extract_pages_with_extraction_engine( chunk_path, - extraction_mode=document_intelligence_extraction_mode + extraction_mode=document_intelligence_extraction_mode, + extraction_engine=extraction_engine, + settings=settings, ) + if engine_fallback_reason: + extraction_engine = engine_used + extraction_engine_reason = engine_fallback_reason + update_callback( + extraction_engine=engine_used, + extraction_engine_reason=engine_fallback_reason, + ) + num_di_pages = len(di_extracted_pages) conceptual_pages = num_di_pages if not is_image else 1 # Image is one conceptual item if not di_extracted_pages and not is_image: - print(f"Warning: Azure DI returned no content pages for {chunk_effective_filename}.") - status_msg = f"Azure DI found no content in {chunk_effective_filename}." + print(f"Warning: {engine_label} returned no content pages for {chunk_effective_filename}.") + status_msg = f"{engine_label} found no content in {chunk_effective_filename}." # Update page count to 0 if nothing found, otherwise keep previous estimate or conceptual count update_callback(number_of_pages=0 if idx == num_file_chunks else conceptual_pages, status=status_msg) elif not di_extracted_pages and is_image: - print(f"Info: Azure DI processed image {chunk_effective_filename}, but extracted no text.") + print(f"Info: {engine_label} processed image {chunk_effective_filename}, but extracted no text.") update_callback(number_of_pages=conceptual_pages, status=f"Processed image {chunk_effective_filename} (no text found).") else: - update_callback(number_of_pages=conceptual_pages, status=f"Received {num_di_pages} content page(s)/slide(s) from Azure DI for {chunk_effective_filename}.") + update_callback(number_of_pages=conceptual_pages, status=f"Received {num_di_pages} content page(s)/slide(s) from {engine_label} for {chunk_effective_filename}.") except Exception as e: - raise Exception(f"Error extracting content from {chunk_effective_filename} with Azure DI: {str(e)}") + raise Exception(f"Error extracting content from {chunk_effective_filename} with {engine_label}: {str(e)}") # --- Multi-Modal Vision Analysis (for images only) - Must happen BEFORE save_chunks --- if is_image and enable_enhanced_citations and idx == 1: # Only run once for first chunk @@ -7250,6 +7431,30 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, final_chunks_to_save = di_extracted_pages else: final_chunks_to_save = [] # No text extracted + # --- Embedded Office image analysis (DOCX/DOC/PPTX) --- + # Neither extraction engine describes figures inside Office files, so embedded images are + # analyzed separately and appended as their own citable chunks. + if (is_word or is_ppt) and not is_legacy_doc and not is_legacy_ppt: + next_chunk_page_number = max( + (int(chunk.get('page_number') or 0) for chunk in final_chunks_to_save), + default=0, + ) + 1 + + embedded_image_chunks, embedded_image_count, embedded_image_engine = _build_office_embedded_image_chunks( + chunk_path, + settings, + update_callback, + starting_page_number=next_chunk_page_number, + ) + + if embedded_image_chunks: + final_chunks_to_save = list(final_chunks_to_save) + embedded_image_chunks + update_callback( + number_of_pages=len(final_chunks_to_save), + office_embedded_image_count=embedded_image_count, + office_embedded_image_engine=embedded_image_engine, + ) + # Save Final Chunks to Search Index num_final_chunks = len(final_chunks_to_save) if not final_chunks_to_save: @@ -7355,12 +7560,12 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, def validate_document_reprocess_source(document_item, user_id=None, group_id=None, public_workspace_id=None): - """Validate that a PDF has a stored source blob available for DI extraction changes.""" + """Validate that a PDF or image has a stored source blob available for extraction changes.""" if not document_item: return False, "Document not found." - if not is_pdf_file_name(document_item.get('file_name')): - return False, "Only PDF documents can change extraction between Standard and Enhanced." + if not is_pdf_or_image_file_name(document_item.get('file_name')): + return False, "Only PDF and image documents can change extraction between Standard and Enhanced." container_name, blob_path = get_document_blob_storage_info( document_item, @@ -7369,13 +7574,13 @@ def validate_document_reprocess_source(document_item, user_id=None, group_id=Non public_workspace_id=public_workspace_id, ) if not container_name or not blob_path: - return False, "Source PDF is unavailable. Re-upload this PDF before changing extraction." + return False, "Source file is unavailable. Re-upload this document before changing extraction." try: if not _blob_exists(container_name, blob_path): - return False, "Stored source PDF was not found in Blob Storage. Re-upload this PDF before changing extraction." + return False, "Stored source file was not found in Blob Storage. Re-upload this document before changing extraction." except Exception as e: - return False, f"Unable to validate stored source PDF: {str(e)}" + return False, f"Unable to validate stored source file: {str(e)}" return True, "" @@ -7388,14 +7593,16 @@ def _download_document_source_to_temp_file(document_item, user_id=None, group_id public_workspace_id=public_workspace_id, ) if not container_name or not blob_path: - raise FileNotFoundError("Source PDF is unavailable.") + raise FileNotFoundError("Source file is unavailable.") + + file_suffix = os.path.splitext(str(document_item.get('file_name') or ''))[-1].lower() or '.pdf' blob_service_client = _get_blob_service_client() blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_path) temp_file_path = None try: - with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file: + with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as temp_file: temp_file_path = temp_file.name download_stream = blob_client.download_blob() for chunk in download_stream.chunks(): @@ -7408,7 +7615,7 @@ def _download_document_source_to_temp_file(document_item, user_id=None, group_id def process_document_reprocess_extraction_background(document_id, user_id, target_extraction_mode, group_id=None, public_workspace_id=None): - """Extract a stored PDF again with an explicit Standard/Enhanced mode.""" + """Extract a stored PDF or image again with an explicit Standard/Enhanced mode.""" is_group = group_id is not None is_public_workspace = public_workspace_id is not None target_mode = normalize_document_intelligence_manual_extraction_mode(target_extraction_mode) @@ -7444,6 +7651,13 @@ def update_doc_callback(**kwargs): raise ValueError(validation_message) original_filename = document_item.get('file_name') or f'{document_id}.pdf' + source_file_ext = os.path.splitext(original_filename)[-1].lower() or '.pdf' + target_engine, target_engine_reason = _resolve_extraction_engine_for_mode(target_mode, get_settings()) + target_engine_label = ( + "Content Understanding" + if target_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + else "Document Intelligence" + ) update_doc_callback( status=f"Queued to extract again with {target_mode_label}", percentage_complete=0, @@ -7454,6 +7668,8 @@ def update_doc_callback(**kwargs): document_intelligence_extraction_mode_requested=target_mode, document_intelligence_auto_sample_pages=get_document_intelligence_auto_sample_pages(get_settings()), document_intelligence_auto_reason='Manual extraction change requested', + extraction_engine=target_engine, + extraction_engine_reason=target_engine_reason, ) temp_file_path = _download_document_source_to_temp_file( @@ -7466,13 +7682,13 @@ def update_doc_callback(**kwargs): update_doc_callback(status=f"Deleting existing chunks before extracting again with {target_mode_label}...") delete_document_chunks(document_id, group_id=group_id, public_workspace_id=public_workspace_id) - update_doc_callback(status=f"Extracting PDF again with Document Intelligence {target_mode_label}...") + update_doc_callback(status=f"Extracting again with {target_engine_label} {target_mode_label}...") result = process_di_document( document_id=document_id, user_id=user_id, temp_file_path=temp_file_path, original_filename=original_filename, - file_ext='.pdf', + file_ext=source_file_ext, enable_enhanced_citations=bool(document_item.get('enhanced_citations')), update_callback=update_doc_callback, group_id=group_id, @@ -7489,7 +7705,7 @@ def update_doc_callback(**kwargs): final_update_args = { "number_of_pages": total_chunks_saved, - "status": _resolve_processing_complete_status(total_chunks_saved, '.pdf', tuple('.' + ext for ext in IMAGE_EXTENSIONS), tuple('.' + ext for ext in TABULAR_EXTENSIONS), 'disabled'), + "status": _resolve_processing_complete_status(total_chunks_saved, source_file_ext, tuple('.' + ext for ext in IMAGE_EXTENSIONS), tuple('.' + ext for ext in TABULAR_EXTENSIONS), 'disabled'), "percentage_complete": 100, "current_file_chunk": None, } @@ -7499,7 +7715,7 @@ def update_doc_callback(**kwargs): final_update_args["embedding_model_deployment_name"] = embedding_model_name update_doc_callback(**final_update_args) - print(f"Document {document_id} extracted again successfully with Document Intelligence {target_mode}.") + print(f"Document {document_id} extracted again successfully with {target_engine_label} {target_mode}.") except Exception as e: print(f"Error extracting document {document_id} again: {repr(e)}\nTraceback:\n{traceback.format_exc()}") try: diff --git a/application/single_app/functions_office_media.py b/application/single_app/functions_office_media.py new file mode 100644 index 000000000..c15ee3f50 --- /dev/null +++ b/application/single_app/functions_office_media.py @@ -0,0 +1,190 @@ +# functions_office_media.py + +"""Embedded media extraction for OOXML Office files. + +Neither Document Intelligence nor Content Understanding describes figures inside Word and +PowerPoint files. Pulling the embedded images out of the OOXML package lets the ingestion pipeline +analyze them individually with whichever engine backs the selected extraction mode. + +This module deliberately keeps its imports light so it stays importable without Azure clients. +""" + +import hashlib +import os +import re +import zipfile +from io import BytesIO + +from defusedxml.ElementTree import ParseError as DefusedParseError +from defusedxml.ElementTree import fromstring as defused_fromstring +from PIL import Image + + +OFFICE_EMBEDDED_IMAGE_MEDIA_PREFIXES = ('word/media/', 'ppt/media/', 'xl/media/') +# Restricted to raster formats that both Document Intelligence and Content Understanding accept. +OFFICE_EMBEDDED_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.bmp', '.tif', '.tiff', '.heif', '.heic') +OFFICE_EMBEDDED_IMAGE_MIN_BYTES = 2048 +# Uploaded Office files are untrusted, so refuse to decompress an oversized embedded image. +OFFICE_EMBEDDED_IMAGE_MAX_BYTES = 64 * 1024 * 1024 +# Slide relationship parts are small XML documents; anything larger is not worth decompressing. +OFFICE_EMBEDDED_RELS_MAX_BYTES = 4 * 1024 * 1024 + +_OFFICE_MEDIA_INDEX_PATTERN = re.compile(r'(\d+)') +_PPTX_SLIDE_RELS_PATTERN = re.compile(r'^ppt/slides/_rels/slide(\d+)\.xml\.rels$', re.IGNORECASE) + + +def _safe_media_base_name(media_name): + """Return a plain file name for a zip entry, defeating path traversal in crafted archives. + + Zip entries are untrusted input and may embed ``..`` or platform separators, so both separator + styles are collapsed before taking the final component. + """ + normalized_name = str(media_name or '').replace('\\', '/') + base_name = normalized_name.rsplit('/', 1)[-1] + base_name = os.path.basename(base_name) + if base_name in ('', '.', '..'): + return '' + return base_name + + +def _office_media_sort_key(media_name): + """Sort embedded media names naturally so image2 precedes image10.""" + base_name = media_name.rsplit('/', 1)[-1] + match = _OFFICE_MEDIA_INDEX_PATTERN.search(base_name) + return (int(match.group(1)) if match else 0, base_name) + + +def _build_pptx_media_slide_map(archive): + """Map ``ppt/media/*`` entries to the slide numbers that reference them. + + Slide relationship parts come from an untrusted upload, so they get the same declared-size + guard as media entries and are parsed with a hardened XML parser. + """ + media_slide_map = {} + + for entry_name in archive.namelist(): + slide_match = _PPTX_SLIDE_RELS_PATTERN.match(entry_name) + if not slide_match: + continue + + slide_number = int(slide_match.group(1)) + try: + entry_info = archive.getinfo(entry_name) + except KeyError: + continue + + # Check the declared size before decompressing so a zip bomb cannot exhaust memory. + if entry_info.file_size > OFFICE_EMBEDDED_RELS_MAX_BYTES: + continue + + try: + rels_root = defused_fromstring(archive.read(entry_name)) + except (KeyError, DefusedParseError, ValueError, zipfile.BadZipFile): + continue + + for relationship in rels_root: + target = str(relationship.attrib.get('Target') or '').replace('\\', '/') + if '/media/' not in target: + continue + media_name = 'ppt/media/' + target.rsplit('/media/', 1)[-1] + # Keep the first slide that references the image so ordering stays stable. + media_slide_map.setdefault(media_name, slide_number) + + return media_slide_map + + +def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_images=25): + """Extract analyzable raster images embedded in an OOXML Office file. + + Args: + file_path (str): Path to the DOCX/PPTX/XLSX file. + output_dir (str): Directory that extracted images are written to. + min_pixels (int): Minimum width and height required for an image to be analyzed. + max_images (int): Maximum number of images to extract. + + Returns: + list: Dicts with ``name``, ``path``, ``width``, ``height``, and ``slide_number`` keys. + """ + if max_images <= 0: + return [] + + extracted_images = [] + seen_digests = set() + + try: + with zipfile.ZipFile(file_path) as archive: + media_slide_map = _build_pptx_media_slide_map(archive) + + candidate_names = [ + entry_name for entry_name in archive.namelist() + if entry_name.lower().startswith(OFFICE_EMBEDDED_IMAGE_MEDIA_PREFIXES) + and entry_name.lower().endswith(OFFICE_EMBEDDED_IMAGE_EXTENSIONS) + ] + + for media_name in sorted(candidate_names, key=_office_media_sort_key): + if len(extracted_images) >= max_images: + break + + base_name = _safe_media_base_name(media_name) + if not base_name: + continue + + try: + entry_info = archive.getinfo(media_name) + except KeyError: + continue + + # Check the declared size before decompressing so a zip bomb cannot exhaust memory. + if entry_info.file_size > OFFICE_EMBEDDED_IMAGE_MAX_BYTES: + continue + if entry_info.file_size < OFFICE_EMBEDDED_IMAGE_MIN_BYTES: + continue + + try: + image_bytes = archive.read(media_name) + except (KeyError, zipfile.BadZipFile): + continue + + # Small assets are almost always icons, bullets, or spacer graphics. + if len(image_bytes) < OFFICE_EMBEDDED_IMAGE_MIN_BYTES: + continue + + digest = hashlib.sha256(image_bytes).hexdigest() + if digest in seen_digests: + continue + + try: + with Image.open(BytesIO(image_bytes)) as embedded_image: + width, height = embedded_image.size + except Exception: + continue + + if width < min_pixels or height < min_pixels: + continue + + # The output name is generated rather than taken from the archive, so a crafted + # entry name can never influence where the file is written. + extension = os.path.splitext(base_name)[1].lower() + if extension not in OFFICE_EMBEDDED_IMAGE_EXTENSIONS: + continue + output_path = os.path.join(output_dir, f"{len(extracted_images) + 1:03d}{extension}") + + seen_digests.add(digest) + + try: + with open(output_path, 'wb') as output_file: + output_file.write(image_bytes) + except OSError: + continue + + extracted_images.append({ + 'name': base_name, + 'path': output_path, + 'width': width, + 'height': height, + 'slide_number': media_slide_map.get(media_name), + }) + except (zipfile.BadZipFile, FileNotFoundError, OSError): + return [] + + return extracted_images diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index b0b8c32da..c65039416 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -90,6 +90,7 @@ "azure_apim_ai_search_subscription_key", "azure_document_intelligence_key", "azure_apim_document_intelligence_subscription_key", + "azure_content_understanding_key", "speech_service_key", "model_endpoint_identity_header_hmac_secret", ) @@ -530,6 +531,23 @@ def is_mixed_source_relevance_candidates_enabled(settings): DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES = {"read", "layout"} DOCUMENT_INTELLIGENCE_AUTO_SAMPLE_PAGES_DEFAULT = 3 DOCUMENT_INTELLIGENCE_AUTO_SAMPLE_PAGES_MAX = 20 + +# Enhanced extraction engines. Standard extraction is always Document Intelligence prebuilt-read. +EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE = "document_intelligence" +EXTRACTION_ENGINE_CONTENT_UNDERSTANDING = "content_understanding" +EXTRACTION_ENGINES = {EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, EXTRACTION_ENGINE_CONTENT_UNDERSTANDING} + +CONTENT_UNDERSTANDING_AUTHENTICATION_TYPES = {"key", "managed_identity"} +CONTENT_UNDERSTANDING_API_VERSION_DEFAULT = "2025-11-01" +CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT = "prebuilt-documentSearch" +CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT = "prebuilt-imageSearch" +# Azure AI Content Understanding is only offered in Azure commercial regions today. +CONTENT_UNDERSTANDING_SUPPORTED_AZURE_ENVIRONMENTS = {"public"} + +OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_DEFAULT = 150 +OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_MAX = 2000 +OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_DEFAULT = 25 +OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_MAX = 200 AGENTS_PAGE_PROMOTED_POPULAR_ORDER_OPTIONS = {"before", "after", "mixed"} AGENTS_PAGE_PROMOTED_POPULAR_WINDOW_OPTIONS = {"all_time", "30_days", "both"} AGENTS_PAGE_PROMOTED_POPULAR_TAG_LABEL_DEFAULT = "Promoted" @@ -701,6 +719,153 @@ def normalize_document_intelligence_manual_extraction_mode(value): return normalized_value +def normalize_enhanced_extraction_enabled(value): + """Normalize the Enhanced extraction master toggle into a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized_value = value.strip().lower() + if normalized_value in {"false", "0", "no", "off", ""}: + return False + if normalized_value in {"true", "1", "yes", "on"}: + return True + return bool(value) + + +def is_enhanced_extraction_enabled(settings): + """Return whether admins have enabled Enhanced extraction beyond Document Intelligence Standard.""" + return normalize_enhanced_extraction_enabled((settings or {}).get('enable_enhanced_extraction')) + + +def get_effective_document_intelligence_pdf_image_extraction_mode(settings): + """Return the extraction mode actually used, forcing Standard when Enhanced is disabled.""" + if not is_enhanced_extraction_enabled(settings): + return "read" + return get_document_intelligence_pdf_image_extraction_mode(settings) + + +def normalize_content_understanding_authentication_type(value): + """Normalize the Content Understanding authentication type to key or managed identity.""" + normalized_value = str(value or "key").strip().lower() + if normalized_value not in CONTENT_UNDERSTANDING_AUTHENTICATION_TYPES: + return "key" + return normalized_value + + +def normalize_content_understanding_endpoint(value): + """Normalize the Content Understanding endpoint by trimming whitespace and trailing slashes.""" + return str(value or "").strip().rstrip("/") + + +def normalize_content_understanding_api_version(value): + """Normalize the Content Understanding API version, falling back to the GA version.""" + normalized_value = str(value or "").strip() + return normalized_value or CONTENT_UNDERSTANDING_API_VERSION_DEFAULT + + +def normalize_content_understanding_analyzer_id(value, default_analyzer_id=None): + """Normalize a Content Understanding analyzer id, falling back to the supplied prebuilt default.""" + fallback_analyzer_id = default_analyzer_id or CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT + normalized_value = str(value or "").strip() + return normalized_value or fallback_analyzer_id + + +def normalize_office_embedded_image_min_pixels(value): + """Normalize the minimum width/height an embedded Office image must have to be analyzed.""" + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_DEFAULT + + return max(1, min(normalized_value, OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_MAX)) + + +def normalize_office_embedded_image_max_per_document(value): + """Normalize how many embedded Office images may be analyzed for a single document.""" + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_DEFAULT + + return max(0, min(normalized_value, OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_MAX)) + + +def is_content_understanding_supported_environment(azure_environment=None): + """Return whether the running Azure cloud offers Azure AI Content Understanding.""" + resolved_environment = str( + azure_environment if azure_environment is not None else AZURE_ENVIRONMENT + ).strip().lower() + return resolved_environment in CONTENT_UNDERSTANDING_SUPPORTED_AZURE_ENVIRONMENTS + + +def get_content_understanding_config(settings): + """Return the normalized Content Understanding connection settings.""" + resolved_settings = settings or {} + return { + 'endpoint': normalize_content_understanding_endpoint( + resolved_settings.get('azure_content_understanding_endpoint') + ), + 'key': str(resolved_settings.get('azure_content_understanding_key') or '').strip(), + 'authentication_type': normalize_content_understanding_authentication_type( + resolved_settings.get('azure_content_understanding_authentication_type') + ), + 'api_version': normalize_content_understanding_api_version( + resolved_settings.get('azure_content_understanding_api_version') + ), + 'analyzer_id': normalize_content_understanding_analyzer_id( + resolved_settings.get('azure_content_understanding_analyzer_id'), + CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + ), + 'image_analyzer_id': normalize_content_understanding_analyzer_id( + resolved_settings.get('azure_content_understanding_image_analyzer_id'), + CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + ), + } + + +def is_content_understanding_configured(settings): + """Return whether Content Understanding has enough configuration to be called.""" + config = get_content_understanding_config(settings) + if not config['endpoint']: + return False + if config['authentication_type'] == 'key' and not config['key']: + return False + return True + + +def resolve_enhanced_extraction_engine(settings, azure_environment=None): + """Resolve which engine backs Enhanced extraction, plus a human-readable reason. + + Enhanced always falls back to Document Intelligence Layout so Enhanced stays usable in + clouds where Content Understanding is unavailable, or before it has been configured. + """ + if not is_content_understanding_supported_environment(azure_environment): + resolved_environment = str( + azure_environment if azure_environment is not None else AZURE_ENVIRONMENT + ).strip().lower() + return ( + EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + f"Content Understanding is not available in the {resolved_environment} cloud, " + "so Enhanced used Document Intelligence Layout", + ) + + if not is_content_understanding_configured(settings): + return ( + EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + "Content Understanding is not configured, so Enhanced used Document Intelligence Layout", + ) + + return (EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, '') + + +def normalize_extraction_engine(value): + """Normalize a persisted extraction engine value, defaulting to Document Intelligence.""" + normalized_value = str(value or "").strip().lower() + if normalized_value not in EXTRACTION_ENGINES: + return EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + return normalized_value + + def normalize_app_role_claims(user_roles): """Normalize app role claims into a flat string list.""" if not user_roles: @@ -1566,6 +1731,20 @@ def get_settings(use_cosmos=False, include_source=False): 'azure_apim_document_intelligence_endpoint': '', 'azure_apim_document_intelligence_subscription_key': '', + # Enhanced extraction. Standard is always Document Intelligence prebuilt-read; Enhanced uses + # Azure AI Content Understanding in Azure commercial clouds and falls back to Document + # Intelligence prebuilt-layout everywhere else. + 'enable_enhanced_extraction': False, + 'azure_content_understanding_endpoint': '', + 'azure_content_understanding_key': '', + 'azure_content_understanding_authentication_type': 'key', + 'azure_content_understanding_api_version': CONTENT_UNDERSTANDING_API_VERSION_DEFAULT, + 'azure_content_understanding_analyzer_id': CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + 'azure_content_understanding_image_analyzer_id': CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + 'enable_office_embedded_image_analysis': True, + 'office_embedded_image_min_pixels': OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_DEFAULT, + 'office_embedded_image_max_per_document': OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_DEFAULT, + # Web search (via Azure AI Foundry agent) 'enable_web_search': False, 'web_search_consent_accepted': False, @@ -1798,6 +1977,13 @@ def _format_result(settings_payload, source): legacy_control_center_schedule = ( 'control_center_auto_refresh_timezone' not in settings_item ) + # Enhanced extraction gained an explicit enable toggle. Deployments that already selected + # Enhanced or Auto must keep it, otherwise they would silently downgrade to Standard and + # then lose the stored mode on the next admin settings save. + legacy_enhanced_extraction = 'enable_enhanced_extraction' not in settings_item + legacy_enhanced_extraction_mode = normalize_document_intelligence_pdf_image_extraction_mode( + settings_item.get('document_intelligence_pdf_image_extraction_mode') + ) legacy_control_center_time = settings_item.get('control_center_auto_refresh_time') if not isinstance(legacy_control_center_time, str): legacy_hour = settings_item.get('control_center_auto_refresh_hour', 6) @@ -1822,6 +2008,10 @@ def _format_result(settings_payload, source): merged['control_center_auto_refresh_timezone'] = 'UTC' merged['control_center_auto_refresh_next_run'] = None control_center_schedule_migration_updated = True + enhanced_extraction_migration_updated = False + if legacy_enhanced_extraction and legacy_enhanced_extraction_mode in ('layout', 'auto'): + merged['enable_enhanced_extraction'] = True + enhanced_extraction_migration_updated = True migration_updated = apply_custom_endpoint_setting_migration(merged) assignment_settings_updated = normalize_group_workflow_assignment_settings(merged) promoted_popular_settings_updated = normalize_agents_page_promoted_popular_settings(merged) @@ -1838,6 +2028,7 @@ def _format_result(settings_payload, source): if ( merge_changed or control_center_schedule_migration_updated + or enhanced_extraction_migration_updated or migration_updated or assignment_settings_updated or promoted_popular_settings_updated diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index ec3261c02..dcb60096e 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -1357,6 +1357,8 @@ def api_reprocess_document_extraction(): if raw_mode not in DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES: return jsonify({'error': 'Extraction mode must be Standard or Enhanced.'}), 400 target_mode = normalize_document_intelligence_manual_extraction_mode(raw_mode) + if target_mode == 'layout' and not is_enhanced_extraction_enabled(get_settings()): + return jsonify({'error': 'Enhanced extraction is disabled. Enable it in Admin Settings first.'}), 400 document_ids = payload.get('document_ids') if not isinstance(document_ids, list): diff --git a/application/single_app/route_backend_group_documents.py b/application/single_app/route_backend_group_documents.py index 5b769623f..607f8a7df 100644 --- a/application/single_app/route_backend_group_documents.py +++ b/application/single_app/route_backend_group_documents.py @@ -1300,6 +1300,8 @@ def api_reprocess_group_document_extraction(): if raw_mode not in DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES: return jsonify({'error': 'Extraction mode must be Standard or Enhanced.'}), 400 target_mode = normalize_document_intelligence_manual_extraction_mode(raw_mode) + if target_mode == 'layout' and not is_enhanced_extraction_enabled(get_settings()): + return jsonify({'error': 'Enhanced extraction is disabled. Enable it in Admin Settings first.'}), 400 document_ids = payload.get('document_ids') if not isinstance(document_ids, list): diff --git a/application/single_app/route_backend_public_documents.py b/application/single_app/route_backend_public_documents.py index 7b7bd6155..3bbc2a445 100644 --- a/application/single_app/route_backend_public_documents.py +++ b/application/single_app/route_backend_public_documents.py @@ -1125,6 +1125,8 @@ def api_reprocess_public_document_extraction(): if raw_mode not in DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES: return jsonify({'error': 'Extraction mode must be Standard or Enhanced.'}), 400 target_mode = normalize_document_intelligence_manual_extraction_mode(raw_mode) + if target_mode == 'layout' and not is_enhanced_extraction_enabled(get_settings()): + return jsonify({'error': 'Enhanced extraction is disabled. Enable it in Admin Settings first.'}), 400 document_ids = payload.get('document_ids') if not isinstance(document_ids, list): diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 24736ed14..9f582f7ec 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -90,6 +90,8 @@ def _resolve_admin_settings_test_secrets(payload): _resolve_test_payload_secret(payload, ('apim', 'subscription_key'), settings, 'azure_apim_document_intelligence_subscription_key') else: _resolve_test_payload_secret(payload, ('direct', 'key'), settings, 'azure_document_intelligence_key') + elif test_type == 'content_understanding': + _resolve_test_payload_secret(payload, ('key',), settings, 'azure_content_understanding_key') elif test_type == 'redis': _resolve_test_payload_secret(payload, ('key',), settings, 'redis_key') elif test_type == 'web_search': @@ -746,6 +748,9 @@ def test_connection(): elif test_type == 'azure_doc_intelligence': return _test_azure_doc_intelligence_connection(data) + elif test_type == 'content_understanding': + return _test_content_understanding_connection(data) + elif test_type == 'multimodal_vision': return _test_multimodal_vision_connection(data) @@ -1997,13 +2002,47 @@ def _test_azure_doc_intelligence_connection(payload): time.sleep(10) if status == "succeeded": - if extraction_mode == "auto": - return jsonify({'message': 'Azure document intelligence Auto connection successful. Auto samples PDFs with Enhanced extraction during ingestion, then finishes with Standard or Enhanced.'}), 200 - extraction_mode_label = "Enhanced" if extraction_mode == "layout" else "Standard" - return jsonify({'message': f'Azure document intelligence {extraction_mode_label} connection successful'}), 200 + if test_extraction_mode == "layout": + return jsonify({'message': ( + 'Azure document intelligence connection successful. Standard extraction and the ' + 'Layout model used for Auto sampling and Enhanced fallback are both reachable.' + )}), 200 + return jsonify({'message': 'Azure document intelligence Standard connection successful'}), 200 else: return jsonify({'error': f"Document Intelligence error: {status}"}), 500 +def _test_content_understanding_connection(payload): + """Attempt to reach Azure AI Content Understanding using ephemeral settings.""" + from functions_content_understanding import test_content_understanding_connection + + config_override = { + 'endpoint': payload.get('endpoint'), + 'key': payload.get('key'), + 'authentication_type': payload.get('authentication_type'), + 'api_version': payload.get('api_version'), + 'analyzer_id': payload.get('analyzer_id'), + 'image_analyzer_id': payload.get('image_analyzer_id'), + } + + sample_file_path = None + if payload.get('run_sample_analysis'): + candidate_path = os.path.join(current_app.root_path, 'static', 'test_files', 'test_document.pdf') + if os.path.exists(candidate_path): + sample_file_path = candidate_path + + try: + is_ok, message = test_content_understanding_connection( + config_override, + sample_file_path=sample_file_path, + ) + except Exception as e: + return jsonify({'error': f'Content Understanding connection error: {str(e)}'}), 500 + + if is_ok: + return jsonify({'message': message}), 200 + return jsonify({'error': message}), 400 + + def _test_key_vault_connection(payload): """Attempt to connect to Azure Key Vault using ephemeral settings.""" vault_name = payload.get('vault_name', '').strip() diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 48f0329d7..0703e81e3 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -1002,6 +1002,10 @@ def admin_settings(): app_settings=settings_for_template, settings=settings_for_template, azure_environment=AZURE_ENVIRONMENT, + content_understanding_supported=is_content_understanding_supported_environment(), + content_understanding_api_version_default=CONTENT_UNDERSTANDING_API_VERSION_DEFAULT, + content_understanding_document_analyzer_default=CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + content_understanding_image_analyzer_default=CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, default_video_indexer_endpoint=video_indexer_endpoint, default_video_indexer_arm_api_version=DEFAULT_VIDEO_INDEXER_ARM_API_VERSION, user_settings=user_settings, @@ -2328,6 +2332,39 @@ def is_valid_url(url): form_data.get('document_intelligence_auto_sample_pages') ) + enable_enhanced_extraction = form_data.get('enable_enhanced_extraction') == 'on' + enhanced_extraction_was_enabled = is_enhanced_extraction_enabled(settings) + if not enable_enhanced_extraction: + # Standard is the only mode available while Enhanced is off. + document_intelligence_pdf_image_extraction_mode = 'read' + elif not enhanced_extraction_was_enabled and document_intelligence_pdf_image_extraction_mode == 'read': + # Turning Enhanced on defaults to Auto so it upgrades only when structure is detected. + document_intelligence_pdf_image_extraction_mode = 'auto' + + azure_content_understanding_endpoint = normalize_content_understanding_endpoint( + form_data.get('azure_content_understanding_endpoint') + ) + azure_content_understanding_authentication_type = normalize_content_understanding_authentication_type( + form_data.get('azure_content_understanding_authentication_type') + ) + azure_content_understanding_api_version = normalize_content_understanding_api_version( + form_data.get('azure_content_understanding_api_version') + ) + azure_content_understanding_analyzer_id = normalize_content_understanding_analyzer_id( + form_data.get('azure_content_understanding_analyzer_id'), + CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + ) + azure_content_understanding_image_analyzer_id = normalize_content_understanding_analyzer_id( + form_data.get('azure_content_understanding_image_analyzer_id'), + CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + ) + office_embedded_image_min_pixels = normalize_office_embedded_image_min_pixels( + form_data.get('office_embedded_image_min_pixels') + ) + office_embedded_image_max_per_document = normalize_office_embedded_image_max_per_document( + form_data.get('office_embedded_image_max_per_document') + ) + # --- Construct new_settings Dictionary --- new_settings = { # Logging @@ -2728,6 +2765,18 @@ def is_valid_url(url): 'azure_apim_document_intelligence_endpoint': form_data.get('azure_apim_document_intelligence_endpoint', '').strip(), 'azure_apim_document_intelligence_subscription_key': admin_secret('azure_apim_document_intelligence_subscription_key'), + # Enhanced extraction (Azure AI Content Understanding, with Doc Intelligence Layout fallback) + 'enable_enhanced_extraction': enable_enhanced_extraction, + 'azure_content_understanding_endpoint': azure_content_understanding_endpoint, + 'azure_content_understanding_key': admin_secret('azure_content_understanding_key'), + 'azure_content_understanding_authentication_type': azure_content_understanding_authentication_type, + 'azure_content_understanding_api_version': azure_content_understanding_api_version, + 'azure_content_understanding_analyzer_id': azure_content_understanding_analyzer_id, + 'azure_content_understanding_image_analyzer_id': azure_content_understanding_image_analyzer_id, + 'enable_office_embedded_image_analysis': form_data.get('enable_office_embedded_image_analysis') == 'on', + 'office_embedded_image_min_pixels': office_embedded_image_min_pixels, + 'office_embedded_image_max_per_document': office_embedded_image_max_per_document, + 'enable_key_vault_secret_storage': form_data.get('enable_key_vault_secret_storage') == 'on', 'key_vault_name': form_data.get('key_vault_name', '').strip(), 'key_vault_identity': form_data.get('key_vault_identity', ''), diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 71adc39dc..4e700465b 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -1317,12 +1317,13 @@ def upload_file(): if file_ext_nodot in (DOCUMENT_EXTENSIONS | {'html'}) or is_image_file: extraction_mode = 'read' if file_ext == '.pdf' or is_image_file: - extraction_mode = get_document_intelligence_pdf_image_extraction_mode(settings) + extraction_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) if extraction_mode == 'auto': extraction_mode = 'layout' if is_image_file else 'read' - extracted_content_raw = extract_content_with_azure_di( + extracted_content_raw, _extraction_engine_used, _extraction_engine_fallback = extract_content_with_extraction_engine( temp_file_path, - extraction_mode=extraction_mode + extraction_mode=extraction_mode, + settings=settings ) # Convert pages_data list to string diff --git a/application/single_app/semantic_kernel_plugins/smart_http_plugin.py b/application/single_app/semantic_kernel_plugins/smart_http_plugin.py index 64e220959..6c148bc57 100644 --- a/application/single_app/semantic_kernel_plugins/smart_http_plugin.py +++ b/application/single_app/semantic_kernel_plugins/smart_http_plugin.py @@ -385,8 +385,8 @@ async def _process_pdf_content(self, pdf_bytes: bytes, uri: str, response) -> st """Process PDF content using Document Intelligence with large PDF support.""" try: # Import here to avoid circular imports - from functions_content import extract_content_with_azure_di - from functions_settings import get_document_intelligence_pdf_image_extraction_mode, get_settings + from functions_content import extract_content_with_extraction_engine + from functions_settings import get_effective_document_intelligence_pdf_image_extraction_mode, get_settings from config import initialize_clients, CLIENTS # Check if pdf_bytes is actually string content (error case) @@ -425,12 +425,13 @@ async def _process_pdf_content(self, pdf_bytes: bytes, uri: str, response) -> st if pdf_size > max_di_size: return f"📄 **PDF TOO LARGE FOR PROCESSING**\n📍 Source: {uri}\n📊 File size: {pdf_size:,} bytes (exceeds {max_di_size:,} byte limit for Document Intelligence)\n\n⚠️ This PDF is too large for automated text extraction. Please try:\n• A smaller PDF document\n• Specific sections of the document\n• Contact the document provider for a text version" - extraction_mode = get_document_intelligence_pdf_image_extraction_mode(settings) + extraction_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) if extraction_mode == 'auto': extraction_mode = 'read' - pages_data = extract_content_with_azure_di( + pages_data, _engine_used, _engine_fallback = extract_content_with_extraction_engine( temp_file_path, - extraction_mode=extraction_mode + extraction_mode=extraction_mode, + settings=settings ) if not pages_data: diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index 3e99fb82d..aefe16ff1 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -6811,6 +6811,59 @@ function setupToggles() { documentIntelligenceAutoSamplePages.addEventListener('input', markFormAsModified); } + const enableEnhancedExtraction = document.getElementById('enable_enhanced_extraction'); + const enhancedExtractionSettings = document.getElementById('enhanced_extraction_settings'); + const updateEnhancedExtractionControls = () => { + if (!enableEnhancedExtraction || !enhancedExtractionSettings) { + return; + } + enhancedExtractionSettings.classList.toggle('d-none', !enableEnhancedExtraction.checked); + }; + if (enableEnhancedExtraction) { + updateEnhancedExtractionControls(); + enableEnhancedExtraction.addEventListener('change', function () { + updateEnhancedExtractionControls(); + // Turning Enhanced on defaults to Auto so it upgrades only when structure is detected. + if (this.checked && documentIntelligenceExtractionMode && documentIntelligenceExtractionMode.value === 'read') { + documentIntelligenceExtractionMode.value = 'auto'; + updateDocumentIntelligenceAutoControls(); + } + markFormAsModified(); + }); + } + + const contentUnderstandingAuthType = document.getElementById('azure_content_understanding_authentication_type'); + const contentUnderstandingKeyContainer = document.getElementById('azure_content_understanding_key_container'); + const updateContentUnderstandingAuthControls = () => { + if (!contentUnderstandingAuthType || !contentUnderstandingKeyContainer) { + return; + } + contentUnderstandingKeyContainer.classList.toggle('d-none', contentUnderstandingAuthType.value === 'managed_identity'); + }; + if (contentUnderstandingAuthType) { + updateContentUnderstandingAuthControls(); + contentUnderstandingAuthType.addEventListener('change', function () { + updateContentUnderstandingAuthControls(); + markFormAsModified(); + }); + } + + const enableOfficeEmbeddedImageAnalysis = document.getElementById('enable_office_embedded_image_analysis'); + const officeEmbeddedImageOptions = document.getElementById('office_embedded_image_options'); + const updateOfficeEmbeddedImageControls = () => { + if (!enableOfficeEmbeddedImageAnalysis || !officeEmbeddedImageOptions) { + return; + } + officeEmbeddedImageOptions.classList.toggle('d-none', !enableOfficeEmbeddedImageAnalysis.checked); + }; + if (enableOfficeEmbeddedImageAnalysis) { + updateOfficeEmbeddedImageControls(); + enableOfficeEmbeddedImageAnalysis.addEventListener('change', function () { + updateOfficeEmbeddedImageControls(); + markFormAsModified(); + }); + } + const enableContentSafetyCheckbox = document.getElementById('enable_content_safety'); if (enableContentSafetyCheckbox) { enableContentSafetyCheckbox.addEventListener('change', function() { @@ -8363,6 +8416,44 @@ function setupTestButtons() { }); } + const testContentUnderstandingBtn = document.getElementById('test_content_understanding_button'); + if (testContentUnderstandingBtn) { + testContentUnderstandingBtn.addEventListener('click', async () => { + const resultDiv = document.getElementById('test_content_understanding_result'); + resultDiv.className = 'mt-2'; + resultDiv.textContent = 'Testing Content Understanding...'; + + const payload = { + test_type: 'content_understanding', + endpoint: document.getElementById('azure_content_understanding_endpoint')?.value || '', + authentication_type: document.getElementById('azure_content_understanding_authentication_type')?.value || 'key', + key: document.getElementById('azure_content_understanding_key')?.value || '', + api_version: document.getElementById('azure_content_understanding_api_version')?.value || '', + analyzer_id: document.getElementById('azure_content_understanding_analyzer_id')?.value || '', + image_analyzer_id: document.getElementById('azure_content_understanding_image_analyzer_id')?.value || '' + }; + + try { + const resp = await fetch('/api/admin/settings/test_connection', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + const data = await resp.json(); + if (resp.ok) { + resultDiv.className = 'mt-2 text-success'; + resultDiv.textContent = data.message; + } else { + resultDiv.className = 'mt-2 text-danger'; + resultDiv.textContent = data.error || 'Error testing Content Understanding'; + } + } catch (err) { + resultDiv.className = 'mt-2 text-danger'; + resultDiv.textContent = `Error: ${err.message}`; + } + }); + } + const testKeyVaultBtn = document.getElementById('test_key_vault_button'); if (testKeyVaultBtn) { testKeyVaultBtn.addEventListener('click', async () => { @@ -9561,6 +9652,7 @@ togglePassword('toggle_image_gen_key', 'azure_openai_image_gen_key'); togglePassword('toggle_content_safety_key', 'content_safety_key'); togglePassword('toggle_search_key', 'azure_ai_search_key'); togglePassword('toggle_docintel_key', 'azure_document_intelligence_key'); +togglePassword('toggle_content_understanding_key', 'azure_content_understanding_key'); togglePassword('toggle_azure_apim_gpt_subscription_key', 'azure_apim_gpt_subscription_key'); togglePassword('toggle_azure_apim_embedding_subscription_key', 'azure_apim_embedding_subscription_key'); togglePassword('toggle_azure_apim_image_gen_subscription_key', 'azure_apim_image_gen_subscription_key'); @@ -10451,7 +10543,17 @@ function setupWalkthroughFieldListeners() { {selector: '#document_intelligence_auto_sample_pages', event: 'input'}, {selector: '#azure_apim_document_intelligence_endpoint', event: 'input'}, {selector: '#azure_apim_document_intelligence_subscription_key', event: 'input'}, - {selector: '#enable_document_intelligence_apim', event: 'change'} + {selector: '#enable_document_intelligence_apim', event: 'change'}, + {selector: '#enable_enhanced_extraction', event: 'change'}, + {selector: '#azure_content_understanding_endpoint', event: 'input'}, + {selector: '#azure_content_understanding_key', event: 'input'}, + {selector: '#azure_content_understanding_authentication_type', event: 'change'}, + {selector: '#azure_content_understanding_api_version', event: 'input'}, + {selector: '#azure_content_understanding_analyzer_id', event: 'input'}, + {selector: '#azure_content_understanding_image_analyzer_id', event: 'input'}, + {selector: '#enable_office_embedded_image_analysis', event: 'change'}, + {selector: '#office_embedded_image_min_pixels', event: 'input'}, + {selector: '#office_embedded_image_max_per_document', event: 'input'} ], 8: [ // Video settings {selector: '#enable_video_file_support', event: 'change'}, diff --git a/application/single_app/static/js/public/public_workspace.js b/application/single_app/static/js/public/public_workspace.js index c75bc2b26..3893d4715 100644 --- a/application/single_app/static/js/public/public_workspace.js +++ b/application/single_app/static/js/public/public_workspace.js @@ -1283,7 +1283,7 @@ function appendPublicDocumentMetaPills(container, doc) { doc.version ? `v${doc.version}` : null, doc.authors ? `By ${doc.authors}` : null, doc.number_of_pages ? `${doc.number_of_pages} pages` : null, - isPublicPdfDocument(doc) ? getPublicDocumentExtractionModeLabel(doc) : null, + supportsPublicExtractionModeChange(doc) ? getPublicDocumentExtractionModeLabel(doc) : null, doc.publication_date ? doc.publication_date : null, ].filter(Boolean); @@ -1297,10 +1297,34 @@ function appendPublicDocumentMetaPills(container, doc) { }); } +const PUBLIC_EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif', '.heic']; +const PUBLIC_EXTRACTION_STANDARD_TOOLTIP = 'Standard extraction uses Document Intelligence Read for faster, lower-cost text extraction. Best for plain text PDFs and images.'; +const PUBLIC_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP = 'Enhanced extraction uses Azure AI Content Understanding to preserve tables, page structure, and checkbox states, and to describe figures, charts, and images. Adds latency and higher cost.'; +const PUBLIC_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP = 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.'; + function isPublicPdfDocument(doc) { return String(doc?.file_name || '').toLowerCase().endsWith('.pdf'); } +function isPublicImageDocument(doc) { + const fileName = String(doc?.file_name || '').toLowerCase(); + return PUBLIC_EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS.some((extension) => fileName.endsWith(extension)); +} + +function supportsPublicExtractionModeChange(doc) { + return isPublicPdfDocument(doc) || isPublicImageDocument(doc); +} + +function getPublicDocumentExtractionEngine(doc) { + return String(doc?.extraction_engine || '').trim().toLowerCase(); +} + +function getPublicDocumentEnhancedTooltipForEngine(engine) { + return engine === 'content_understanding' + ? PUBLIC_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP + : PUBLIC_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP; +} + function getPublicDocumentExtractionModeLabel(doc) { const mode = String(doc?.document_intelligence_extraction_mode || '').trim().toLowerCase(); return mode === 'layout' ? 'Enhanced' : 'Standard'; @@ -1319,17 +1343,19 @@ function getPublicDocumentExtractionModeLabelFromMode(mode) { return mode === 'layout' ? 'Enhanced' : 'Standard'; } -function getPublicDocumentExtractionChangeTooltip(targetMode) { +function getPublicDocumentExtractionChangeTooltip(targetMode, engine) { return targetMode === 'layout' - ? 'Extract again with Enhanced extraction. Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.' - : 'Extract again with Standard extraction. Standard extraction uses Document Intelligence Read for faster text extraction. Best for plain text PDFs and images.'; + ? `Extract again with Enhanced extraction. ${getPublicDocumentEnhancedTooltipForEngine(engine)}` + : `Extract again with Standard extraction. ${PUBLIC_EXTRACTION_STANDARD_TOOLTIP}`; } function getPublicDocumentExtractionModeTooltip(doc) { const mode = String(doc?.document_intelligence_extraction_mode || '').trim().toLowerCase(); - return mode === 'layout' - ? 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.' - : 'Standard extraction uses Document Intelligence Read for faster text extraction. Best for plain text PDFs and images.'; + const tooltip = mode === 'layout' + ? getPublicDocumentEnhancedTooltipForEngine(getPublicDocumentExtractionEngine(doc)) + : PUBLIC_EXTRACTION_STANDARD_TOOLTIP; + const reason = String(doc?.extraction_engine_reason || '').trim(); + return reason ? `${tooltip} ${reason}.` : tooltip; } function getPublicDocumentCitationTooltip(doc) { @@ -1339,7 +1365,7 @@ function getPublicDocumentCitationTooltip(doc) { } function createPublicDocumentExtractionModeBadge(doc) { - if (!isPublicPdfDocument(doc)) { + if (!supportsPublicExtractionModeChange(doc)) { return null; } @@ -1355,7 +1381,7 @@ function createPublicDocumentExtractionModeBadge(doc) { } function getPublicDocumentExtractionModeBadgeHtml(doc) { - if (!isPublicPdfDocument(doc)) { + if (!supportsPublicExtractionModeChange(doc)) { return ''; } @@ -1501,11 +1527,11 @@ function createPublicDocumentCard(doc) { if (canManage) { dropdownItems.push(createPublicDropdownItem('bi-pencil-fill', 'Edit Metadata', () => window.onEditPublicDocument(docId))); dropdownItems.push(createPublicDropdownItem('bi-magic', 'Extract Metadata', () => window.onExtractPublicMetadata(docId, null))); - if (isPublicPdfDocument(doc)) { + if (supportsPublicExtractionModeChange(doc)) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); dropdownItems.push(createPublicDropdownDivider()); dropdownItems.push(createPublicDropdownHeader('Change Extraction')); dropdownItems.push(createPublicDropdownItem(extractionActionIcon, `Change to ${extractionActionLabel}`, () => window.reprocessPublicDocumentExtraction(docId, extractionActionMode, null), false, extractionActionTooltip)); @@ -1672,7 +1698,7 @@ function renderPublicDocumentRow(doc) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); actionsDropdown += `
  • @@ -1681,7 +1707,7 @@ function renderPublicDocumentRow(doc) {
  • Extract Metadata
    • - ${isPublicPdfDocument(doc) ? ` + ${supportsPublicExtractionModeChange(doc) ? `
  • Change Extraction
  • @@ -1749,7 +1775,7 @@ function renderPublicDocumentRow(doc) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); detailsRow.innerHTML = ` @@ -1759,7 +1785,7 @@ function renderPublicDocumentRow(doc) {

    Version: ${escapeHtml(doc.version || '1')}

    Authors: ${escapeHtml(doc.authors || 'N/A')}

    Pages/Chunks: ${escapeHtml(doc.number_of_pages || 'N/A')}

    - ${isPublicPdfDocument(doc) ? `

    Extraction: ${getPublicDocumentExtractionModeBadgeHtml(doc)}

    ` : ''} + ${supportsPublicExtractionModeChange(doc) ? `

    Extraction: ${getPublicDocumentExtractionModeBadgeHtml(doc)}

    ` : ''}

    Citations: ${getCitationBadge(doc.enhanced_citations)}

    Publication Date: ${escapeHtml(doc.publication_date || 'N/A')}

    Keywords: ${escapeHtml(doc.keywords || 'N/A')}

    @@ -1774,7 +1800,7 @@ function renderPublicDocumentRow(doc) { - ${isPublicPdfDocument(doc) ? ` + ${supportsPublicExtractionModeChange(doc) ? ` ` : ''} @@ -3341,10 +3367,10 @@ function buildPublicFolderDocumentsTable(docs) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); actionsHtml += `
  • Edit Metadata
  • Extract Metadata
    • - ${isPublicPdfDocument(doc) ? `
    • + ${supportsPublicExtractionModeChange(doc) ? `
  • Change Extraction
  • Change to ${extractionActionLabel}
    • ` : ''}
    • diff --git a/application/single_app/static/js/workspace/workspace-documents.js b/application/single_app/static/js/workspace/workspace-documents.js index 76c53f539..a887786f5 100644 --- a/application/single_app/static/js/workspace/workspace-documents.js +++ b/application/single_app/static/js/workspace/workspace-documents.js @@ -392,12 +392,39 @@ function getDocumentClassificationBadge(doc) { return 'None'; } +const EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif', '.heic']; + function isPdfDocument(doc) { return String(doc?.file_name || '').toLowerCase().endsWith('.pdf'); } -const DOCUMENT_EXTRACTION_STANDARD_TOOLTIP = 'Standard extraction uses Document Intelligence Read for faster text extraction. Best for plain text PDFs and images.'; -const DOCUMENT_EXTRACTION_ENHANCED_TOOLTIP = 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.'; +function isImageDocument(doc) { + const fileName = String(doc?.file_name || '').toLowerCase(); + return EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS.some((extension) => fileName.endsWith(extension)); +} + +function supportsExtractionModeChange(doc) { + return isPdfDocument(doc) || isImageDocument(doc); +} + +const DOCUMENT_EXTRACTION_STANDARD_TOOLTIP = 'Standard extraction uses Document Intelligence Read for faster, lower-cost text extraction. Best for plain text PDFs and images.'; +const DOCUMENT_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP = 'Enhanced extraction uses Azure AI Content Understanding to preserve tables, page structure, and checkbox states, and to describe figures, charts, and images. Adds latency and higher cost.'; +const DOCUMENT_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP = 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.'; + +function getDocumentExtractionEngine(doc) { + return String(doc?.extraction_engine || '').trim().toLowerCase(); +} + +function getDocumentExtractionEngineReason(doc) { + return String(doc?.extraction_engine_reason || '').trim(); +} + +function getDocumentEnhancedTooltipForEngine(engine) { + return engine === 'content_understanding' + ? DOCUMENT_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP + : DOCUMENT_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP; +} + const DOCUMENT_CITATION_STANDARD_TOOLTIP = 'Standard citations reference indexed text chunks.'; const DOCUMENT_CITATION_ENHANCED_TOOLTIP = 'Enhanced citations preserve source-file context for richer citation previews and supported file workflows.'; @@ -405,8 +432,10 @@ function getDocumentExtractionModeLabelFromMode(mode) { return mode === 'layout' ? 'Enhanced' : 'Standard'; } -function getDocumentExtractionModeTooltipFromMode(mode) { - return mode === 'layout' ? DOCUMENT_EXTRACTION_ENHANCED_TOOLTIP : DOCUMENT_EXTRACTION_STANDARD_TOOLTIP; +function getDocumentExtractionModeTooltipFromMode(mode, engine) { + return mode === 'layout' + ? getDocumentEnhancedTooltipForEngine(engine) + : DOCUMENT_EXTRACTION_STANDARD_TOOLTIP; } function getDocumentExtractionModeIcon(mode) { @@ -418,9 +447,9 @@ function getDocumentTargetExtractionMode(doc) { return currentMode === 'layout' ? 'read' : 'layout'; } -function getDocumentExtractionChangeTooltip(targetMode) { +function getDocumentExtractionChangeTooltip(targetMode, engine) { return targetMode === 'layout' - ? `Extract again with Enhanced extraction. ${DOCUMENT_EXTRACTION_ENHANCED_TOOLTIP}` + ? `Extract again with Enhanced extraction. ${getDocumentEnhancedTooltipForEngine(engine)}` : `Extract again with Standard extraction. ${DOCUMENT_EXTRACTION_STANDARD_TOOLTIP}`; } @@ -431,7 +460,9 @@ function getDocumentExtractionModeLabel(doc) { function getDocumentExtractionModeTooltip(doc) { const mode = String(doc?.document_intelligence_extraction_mode || '').trim().toLowerCase(); - return getDocumentExtractionModeTooltipFromMode(mode); + const tooltip = getDocumentExtractionModeTooltipFromMode(mode, getDocumentExtractionEngine(doc)); + const reason = getDocumentExtractionEngineReason(doc); + return reason ? `${tooltip} ${reason}.` : tooltip; } function getDocumentCitationTooltip(doc) { @@ -439,7 +470,7 @@ function getDocumentCitationTooltip(doc) { } function getDocumentExtractionModeBadge(doc) { - if (!isPdfDocument(doc)) { + if (!supportsExtractionModeChange(doc)) { return ''; } @@ -450,7 +481,7 @@ function getDocumentExtractionModeBadge(doc) { } function getDocumentReprocessDropdownItems(doc) { - if (!isPdfDocument(doc)) { + if (!supportsExtractionModeChange(doc)) { return ''; } @@ -458,7 +489,7 @@ function getDocumentReprocessDropdownItems(doc) { const targetMode = getDocumentTargetExtractionMode(doc); const targetLabel = getDocumentExtractionModeLabelFromMode(targetMode); const targetIcon = getDocumentExtractionModeIcon(targetMode); - const targetTooltip = getDocumentExtractionChangeTooltip(targetMode); + const targetTooltip = getDocumentExtractionChangeTooltip(targetMode, getDocumentExtractionEngine(doc)); return `
    • @@ -485,7 +516,7 @@ function getDocumentMetaPills(doc) { if (doc.number_of_pages) { pills.push(`${escapeHtml(String(doc.number_of_pages))} pages`); } - if (isPdfDocument(doc)) { + if (supportsExtractionModeChange(doc)) { pills.push(`${getDocumentExtractionModeLabel(doc)}`); } if (authors.length) { @@ -1956,7 +1987,7 @@ function renderDocumentRow(doc) {

    Version: ${escapeHtml(doc.version || "N/A")}

    Authors: ${escapeHtml(Array.isArray(doc.authors) ? doc.authors.join(", ") : doc.authors || "N/A")}

    Pages: ${escapeHtml(doc.number_of_pages || "N/A")}

    - ${isPdfDocument(doc) ? `

    Extraction: ${getDocumentExtractionModeBadge(doc)}

    ` : ''} + ${supportsExtractionModeChange(doc) ? `

    Extraction: ${getDocumentExtractionModeBadge(doc)}

    ` : ''}

    Citations: ${doc.enhanced_citations ? `Enhanced` : `Standard`}

    Publication Date: ${escapeHtml(doc.publication_date || "N/A")}

    Keywords: ${escapeHtml(Array.isArray(doc.keywords) ? doc.keywords.join(", ") : doc.keywords || "N/A")}

    @@ -1978,12 +2009,12 @@ function renderDocumentRow(doc) { `; } - if (isOwner && isPdfDocument(doc)) { + if (isOwner && supportsExtractionModeChange(doc)) { const reprocessDocId = escapeHtml(String(docId || '')); const extractionActionMode = getDocumentTargetExtractionMode(doc); const extractionActionLabel = getDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getDocumentExtractionChangeTooltip(extractionActionMode, getDocumentExtractionEngine(doc)); detailsHtml += ` + + +
    + {% if content_understanding_supported %} +
    + Enhanced extraction uses Azure AI Content Understanding, which returns tables, + page structure, checkbox states, and AI-generated descriptions of figures, charts, and images. + Configure the connection below. +
    + {% else %} +
    + Azure AI Content Understanding is not available in the + {{ azure_environment }} cloud, so Enhanced extraction uses + Document Intelligence Layout here. There is nothing more to configure — + Enhanced still captures tables, page structure, forms, and checkbox states. +
    + {% endif %} + +
    + +
    + Enhanced captures more document detail for PDFs and images, including tables, page structure, and checked or unchecked marks. It adds latency and has a 6X increase for every 1000 pages when selected. +
    +
    + + +
    + Auto samples this many first PDF pages with Document Intelligence Layout. If it detects tables, selection marks, or figures, the full PDF uses Enhanced; otherwise it finishes with Standard. Images use Enhanced in Auto mode. +
    +
    +
    +
    Standard: Document Intelligence Read. Fastest and lowest-cost path for plain text PDFs and images.
    +
    Enhanced: best for tables, section structure, page layout, forms, checkbox states, and figure descriptions. Expect more latency and higher cost.
    +
    Auto: samples the first pages with Document Intelligence Layout, then uses Enhanced only when the sample shows tables, selection marks, or figures.
    +
    +
    + + {% if content_understanding_supported %} +
    +
    +
    Azure AI Content Understanding
    + +
    +

    + Required for Enhanced extraction. Deployed from a Microsoft Foundry resource. + If this is left blank, Enhanced falls back to Document Intelligence Layout. +

    + +
    + + +
    + Your Microsoft Foundry resource endpoint, without a trailing path. +
    +
    + +
    + + +
    + Managed identity requires the Cognitive Services User role on the Foundry resource. +
    +
    + +
    + +
    + + +
    +
    + +
    +
    + + +
    Default: {{ content_understanding_api_version_default }}
    +
    +
    + + +
    Default: {{ content_understanding_document_analyzer_default }}
    +
    +
    + + +
    Default: {{ content_understanding_image_analyzer_default }}
    +
    +
    + +
    - -
    - Enhanced captures more document detail for PDFs and images, including tables, page structure, and checked or unchecked marks. It adds latency and has a 6X increase for every 1000 pages when selected. -
    -
    - + {% endif %} +
    + +
    +
    Images Inside Office Files
    +

    + Neither extraction engine describes figures inside Word and PowerPoint files. When this is on, + embedded images are pulled out of the file, analyzed with whichever engine backs the selected + extraction mode, and indexed as their own citable chunks. This works with Standard extraction + too, using Document Intelligence. +

    +
    -
    - Auto samples this many first PDF pages with Enhanced extraction. If it detects tables or selection marks, the full PDF uses Enhanced; otherwise it finishes with Standard. Images use Enhanced in Auto mode. -
    +
    -
    -
    Standard: fastest and lowest-cost path for plain text PDFs and images.
    -
    Enhanced: best for tables, section structure, page layout, forms, and checkbox states. Expect more latency and a 6X increase for every 1000 pages.
    -
    Auto: samples the first pages, then uses Enhanced only when the sample shows tables or selection marks.
    +
    +
    + + +
    Images narrower or shorter than this are skipped as icons or spacers.
    +
    +
    + + +
    Caps per-document cost. Duplicate images are analyzed once.
    +
    + +
    + +