diff --git a/application/single_app/config.py b/application/single_app/config.py index cfe20b596..959e239f3 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.220" +VERSION = "0.250.221" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index 563544adf..50051cf16 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -10,6 +10,7 @@ from functions_debug import debug_print from config import * +from functions_office_media import extract_office_embedded_images import functions_settings from functions_settings import * from functions_logging import * @@ -607,6 +608,85 @@ def extract_content_with_azure_di(file_path, extraction_mode='read', pages=None) raise e +def resolve_extraction_engine_for_mode(extraction_mode, settings): + """Resolve which engine backs the given extraction mode, plus a human-readable reason.""" + if extraction_mode != 'layout': + return functions_settings.EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, '' + return functions_settings.resolve_enhanced_extraction_engine(settings) + + +def extract_content_with_extraction_engine( + file_path, + extraction_mode, + extraction_engine=None, + settings=None, + pages=None, +): + """Extract page content with the engine that backs the requested extraction mode. + + Standard extraction always uses Document Intelligence ``prebuilt-read``. Enhanced extraction + prefers Azure AI Content Understanding, but always falls back to Document Intelligence + ``prebuilt-layout`` so a Content Understanding outage never blocks ingestion. + + Returns ``(pages_data, engine_used, fallback_reason)``. + """ + resolved_settings = settings if settings is not None else get_settings() + if extraction_engine is None: + extraction_engine, _ = resolve_extraction_engine_for_mode(extraction_mode, resolved_settings) + + if ( + extraction_mode != 'layout' + or extraction_engine != functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + ): + return ( + extract_content_with_azure_di(file_path, extraction_mode=extraction_mode, pages=pages), + functions_settings.EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + '', + ) + + from functions_content_understanding import extract_content_with_content_understanding + + try: + content_understanding_pages = extract_content_with_content_understanding( + file_path, + pages=pages, + settings=resolved_settings, + ) + if content_understanding_pages: + return ( + content_understanding_pages, + functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, + '', + ) + fallback_reason = ( + 'Content Understanding returned no content, so Document Intelligence Layout was used' + ) + log_event( + f"[EXTRACTION_ENGINE] Content Understanding returned no content for " + f"{os.path.basename(file_path)}; falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + except Exception as content_understanding_error: + # The reason is persisted and rendered in workspace tooltips, so keep it short. + error_summary = " ".join(str(content_understanding_error).split())[:240] + fallback_reason = ( + f'Content Understanding failed ({error_summary}), ' + 'so Document Intelligence Layout was used' + ) + log_event( + f"[EXTRACTION_ENGINE] Content Understanding failed for " + f"{os.path.basename(file_path)}: {content_understanding_error}. " + "Falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + + return ( + extract_content_with_azure_di(file_path, extraction_mode='layout', pages=pages), + functions_settings.EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + fallback_reason, + ) + + def extract_table_file(file_path, file_ext): try: if file_ext == '.csv': diff --git a/application/single_app/functions_content_understanding.py b/application/single_app/functions_content_understanding.py new file mode 100644 index 000000000..c78098852 --- /dev/null +++ b/application/single_app/functions_content_understanding.py @@ -0,0 +1,620 @@ +# functions_content_understanding.py + +"""Azure AI Content Understanding client used for Enhanced document extraction. + +Standard extraction always uses Azure Document Intelligence ``prebuilt-read``. Enhanced extraction +uses Azure AI Content Understanding when the app runs in an Azure commercial cloud and the service is +configured; otherwise callers fall back to Document Intelligence ``prebuilt-layout``. + +The public extraction helper intentionally mirrors the return contract of +``functions_content.extract_content_with_azure_di`` so the ingestion pipeline can swap engines +without reshaping downstream chunking. +""" + +import json +import logging +import os +import time + +import requests +from azure.identity import DefaultAzureCredential + +from config import AZURE_ENVIRONMENT, cognitive_services_scope +from functions_appinsights import log_event +from functions_debug import debug_print +import functions_settings + + +CONTENT_UNDERSTANDING_ANALYZE_TIMEOUT_SECONDS = 600 +CONTENT_UNDERSTANDING_POLL_INTERVAL_SECONDS = 3 +CONTENT_UNDERSTANDING_SUBMIT_TIMEOUT_SECONDS = 300 +CONTENT_UNDERSTANDING_POLL_TIMEOUT_SECONDS = 60 +CONTENT_UNDERSTANDING_TOKEN_REFRESH_BUFFER_SECONDS = 300 + +# Content Understanding refuses LLM-backed analyzers until the Foundry resource has default model +# deployments configured. That failure is common enough during setup to deserve its own guidance. +CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_HINTS = ( + 'model deployment', + 'modeldeployment', + 'no default', + 'default model', + 'deployment not found', + 'deploymentnotfound', +) + +CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_MESSAGE = ( + "Content Understanding rejected the request because the Foundry resource has no default model " + "deployments. Deploy a completion model and an embedding model, then set them as Content " + "Understanding defaults before using Enhanced extraction." +) + + +class ContentUnderstandingError(Exception): + """Raised when a Content Understanding request cannot be completed.""" + + +class ContentUnderstandingNotConfiguredError(ContentUnderstandingError): + """Raised when Content Understanding is unavailable or missing required configuration.""" + + +def _get_settings(): + return functions_settings.get_settings() + + +def _resolve_config(settings=None, config_override=None): + """Return normalized Content Understanding configuration from settings or an explicit override.""" + if config_override: + return { + 'endpoint': functions_settings.normalize_content_understanding_endpoint( + config_override.get('endpoint') + ), + 'key': str(config_override.get('key') or '').strip(), + 'authentication_type': functions_settings.normalize_content_understanding_authentication_type( + config_override.get('authentication_type') + ), + 'api_version': functions_settings.normalize_content_understanding_api_version( + config_override.get('api_version') + ), + 'analyzer_id': functions_settings.normalize_content_understanding_analyzer_id( + config_override.get('analyzer_id'), + functions_settings.CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + ), + 'image_analyzer_id': functions_settings.normalize_content_understanding_analyzer_id( + config_override.get('image_analyzer_id'), + functions_settings.CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + ), + } + + return functions_settings.get_content_understanding_config( + settings if settings is not None else _get_settings() + ) + + +_TOKEN_CACHE = {'token': None, 'expires_on': 0} + + +def _get_bearer_token(): + """Return a cached Entra ID access token for Content Understanding.""" + now = time.time() + cached_token = _TOKEN_CACHE.get('token') + cached_expiry = _TOKEN_CACHE.get('expires_on') or 0 + if cached_token and cached_expiry - CONTENT_UNDERSTANDING_TOKEN_REFRESH_BUFFER_SECONDS > now: + return cached_token + + credential = DefaultAzureCredential() + access_token = credential.get_token(cognitive_services_scope) + _TOKEN_CACHE['token'] = access_token.token + _TOKEN_CACHE['expires_on'] = getattr(access_token, 'expires_on', 0) or 0 + return access_token.token + + +def _build_auth_headers(config): + """Build authentication headers for either key or managed identity access.""" + if config.get('authentication_type') == 'managed_identity': + return {'Authorization': f"Bearer {_get_bearer_token()}"} + + key = config.get('key') + if not key: + raise ContentUnderstandingNotConfiguredError( + "Content Understanding key authentication is selected but no key is configured." + ) + return {'Ocp-Apim-Subscription-Key': key} + + +def _validate_config(config): + """Raise when the environment or configuration cannot support a Content Understanding call.""" + if not functions_settings.is_content_understanding_supported_environment(): + raise ContentUnderstandingNotConfiguredError( + f"Content Understanding is not available in the {AZURE_ENVIRONMENT} cloud." + ) + if not config.get('endpoint'): + raise ContentUnderstandingNotConfiguredError( + "Content Understanding endpoint is not configured." + ) + + +def _build_analyze_binary_url(config, analyzer_id, page_range=None): + url = ( + f"{config['endpoint']}/contentunderstanding/analyzers/{analyzer_id}:analyzeBinary" + f"?api-version={config['api_version']}" + ) + if page_range: + url = f"{url}&range={page_range}" + return url + + +def _describe_http_error(response): + """Return a readable message for a failed Content Understanding HTTP response.""" + detail = '' + try: + payload = response.json() + error_payload = payload.get('error') if isinstance(payload, dict) else None + if isinstance(error_payload, dict): + detail = str(error_payload.get('message') or error_payload.get('code') or '') + if not detail and isinstance(payload, dict): + detail = str(payload.get('message') or '') + if not detail: + detail = json.dumps(payload)[:500] + except (ValueError, TypeError): + detail = (response.text or '')[:500] + + if _looks_like_missing_model_deployment(detail): + return CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_MESSAGE + + return f"Content Understanding request failed with HTTP {response.status_code}: {detail}".strip() + + +def _looks_like_missing_model_deployment(message): + normalized_message = str(message or '').lower() + return any(hint in normalized_message for hint in CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_HINTS) + + +def analyze_file_with_content_understanding( + file_path, + analyzer_id=None, + page_range=None, + settings=None, + config_override=None, + max_wait_seconds=CONTENT_UNDERSTANDING_ANALYZE_TIMEOUT_SECONDS, +): + """Submit a local file to Content Understanding and return the completed ``result`` payload.""" + config = _resolve_config(settings=settings, config_override=config_override) + _validate_config(config) + + resolved_analyzer_id = functions_settings.normalize_content_understanding_analyzer_id( + analyzer_id, + config['analyzer_id'], + ) + + with open(file_path, 'rb') as file_handle: + file_bytes = file_handle.read() + + if not file_bytes: + raise ContentUnderstandingError(f"Cannot analyze empty file: {os.path.basename(file_path)}") + + headers = {'Content-Type': 'application/octet-stream'} + headers.update(_build_auth_headers(config)) + + submit_url = _build_analyze_binary_url(config, resolved_analyzer_id, page_range=page_range) + debug_print( + f"[CONTENT_UNDERSTANDING] Submitting {os.path.basename(file_path)} " + f"({len(file_bytes)} bytes) to analyzer {resolved_analyzer_id}" + + (f" range={page_range}" if page_range else "") + ) + + response = requests.post( + submit_url, + headers=headers, + data=file_bytes, + timeout=CONTENT_UNDERSTANDING_SUBMIT_TIMEOUT_SECONDS, + ) + + if response.status_code >= 400: + raise ContentUnderstandingError(_describe_http_error(response)) + + # Content extraction analyzers can answer synchronously for small inputs. + if response.status_code == 200: + payload = response.json() + if isinstance(payload, dict) and payload.get('result'): + return payload['result'] + if isinstance(payload, dict) and payload.get('contents'): + return payload + + operation_location = response.headers.get('Operation-Location') or response.headers.get('operation-location') + if not operation_location: + raise ContentUnderstandingError( + "Content Understanding did not return an Operation-Location header for the analysis request." + ) + + return _poll_analysis_result( + operation_location, + config, + max_wait_seconds=max_wait_seconds, + ) + + +def _poll_analysis_result(operation_location, config, max_wait_seconds): + """Poll a Content Understanding operation until it succeeds, fails, or times out.""" + poll_headers = _build_auth_headers(config) + start_time = time.time() + + while True: + elapsed_seconds = time.time() - start_time + if elapsed_seconds > max_wait_seconds: + raise TimeoutError( + f"Content Understanding analysis did not finish within {max_wait_seconds} seconds." + ) + + time.sleep(CONTENT_UNDERSTANDING_POLL_INTERVAL_SECONDS) + + poll_response = requests.get( + operation_location, + headers=poll_headers, + timeout=CONTENT_UNDERSTANDING_POLL_TIMEOUT_SECONDS, + ) + if poll_response.status_code >= 400: + raise ContentUnderstandingError(_describe_http_error(poll_response)) + + payload = poll_response.json() + status = str(payload.get('status') or '').strip().lower() + + if status == 'succeeded': + result = payload.get('result') + if not result: + raise ContentUnderstandingError( + "Content Understanding reported success but returned no result payload." + ) + return result + + if status == 'failed': + error_payload = payload.get('error') or {} + message = str(error_payload.get('message') or 'Content Understanding analysis failed.') + if _looks_like_missing_model_deployment(message): + raise ContentUnderstandingError(CONTENT_UNDERSTANDING_MISSING_DEPLOYMENT_MESSAGE) + raise ContentUnderstandingError(f"Content Understanding analysis failed: {message}") + + if status == 'canceled': + raise ContentUnderstandingError("Content Understanding analysis was canceled.") + + +def _iter_document_contents(result): + """Yield document content entries from a Content Understanding result payload.""" + contents = (result or {}).get('contents') + if not isinstance(contents, list): + return [] + return [content for content in contents if isinstance(content, dict)] + + +def _slice_markdown_by_spans(markdown_text, spans): + """Return the markdown covered by a page's spans.""" + if not markdown_text or not spans: + return '' + + slices = [] + for span in spans: + if not isinstance(span, dict): + continue + try: + offset = int(span.get('offset', 0)) + length = int(span.get('length', 0)) + except (TypeError, ValueError): + continue + if length <= 0: + continue + slices.append(markdown_text[offset:offset + length]) + + return ''.join(slices) + + +def _build_figure_summaries(content): + """Map page-agnostic figure offsets to readable description blocks. + + The description and any structured diagram source are kept separate so de-duplication can drop + a description that Content Understanding already inlined into the markdown without discarding + the machine-readable diagram alongside it. + """ + figures = content.get('figures') + if not isinstance(figures, list): + return [] + + summaries = [] + for figure in figures: + if not isinstance(figure, dict): + continue + + description = str(figure.get('description') or '').strip() + figure_content = figure.get('content') + figure_kind = str(figure.get('kind') or '').strip().lower() + + diagram_block = '' + if figure_kind == 'mermaid' and isinstance(figure_content, str) and figure_content.strip(): + diagram_block = f"```mermaid\n{figure_content.strip()}\n```" + elif figure_kind == 'chart' and isinstance(figure_content, (dict, list)): + try: + diagram_block = f"```json\n{json.dumps(figure_content, ensure_ascii=False)}\n```" + except (TypeError, ValueError): + diagram_block = '' + + caption = figure.get('caption') + caption_text = '' + if isinstance(caption, dict): + caption_text = str(caption.get('content') or '').strip() + elif isinstance(caption, str): + caption_text = caption.strip() + + # Descriptions are optional in the API schema, so a caption alone is still worth indexing. + if not description and not diagram_block and not caption_text: + continue + + label = caption_text or str(figure.get('id') or '').strip() or 'Figure' + + span = figure.get('span') + offset = None + if isinstance(span, dict): + try: + offset = int(span.get('offset')) + except (TypeError, ValueError): + offset = None + + summaries.append({ + 'offset': offset, + 'label': label, + 'description': description, + 'diagram_block': diagram_block, + }) + + return summaries + + +def _render_figure_summary(summary, page_text=''): + """Render a figure summary block, omitting parts already present in the page markdown.""" + description = summary.get('description') or '' + diagram_block = summary.get('diagram_block') or '' + label = summary.get('label') or 'Figure' + + lines = [] + # prebuilt-documentSearch already inlines most descriptions into the markdown. + if description and description not in page_text: + lines.append(description) + if diagram_block and diagram_block not in page_text: + lines.append(diagram_block) + + if not lines: + # A caption-only figure still earns a line when nothing else survived de-duplication. + if label and label not in page_text: + return f"Figure ({label})" + return '' + + # The label goes on its own line so a leading code fence still starts at column zero. + return f"Figure ({label}):\n" + "\n".join(lines) + + +def _assign_figures_to_pages(figure_summaries, page_ranges): + """Group figure summary blocks by the page whose span range contains them.""" + figures_by_page = {} + for summary in figure_summaries: + offset = summary.get('offset') + target_page = None + + if offset is not None: + for page_number, start, end in page_ranges: + if start <= offset < end: + target_page = page_number + break + + if target_page is None: + target_page = page_ranges[0][0] if page_ranges else 1 + + figures_by_page.setdefault(target_page, []).append(summary) + + return figures_by_page + + +def build_pages_from_content_understanding_result(result): + """Convert a Content Understanding result into ``[{page_number, content}]`` entries. + + Page text is reconstructed by slicing the content-level ``markdown`` with each page's spans, which + is the documented way to recover per-page content. Figure descriptions are appended to the page + that contains them when they are not already present in that page's markdown. + """ + pages_data = [] + + for content in _iter_document_contents(result): + markdown_text = str(content.get('markdown') or '') + pages = content.get('pages') + start_page_number = content.get('startPageNumber') + try: + start_page_number = int(start_page_number) + except (TypeError, ValueError): + start_page_number = 1 + + if not isinstance(pages, list) or not pages: + fallback_content = markdown_text.strip() + # Figures still carry descriptions even when the response has no per-page spans. + fallback_blocks = [] + for summary in _build_figure_summaries(content): + rendered_summary = _render_figure_summary(summary, fallback_content) + if rendered_summary: + fallback_blocks.append(rendered_summary) + if fallback_blocks: + fallback_content = ( + f"{fallback_content.rstrip()}\n\n" + "\n\n".join(fallback_blocks) + ).strip() + if fallback_content: + pages_data.append({ + 'page_number': start_page_number or 1, + 'content': fallback_content, + }) + continue + + page_ranges = [] + page_markdown = {} + for page in pages: + if not isinstance(page, dict): + continue + try: + page_number = int(page.get('pageNumber')) + except (TypeError, ValueError): + page_number = len(page_markdown) + start_page_number + + spans = page.get('spans') if isinstance(page.get('spans'), list) else [] + page_markdown[page_number] = _slice_markdown_by_spans(markdown_text, spans) + + span_starts = [] + span_ends = [] + for span in spans: + if not isinstance(span, dict): + continue + try: + offset = int(span.get('offset', 0)) + length = int(span.get('length', 0)) + except (TypeError, ValueError): + continue + span_starts.append(offset) + span_ends.append(offset + length) + + if span_starts: + page_ranges.append((page_number, min(span_starts), max(span_ends))) + + figures_by_page = _assign_figures_to_pages(_build_figure_summaries(content), page_ranges) + + for page_number in sorted(page_markdown.keys()): + page_text = page_markdown.get(page_number, '') + figure_blocks = [] + for summary in figures_by_page.get(page_number, []): + rendered_summary = _render_figure_summary(summary, page_text) + if rendered_summary: + figure_blocks.append(rendered_summary) + + if figure_blocks: + page_text = f"{page_text.rstrip()}\n\n" + "\n\n".join(figure_blocks) + + page_text = page_text.strip() + if not page_text: + continue + + pages_data.append({ + 'page_number': page_number, + 'content': page_text, + }) + + return pages_data + + +def extract_content_with_content_understanding(file_path, pages=None, settings=None): + """Extract page-by-page content with Content Understanding. + + Mirrors the return contract of ``functions_content.extract_content_with_azure_di`` so the + ingestion pipeline can switch engines without reshaping downstream chunking. + """ + result = analyze_file_with_content_understanding( + file_path, + page_range=str(pages) if pages else None, + settings=settings, + ) + + pages_data = build_pages_from_content_understanding_result(result) + debug_print( + f"[CONTENT_UNDERSTANDING] Extracted {len(pages_data)} page(s) from " + f"{os.path.basename(file_path)}" + ) + return pages_data + + +def analyze_image_with_content_understanding(image_path, settings=None): + """Analyze a standalone image and return its description text, or an empty string.""" + resolved_settings = settings if settings is not None else _get_settings() + config = _resolve_config(settings=resolved_settings) + + result = analyze_file_with_content_understanding( + image_path, + analyzer_id=config['image_analyzer_id'], + settings=resolved_settings, + ) + + text_blocks = [] + for content in _iter_document_contents(result): + markdown_text = str(content.get('markdown') or '').strip() + if markdown_text: + text_blocks.append(markdown_text) + + for summary in _build_figure_summaries(content): + rendered_summary = _render_figure_summary(summary, markdown_text) + if rendered_summary: + text_blocks.append(rendered_summary) + + return "\n\n".join(block for block in text_blocks if block).strip() + + +def test_content_understanding_connection(config_override, sample_file_path=None): + """Validate Content Understanding connectivity and return ``(ok, message)``.""" + if not functions_settings.is_content_understanding_supported_environment(): + return False, ( + f"Content Understanding is not available in the {AZURE_ENVIRONMENT} cloud. " + "Enhanced extraction uses Document Intelligence Layout here instead." + ) + + config = _resolve_config(config_override=config_override) + + if not config['endpoint']: + return False, "Content Understanding endpoint is required." + if config['authentication_type'] == 'key' and not config['key']: + return False, "Content Understanding key is required when key authentication is selected." + + try: + headers = _build_auth_headers(config) + except ContentUnderstandingError as auth_error: + return False, str(auth_error) + + analyzer_id = config['analyzer_id'] + analyzer_url = ( + f"{config['endpoint']}/contentunderstanding/analyzers/{analyzer_id}" + f"?api-version={config['api_version']}" + ) + + try: + response = requests.get( + analyzer_url, + headers=headers, + timeout=CONTENT_UNDERSTANDING_POLL_TIMEOUT_SECONDS, + ) + except requests.RequestException as request_error: + return False, f"Content Understanding connection error: {request_error}" + + if response.status_code == 404: + return False, ( + f"Content Understanding reached the endpoint but analyzer '{analyzer_id}' was not found. " + "Verify the analyzer id and API version." + ) + if response.status_code in (401, 403): + return False, ( + "Content Understanding rejected the credentials. For managed identity, assign the " + "Cognitive Services User role on the Foundry resource." + ) + if response.status_code >= 400: + return False, _describe_http_error(response) + + if not sample_file_path or not os.path.exists(sample_file_path): + return True, ( + f"Content Understanding connection successful. Analyzer '{analyzer_id}' is reachable." + ) + + try: + analyze_file_with_content_understanding( + sample_file_path, + analyzer_id=analyzer_id, + config_override=config, + max_wait_seconds=180, + ) + except (ContentUnderstandingError, TimeoutError) as analyze_error: + return False, str(analyze_error) + except Exception as analyze_error: # noqa: BLE001 - surface any client failure to the admin + log_event( + f"[CONTENT_UNDERSTANDING] Test analysis failed: {analyze_error}", + level=logging.WARNING, + ) + return False, f"Content Understanding test analysis failed: {analyze_error}" + + return True, ( + f"Content Understanding connection successful. Analyzer '{analyzer_id}' analyzed the test document." + ) diff --git a/application/single_app/functions_document_access_index.py b/application/single_app/functions_document_access_index.py index f7ba2d743..fa53e9c05 100644 --- a/application/single_app/functions_document_access_index.py +++ b/application/single_app/functions_document_access_index.py @@ -1739,6 +1739,8 @@ def _build_base_row(document_item, source_scope, scope_type, scope_id, access_ro 'publication_date': document_item.get('publication_date'), 'enhanced_citations': _has_persisted_blob_reference(document_item), 'document_intelligence_extraction_mode': document_item.get('document_intelligence_extraction_mode'), + 'extraction_engine': document_item.get('extraction_engine'), + 'extraction_engine_reason': document_item.get('extraction_engine_reason'), 'generated_artifact_promotion_status': document_item.get('generated_artifact_promotion_status'), 'generated_artifact_requested_by_user_id': document_item.get('generated_artifact_requested_by_user_id'), 'file_sync': document_item.get('file_sync'), @@ -2153,6 +2155,7 @@ def _query_candidate_projection_rows_for_scope(scope_key, source_scope): 'c.shared_user_ids, c.shared_group_ids, c.file_name, c.title, c.document_classification, c.tags, c.authors, c.keywords, ' 'c.abstract, c.status, c.percentage_complete, c.number_of_pages, c.publication_date, ' 'c.enhanced_citations, c.document_intelligence_extraction_mode, ' + 'c.extraction_engine, c.extraction_engine_reason, ' 'c.generated_artifact_promotion_status, c.generated_artifact_requested_by_user_id, ' 'c.file_sync, c.created_from_chat_upload, ' 'c.conversation_id, c.conversation_title_at_upload, c.upload_date, c.last_updated, ' @@ -2407,6 +2410,8 @@ def _projection_row_to_document(row, source_scope): 'publication_date': row.get('publication_date'), 'enhanced_citations': row.get('enhanced_citations'), 'document_intelligence_extraction_mode': row.get('document_intelligence_extraction_mode'), + 'extraction_engine': row.get('extraction_engine'), + 'extraction_engine_reason': row.get('extraction_engine_reason'), 'generated_artifact_promotion_status': row.get('generated_artifact_promotion_status'), 'generated_artifact_requested_by_user_id': row.get('generated_artifact_requested_by_user_id'), 'file_sync': row.get('file_sync'), diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index ecf6afa39..f0c735eda 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -26,6 +26,7 @@ ) from functions_visio import build_visio_page_markdown, parse_vsdx_pages from functions_content import * +from functions_content_understanding import analyze_image_with_content_understanding from functions_settings import * from functions_search import * from functions_logging import * @@ -252,6 +253,10 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): ) DI_MARKDOWN_TABLE_SEPARATOR_PATTERN = re.compile(r'(?m)^\s*\|(?:\s*:?-{3,}:?\s*\|)+\s*$') DI_MARKDOWN_TABLE_ROW_PATTERN = re.compile(r'(?m)^\s*\|.+\|\s*$') +# Document Intelligence Layout emits figures as markdown
blocks or image tags. Auto mode +# treats them as a signal that Enhanced extraction is worth the extra cost, because Content +# Understanding is the only engine that describes figures. +DI_MARKDOWN_FIGURE_PATTERN = re.compile(r'(|!\[[^\]]*\]\()', re.IGNORECASE) def is_pdf_file_name(file_name): @@ -288,12 +293,155 @@ def _get_document_intelligence_auto_layout_reason(sampled_pages): return 'selection marks or checkbox states detected in the sampled pages' if DI_MARKDOWN_TABLE_ROW_PATTERN.search(sampled_text) and DI_MARKDOWN_TABLE_SEPARATOR_PATTERN.search(sampled_text): return 'table structure detected in the sampled pages' + if DI_MARKDOWN_FIGURE_PATTERN.search(sampled_text): + return 'figures or images detected in the sampled pages' return '' +def _analyze_single_embedded_image(image_path, extraction_engine, image_extraction_mode, settings): + """Analyze one extracted Office image with the active engine and return its text.""" + if extraction_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING: + return analyze_image_with_content_understanding(image_path, settings=settings) + + image_pages = extract_content_with_azure_di(image_path, extraction_mode=image_extraction_mode) + return "\n\n".join( + str(page.get('content', '') or '').strip() + for page in image_pages or [] + if isinstance(page, dict) and str(page.get('content', '') or '').strip() + ).strip() + + +def _build_office_embedded_image_chunks( + file_path, + settings, + update_callback, + starting_page_number=1, +): + """Analyze images embedded in an Office file and return them as labelled content chunks. + + Neither engine describes figures inside Office files, so the images are pulled out of the OOXML + package and analyzed individually with whichever engine backs the admin's selected mode. + + Returns ``(chunks, analyzed_count, extraction_engine)``. + """ + if not settings.get('enable_office_embedded_image_analysis', True): + return [], 0, EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + + min_pixels = normalize_office_embedded_image_min_pixels( + settings.get('office_embedded_image_min_pixels') + ) + max_images = normalize_office_embedded_image_max_per_document( + settings.get('office_embedded_image_max_per_document') + ) + if max_images <= 0: + return [], 0, EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + + # Embedded images are images, so they follow the image side of the admin's configured mode. + admin_extraction_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) + image_extraction_mode = 'read' if admin_extraction_mode == 'read' else 'layout' + extraction_engine, _ = _resolve_extraction_engine_for_mode(image_extraction_mode, settings) + + chunks = [] + analyzed_count = 0 + temp_image_dir = None + + try: + temp_image_dir = tempfile.mkdtemp(prefix='office_images_') + embedded_images = extract_office_embedded_images( + file_path, + temp_image_dir, + min_pixels=min_pixels, + max_images=max_images, + ) + + if not embedded_images: + return [], 0, extraction_engine + + engine_label = ( + "Content Understanding" + if extraction_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + else "Document Intelligence" + ) + total_images = len(embedded_images) + update_callback( + status=f"Analyzing {total_images} embedded image(s) with {engine_label}..." + ) + + for image_index, embedded_image in enumerate(embedded_images, start=1): + try: + analysis_text = _analyze_single_embedded_image( + embedded_image['path'], + extraction_engine, + image_extraction_mode, + settings, + ) + except Exception as image_error: + log_event( + f"[OFFICE_EMBEDDED_IMAGES] Failed to analyze {embedded_image.get('name')}: {image_error}", + level=logging.WARNING, + ) + continue + + if not analysis_text: + continue + + location_label = '' + slide_number = embedded_image.get('slide_number') + if slide_number: + location_label = f" on slide {slide_number}" + + heading = ( + f"### Embedded image {image_index} of {total_images}: " + f"{embedded_image.get('name')}{location_label}" + ) + chunks.append({ + 'page_number': starting_page_number + len(chunks), + 'content': f"{heading}\n\n{analysis_text}", + }) + analyzed_count += 1 + + if analyzed_count: + update_callback( + status=f"Analyzed {analyzed_count} embedded image(s) with {engine_label}." + ) + except Exception as embedded_image_error: + log_event( + f"[OFFICE_EMBEDDED_IMAGES] Embedded image analysis failed for " + f"{os.path.basename(file_path)}: {embedded_image_error}", + level=logging.WARNING, + ) + finally: + if temp_image_dir and os.path.isdir(temp_image_dir): + shutil.rmtree(temp_image_dir, ignore_errors=True) + + return chunks, analyzed_count, extraction_engine + + +def _resolve_extraction_engine_for_mode(extraction_mode, settings): + """Resolve which engine backs the given extraction mode, plus a human-readable reason.""" + return resolve_extraction_engine_for_mode(extraction_mode, settings) + + +def _extract_pages_with_extraction_engine( + file_path, + extraction_mode, + extraction_engine, + settings=None, + pages=None, +): + """Extract page content with the resolved engine, falling back to Document Intelligence Layout.""" + return extract_content_with_extraction_engine( + file_path, + extraction_mode=extraction_mode, + extraction_engine=extraction_engine, + settings=settings, + pages=pages, + ) + + def _resolve_document_intelligence_auto_mode(temp_file_path, is_pdf, is_image, page_count, sample_pages, update_callback): if is_image: - return 'layout', 'image input benefits from Enhanced extraction for spatial structure and selection marks' + return 'layout', 'image input benefits from Enhanced extraction for figures, spatial structure, and selection marks' if not is_pdf: return 'read', 'Auto mode is only evaluated for PDFs and images' @@ -7004,18 +7152,27 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, print(f"Warning: Failed to extract initial metadata for {original_filename}: {e}") # Continue processing even if metadata fails - # --- DI Processing Logic --- + # --- Extraction Engine Resolution --- + # Standard extraction is always Document Intelligence prebuilt-read. Enhanced prefers Content + # Understanding, falling back to Document Intelligence prebuilt-layout when it is unavailable. settings = get_settings() # Assuming get_settings is accessible chunk_config = get_chunk_size_config(settings) + enhanced_extraction_enabled = is_enhanced_extraction_enabled(settings) document_intelligence_extraction_mode = 'read' document_intelligence_requested_mode = 'read' document_intelligence_auto_sample_pages = get_document_intelligence_auto_sample_pages(settings) document_intelligence_auto_reason = '' + extraction_engine = EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + extraction_engine_reason = '' if is_pdf or is_image: if extraction_mode_override: document_intelligence_requested_mode = normalize_document_intelligence_manual_extraction_mode(extraction_mode_override) else: - document_intelligence_requested_mode = get_document_intelligence_pdf_image_extraction_mode(settings) + document_intelligence_requested_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) + + if document_intelligence_requested_mode != 'read' and not enhanced_extraction_enabled: + document_intelligence_requested_mode = 'read' + extraction_engine_reason = 'Enhanced extraction is disabled, so Standard extraction was used' if document_intelligence_requested_mode == 'auto': document_intelligence_extraction_mode, document_intelligence_auto_reason = _resolve_document_intelligence_auto_mode( @@ -7030,11 +7187,20 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, document_intelligence_extraction_mode = document_intelligence_requested_mode document_intelligence_auto_reason = '' + resolved_engine, resolved_engine_reason = _resolve_extraction_engine_for_mode( + document_intelligence_extraction_mode, + settings, + ) + extraction_engine = resolved_engine + extraction_engine_reason = resolved_engine_reason or extraction_engine_reason + update_callback( document_intelligence_extraction_mode=document_intelligence_extraction_mode, document_intelligence_extraction_mode_requested=document_intelligence_requested_mode, document_intelligence_auto_sample_pages=document_intelligence_auto_sample_pages, document_intelligence_auto_reason=document_intelligence_auto_reason, + extraction_engine=extraction_engine, + extraction_engine_reason=extraction_engine_reason, ) di_limit_bytes = 500 * 1024 * 1024 @@ -7139,29 +7305,44 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, except Exception as e: raise Exception(f"Error extracting content from {chunk_effective_filename} with the legacy PowerPoint extractor: {str(e)}") else: - # Send chunk to Azure DI - update_callback(status=f"Sending {chunk_effective_filename} to Azure Document Intelligence...") + # Send chunk to the resolved extraction engine + engine_label = ( + "Azure AI Content Understanding" + if extraction_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + else "Azure Document Intelligence" + ) + update_callback(status=f"Sending {chunk_effective_filename} to {engine_label}...") try: - di_extracted_pages = extract_content_with_azure_di( + di_extracted_pages, engine_used, engine_fallback_reason = _extract_pages_with_extraction_engine( chunk_path, - extraction_mode=document_intelligence_extraction_mode + extraction_mode=document_intelligence_extraction_mode, + extraction_engine=extraction_engine, + settings=settings, ) + if engine_fallback_reason: + extraction_engine = engine_used + extraction_engine_reason = engine_fallback_reason + update_callback( + extraction_engine=engine_used, + extraction_engine_reason=engine_fallback_reason, + ) + num_di_pages = len(di_extracted_pages) conceptual_pages = num_di_pages if not is_image else 1 # Image is one conceptual item if not di_extracted_pages and not is_image: - print(f"Warning: Azure DI returned no content pages for {chunk_effective_filename}.") - status_msg = f"Azure DI found no content in {chunk_effective_filename}." + print(f"Warning: {engine_label} returned no content pages for {chunk_effective_filename}.") + status_msg = f"{engine_label} found no content in {chunk_effective_filename}." # Update page count to 0 if nothing found, otherwise keep previous estimate or conceptual count update_callback(number_of_pages=0 if idx == num_file_chunks else conceptual_pages, status=status_msg) elif not di_extracted_pages and is_image: - print(f"Info: Azure DI processed image {chunk_effective_filename}, but extracted no text.") + print(f"Info: {engine_label} processed image {chunk_effective_filename}, but extracted no text.") update_callback(number_of_pages=conceptual_pages, status=f"Processed image {chunk_effective_filename} (no text found).") else: - update_callback(number_of_pages=conceptual_pages, status=f"Received {num_di_pages} content page(s)/slide(s) from Azure DI for {chunk_effective_filename}.") + update_callback(number_of_pages=conceptual_pages, status=f"Received {num_di_pages} content page(s)/slide(s) from {engine_label} for {chunk_effective_filename}.") except Exception as e: - raise Exception(f"Error extracting content from {chunk_effective_filename} with Azure DI: {str(e)}") + raise Exception(f"Error extracting content from {chunk_effective_filename} with {engine_label}: {str(e)}") # --- Multi-Modal Vision Analysis (for images only) - Must happen BEFORE save_chunks --- if is_image and enable_enhanced_citations and idx == 1: # Only run once for first chunk @@ -7250,6 +7431,30 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, final_chunks_to_save = di_extracted_pages else: final_chunks_to_save = [] # No text extracted + # --- Embedded Office image analysis (DOCX/DOC/PPTX) --- + # Neither extraction engine describes figures inside Office files, so embedded images are + # analyzed separately and appended as their own citable chunks. + if (is_word or is_ppt) and not is_legacy_doc and not is_legacy_ppt: + next_chunk_page_number = max( + (int(chunk.get('page_number') or 0) for chunk in final_chunks_to_save), + default=0, + ) + 1 + + embedded_image_chunks, embedded_image_count, embedded_image_engine = _build_office_embedded_image_chunks( + chunk_path, + settings, + update_callback, + starting_page_number=next_chunk_page_number, + ) + + if embedded_image_chunks: + final_chunks_to_save = list(final_chunks_to_save) + embedded_image_chunks + update_callback( + number_of_pages=len(final_chunks_to_save), + office_embedded_image_count=embedded_image_count, + office_embedded_image_engine=embedded_image_engine, + ) + # Save Final Chunks to Search Index num_final_chunks = len(final_chunks_to_save) if not final_chunks_to_save: @@ -7355,12 +7560,12 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, def validate_document_reprocess_source(document_item, user_id=None, group_id=None, public_workspace_id=None): - """Validate that a PDF has a stored source blob available for DI extraction changes.""" + """Validate that a PDF or image has a stored source blob available for extraction changes.""" if not document_item: return False, "Document not found." - if not is_pdf_file_name(document_item.get('file_name')): - return False, "Only PDF documents can change extraction between Standard and Enhanced." + if not is_pdf_or_image_file_name(document_item.get('file_name')): + return False, "Only PDF and image documents can change extraction between Standard and Enhanced." container_name, blob_path = get_document_blob_storage_info( document_item, @@ -7369,13 +7574,13 @@ def validate_document_reprocess_source(document_item, user_id=None, group_id=Non public_workspace_id=public_workspace_id, ) if not container_name or not blob_path: - return False, "Source PDF is unavailable. Re-upload this PDF before changing extraction." + return False, "Source file is unavailable. Re-upload this document before changing extraction." try: if not _blob_exists(container_name, blob_path): - return False, "Stored source PDF was not found in Blob Storage. Re-upload this PDF before changing extraction." + return False, "Stored source file was not found in Blob Storage. Re-upload this document before changing extraction." except Exception as e: - return False, f"Unable to validate stored source PDF: {str(e)}" + return False, f"Unable to validate stored source file: {str(e)}" return True, "" @@ -7388,14 +7593,16 @@ def _download_document_source_to_temp_file(document_item, user_id=None, group_id public_workspace_id=public_workspace_id, ) if not container_name or not blob_path: - raise FileNotFoundError("Source PDF is unavailable.") + raise FileNotFoundError("Source file is unavailable.") + + file_suffix = os.path.splitext(str(document_item.get('file_name') or ''))[-1].lower() or '.pdf' blob_service_client = _get_blob_service_client() blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_path) temp_file_path = None try: - with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file: + with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as temp_file: temp_file_path = temp_file.name download_stream = blob_client.download_blob() for chunk in download_stream.chunks(): @@ -7408,7 +7615,7 @@ def _download_document_source_to_temp_file(document_item, user_id=None, group_id def process_document_reprocess_extraction_background(document_id, user_id, target_extraction_mode, group_id=None, public_workspace_id=None): - """Extract a stored PDF again with an explicit Standard/Enhanced mode.""" + """Extract a stored PDF or image again with an explicit Standard/Enhanced mode.""" is_group = group_id is not None is_public_workspace = public_workspace_id is not None target_mode = normalize_document_intelligence_manual_extraction_mode(target_extraction_mode) @@ -7444,6 +7651,13 @@ def update_doc_callback(**kwargs): raise ValueError(validation_message) original_filename = document_item.get('file_name') or f'{document_id}.pdf' + source_file_ext = os.path.splitext(original_filename)[-1].lower() or '.pdf' + target_engine, target_engine_reason = _resolve_extraction_engine_for_mode(target_mode, get_settings()) + target_engine_label = ( + "Content Understanding" + if target_engine == EXTRACTION_ENGINE_CONTENT_UNDERSTANDING + else "Document Intelligence" + ) update_doc_callback( status=f"Queued to extract again with {target_mode_label}", percentage_complete=0, @@ -7454,6 +7668,8 @@ def update_doc_callback(**kwargs): document_intelligence_extraction_mode_requested=target_mode, document_intelligence_auto_sample_pages=get_document_intelligence_auto_sample_pages(get_settings()), document_intelligence_auto_reason='Manual extraction change requested', + extraction_engine=target_engine, + extraction_engine_reason=target_engine_reason, ) temp_file_path = _download_document_source_to_temp_file( @@ -7466,13 +7682,13 @@ def update_doc_callback(**kwargs): update_doc_callback(status=f"Deleting existing chunks before extracting again with {target_mode_label}...") delete_document_chunks(document_id, group_id=group_id, public_workspace_id=public_workspace_id) - update_doc_callback(status=f"Extracting PDF again with Document Intelligence {target_mode_label}...") + update_doc_callback(status=f"Extracting again with {target_engine_label} {target_mode_label}...") result = process_di_document( document_id=document_id, user_id=user_id, temp_file_path=temp_file_path, original_filename=original_filename, - file_ext='.pdf', + file_ext=source_file_ext, enable_enhanced_citations=bool(document_item.get('enhanced_citations')), update_callback=update_doc_callback, group_id=group_id, @@ -7489,7 +7705,7 @@ def update_doc_callback(**kwargs): final_update_args = { "number_of_pages": total_chunks_saved, - "status": _resolve_processing_complete_status(total_chunks_saved, '.pdf', tuple('.' + ext for ext in IMAGE_EXTENSIONS), tuple('.' + ext for ext in TABULAR_EXTENSIONS), 'disabled'), + "status": _resolve_processing_complete_status(total_chunks_saved, source_file_ext, tuple('.' + ext for ext in IMAGE_EXTENSIONS), tuple('.' + ext for ext in TABULAR_EXTENSIONS), 'disabled'), "percentage_complete": 100, "current_file_chunk": None, } @@ -7499,7 +7715,7 @@ def update_doc_callback(**kwargs): final_update_args["embedding_model_deployment_name"] = embedding_model_name update_doc_callback(**final_update_args) - print(f"Document {document_id} extracted again successfully with Document Intelligence {target_mode}.") + print(f"Document {document_id} extracted again successfully with {target_engine_label} {target_mode}.") except Exception as e: print(f"Error extracting document {document_id} again: {repr(e)}\nTraceback:\n{traceback.format_exc()}") try: diff --git a/application/single_app/functions_office_media.py b/application/single_app/functions_office_media.py new file mode 100644 index 000000000..1b9fca28e --- /dev/null +++ b/application/single_app/functions_office_media.py @@ -0,0 +1,240 @@ +# functions_office_media.py + +"""Embedded media extraction for OOXML Office files. + +Neither Document Intelligence nor Content Understanding describes figures inside Word and +PowerPoint files. Pulling the embedded images out of the OOXML package lets the ingestion pipeline +analyze them individually with whichever engine backs the selected extraction mode. + +This module deliberately keeps its imports light so it stays importable without Azure clients. +""" + +import hashlib +import os +import re +import zipfile +from io import BytesIO + +from defusedxml.ElementTree import ParseError as DefusedParseError +from defusedxml.ElementTree import fromstring as defused_fromstring +from PIL import Image + + +OFFICE_EMBEDDED_IMAGE_MEDIA_PREFIXES = ('word/media/', 'ppt/media/', 'xl/media/') +# Restricted to raster formats that both Document Intelligence and Content Understanding accept. +OFFICE_EMBEDDED_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.bmp', '.tif', '.tiff', '.heif', '.heic') +OFFICE_EMBEDDED_IMAGE_MIN_BYTES = 2048 +# Uploaded Office files are untrusted, so refuse to decompress an oversized embedded image. +OFFICE_EMBEDDED_IMAGE_MAX_BYTES = 64 * 1024 * 1024 +# Slide relationship parts are small XML documents; anything larger is not worth decompressing. +OFFICE_EMBEDDED_RELS_MAX_BYTES = 4 * 1024 * 1024 +# A zip header can understate the uncompressed size, so entries are streamed in bounded chunks. +OFFICE_ZIP_READ_CHUNK_BYTES = 256 * 1024 +# Caps on how much of a crafted archive is inspected at all. +OFFICE_ZIP_MAX_ENTRIES = 5000 +OFFICE_ZIP_MAX_RELS_ENTRIES = 500 +# Only the compression methods real OOXML packages use. +OFFICE_ZIP_ALLOWED_COMPRESSION = (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) + +_OFFICE_MEDIA_INDEX_PATTERN = re.compile(r'(\d+)') +_PPTX_SLIDE_RELS_PATTERN = re.compile(r'^ppt/slides/_rels/slide(\d+)\.xml\.rels$', re.IGNORECASE) + + +def _safe_media_base_name(media_name): + """Return a plain file name for a zip entry, defeating path traversal in crafted archives. + + Zip entries are untrusted input and may embed ``..`` or platform separators, so both separator + styles are collapsed before taking the final component. + """ + normalized_name = str(media_name or '').replace('\\', '/') + base_name = normalized_name.rsplit('/', 1)[-1] + base_name = os.path.basename(base_name) + if base_name in ('', '.', '..'): + return '' + return base_name + + +def _read_zip_entry_bounded(archive, entry_name, max_bytes): + """Read a zip entry without trusting its declared uncompressed size. + + ``ZipInfo.file_size`` comes from the archive header and can be forged. CPython decompresses a + chunk before truncating it to the declared size, so a 64 KB entry claiming to be 4 KB can still + spike memory into the hundreds of megabytes. Streaming with a bounded per-chunk read keeps the + decompressor's output capped, so the only safe limit is enforced here rather than from the header. + + Returns the entry bytes, or ``None`` when the entry is unreadable or exceeds ``max_bytes``. + """ + try: + entry_info = archive.getinfo(entry_name) + except KeyError: + return None + + if entry_info.compress_type not in OFFICE_ZIP_ALLOWED_COMPRESSION: + return None + + # Cheap pre-filter for honest archives; never the only bound. + if entry_info.file_size > max_bytes: + return None + + chunks = [] + total_bytes = 0 + try: + with archive.open(entry_name) as entry_file: + while True: + chunk = entry_file.read(OFFICE_ZIP_READ_CHUNK_BYTES) + if not chunk: + break + total_bytes += len(chunk) + if total_bytes > max_bytes: + return None + chunks.append(chunk) + except (KeyError, ValueError, EOFError, zipfile.BadZipFile, zipfile.LargeZipFile, NotImplementedError): + return None + + return b''.join(chunks) + + +def _office_media_sort_key(media_name): + """Sort embedded media names naturally so image2 precedes image10.""" + base_name = media_name.rsplit('/', 1)[-1] + match = _OFFICE_MEDIA_INDEX_PATTERN.search(base_name) + return (int(match.group(1)) if match else 0, base_name) + + +def _build_pptx_media_slide_map(archive): + """Map ``ppt/media/*`` entries to the slide numbers that reference them. + + Slide relationship parts come from an untrusted upload, so they get the same declared-size + guard as media entries and are parsed with a hardened XML parser. + """ + media_slide_map = {} + inspected_rels = 0 + + for entry_name in archive.namelist(): + if inspected_rels >= OFFICE_ZIP_MAX_RELS_ENTRIES: + break + + slide_match = _PPTX_SLIDE_RELS_PATTERN.match(entry_name) + if not slide_match: + continue + + inspected_rels += 1 + slide_number = int(slide_match.group(1)) + + rels_bytes = _read_zip_entry_bounded(archive, entry_name, OFFICE_EMBEDDED_RELS_MAX_BYTES) + if rels_bytes is None: + continue + + try: + rels_root = defused_fromstring(rels_bytes) + except (DefusedParseError, ValueError): + continue + + for relationship in rels_root: + target = str(relationship.attrib.get('Target') or '').replace('\\', '/') + if '/media/' not in target: + continue + media_name = 'ppt/media/' + target.rsplit('/media/', 1)[-1] + # Keep the first slide that references the image so ordering stays stable. + media_slide_map.setdefault(media_name, slide_number) + + return media_slide_map + + +def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_images=25): + """Extract analyzable raster images embedded in an OOXML Office file. + + Args: + file_path (str): Path to the DOCX/PPTX/XLSX file. + output_dir (str): Directory that extracted images are written to. + min_pixels (int): Minimum width and height required for an image to be analyzed. + max_images (int): Maximum number of images to extract. + + Returns: + list: Dicts with ``name``, ``path``, ``width``, ``height``, and ``slide_number`` keys. + """ + if max_images <= 0: + return [] + + extracted_images = [] + seen_digests = set() + + try: + with zipfile.ZipFile(file_path) as archive: + entry_names = archive.namelist() + if len(entry_names) > OFFICE_ZIP_MAX_ENTRIES: + # A real Office package never has this many parts; refuse to walk a crafted archive. + return [] + + media_slide_map = _build_pptx_media_slide_map(archive) + + candidate_names = [ + entry_name for entry_name in entry_names + if entry_name.lower().startswith(OFFICE_EMBEDDED_IMAGE_MEDIA_PREFIXES) + and entry_name.lower().endswith(OFFICE_EMBEDDED_IMAGE_EXTENSIONS) + ] + + inspected_media = 0 + for media_name in sorted(candidate_names, key=_office_media_sort_key): + if len(extracted_images) >= max_images: + break + # Bound the work a malformed archive can cause, not just successful extractions. + if inspected_media >= max_images * 10: + break + inspected_media += 1 + + base_name = _safe_media_base_name(media_name) + if not base_name: + continue + + image_bytes = _read_zip_entry_bounded( + archive, + media_name, + OFFICE_EMBEDDED_IMAGE_MAX_BYTES, + ) + if image_bytes is None: + continue + + # Small assets are almost always icons, bullets, or spacer graphics. + if len(image_bytes) < OFFICE_EMBEDDED_IMAGE_MIN_BYTES: + continue + + digest = hashlib.sha256(image_bytes).hexdigest() + if digest in seen_digests: + continue + + try: + with Image.open(BytesIO(image_bytes)) as embedded_image: + width, height = embedded_image.size + except Exception: + continue + + if width < min_pixels or height < min_pixels: + continue + + # The output name is generated rather than taken from the archive, so a crafted + # entry name can never influence where the file is written. + extension = os.path.splitext(base_name)[1].lower() + if extension not in OFFICE_EMBEDDED_IMAGE_EXTENSIONS: + continue + output_path = os.path.join(output_dir, f"{len(extracted_images) + 1:03d}{extension}") + + seen_digests.add(digest) + + try: + with open(output_path, 'wb') as output_file: + output_file.write(image_bytes) + except OSError: + continue + + extracted_images.append({ + 'name': base_name, + 'path': output_path, + 'width': width, + 'height': height, + 'slide_number': media_slide_map.get(media_name), + }) + except (zipfile.BadZipFile, FileNotFoundError, OSError): + return [] + + return extracted_images diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index b0b8c32da..c65039416 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -90,6 +90,7 @@ "azure_apim_ai_search_subscription_key", "azure_document_intelligence_key", "azure_apim_document_intelligence_subscription_key", + "azure_content_understanding_key", "speech_service_key", "model_endpoint_identity_header_hmac_secret", ) @@ -530,6 +531,23 @@ def is_mixed_source_relevance_candidates_enabled(settings): DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES = {"read", "layout"} DOCUMENT_INTELLIGENCE_AUTO_SAMPLE_PAGES_DEFAULT = 3 DOCUMENT_INTELLIGENCE_AUTO_SAMPLE_PAGES_MAX = 20 + +# Enhanced extraction engines. Standard extraction is always Document Intelligence prebuilt-read. +EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE = "document_intelligence" +EXTRACTION_ENGINE_CONTENT_UNDERSTANDING = "content_understanding" +EXTRACTION_ENGINES = {EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, EXTRACTION_ENGINE_CONTENT_UNDERSTANDING} + +CONTENT_UNDERSTANDING_AUTHENTICATION_TYPES = {"key", "managed_identity"} +CONTENT_UNDERSTANDING_API_VERSION_DEFAULT = "2025-11-01" +CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT = "prebuilt-documentSearch" +CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT = "prebuilt-imageSearch" +# Azure AI Content Understanding is only offered in Azure commercial regions today. +CONTENT_UNDERSTANDING_SUPPORTED_AZURE_ENVIRONMENTS = {"public"} + +OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_DEFAULT = 150 +OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_MAX = 2000 +OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_DEFAULT = 25 +OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_MAX = 200 AGENTS_PAGE_PROMOTED_POPULAR_ORDER_OPTIONS = {"before", "after", "mixed"} AGENTS_PAGE_PROMOTED_POPULAR_WINDOW_OPTIONS = {"all_time", "30_days", "both"} AGENTS_PAGE_PROMOTED_POPULAR_TAG_LABEL_DEFAULT = "Promoted" @@ -701,6 +719,153 @@ def normalize_document_intelligence_manual_extraction_mode(value): return normalized_value +def normalize_enhanced_extraction_enabled(value): + """Normalize the Enhanced extraction master toggle into a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized_value = value.strip().lower() + if normalized_value in {"false", "0", "no", "off", ""}: + return False + if normalized_value in {"true", "1", "yes", "on"}: + return True + return bool(value) + + +def is_enhanced_extraction_enabled(settings): + """Return whether admins have enabled Enhanced extraction beyond Document Intelligence Standard.""" + return normalize_enhanced_extraction_enabled((settings or {}).get('enable_enhanced_extraction')) + + +def get_effective_document_intelligence_pdf_image_extraction_mode(settings): + """Return the extraction mode actually used, forcing Standard when Enhanced is disabled.""" + if not is_enhanced_extraction_enabled(settings): + return "read" + return get_document_intelligence_pdf_image_extraction_mode(settings) + + +def normalize_content_understanding_authentication_type(value): + """Normalize the Content Understanding authentication type to key or managed identity.""" + normalized_value = str(value or "key").strip().lower() + if normalized_value not in CONTENT_UNDERSTANDING_AUTHENTICATION_TYPES: + return "key" + return normalized_value + + +def normalize_content_understanding_endpoint(value): + """Normalize the Content Understanding endpoint by trimming whitespace and trailing slashes.""" + return str(value or "").strip().rstrip("/") + + +def normalize_content_understanding_api_version(value): + """Normalize the Content Understanding API version, falling back to the GA version.""" + normalized_value = str(value or "").strip() + return normalized_value or CONTENT_UNDERSTANDING_API_VERSION_DEFAULT + + +def normalize_content_understanding_analyzer_id(value, default_analyzer_id=None): + """Normalize a Content Understanding analyzer id, falling back to the supplied prebuilt default.""" + fallback_analyzer_id = default_analyzer_id or CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT + normalized_value = str(value or "").strip() + return normalized_value or fallback_analyzer_id + + +def normalize_office_embedded_image_min_pixels(value): + """Normalize the minimum width/height an embedded Office image must have to be analyzed.""" + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_DEFAULT + + return max(1, min(normalized_value, OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_MAX)) + + +def normalize_office_embedded_image_max_per_document(value): + """Normalize how many embedded Office images may be analyzed for a single document.""" + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_DEFAULT + + return max(0, min(normalized_value, OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_MAX)) + + +def is_content_understanding_supported_environment(azure_environment=None): + """Return whether the running Azure cloud offers Azure AI Content Understanding.""" + resolved_environment = str( + azure_environment if azure_environment is not None else AZURE_ENVIRONMENT + ).strip().lower() + return resolved_environment in CONTENT_UNDERSTANDING_SUPPORTED_AZURE_ENVIRONMENTS + + +def get_content_understanding_config(settings): + """Return the normalized Content Understanding connection settings.""" + resolved_settings = settings or {} + return { + 'endpoint': normalize_content_understanding_endpoint( + resolved_settings.get('azure_content_understanding_endpoint') + ), + 'key': str(resolved_settings.get('azure_content_understanding_key') or '').strip(), + 'authentication_type': normalize_content_understanding_authentication_type( + resolved_settings.get('azure_content_understanding_authentication_type') + ), + 'api_version': normalize_content_understanding_api_version( + resolved_settings.get('azure_content_understanding_api_version') + ), + 'analyzer_id': normalize_content_understanding_analyzer_id( + resolved_settings.get('azure_content_understanding_analyzer_id'), + CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + ), + 'image_analyzer_id': normalize_content_understanding_analyzer_id( + resolved_settings.get('azure_content_understanding_image_analyzer_id'), + CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + ), + } + + +def is_content_understanding_configured(settings): + """Return whether Content Understanding has enough configuration to be called.""" + config = get_content_understanding_config(settings) + if not config['endpoint']: + return False + if config['authentication_type'] == 'key' and not config['key']: + return False + return True + + +def resolve_enhanced_extraction_engine(settings, azure_environment=None): + """Resolve which engine backs Enhanced extraction, plus a human-readable reason. + + Enhanced always falls back to Document Intelligence Layout so Enhanced stays usable in + clouds where Content Understanding is unavailable, or before it has been configured. + """ + if not is_content_understanding_supported_environment(azure_environment): + resolved_environment = str( + azure_environment if azure_environment is not None else AZURE_ENVIRONMENT + ).strip().lower() + return ( + EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + f"Content Understanding is not available in the {resolved_environment} cloud, " + "so Enhanced used Document Intelligence Layout", + ) + + if not is_content_understanding_configured(settings): + return ( + EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE, + "Content Understanding is not configured, so Enhanced used Document Intelligence Layout", + ) + + return (EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, '') + + +def normalize_extraction_engine(value): + """Normalize a persisted extraction engine value, defaulting to Document Intelligence.""" + normalized_value = str(value or "").strip().lower() + if normalized_value not in EXTRACTION_ENGINES: + return EXTRACTION_ENGINE_DOCUMENT_INTELLIGENCE + return normalized_value + + def normalize_app_role_claims(user_roles): """Normalize app role claims into a flat string list.""" if not user_roles: @@ -1566,6 +1731,20 @@ def get_settings(use_cosmos=False, include_source=False): 'azure_apim_document_intelligence_endpoint': '', 'azure_apim_document_intelligence_subscription_key': '', + # Enhanced extraction. Standard is always Document Intelligence prebuilt-read; Enhanced uses + # Azure AI Content Understanding in Azure commercial clouds and falls back to Document + # Intelligence prebuilt-layout everywhere else. + 'enable_enhanced_extraction': False, + 'azure_content_understanding_endpoint': '', + 'azure_content_understanding_key': '', + 'azure_content_understanding_authentication_type': 'key', + 'azure_content_understanding_api_version': CONTENT_UNDERSTANDING_API_VERSION_DEFAULT, + 'azure_content_understanding_analyzer_id': CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + 'azure_content_understanding_image_analyzer_id': CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + 'enable_office_embedded_image_analysis': True, + 'office_embedded_image_min_pixels': OFFICE_EMBEDDED_IMAGE_MIN_PIXELS_DEFAULT, + 'office_embedded_image_max_per_document': OFFICE_EMBEDDED_IMAGE_MAX_PER_DOCUMENT_DEFAULT, + # Web search (via Azure AI Foundry agent) 'enable_web_search': False, 'web_search_consent_accepted': False, @@ -1798,6 +1977,13 @@ def _format_result(settings_payload, source): legacy_control_center_schedule = ( 'control_center_auto_refresh_timezone' not in settings_item ) + # Enhanced extraction gained an explicit enable toggle. Deployments that already selected + # Enhanced or Auto must keep it, otherwise they would silently downgrade to Standard and + # then lose the stored mode on the next admin settings save. + legacy_enhanced_extraction = 'enable_enhanced_extraction' not in settings_item + legacy_enhanced_extraction_mode = normalize_document_intelligence_pdf_image_extraction_mode( + settings_item.get('document_intelligence_pdf_image_extraction_mode') + ) legacy_control_center_time = settings_item.get('control_center_auto_refresh_time') if not isinstance(legacy_control_center_time, str): legacy_hour = settings_item.get('control_center_auto_refresh_hour', 6) @@ -1822,6 +2008,10 @@ def _format_result(settings_payload, source): merged['control_center_auto_refresh_timezone'] = 'UTC' merged['control_center_auto_refresh_next_run'] = None control_center_schedule_migration_updated = True + enhanced_extraction_migration_updated = False + if legacy_enhanced_extraction and legacy_enhanced_extraction_mode in ('layout', 'auto'): + merged['enable_enhanced_extraction'] = True + enhanced_extraction_migration_updated = True migration_updated = apply_custom_endpoint_setting_migration(merged) assignment_settings_updated = normalize_group_workflow_assignment_settings(merged) promoted_popular_settings_updated = normalize_agents_page_promoted_popular_settings(merged) @@ -1838,6 +2028,7 @@ def _format_result(settings_payload, source): if ( merge_changed or control_center_schedule_migration_updated + or enhanced_extraction_migration_updated or migration_updated or assignment_settings_updated or promoted_popular_settings_updated diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index ec3261c02..dcb60096e 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -1357,6 +1357,8 @@ def api_reprocess_document_extraction(): if raw_mode not in DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES: return jsonify({'error': 'Extraction mode must be Standard or Enhanced.'}), 400 target_mode = normalize_document_intelligence_manual_extraction_mode(raw_mode) + if target_mode == 'layout' and not is_enhanced_extraction_enabled(get_settings()): + return jsonify({'error': 'Enhanced extraction is disabled. Enable it in Admin Settings first.'}), 400 document_ids = payload.get('document_ids') if not isinstance(document_ids, list): diff --git a/application/single_app/route_backend_group_documents.py b/application/single_app/route_backend_group_documents.py index 5b769623f..607f8a7df 100644 --- a/application/single_app/route_backend_group_documents.py +++ b/application/single_app/route_backend_group_documents.py @@ -1300,6 +1300,8 @@ def api_reprocess_group_document_extraction(): if raw_mode not in DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES: return jsonify({'error': 'Extraction mode must be Standard or Enhanced.'}), 400 target_mode = normalize_document_intelligence_manual_extraction_mode(raw_mode) + if target_mode == 'layout' and not is_enhanced_extraction_enabled(get_settings()): + return jsonify({'error': 'Enhanced extraction is disabled. Enable it in Admin Settings first.'}), 400 document_ids = payload.get('document_ids') if not isinstance(document_ids, list): diff --git a/application/single_app/route_backend_public_documents.py b/application/single_app/route_backend_public_documents.py index 7b7bd6155..3bbc2a445 100644 --- a/application/single_app/route_backend_public_documents.py +++ b/application/single_app/route_backend_public_documents.py @@ -1125,6 +1125,8 @@ def api_reprocess_public_document_extraction(): if raw_mode not in DOCUMENT_INTELLIGENCE_MANUAL_EXTRACTION_MODES: return jsonify({'error': 'Extraction mode must be Standard or Enhanced.'}), 400 target_mode = normalize_document_intelligence_manual_extraction_mode(raw_mode) + if target_mode == 'layout' and not is_enhanced_extraction_enabled(get_settings()): + return jsonify({'error': 'Enhanced extraction is disabled. Enable it in Admin Settings first.'}), 400 document_ids = payload.get('document_ids') if not isinstance(document_ids, list): diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 24736ed14..9f582f7ec 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -90,6 +90,8 @@ def _resolve_admin_settings_test_secrets(payload): _resolve_test_payload_secret(payload, ('apim', 'subscription_key'), settings, 'azure_apim_document_intelligence_subscription_key') else: _resolve_test_payload_secret(payload, ('direct', 'key'), settings, 'azure_document_intelligence_key') + elif test_type == 'content_understanding': + _resolve_test_payload_secret(payload, ('key',), settings, 'azure_content_understanding_key') elif test_type == 'redis': _resolve_test_payload_secret(payload, ('key',), settings, 'redis_key') elif test_type == 'web_search': @@ -746,6 +748,9 @@ def test_connection(): elif test_type == 'azure_doc_intelligence': return _test_azure_doc_intelligence_connection(data) + elif test_type == 'content_understanding': + return _test_content_understanding_connection(data) + elif test_type == 'multimodal_vision': return _test_multimodal_vision_connection(data) @@ -1997,13 +2002,47 @@ def _test_azure_doc_intelligence_connection(payload): time.sleep(10) if status == "succeeded": - if extraction_mode == "auto": - return jsonify({'message': 'Azure document intelligence Auto connection successful. Auto samples PDFs with Enhanced extraction during ingestion, then finishes with Standard or Enhanced.'}), 200 - extraction_mode_label = "Enhanced" if extraction_mode == "layout" else "Standard" - return jsonify({'message': f'Azure document intelligence {extraction_mode_label} connection successful'}), 200 + if test_extraction_mode == "layout": + return jsonify({'message': ( + 'Azure document intelligence connection successful. Standard extraction and the ' + 'Layout model used for Auto sampling and Enhanced fallback are both reachable.' + )}), 200 + return jsonify({'message': 'Azure document intelligence Standard connection successful'}), 200 else: return jsonify({'error': f"Document Intelligence error: {status}"}), 500 +def _test_content_understanding_connection(payload): + """Attempt to reach Azure AI Content Understanding using ephemeral settings.""" + from functions_content_understanding import test_content_understanding_connection + + config_override = { + 'endpoint': payload.get('endpoint'), + 'key': payload.get('key'), + 'authentication_type': payload.get('authentication_type'), + 'api_version': payload.get('api_version'), + 'analyzer_id': payload.get('analyzer_id'), + 'image_analyzer_id': payload.get('image_analyzer_id'), + } + + sample_file_path = None + if payload.get('run_sample_analysis'): + candidate_path = os.path.join(current_app.root_path, 'static', 'test_files', 'test_document.pdf') + if os.path.exists(candidate_path): + sample_file_path = candidate_path + + try: + is_ok, message = test_content_understanding_connection( + config_override, + sample_file_path=sample_file_path, + ) + except Exception as e: + return jsonify({'error': f'Content Understanding connection error: {str(e)}'}), 500 + + if is_ok: + return jsonify({'message': message}), 200 + return jsonify({'error': message}), 400 + + def _test_key_vault_connection(payload): """Attempt to connect to Azure Key Vault using ephemeral settings.""" vault_name = payload.get('vault_name', '').strip() diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 48f0329d7..0703e81e3 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -1002,6 +1002,10 @@ def admin_settings(): app_settings=settings_for_template, settings=settings_for_template, azure_environment=AZURE_ENVIRONMENT, + content_understanding_supported=is_content_understanding_supported_environment(), + content_understanding_api_version_default=CONTENT_UNDERSTANDING_API_VERSION_DEFAULT, + content_understanding_document_analyzer_default=CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + content_understanding_image_analyzer_default=CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, default_video_indexer_endpoint=video_indexer_endpoint, default_video_indexer_arm_api_version=DEFAULT_VIDEO_INDEXER_ARM_API_VERSION, user_settings=user_settings, @@ -2328,6 +2332,39 @@ def is_valid_url(url): form_data.get('document_intelligence_auto_sample_pages') ) + enable_enhanced_extraction = form_data.get('enable_enhanced_extraction') == 'on' + enhanced_extraction_was_enabled = is_enhanced_extraction_enabled(settings) + if not enable_enhanced_extraction: + # Standard is the only mode available while Enhanced is off. + document_intelligence_pdf_image_extraction_mode = 'read' + elif not enhanced_extraction_was_enabled and document_intelligence_pdf_image_extraction_mode == 'read': + # Turning Enhanced on defaults to Auto so it upgrades only when structure is detected. + document_intelligence_pdf_image_extraction_mode = 'auto' + + azure_content_understanding_endpoint = normalize_content_understanding_endpoint( + form_data.get('azure_content_understanding_endpoint') + ) + azure_content_understanding_authentication_type = normalize_content_understanding_authentication_type( + form_data.get('azure_content_understanding_authentication_type') + ) + azure_content_understanding_api_version = normalize_content_understanding_api_version( + form_data.get('azure_content_understanding_api_version') + ) + azure_content_understanding_analyzer_id = normalize_content_understanding_analyzer_id( + form_data.get('azure_content_understanding_analyzer_id'), + CONTENT_UNDERSTANDING_DOCUMENT_ANALYZER_DEFAULT, + ) + azure_content_understanding_image_analyzer_id = normalize_content_understanding_analyzer_id( + form_data.get('azure_content_understanding_image_analyzer_id'), + CONTENT_UNDERSTANDING_IMAGE_ANALYZER_DEFAULT, + ) + office_embedded_image_min_pixels = normalize_office_embedded_image_min_pixels( + form_data.get('office_embedded_image_min_pixels') + ) + office_embedded_image_max_per_document = normalize_office_embedded_image_max_per_document( + form_data.get('office_embedded_image_max_per_document') + ) + # --- Construct new_settings Dictionary --- new_settings = { # Logging @@ -2728,6 +2765,18 @@ def is_valid_url(url): 'azure_apim_document_intelligence_endpoint': form_data.get('azure_apim_document_intelligence_endpoint', '').strip(), 'azure_apim_document_intelligence_subscription_key': admin_secret('azure_apim_document_intelligence_subscription_key'), + # Enhanced extraction (Azure AI Content Understanding, with Doc Intelligence Layout fallback) + 'enable_enhanced_extraction': enable_enhanced_extraction, + 'azure_content_understanding_endpoint': azure_content_understanding_endpoint, + 'azure_content_understanding_key': admin_secret('azure_content_understanding_key'), + 'azure_content_understanding_authentication_type': azure_content_understanding_authentication_type, + 'azure_content_understanding_api_version': azure_content_understanding_api_version, + 'azure_content_understanding_analyzer_id': azure_content_understanding_analyzer_id, + 'azure_content_understanding_image_analyzer_id': azure_content_understanding_image_analyzer_id, + 'enable_office_embedded_image_analysis': form_data.get('enable_office_embedded_image_analysis') == 'on', + 'office_embedded_image_min_pixels': office_embedded_image_min_pixels, + 'office_embedded_image_max_per_document': office_embedded_image_max_per_document, + 'enable_key_vault_secret_storage': form_data.get('enable_key_vault_secret_storage') == 'on', 'key_vault_name': form_data.get('key_vault_name', '').strip(), 'key_vault_identity': form_data.get('key_vault_identity', ''), diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 71adc39dc..4e700465b 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -1317,12 +1317,13 @@ def upload_file(): if file_ext_nodot in (DOCUMENT_EXTENSIONS | {'html'}) or is_image_file: extraction_mode = 'read' if file_ext == '.pdf' or is_image_file: - extraction_mode = get_document_intelligence_pdf_image_extraction_mode(settings) + extraction_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) if extraction_mode == 'auto': extraction_mode = 'layout' if is_image_file else 'read' - extracted_content_raw = extract_content_with_azure_di( + extracted_content_raw, _extraction_engine_used, _extraction_engine_fallback = extract_content_with_extraction_engine( temp_file_path, - extraction_mode=extraction_mode + extraction_mode=extraction_mode, + settings=settings ) # Convert pages_data list to string diff --git a/application/single_app/semantic_kernel_plugins/smart_http_plugin.py b/application/single_app/semantic_kernel_plugins/smart_http_plugin.py index 64e220959..6c148bc57 100644 --- a/application/single_app/semantic_kernel_plugins/smart_http_plugin.py +++ b/application/single_app/semantic_kernel_plugins/smart_http_plugin.py @@ -385,8 +385,8 @@ async def _process_pdf_content(self, pdf_bytes: bytes, uri: str, response) -> st """Process PDF content using Document Intelligence with large PDF support.""" try: # Import here to avoid circular imports - from functions_content import extract_content_with_azure_di - from functions_settings import get_document_intelligence_pdf_image_extraction_mode, get_settings + from functions_content import extract_content_with_extraction_engine + from functions_settings import get_effective_document_intelligence_pdf_image_extraction_mode, get_settings from config import initialize_clients, CLIENTS # Check if pdf_bytes is actually string content (error case) @@ -425,12 +425,13 @@ async def _process_pdf_content(self, pdf_bytes: bytes, uri: str, response) -> st if pdf_size > max_di_size: return f"📄 **PDF TOO LARGE FOR PROCESSING**\n📍 Source: {uri}\n📊 File size: {pdf_size:,} bytes (exceeds {max_di_size:,} byte limit for Document Intelligence)\n\n⚠️ This PDF is too large for automated text extraction. Please try:\n• A smaller PDF document\n• Specific sections of the document\n• Contact the document provider for a text version" - extraction_mode = get_document_intelligence_pdf_image_extraction_mode(settings) + extraction_mode = get_effective_document_intelligence_pdf_image_extraction_mode(settings) if extraction_mode == 'auto': extraction_mode = 'read' - pages_data = extract_content_with_azure_di( + pages_data, _engine_used, _engine_fallback = extract_content_with_extraction_engine( temp_file_path, - extraction_mode=extraction_mode + extraction_mode=extraction_mode, + settings=settings ) if not pages_data: diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index 3e99fb82d..aefe16ff1 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -6811,6 +6811,59 @@ function setupToggles() { documentIntelligenceAutoSamplePages.addEventListener('input', markFormAsModified); } + const enableEnhancedExtraction = document.getElementById('enable_enhanced_extraction'); + const enhancedExtractionSettings = document.getElementById('enhanced_extraction_settings'); + const updateEnhancedExtractionControls = () => { + if (!enableEnhancedExtraction || !enhancedExtractionSettings) { + return; + } + enhancedExtractionSettings.classList.toggle('d-none', !enableEnhancedExtraction.checked); + }; + if (enableEnhancedExtraction) { + updateEnhancedExtractionControls(); + enableEnhancedExtraction.addEventListener('change', function () { + updateEnhancedExtractionControls(); + // Turning Enhanced on defaults to Auto so it upgrades only when structure is detected. + if (this.checked && documentIntelligenceExtractionMode && documentIntelligenceExtractionMode.value === 'read') { + documentIntelligenceExtractionMode.value = 'auto'; + updateDocumentIntelligenceAutoControls(); + } + markFormAsModified(); + }); + } + + const contentUnderstandingAuthType = document.getElementById('azure_content_understanding_authentication_type'); + const contentUnderstandingKeyContainer = document.getElementById('azure_content_understanding_key_container'); + const updateContentUnderstandingAuthControls = () => { + if (!contentUnderstandingAuthType || !contentUnderstandingKeyContainer) { + return; + } + contentUnderstandingKeyContainer.classList.toggle('d-none', contentUnderstandingAuthType.value === 'managed_identity'); + }; + if (contentUnderstandingAuthType) { + updateContentUnderstandingAuthControls(); + contentUnderstandingAuthType.addEventListener('change', function () { + updateContentUnderstandingAuthControls(); + markFormAsModified(); + }); + } + + const enableOfficeEmbeddedImageAnalysis = document.getElementById('enable_office_embedded_image_analysis'); + const officeEmbeddedImageOptions = document.getElementById('office_embedded_image_options'); + const updateOfficeEmbeddedImageControls = () => { + if (!enableOfficeEmbeddedImageAnalysis || !officeEmbeddedImageOptions) { + return; + } + officeEmbeddedImageOptions.classList.toggle('d-none', !enableOfficeEmbeddedImageAnalysis.checked); + }; + if (enableOfficeEmbeddedImageAnalysis) { + updateOfficeEmbeddedImageControls(); + enableOfficeEmbeddedImageAnalysis.addEventListener('change', function () { + updateOfficeEmbeddedImageControls(); + markFormAsModified(); + }); + } + const enableContentSafetyCheckbox = document.getElementById('enable_content_safety'); if (enableContentSafetyCheckbox) { enableContentSafetyCheckbox.addEventListener('change', function() { @@ -8363,6 +8416,44 @@ function setupTestButtons() { }); } + const testContentUnderstandingBtn = document.getElementById('test_content_understanding_button'); + if (testContentUnderstandingBtn) { + testContentUnderstandingBtn.addEventListener('click', async () => { + const resultDiv = document.getElementById('test_content_understanding_result'); + resultDiv.className = 'mt-2'; + resultDiv.textContent = 'Testing Content Understanding...'; + + const payload = { + test_type: 'content_understanding', + endpoint: document.getElementById('azure_content_understanding_endpoint')?.value || '', + authentication_type: document.getElementById('azure_content_understanding_authentication_type')?.value || 'key', + key: document.getElementById('azure_content_understanding_key')?.value || '', + api_version: document.getElementById('azure_content_understanding_api_version')?.value || '', + analyzer_id: document.getElementById('azure_content_understanding_analyzer_id')?.value || '', + image_analyzer_id: document.getElementById('azure_content_understanding_image_analyzer_id')?.value || '' + }; + + try { + const resp = await fetch('/api/admin/settings/test_connection', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + const data = await resp.json(); + if (resp.ok) { + resultDiv.className = 'mt-2 text-success'; + resultDiv.textContent = data.message; + } else { + resultDiv.className = 'mt-2 text-danger'; + resultDiv.textContent = data.error || 'Error testing Content Understanding'; + } + } catch (err) { + resultDiv.className = 'mt-2 text-danger'; + resultDiv.textContent = `Error: ${err.message}`; + } + }); + } + const testKeyVaultBtn = document.getElementById('test_key_vault_button'); if (testKeyVaultBtn) { testKeyVaultBtn.addEventListener('click', async () => { @@ -9561,6 +9652,7 @@ togglePassword('toggle_image_gen_key', 'azure_openai_image_gen_key'); togglePassword('toggle_content_safety_key', 'content_safety_key'); togglePassword('toggle_search_key', 'azure_ai_search_key'); togglePassword('toggle_docintel_key', 'azure_document_intelligence_key'); +togglePassword('toggle_content_understanding_key', 'azure_content_understanding_key'); togglePassword('toggle_azure_apim_gpt_subscription_key', 'azure_apim_gpt_subscription_key'); togglePassword('toggle_azure_apim_embedding_subscription_key', 'azure_apim_embedding_subscription_key'); togglePassword('toggle_azure_apim_image_gen_subscription_key', 'azure_apim_image_gen_subscription_key'); @@ -10451,7 +10543,17 @@ function setupWalkthroughFieldListeners() { {selector: '#document_intelligence_auto_sample_pages', event: 'input'}, {selector: '#azure_apim_document_intelligence_endpoint', event: 'input'}, {selector: '#azure_apim_document_intelligence_subscription_key', event: 'input'}, - {selector: '#enable_document_intelligence_apim', event: 'change'} + {selector: '#enable_document_intelligence_apim', event: 'change'}, + {selector: '#enable_enhanced_extraction', event: 'change'}, + {selector: '#azure_content_understanding_endpoint', event: 'input'}, + {selector: '#azure_content_understanding_key', event: 'input'}, + {selector: '#azure_content_understanding_authentication_type', event: 'change'}, + {selector: '#azure_content_understanding_api_version', event: 'input'}, + {selector: '#azure_content_understanding_analyzer_id', event: 'input'}, + {selector: '#azure_content_understanding_image_analyzer_id', event: 'input'}, + {selector: '#enable_office_embedded_image_analysis', event: 'change'}, + {selector: '#office_embedded_image_min_pixels', event: 'input'}, + {selector: '#office_embedded_image_max_per_document', event: 'input'} ], 8: [ // Video settings {selector: '#enable_video_file_support', event: 'change'}, diff --git a/application/single_app/static/js/public/public_workspace.js b/application/single_app/static/js/public/public_workspace.js index c75bc2b26..3893d4715 100644 --- a/application/single_app/static/js/public/public_workspace.js +++ b/application/single_app/static/js/public/public_workspace.js @@ -1283,7 +1283,7 @@ function appendPublicDocumentMetaPills(container, doc) { doc.version ? `v${doc.version}` : null, doc.authors ? `By ${doc.authors}` : null, doc.number_of_pages ? `${doc.number_of_pages} pages` : null, - isPublicPdfDocument(doc) ? getPublicDocumentExtractionModeLabel(doc) : null, + supportsPublicExtractionModeChange(doc) ? getPublicDocumentExtractionModeLabel(doc) : null, doc.publication_date ? doc.publication_date : null, ].filter(Boolean); @@ -1297,10 +1297,34 @@ function appendPublicDocumentMetaPills(container, doc) { }); } +const PUBLIC_EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif', '.heic']; +const PUBLIC_EXTRACTION_STANDARD_TOOLTIP = 'Standard extraction uses Document Intelligence Read for faster, lower-cost text extraction. Best for plain text PDFs and images.'; +const PUBLIC_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP = 'Enhanced extraction uses Azure AI Content Understanding to preserve tables, page structure, and checkbox states, and to describe figures, charts, and images. Adds latency and higher cost.'; +const PUBLIC_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP = 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.'; + function isPublicPdfDocument(doc) { return String(doc?.file_name || '').toLowerCase().endsWith('.pdf'); } +function isPublicImageDocument(doc) { + const fileName = String(doc?.file_name || '').toLowerCase(); + return PUBLIC_EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS.some((extension) => fileName.endsWith(extension)); +} + +function supportsPublicExtractionModeChange(doc) { + return isPublicPdfDocument(doc) || isPublicImageDocument(doc); +} + +function getPublicDocumentExtractionEngine(doc) { + return String(doc?.extraction_engine || '').trim().toLowerCase(); +} + +function getPublicDocumentEnhancedTooltipForEngine(engine) { + return engine === 'content_understanding' + ? PUBLIC_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP + : PUBLIC_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP; +} + function getPublicDocumentExtractionModeLabel(doc) { const mode = String(doc?.document_intelligence_extraction_mode || '').trim().toLowerCase(); return mode === 'layout' ? 'Enhanced' : 'Standard'; @@ -1319,17 +1343,19 @@ function getPublicDocumentExtractionModeLabelFromMode(mode) { return mode === 'layout' ? 'Enhanced' : 'Standard'; } -function getPublicDocumentExtractionChangeTooltip(targetMode) { +function getPublicDocumentExtractionChangeTooltip(targetMode, engine) { return targetMode === 'layout' - ? 'Extract again with Enhanced extraction. Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.' - : 'Extract again with Standard extraction. Standard extraction uses Document Intelligence Read for faster text extraction. Best for plain text PDFs and images.'; + ? `Extract again with Enhanced extraction. ${getPublicDocumentEnhancedTooltipForEngine(engine)}` + : `Extract again with Standard extraction. ${PUBLIC_EXTRACTION_STANDARD_TOOLTIP}`; } function getPublicDocumentExtractionModeTooltip(doc) { const mode = String(doc?.document_intelligence_extraction_mode || '').trim().toLowerCase(); - return mode === 'layout' - ? 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.' - : 'Standard extraction uses Document Intelligence Read for faster text extraction. Best for plain text PDFs and images.'; + const tooltip = mode === 'layout' + ? getPublicDocumentEnhancedTooltipForEngine(getPublicDocumentExtractionEngine(doc)) + : PUBLIC_EXTRACTION_STANDARD_TOOLTIP; + const reason = String(doc?.extraction_engine_reason || '').trim(); + return reason ? `${tooltip} ${reason}.` : tooltip; } function getPublicDocumentCitationTooltip(doc) { @@ -1339,7 +1365,7 @@ function getPublicDocumentCitationTooltip(doc) { } function createPublicDocumentExtractionModeBadge(doc) { - if (!isPublicPdfDocument(doc)) { + if (!supportsPublicExtractionModeChange(doc)) { return null; } @@ -1355,7 +1381,7 @@ function createPublicDocumentExtractionModeBadge(doc) { } function getPublicDocumentExtractionModeBadgeHtml(doc) { - if (!isPublicPdfDocument(doc)) { + if (!supportsPublicExtractionModeChange(doc)) { return ''; } @@ -1501,11 +1527,11 @@ function createPublicDocumentCard(doc) { if (canManage) { dropdownItems.push(createPublicDropdownItem('bi-pencil-fill', 'Edit Metadata', () => window.onEditPublicDocument(docId))); dropdownItems.push(createPublicDropdownItem('bi-magic', 'Extract Metadata', () => window.onExtractPublicMetadata(docId, null))); - if (isPublicPdfDocument(doc)) { + if (supportsPublicExtractionModeChange(doc)) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); dropdownItems.push(createPublicDropdownDivider()); dropdownItems.push(createPublicDropdownHeader('Change Extraction')); dropdownItems.push(createPublicDropdownItem(extractionActionIcon, `Change to ${extractionActionLabel}`, () => window.reprocessPublicDocumentExtraction(docId, extractionActionMode, null), false, extractionActionTooltip)); @@ -1672,7 +1698,7 @@ function renderPublicDocumentRow(doc) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); actionsDropdown += `
  • @@ -1681,7 +1707,7 @@ function renderPublicDocumentRow(doc) {
  • Extract Metadata
  • - ${isPublicPdfDocument(doc) ? ` + ${supportsPublicExtractionModeChange(doc) ? `
  • @@ -1749,7 +1775,7 @@ function renderPublicDocumentRow(doc) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); detailsRow.innerHTML = ` @@ -1759,7 +1785,7 @@ function renderPublicDocumentRow(doc) {

    Version: ${escapeHtml(doc.version || '1')}

    Authors: ${escapeHtml(doc.authors || 'N/A')}

    Pages/Chunks: ${escapeHtml(doc.number_of_pages || 'N/A')}

    - ${isPublicPdfDocument(doc) ? `

    Extraction: ${getPublicDocumentExtractionModeBadgeHtml(doc)}

    ` : ''} + ${supportsPublicExtractionModeChange(doc) ? `

    Extraction: ${getPublicDocumentExtractionModeBadgeHtml(doc)}

    ` : ''}

    Citations: ${getCitationBadge(doc.enhanced_citations)}

    Publication Date: ${escapeHtml(doc.publication_date || 'N/A')}

    Keywords: ${escapeHtml(doc.keywords || 'N/A')}

    @@ -1774,7 +1800,7 @@ function renderPublicDocumentRow(doc) { - ${isPublicPdfDocument(doc) ? ` + ${supportsPublicExtractionModeChange(doc) ? ` ` : ''} @@ -3341,10 +3367,10 @@ function buildPublicFolderDocumentsTable(docs) { const extractionActionMode = getPublicDocumentTargetExtractionMode(doc); const extractionActionLabel = getPublicDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getPublicDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getPublicDocumentExtractionChangeTooltip(extractionActionMode, getPublicDocumentExtractionEngine(doc)); actionsHtml += `
  • Edit Metadata
  • Extract Metadata
  • - ${isPublicPdfDocument(doc) ? `
  • + ${supportsPublicExtractionModeChange(doc) ? `
  • Change to ${extractionActionLabel}
  • ` : ''}
  • diff --git a/application/single_app/static/js/workspace/workspace-documents.js b/application/single_app/static/js/workspace/workspace-documents.js index 76c53f539..a887786f5 100644 --- a/application/single_app/static/js/workspace/workspace-documents.js +++ b/application/single_app/static/js/workspace/workspace-documents.js @@ -392,12 +392,39 @@ function getDocumentClassificationBadge(doc) { return 'None'; } +const EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif', '.heic']; + function isPdfDocument(doc) { return String(doc?.file_name || '').toLowerCase().endsWith('.pdf'); } -const DOCUMENT_EXTRACTION_STANDARD_TOOLTIP = 'Standard extraction uses Document Intelligence Read for faster text extraction. Best for plain text PDFs and images.'; -const DOCUMENT_EXTRACTION_ENHANCED_TOOLTIP = 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.'; +function isImageDocument(doc) { + const fileName = String(doc?.file_name || '').toLowerCase(); + return EXTRACTION_MODE_CHANGE_IMAGE_EXTENSIONS.some((extension) => fileName.endsWith(extension)); +} + +function supportsExtractionModeChange(doc) { + return isPdfDocument(doc) || isImageDocument(doc); +} + +const DOCUMENT_EXTRACTION_STANDARD_TOOLTIP = 'Standard extraction uses Document Intelligence Read for faster, lower-cost text extraction. Best for plain text PDFs and images.'; +const DOCUMENT_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP = 'Enhanced extraction uses Azure AI Content Understanding to preserve tables, page structure, and checkbox states, and to describe figures, charts, and images. Adds latency and higher cost.'; +const DOCUMENT_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP = 'Enhanced extraction uses Document Intelligence Layout to preserve tables, page structure, forms, and checkbox states. Adds latency and higher cost.'; + +function getDocumentExtractionEngine(doc) { + return String(doc?.extraction_engine || '').trim().toLowerCase(); +} + +function getDocumentExtractionEngineReason(doc) { + return String(doc?.extraction_engine_reason || '').trim(); +} + +function getDocumentEnhancedTooltipForEngine(engine) { + return engine === 'content_understanding' + ? DOCUMENT_EXTRACTION_ENHANCED_CONTENT_UNDERSTANDING_TOOLTIP + : DOCUMENT_EXTRACTION_ENHANCED_DOCUMENT_INTELLIGENCE_TOOLTIP; +} + const DOCUMENT_CITATION_STANDARD_TOOLTIP = 'Standard citations reference indexed text chunks.'; const DOCUMENT_CITATION_ENHANCED_TOOLTIP = 'Enhanced citations preserve source-file context for richer citation previews and supported file workflows.'; @@ -405,8 +432,10 @@ function getDocumentExtractionModeLabelFromMode(mode) { return mode === 'layout' ? 'Enhanced' : 'Standard'; } -function getDocumentExtractionModeTooltipFromMode(mode) { - return mode === 'layout' ? DOCUMENT_EXTRACTION_ENHANCED_TOOLTIP : DOCUMENT_EXTRACTION_STANDARD_TOOLTIP; +function getDocumentExtractionModeTooltipFromMode(mode, engine) { + return mode === 'layout' + ? getDocumentEnhancedTooltipForEngine(engine) + : DOCUMENT_EXTRACTION_STANDARD_TOOLTIP; } function getDocumentExtractionModeIcon(mode) { @@ -418,9 +447,9 @@ function getDocumentTargetExtractionMode(doc) { return currentMode === 'layout' ? 'read' : 'layout'; } -function getDocumentExtractionChangeTooltip(targetMode) { +function getDocumentExtractionChangeTooltip(targetMode, engine) { return targetMode === 'layout' - ? `Extract again with Enhanced extraction. ${DOCUMENT_EXTRACTION_ENHANCED_TOOLTIP}` + ? `Extract again with Enhanced extraction. ${getDocumentEnhancedTooltipForEngine(engine)}` : `Extract again with Standard extraction. ${DOCUMENT_EXTRACTION_STANDARD_TOOLTIP}`; } @@ -431,7 +460,9 @@ function getDocumentExtractionModeLabel(doc) { function getDocumentExtractionModeTooltip(doc) { const mode = String(doc?.document_intelligence_extraction_mode || '').trim().toLowerCase(); - return getDocumentExtractionModeTooltipFromMode(mode); + const tooltip = getDocumentExtractionModeTooltipFromMode(mode, getDocumentExtractionEngine(doc)); + const reason = getDocumentExtractionEngineReason(doc); + return reason ? `${tooltip} ${reason}.` : tooltip; } function getDocumentCitationTooltip(doc) { @@ -439,7 +470,7 @@ function getDocumentCitationTooltip(doc) { } function getDocumentExtractionModeBadge(doc) { - if (!isPdfDocument(doc)) { + if (!supportsExtractionModeChange(doc)) { return ''; } @@ -450,7 +481,7 @@ function getDocumentExtractionModeBadge(doc) { } function getDocumentReprocessDropdownItems(doc) { - if (!isPdfDocument(doc)) { + if (!supportsExtractionModeChange(doc)) { return ''; } @@ -458,7 +489,7 @@ function getDocumentReprocessDropdownItems(doc) { const targetMode = getDocumentTargetExtractionMode(doc); const targetLabel = getDocumentExtractionModeLabelFromMode(targetMode); const targetIcon = getDocumentExtractionModeIcon(targetMode); - const targetTooltip = getDocumentExtractionChangeTooltip(targetMode); + const targetTooltip = getDocumentExtractionChangeTooltip(targetMode, getDocumentExtractionEngine(doc)); return `
  • @@ -485,7 +516,7 @@ function getDocumentMetaPills(doc) { if (doc.number_of_pages) { pills.push(`${escapeHtml(String(doc.number_of_pages))} pages`); } - if (isPdfDocument(doc)) { + if (supportsExtractionModeChange(doc)) { pills.push(`${getDocumentExtractionModeLabel(doc)}`); } if (authors.length) { @@ -1956,7 +1987,7 @@ function renderDocumentRow(doc) {

    Version: ${escapeHtml(doc.version || "N/A")}

    Authors: ${escapeHtml(Array.isArray(doc.authors) ? doc.authors.join(", ") : doc.authors || "N/A")}

    Pages: ${escapeHtml(doc.number_of_pages || "N/A")}

    - ${isPdfDocument(doc) ? `

    Extraction: ${getDocumentExtractionModeBadge(doc)}

    ` : ''} + ${supportsExtractionModeChange(doc) ? `

    Extraction: ${getDocumentExtractionModeBadge(doc)}

    ` : ''}

    Citations: ${doc.enhanced_citations ? `Enhanced` : `Standard`}

    Publication Date: ${escapeHtml(doc.publication_date || "N/A")}

    Keywords: ${escapeHtml(Array.isArray(doc.keywords) ? doc.keywords.join(", ") : doc.keywords || "N/A")}

    @@ -1978,12 +2009,12 @@ function renderDocumentRow(doc) { `; } - if (isOwner && isPdfDocument(doc)) { + if (isOwner && supportsExtractionModeChange(doc)) { const reprocessDocId = escapeHtml(String(docId || '')); const extractionActionMode = getDocumentTargetExtractionMode(doc); const extractionActionLabel = getDocumentExtractionModeLabelFromMode(extractionActionMode); const extractionActionIcon = getDocumentExtractionModeIcon(extractionActionMode); - const extractionActionTooltip = getDocumentExtractionChangeTooltip(extractionActionMode); + const extractionActionTooltip = getDocumentExtractionChangeTooltip(extractionActionMode, getDocumentExtractionEngine(doc)); detailsHtml += ` + + +
    + {% if content_understanding_supported %} +
    + Enhanced extraction uses Azure AI Content Understanding, which returns tables, + page structure, checkbox states, and AI-generated descriptions of figures, charts, and images. + Configure the connection below. +
    + {% else %} +
    + Azure AI Content Understanding is not available in the + {{ azure_environment }} cloud, so Enhanced extraction uses + Document Intelligence Layout here. There is nothing more to configure — + Enhanced still captures tables, page structure, forms, and checkbox states. +
    + {% endif %} + +
    + +
    + Enhanced captures more document detail for PDFs and images, including tables, page structure, and checked or unchecked marks. It adds latency and has a 6X increase for every 1000 pages when selected. +
    +
    + + +
    + Auto samples this many first PDF pages with Document Intelligence Layout. If it detects tables, selection marks, or figures, the full PDF uses Enhanced; otherwise it finishes with Standard. Images use Enhanced in Auto mode. +
    +
    +
    +
    Standard: Document Intelligence Read. Fastest and lowest-cost path for plain text PDFs and images.
    +
    Enhanced: best for tables, section structure, page layout, forms, checkbox states, and figure descriptions. Expect more latency and higher cost.
    +
    Auto: samples the first pages with Document Intelligence Layout, then uses Enhanced only when the sample shows tables, selection marks, or figures.
    +
    +
    + + {% if content_understanding_supported %} +
    +
    +
    Azure AI Content Understanding
    + +
    +

    + Required for Enhanced extraction. Deployed from a Microsoft Foundry resource. + If this is left blank, Enhanced falls back to Document Intelligence Layout. +

    + +
    + + +
    + Your Microsoft Foundry resource endpoint, without a trailing path. +
    +
    + +
    + + +
    + Managed identity requires the Cognitive Services User role on the Foundry resource. +
    +
    + +
    + +
    + + +
    +
    + +
    +
    + + +
    Default: {{ content_understanding_api_version_default }}
    +
    +
    + + +
    Default: {{ content_understanding_document_analyzer_default }}
    +
    +
    + + +
    Default: {{ content_understanding_image_analyzer_default }}
    +
    +
    + +
    - -
    - Enhanced captures more document detail for PDFs and images, including tables, page structure, and checked or unchecked marks. It adds latency and has a 6X increase for every 1000 pages when selected. -
    -
    - + {% endif %} +
    + +
    +
    Images Inside Office Files
    +

    + Neither extraction engine describes figures inside Word and PowerPoint files. When this is on, + embedded images are pulled out of the file, analyzed with whichever engine backs the selected + extraction mode, and indexed as their own citable chunks. This works with Standard extraction + too, using Document Intelligence. +

    +
    -
    - Auto samples this many first PDF pages with Enhanced extraction. If it detects tables or selection marks, the full PDF uses Enhanced; otherwise it finishes with Standard. Images use Enhanced in Auto mode. -
    +
    -
    -
    Standard: fastest and lowest-cost path for plain text PDFs and images.
    -
    Enhanced: best for tables, section structure, page layout, forms, and checkbox states. Expect more latency and a 6X increase for every 1000 pages.
    -
    Auto: samples the first pages, then uses Enhanced only when the sample shows tables or selection marks.
    +
    +
    + + +
    Images narrower or shorter than this are skipped as icons or spacers.
    +
    +
    + + +
    Caps per-document cost. Duplicate images are analyzed once.
    +
    + +
    + +