Merge pull request #1743 from microsoft/paullizer-document-analysis-validation

Fix Markdown escape differences in document evidence validation

Author: paullizer

application/single_app/config.py

@@ -101,7 +101,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.315"
+VERSION = "0.261.316"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 # Opt-out for deployments where App Service Easy Auth is active but the platform

application/single_app/functions_document_analysis_results.py

@@ -5,6 +5,7 @@
 import html
 import json
 import re
+import string
 import unicodedata
 from copy import deepcopy
 
@@ -398,17 +399,68 @@ def _evidence_word_character(character):
     return character.isalnum() or category[0] == 'M' or category == 'Pc'
 
 
-def _evidence_markup(text):
+def _evidence_literals(text):
+    """Map Markdown escapes and code content to literal original spans, before markup removal."""
+    literals = {}
+    index = 0
+    while index + 1 < len(text):
+        if text[index] == '\\' and text[index + 1] in string.punctuation:
+            unit = (text[index + 1], index, index + 2)
+            literals[index] = literals[index + 1] = unit
+            index += 2
+        else:
+            index += 1
+
+    def protect(start, end):
+        for offset in range(start, end):
+            literals[offset] = (text[offset], offset, offset + 1)
+
+    fence = None
+    for line in _EVIDENCE_LINE.finditer(text):
+        marker = re.match(r' {0,3}(`{3,}|~{3,})(.*)$', line.group(0))
+        if fence is not None:
+            protect(line.start(), line.end())
+            if marker and marker[1][0] == fence[0] and len(marker[1]) >= len(fence) and not marker[2].strip():
+                fence = None
+        elif marker and (marker[1][0] != '`' or '`' not in marker[2]):
+            fence = marker[1]
+            protect(line.start(), line.end())
+        elif line.group(0).startswith(('    ', '\t')):
+            protect(line.start(), line.end())
+
+    runs = [match for match in re.finditer(r'`+', text) if match.start() not in literals]
+    following, closers = {}, {}
+    for index in range(len(runs) - 1, -1, -1):
+        length = len(runs[index].group(0))
+        closers[index] = following.get(length)
+        following[length] = index
+    index = 0
+    while index < len(runs):
+        closer = closers[index]
+        if closer is not None:
+            protect(runs[index].end(), runs[closer].start())
+            index = closer + 1
+        else:
+            index += 1
+    for match in re.finditer(r'<(code|pre)\b[^>]*>(.*?)(?:</\1\s*>|\Z)', text, re.DOTALL | re.IGNORECASE):
+        if match.start() not in literals:
+            protect(match.start(2), match.end(2))
+    return literals
+
+
+def _evidence_markup(text, literals):
     """Presentation markup as (start, end, separates): tags, comments and table rule lines."""
     spans = [
         (match.start(), match.end(), (match.group(1) or '').lower() not in _EVIDENCE_INLINE_TAGS)
         for match in _EVIDENCE_MARKUP.finditer(text)
+        if match.start() not in literals
     ]
     for line in _EVIDENCE_LINE.finditer(text):
         content = line.group(0).strip()
         if (
             content and '-' in content and not content.strip('|:- \t')
             and ('|' in content or content.count('-') >= 3)
+            and not any(offset in literals for offset in range(line.start(), line.end()))
         ):
             spans.append((line.start(), line.end(), True))
     merged = []
@@ -430,51 +482,54 @@ def _evidence_form(text):
     """
     units = []
 
-    def add(character, start, end):
+    def add(character, start, end, literal=False):
         character = character.translate(_EVIDENCE_FOLDS)
         if character in _EVIDENCE_INVISIBLE:
             return
         # Marks, including vowel signs with no combining class, stay with their base character.
-        if units and units[-1][0] == 'text' and unicodedata.category(character)[0] == 'M':
+        kind = 'literal' if literal else 'text'
+        if units and units[-1][0] != 'gap' and unicodedata.category(character)[0] == 'M':
             units[-1][1] += character
             units[-1][3] = end
         else:
-            units.append(['text', character, start, end])
+            units.append([kind, character, start, end])
 
+    literals = _evidence_literals(text)
     position = 0
-    for start, end, separates in [*_evidence_markup(text), (len(text), len(text), False)]:
+    for start, end, separates in [*_evidence_markup(text, literals), (len(text), len(text), False)]:
         if start > position:
             index = position
             # Entities are decoded only after markup removal, so escaped text never becomes markup.
-            for entity in _EVIDENCE_ENTITY.finditer(text, position, start):
-                for offset in range(index, entity.start()):
-                    add(text[offset], offset, offset + 1)
-                value = html.unescape(entity.group(0))
-                if value == entity.group(0):
-                    for offset in range(entity.start(), entity.end()):
-                        add(text[offset], offset, offset + 1)
+            while index < start:
+                if index in literals:
+                    character, literal_start, literal_end = literals[index]
+                    add(character, literal_start, literal_end, literal=True)
+                    index = literal_end
+                    continue
+                entity = _EVIDENCE_ENTITY.match(text, index, start)
+                if entity is not None and html.unescape(entity.group(0)) != entity.group(0):
+                    for character in html.unescape(entity.group(0)):
+                        add(character, entity.start(), entity.end(), literal=True)
+                    index = entity.end()
                 else:
-                    for character in value:
-                        add(character, entity.start(), entity.end())
-                index = entity.end()
-            for offset in range(index, start):
-                add(text[offset], offset, offset + 1)
+                    add(text[index], index, index + 1)
+                    index += 1
         if separates:
             units.append(['gap', ' ', start, end])
         position = max(position, end)
 
     items = []
     for kind, characters, start, end in units:
-        if kind == 'text' and not characters.isascii():
+        if kind != 'gap' and not characters.isascii():
             kept = any(
                 unicodedata.decomposition(character).startswith(_EVIDENCE_KEPT_FORMS)
                 for character in characters
             )
             characters = unicodedata.normalize('NFC' if kept else 'NFKC', characters).translate(_EVIDENCE_FOLDS)
         for character in characters:
-            if kind == 'gap' or character.isspace() or character == '|':
+            if kind == 'gap' or character.isspace() or (character == '|' and kind != 'literal'):
                 items.append(('gap', ' ', start, end))
-            elif character in _EVIDENCE_EMPHASIS:
+            elif character in _EVIDENCE_EMPHASIS and kind != 'literal':
                 items.append(('mark', character, start, end))
             elif character not in _EVIDENCE_INVISIBLE:
                 items.append(('char', character, start, end))

docs/explanation/fixes/DOCUMENT_ANALYSIS_MARKDOWN_ESCAPE_FIX.md

@@ -0,0 +1,92 @@
+# Document analysis Markdown escape fix
+
+**Version: 0.261.316**
+
+Fixed in version: **0.261.316**, recorded in
+`application/single_app/config.py`.
+
+## Issue and root cause
+
+A fresh PDF analysis read its entire source window but rejected two of six
+findings. The extracted Markdown contained `2\. Partnership` and
+`6\) TELEPHONE NUMBER`; the model quoted the rendered punctuation as
+`2. Partnership` and `6) TELEPHONE NUMBER`. The shared evidence locator
+treated the formatting backslashes as written content.
+
+Each affected finding also had valid citations, but one unmatched citation makes
+the finding unresolved. The final analysis therefore retained four findings and
+was partial. Complete-only Compose correctly refused that input, before Word
+rendering started. Ordinary search-backed chat does not use this same narrative
+Analyze validation path.
+
+## General formatting comparison
+
+The shared normalized comparison now decodes Markdown backslash escapes of ASCII
+punctuation in both the quote and source. It retains the punctuation itself and
+maps it back to the original two-character source span. For example, `6\)`
+can match `6)`, while saved evidence still contains `6\)` and its original
+character offsets.
+
+This is deterministic formatting normalization, not a document-specific
+exception, an additional model evaluation, or a retry strategy. Exact matching
+still runs first, preserving the [literal evidence fix](DOCUMENT_ANALYSIS_LITERAL_EVIDENCE_FIX.md).
+The result contract, tiers, and `evidence-matcher-v2` identifier are unchanged.
+
+Escaped punctuation stays literal after decoding: escaped asterisks and backticks
+do not become emphasis, escaped angle brackets do not become HTML tags, escaped
+ampersands do not begin entities, and escaped pipes do not become table separators.
+Decoded character entities receive the same literal treatment.
+
+## Safeguards and limitations
+
+Backslash pairs are processed left to right. A doubled backslash represents a
+literal backslash; a backslash before a letter is not a punctuation escape.
+Backslashes inside recognized inline code, top-level backtick or tilde fences,
+indented code lines, and HTML `code` or `pre` content remain literal. Unclosed
+fences and HTML code regions are protected through the end of the text.
+This is a conservative evidence normalizer, not a complete Markdown renderer;
+it does not add support for every nested Markdown construct.
+
+Punctuation, numbers, signs, and checkbox states must still agree. Missing,
+wrong-location, ambiguous, paraphrased, and unsupported citations remain
+unresolved. Complete-only Compose, source coverage, access checks, and findings
+validation are unchanged. No citation is silently dropped to make a report pass.
+
+Matching proves that a quotation can be located, not that every claim in a
+finding is factually entailed by the quotation. No factual-accuracy evaluation
+was added. Old failed attempts are not repaired or migrated; use a fresh request
+after deployment. This change adds no settings, routes, dependencies, or
+source-reprocessing requirements.
+
+## Files and regression coverage
+
+- `application/single_app/functions_document_analysis_results.py`: literal-span
+  handling before markup and entity normalization.
+- `functional_tests/test_document_analysis_evidence_matching.py`: punctuation
+  escapes, backslash parity, literal code, source offsets, changed values,
+  checkbox states, citation locations, and real Analyze finalization without
+  correction calls.
+- `functional_tests/test_orchestration_document_derivation_reliability.py`:
+  escaped source evidence through real Analyze, Compose, Word rendering,
+  publication, and download, using the existing offline harness.
+- `application/single_app/config.py`: patch version update.
+
+## Validation
+
+The unchanged saved response was replayed offline against the original source,
+with its content fingerprint verified. Before the fix, four findings were
+accepted, two unresolved, and 23 evidence passages retained. After the fix,
+all six findings were accepted, none unresolved, and all 25 evidence passages
+retained. Source coverage remained complete. Normal and optimized Python
+produced the same result, with no model calls or Azure writes.
+
+The earlier footer incident also remains valid: all 25 findings pass unchanged.
+The automated regressions exercise both positive matches and rejected content
+changes, and the Word publication cases require exactly the existing two
+analysis calls plus one composition call, with no response correction.
+
+The targeted matcher, finalization, derivation, saved-result, workflow-publication,
+and attempt-fence suites passed **379 tests and 272 subtests**. Documentation
+surface coverage and site quality checks also passed. Validation used isolated
+test dependencies; no shared Python packages or application manifests were
+modified. No live deployment was performed.

docs/explanation/fixes/V2_COMPARISON_ANALYZE_EVIDENCE_MATCHING_FIX.md

@@ -119,6 +119,11 @@ renders as nothing:
   marks (`*`, `_` and backticks) at the edge of a word, as in `**Note**:`.
 - Character entities are decoded only after tags are handled, so escaped text is never
   read as markup.
+- As of **0.261.316**, Markdown backslash escapes of ASCII punctuation compare
+  with their rendered punctuation, while evidence retains the original source
+  spans. Escaped symbols remain literal rather than becoming markup, and
+  backslashes in recognized code contexts are preserved. See the
+  [Markdown escape fix](DOCUMENT_ANALYSIS_MARKDOWN_ESCAPE_FIX.md).
 - Typographic quotes, and the prime and double prime, fold to `'` or `"`. An acute
   accent typed as an apostrophe, as in `can´t`, also folds to `'`. Hyphen, dash and
   minus variants fold to `-`.