feat: Claude Vision OCR fallback for unreadable documents

When the CRM's readDocument returns success:false (scanned PDFs,
image-only DOCX, corrupted files), shira-hermes now:

1. Fetches the raw file bytes via new getDocumentBytes endpoint
2. Sends them to Claude Vision through ai-gateway:
   - Images (PNG/JPG/TIFF/etc): direct vision call
   - PDFs: render each page at 180 DPI via PyMuPDF, vision call per page
   - DOCX: extract images from word/media/, vision call per image

New tool: read_document_ocr (explicit OCR request)
Existing tool read_document auto-falls-back to OCR on extraction failure.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-19 11:37:51 +00:00
parent 5086fd48b8
commit 51c76c1cf9
3 changed files with 173 additions and 2 deletions
+41 -2
View File
@@ -96,15 +96,42 @@ def register_document_tools(
except Exception as e:
return fail(f"שגיאה בשמירת נתיב: {e}")
async def _ocr_fallback(filePath: str, reason: str) -> str:
"""Fetch file bytes and OCR via Claude Vision."""
from api.services.ocr import extract_text_from_bytes
try:
bytes_result = await crm.post("SmartAssistant/action/getDocumentBytes", {"filePath": filePath})
except Exception as e:
return fail(f'לא ניתן לחלץ טקסט מ-"{filePath}" ({reason}). כשל ב-getDocumentBytes: {e}')
if not bytes_result.get("success") or not bytes_result.get("base64"):
err = bytes_result.get("error", "unknown")
return fail(f'לא ניתן לחלץ טקסט מ-"{filePath}" ({reason}). בייטי הקובץ: {err}')
file_name = bytes_result.get("fileName", filePath)
mime_type = bytes_result.get("mimeType", "")
try:
text = await extract_text_from_bytes(
bytes_result["base64"], mime_type, file_name
)
return ok(f'=== {file_name} (OCR via Vision) ===\n\n{text}')
except Exception as e:
return fail(f'שגיאה ב-OCR של "{file_name}": {e}')
async def read_document(filePath: str) -> str:
try:
result = await crm.post("SmartAssistant/action/readDocument", {"filePath": filePath})
if not result.get("success") or not result.get("text"):
return fail(f'לא ניתן לחלץ טקסט מ-"{result.get("fileName", filePath)}"')
# Fallback to OCR via Claude Vision
return await _ocr_fallback(filePath, "חילוץ טקסט רגיל נכשל")
return ok(f'=== {result["fileName"]} ({result.get("charCount", "?")} תווים) ===\n\n{result["text"]}')
except Exception as e:
return fail(f"שגיאה בקריאת מסמך: {e}")
async def read_document_ocr(filePath: str) -> str:
"""Force OCR via Claude Vision — skip regular text extraction."""
return await _ocr_fallback(filePath, "OCR מבוקש במפורש")
async def read_multiple_documents(filePaths: list[str], maxCharsPerFile: int = 50000) -> str:
try:
result = await crm.post("SmartAssistant/action/readMultipleDocuments", {
@@ -206,7 +233,7 @@ def register_document_tools(
}
tools["read_document"] = {
"description": "Read and extract text content from a document (PDF, DOCX, TXT). Requires the file path from list_documents.",
"description": "Read and extract text content from a document (PDF, DOCX, TXT, images). Requires the file path from list_documents. Automatically falls back to Claude Vision OCR if regular extraction fails (e.g. scanned PDFs, image-only docx).",
"parameters": {
"type": "object",
"properties": {
@@ -217,6 +244,18 @@ def register_document_tools(
"handler": read_document,
}
tools["read_document_ocr"] = {
"description": "Force OCR extraction via Claude Vision — for images (PNG/JPG/TIFF), scanned PDFs, or any file where you want AI vision to read it. Skips regular text extraction and sends the file straight to Claude Vision.",
"parameters": {
"type": "object",
"properties": {
"filePath": {"type": "string", "description": "Full path of the document (from list_documents results)"},
},
"required": ["filePath"],
},
"handler": read_document_ocr,
}
tools["read_multiple_documents"] = {
"description": "Read and extract text from multiple documents at once.",
"parameters": {