feat: Claude Vision OCR fallback for unreadable documents
When the CRM's readDocument returns success:false (scanned PDFs, image-only DOCX, corrupted files), shira-hermes now: 1. Fetches the raw file bytes via new getDocumentBytes endpoint 2. Sends them to Claude Vision through ai-gateway: - Images (PNG/JPG/TIFF/etc): direct vision call - PDFs: render each page at 180 DPI via PyMuPDF, vision call per page - DOCX: extract images from word/media/, vision call per image New tool: read_document_ocr (explicit OCR request) Existing tool read_document auto-falls-back to OCR on extraction failure. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -96,15 +96,42 @@ def register_document_tools(
|
||||
except Exception as e:
|
||||
return fail(f"שגיאה בשמירת נתיב: {e}")
|
||||
|
||||
async def _ocr_fallback(filePath: str, reason: str) -> str:
|
||||
"""Fetch file bytes and OCR via Claude Vision."""
|
||||
from api.services.ocr import extract_text_from_bytes
|
||||
try:
|
||||
bytes_result = await crm.post("SmartAssistant/action/getDocumentBytes", {"filePath": filePath})
|
||||
except Exception as e:
|
||||
return fail(f'לא ניתן לחלץ טקסט מ-"{filePath}" ({reason}). כשל ב-getDocumentBytes: {e}')
|
||||
|
||||
if not bytes_result.get("success") or not bytes_result.get("base64"):
|
||||
err = bytes_result.get("error", "unknown")
|
||||
return fail(f'לא ניתן לחלץ טקסט מ-"{filePath}" ({reason}). בייטי הקובץ: {err}')
|
||||
|
||||
file_name = bytes_result.get("fileName", filePath)
|
||||
mime_type = bytes_result.get("mimeType", "")
|
||||
try:
|
||||
text = await extract_text_from_bytes(
|
||||
bytes_result["base64"], mime_type, file_name
|
||||
)
|
||||
return ok(f'=== {file_name} (OCR via Vision) ===\n\n{text}')
|
||||
except Exception as e:
|
||||
return fail(f'שגיאה ב-OCR של "{file_name}": {e}')
|
||||
|
||||
async def read_document(filePath: str) -> str:
|
||||
try:
|
||||
result = await crm.post("SmartAssistant/action/readDocument", {"filePath": filePath})
|
||||
if not result.get("success") or not result.get("text"):
|
||||
return fail(f'לא ניתן לחלץ טקסט מ-"{result.get("fileName", filePath)}"')
|
||||
# Fallback to OCR via Claude Vision
|
||||
return await _ocr_fallback(filePath, "חילוץ טקסט רגיל נכשל")
|
||||
return ok(f'=== {result["fileName"]} ({result.get("charCount", "?")} תווים) ===\n\n{result["text"]}')
|
||||
except Exception as e:
|
||||
return fail(f"שגיאה בקריאת מסמך: {e}")
|
||||
|
||||
async def read_document_ocr(filePath: str) -> str:
|
||||
"""Force OCR via Claude Vision — skip regular text extraction."""
|
||||
return await _ocr_fallback(filePath, "OCR מבוקש במפורש")
|
||||
|
||||
async def read_multiple_documents(filePaths: list[str], maxCharsPerFile: int = 50000) -> str:
|
||||
try:
|
||||
result = await crm.post("SmartAssistant/action/readMultipleDocuments", {
|
||||
@@ -206,7 +233,7 @@ def register_document_tools(
|
||||
}
|
||||
|
||||
tools["read_document"] = {
|
||||
"description": "Read and extract text content from a document (PDF, DOCX, TXT). Requires the file path from list_documents.",
|
||||
"description": "Read and extract text content from a document (PDF, DOCX, TXT, images). Requires the file path from list_documents. Automatically falls back to Claude Vision OCR if regular extraction fails (e.g. scanned PDFs, image-only docx).",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -217,6 +244,18 @@ def register_document_tools(
|
||||
"handler": read_document,
|
||||
}
|
||||
|
||||
tools["read_document_ocr"] = {
|
||||
"description": "Force OCR extraction via Claude Vision — for images (PNG/JPG/TIFF), scanned PDFs, or any file where you want AI vision to read it. Skips regular text extraction and sends the file straight to Claude Vision.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filePath": {"type": "string", "description": "Full path of the document (from list_documents results)"},
|
||||
},
|
||||
"required": ["filePath"],
|
||||
},
|
||||
"handler": read_document_ocr,
|
||||
}
|
||||
|
||||
tools["read_multiple_documents"] = {
|
||||
"description": "Read and extract text from multiple documents at once.",
|
||||
"parameters": {
|
||||
|
||||
Reference in New Issue
Block a user