From 51c76c1cf9d9b661ea57bc92f8b9c305ff0d4cda Mon Sep 17 00:00:00 2001 From: Chaim Date: Sun, 19 Apr 2026 11:37:51 +0000 Subject: [PATCH] feat: Claude Vision OCR fallback for unreadable documents When the CRM's readDocument returns success:false (scanned PDFs, image-only DOCX, corrupted files), shira-hermes now: 1. Fetches the raw file bytes via new getDocumentBytes endpoint 2. Sends them to Claude Vision through ai-gateway: - Images (PNG/JPG/TIFF/etc): direct vision call - PDFs: render each page at 180 DPI via PyMuPDF, vision call per page - DOCX: extract images from word/media/, vision call per image New tool: read_document_ocr (explicit OCR request) Existing tool read_document auto-falls-back to OCR on extraction failure. Co-Authored-By: Claude Opus 4.6 (1M context) --- api/services/ocr.py | 131 +++++++++++++++++++++++++++++ mcp_server/tools/document_tools.py | 43 +++++++++- pyproject.toml | 1 + 3 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 api/services/ocr.py diff --git a/api/services/ocr.py b/api/services/ocr.py new file mode 100644 index 0000000..0637f52 --- /dev/null +++ b/api/services/ocr.py @@ -0,0 +1,131 @@ +"""OCR fallback — extract text from images and scanned PDFs via Claude Vision.""" + +from __future__ import annotations + +import base64 +import io +import logging +import os + +from openai import AsyncOpenAI + +logger = logging.getLogger("shira.ocr") + +OCR_PROMPT = ( + "Extract ALL visible text from this image. Return ONLY the raw text content — no commentary, " + "no formatting notes, no headers like 'Here is the text:'. Preserve the original language " + "(Hebrew, English, Arabic, etc.) and line breaks. If the image contains a form or table, " + "preserve the structure with tabs or spaces." +) + +MAX_PDF_PAGES = 20 +PDF_RENDER_DPI = 180 + + +def _build_vision_messages(images_b64: list[tuple[str, str]], prompt: str = OCR_PROMPT) -> list[dict]: + """Build OpenAI-format messages with vision content. images_b64 is list of (mime, base64).""" + content = [{"type": "text", "text": prompt}] + for mime, b64 in images_b64: + content.append({ + "type": "image_url", + "image_url": {"url": f"data:{mime};base64,{b64}"}, + }) + return [{"role": "user", "content": content}] + + +async def _call_vision(images_b64: list[tuple[str, str]], prompt: str = OCR_PROMPT) -> str: + """Send images to Claude Vision via ai-gateway and return extracted text.""" + client = AsyncOpenAI( + base_url=os.environ.get("AI_GATEWAY_URL", "http://localhost:3000").rstrip("/") + "/v1", + api_key=os.environ.get("AI_GATEWAY_API_KEY", ""), + ) + response = await client.chat.completions.create( + model=os.environ.get("CLAUDE_MODEL", "sonnet"), + messages=_build_vision_messages(images_b64, prompt), + max_tokens=4096, + ) + return (response.choices[0].message.content or "").strip() + + +async def ocr_image(base64_data: str, mime_type: str) -> str: + """OCR a single image via Claude Vision.""" + logger.info("[ocr] image mime=%s size=%d", mime_type, len(base64_data)) + return await _call_vision([(mime_type, base64_data)]) + + +async def ocr_pdf(base64_data: str) -> str: + """Render PDF pages as images and OCR each via Claude Vision. Returns combined text.""" + try: + import fitz # PyMuPDF + except ImportError: + return "❌ PyMuPDF לא מותקן — לא ניתן לסרוק PDF כתמונה." + + pdf_bytes = base64.b64decode(base64_data) + doc = fitz.open(stream=pdf_bytes, filetype="pdf") + page_count = min(len(doc), MAX_PDF_PAGES) + logger.info("[ocr] pdf pages=%d (limit=%d)", len(doc), MAX_PDF_PAGES) + + pages_text = [] + for i in range(page_count): + page = doc[i] + pix = page.get_pixmap(dpi=PDF_RENDER_DPI) + img_bytes = pix.tobytes("png") + img_b64 = base64.b64encode(img_bytes).decode("ascii") + try: + text = await _call_vision([("image/png", img_b64)]) + pages_text.append(f"=== עמוד {i + 1} ===\n{text}") + except Exception as e: + logger.error("[ocr] page %d failed: %s", i + 1, e) + pages_text.append(f"=== עמוד {i + 1} ===\n[שגיאה: {e}]") + + doc.close() + if len(doc) > MAX_PDF_PAGES: + pages_text.append(f"\n[הוגבל ל-{MAX_PDF_PAGES} עמודים מתוך {len(doc)}]") + return "\n\n".join(pages_text) + + +async def ocr_docx_images(base64_data: str) -> str: + """Extract images from a DOCX archive and OCR each via Claude Vision.""" + import zipfile + + docx_bytes = base64.b64decode(base64_data) + image_extracts = [] + try: + with zipfile.ZipFile(io.BytesIO(docx_bytes)) as z: + media = [n for n in z.namelist() if n.startswith("word/media/")] + logger.info("[ocr] docx images found=%d", len(media)) + for i, name in enumerate(media[:MAX_PDF_PAGES]): + ext = name.rsplit(".", 1)[-1].lower() + mime = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "gif": "image/gif"}.get(ext, "image/png") + img_bytes = z.read(name) + img_b64 = base64.b64encode(img_bytes).decode("ascii") + try: + text = await _call_vision([(mime, img_b64)]) + image_extracts.append(f"=== תמונה {i + 1} ({name}) ===\n{text}") + except Exception as e: + logger.error("[ocr] docx image %s failed: %s", name, e) + except zipfile.BadZipFile: + return "❌ הקובץ אינו DOCX תקין." + + if not image_extracts: + return "❌ לא נמצאו תמונות ב-DOCX לחילוץ טקסט." + return "\n\n".join(image_extracts) + + +async def extract_text_from_bytes(base64_data: str, mime_type: str, file_name: str = "") -> str: + """Route to the right OCR method based on mime type. Returns extracted text.""" + mt = (mime_type or "").lower() + + if mt.startswith("image/"): + return await ocr_image(base64_data, mt) + + if mt == "application/pdf": + return await ocr_pdf(base64_data) + + if mt in ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + ) or file_name.lower().endswith((".docx", ".doc")): + return await ocr_docx_images(base64_data) + + return f"❌ סוג קובץ לא נתמך ל-OCR: {mime_type}" diff --git a/mcp_server/tools/document_tools.py b/mcp_server/tools/document_tools.py index d327c5c..eb29ee9 100644 --- a/mcp_server/tools/document_tools.py +++ b/mcp_server/tools/document_tools.py @@ -96,15 +96,42 @@ def register_document_tools( except Exception as e: return fail(f"שגיאה בשמירת נתיב: {e}") + async def _ocr_fallback(filePath: str, reason: str) -> str: + """Fetch file bytes and OCR via Claude Vision.""" + from api.services.ocr import extract_text_from_bytes + try: + bytes_result = await crm.post("SmartAssistant/action/getDocumentBytes", {"filePath": filePath}) + except Exception as e: + return fail(f'לא ניתן לחלץ טקסט מ-"{filePath}" ({reason}). כשל ב-getDocumentBytes: {e}') + + if not bytes_result.get("success") or not bytes_result.get("base64"): + err = bytes_result.get("error", "unknown") + return fail(f'לא ניתן לחלץ טקסט מ-"{filePath}" ({reason}). בייטי הקובץ: {err}') + + file_name = bytes_result.get("fileName", filePath) + mime_type = bytes_result.get("mimeType", "") + try: + text = await extract_text_from_bytes( + bytes_result["base64"], mime_type, file_name + ) + return ok(f'=== {file_name} (OCR via Vision) ===\n\n{text}') + except Exception as e: + return fail(f'שגיאה ב-OCR של "{file_name}": {e}') + async def read_document(filePath: str) -> str: try: result = await crm.post("SmartAssistant/action/readDocument", {"filePath": filePath}) if not result.get("success") or not result.get("text"): - return fail(f'לא ניתן לחלץ טקסט מ-"{result.get("fileName", filePath)}"') + # Fallback to OCR via Claude Vision + return await _ocr_fallback(filePath, "חילוץ טקסט רגיל נכשל") return ok(f'=== {result["fileName"]} ({result.get("charCount", "?")} תווים) ===\n\n{result["text"]}') except Exception as e: return fail(f"שגיאה בקריאת מסמך: {e}") + async def read_document_ocr(filePath: str) -> str: + """Force OCR via Claude Vision — skip regular text extraction.""" + return await _ocr_fallback(filePath, "OCR מבוקש במפורש") + async def read_multiple_documents(filePaths: list[str], maxCharsPerFile: int = 50000) -> str: try: result = await crm.post("SmartAssistant/action/readMultipleDocuments", { @@ -206,7 +233,7 @@ def register_document_tools( } tools["read_document"] = { - "description": "Read and extract text content from a document (PDF, DOCX, TXT). Requires the file path from list_documents.", + "description": "Read and extract text content from a document (PDF, DOCX, TXT, images). Requires the file path from list_documents. Automatically falls back to Claude Vision OCR if regular extraction fails (e.g. scanned PDFs, image-only docx).", "parameters": { "type": "object", "properties": { @@ -217,6 +244,18 @@ def register_document_tools( "handler": read_document, } + tools["read_document_ocr"] = { + "description": "Force OCR extraction via Claude Vision — for images (PNG/JPG/TIFF), scanned PDFs, or any file where you want AI vision to read it. Skips regular text extraction and sends the file straight to Claude Vision.", + "parameters": { + "type": "object", + "properties": { + "filePath": {"type": "string", "description": "Full path of the document (from list_documents results)"}, + }, + "required": ["filePath"], + }, + "handler": read_document_ocr, + } + tools["read_multiple_documents"] = { "description": "Read and extract text from multiple documents at once.", "parameters": { diff --git a/pyproject.toml b/pyproject.toml index bb435d9..ab8f044 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "pydantic>=2.10.0", "openai>=1.50.0", "pyyaml>=6.0", + "pymupdf>=1.24.0", ] [project.optional-dependencies]