From 75ee79bbfdcc86c752a6feadfd45cece0aa28594 Mon Sep 17 00:00:00 2001 From: Nitzan Pomerantz <9297302+nitzpo@users.noreply.github.com> Date: Sat, 29 Nov 2025 14:44:14 +0200 Subject: [PATCH] Add percentage-based backup outlier filtering for heterogeneous data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: IQR filtering fails with wide distributions (mixed room counts, property types). Example: 9,459 NIS/sqm deal (68% below average) passed IQR with k=1.0 due to wide IQR. Solution: Dual filtering approach - IQR + percentage backup Changes: - Add ANALYSIS_USE_PERCENTAGE_BACKUP config (default: true) - Add ANALYSIS_PERCENTAGE_THRESHOLD config (default: 0.5 = 50%) - Apply percentage backup after IQR when method=iqr - Update outlier report with percentage backup parameters - Update CLAUDE.md docs with dual filtering explanation Behavior: 1. Hard bounds filter (1K-100K/sqm, 100K min total) 2. IQR filter (k=1.0) - catches mild outliers 3. Percentage filter (50% from median) - catches extreme outliers 4. Deal removed if flagged by ANY method Example: 12K/sqm deal (60% below 30K median) now caught by percentage backup even when IQR bounds are permissive due to heterogeneous data. Tests: All 326 tests pass, manual verification confirms extreme outliers removed. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- CLAUDE.md | 10 +++++-- nadlan_mcp/config.py | 11 ++++++++ nadlan_mcp/govmap/outlier_detection.py | 36 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e22d3c4..7fd5295 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -244,6 +244,7 @@ The MCP now includes configurable outlier detection and robust statistical measu **Key Features:** - **IQR-based outlier detection**: Uses Interquartile Range (robust to skewed data) +- **Percentage backup filtering**: Catches extreme outliers (>50% from median) in heterogeneous data - **Hard bounds filtering**: Removes obvious errors (price_per_sqm < 1K or > 100K NIS/sqm, deals < 100K NIS) - **Robust volatility**: Investment analysis uses IQR instead of std_dev for stability ratings - **Transparent reporting**: Returns both filtered and unfiltered statistics with outlier reports @@ -255,6 +256,10 @@ ANALYSIS_OUTLIER_METHOD=iqr # Options: iqr, percent, none (default: iqr ANALYSIS_IQR_MULTIPLIER=1.0 # 1.0=aggressive, 1.5=moderate, 3.0=conservative (default: 1.0) ANALYSIS_MIN_DEALS_FOR_OUTLIER_DETECTION=10 # Minimum deals needed (default: 10) +# Percentage-based Backup Filtering (catches extreme outliers in heterogeneous data) +ANALYSIS_USE_PERCENTAGE_BACKUP=true # Enable percentage backup (default: true) +ANALYSIS_PERCENTAGE_THRESHOLD=0.5 # Remove deals >50% from median (default: 0.5) + # Hard Bounds (catches obvious data errors) ANALYSIS_PRICE_PER_SQM_MIN=1000 # 1K NIS/sqm minimum (default: 1000) ANALYSIS_PRICE_PER_SQM_MAX=100000 # 100K NIS/sqm maximum (default: 100000) @@ -296,9 +301,10 @@ original_mean = stats.price_per_sqm_statistics["mean"] ``` **Design Principles:** -- **Enabled by default**: Aggressive filtering (k=1.0) to catch common errors & suspicious deals +- **Dual filtering approach**: IQR (k=1.0) + percentage backup (50%) catch both mild and extreme outliers +- **Handles heterogeneous data**: Percentage backup catches outliers when IQR becomes too permissive (mixed room counts, property types) - **Transparent**: Both filtered and unfiltered statistics returned -- **Conservative with real data**: Hard bounds + IQR preserve legitimate high-end properties +- **Conservative with real data**: Hard bounds + IQR + percentage preserve legitimate high-end properties - **Configurable**: Adjust sensitivity via environment variables or function parameters - **MCP provides data, LLM provides intelligence**: Outlier detection improves data quality; LLM interprets results diff --git a/nadlan_mcp/config.py b/nadlan_mcp/config.py index 7602284..2c728ae 100644 --- a/nadlan_mcp/config.py +++ b/nadlan_mcp/config.py @@ -67,6 +67,15 @@ class GovmapConfig: default_factory=lambda: int(os.getenv("ANALYSIS_MIN_DEALS_FOR_OUTLIER_DETECTION", "10")) ) + # Percentage-based backup filtering (catches extreme outliers in heterogeneous data) + analysis_use_percentage_backup: bool = field( + default_factory=lambda: os.getenv("ANALYSIS_USE_PERCENTAGE_BACKUP", "true").lower() + == "true" + ) + analysis_percentage_threshold: float = field( + default_factory=lambda: float(os.getenv("ANALYSIS_PERCENTAGE_THRESHOLD", "0.5")) + ) + # Hard Bounds for Price per Sqm (catches obvious data errors) analysis_price_per_sqm_min: float = field( default_factory=lambda: float(os.getenv("ANALYSIS_PRICE_PER_SQM_MIN", "1000")) @@ -146,6 +155,8 @@ class GovmapConfig: raise ValueError("analysis_iqr_multiplier must be positive") if self.analysis_min_deals_for_outlier_detection < 0: raise ValueError("analysis_min_deals_for_outlier_detection must be non-negative") + if self.analysis_percentage_threshold <= 0 or self.analysis_percentage_threshold >= 1: + raise ValueError("analysis_percentage_threshold must be between 0 and 1") if self.analysis_price_per_sqm_min <= 0: raise ValueError("analysis_price_per_sqm_min must be positive") if self.analysis_price_per_sqm_max <= self.analysis_price_per_sqm_min: diff --git a/nadlan_mcp/govmap/outlier_detection.py b/nadlan_mcp/govmap/outlier_detection.py index 518e3be..bd1640c 100644 --- a/nadlan_mcp/govmap/outlier_detection.py +++ b/nadlan_mcp/govmap/outlier_detection.py @@ -300,6 +300,36 @@ def filter_deals_for_analysis( if is_outlier: filters_to_remove[value_indices[i]] = True + # Step 4: Apply percentage-based backup filtering (catches extreme outliers in heterogeneous data) + # This runs in addition to IQR when enabled, providing a safety net for wide distributions + if config.analysis_use_percentage_backup and config.analysis_outlier_method == "iqr": + # Extract values for the specified metric (same logic as above) + if metric == "price_per_sqm": + values = [ + deal.price_per_sqm + for i, deal in enumerate(deals) + if deal.price_per_sqm is not None and not filters_to_remove[i] + ] + value_indices = [ + i + for i, deal in enumerate(deals) + if deal.price_per_sqm is not None and not filters_to_remove[i] + ] + elif metric == "deal_amount": + values = [deal.deal_amount for i, deal in enumerate(deals) if not filters_to_remove[i]] + value_indices = [i for i in range(len(deals)) if not filters_to_remove[i]] + else: + values = [] + value_indices = [] + + if values: + percentage_outliers = detect_outliers_percent( + values, config.analysis_percentage_threshold + ) + for i, is_outlier in enumerate(percentage_outliers): + if is_outlier: + filters_to_remove[value_indices[i]] = True + # Filter deals filtered_deals = [deal for i, deal in enumerate(deals) if not filters_to_remove[i]] outlier_indices = [i for i, should_remove in enumerate(filters_to_remove) if should_remove] @@ -314,6 +344,12 @@ def filter_deals_for_analysis( "iqr_multiplier": effective_iqr_multiplier if config.analysis_outlier_method == "iqr" else None, + "percentage_backup_enabled": config.analysis_use_percentage_backup + if config.analysis_outlier_method == "iqr" + else None, + "percentage_threshold": config.analysis_percentage_threshold + if config.analysis_use_percentage_backup and config.analysis_outlier_method == "iqr" + else None, "metric": metric, "price_per_sqm_min": config.analysis_price_per_sqm_min, "price_per_sqm_max": config.analysis_price_per_sqm_max,