From 8e591423da511f0fb2d59fcd323e53b6379b8bf8 Mon Sep 17 00:00:00 2001 From: Nitzan Pomerantz <9297302+nitzpo@users.noreply.github.com> Date: Fri, 28 Nov 2025 22:40:29 +0200 Subject: [PATCH] Change default IQR multiplier to 1.0 for more aggressive outlier filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change ANALYSIS_IQR_MULTIPLIER default from 1.5 to 1.0 in config.py - Add iqr_multiplier parameter to all filtering & statistics functions - Allow runtime override via MCP tools (get_valuation_comparables, get_deal_statistics) - Update CLAUDE.md docs with new default & override examples Rationale: k=1.0 catches more suspicious deals (e.g. 43% below median) while still preserving legitimate edge cases via hard bounds. Users can override per-call for more conservative filtering (k=1.5) if needed. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- CLAUDE.md | 12 ++++++++---- nadlan_mcp/config.py | 2 +- nadlan_mcp/fastmcp_server.py | 19 ++++++++++++++----- nadlan_mcp/govmap/client.py | 7 +++++-- nadlan_mcp/govmap/outlier_detection.py | 15 ++++++++++++--- nadlan_mcp/govmap/statistics.py | 5 +++-- 6 files changed, 43 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3ed4dd9..e22d3c4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -252,7 +252,7 @@ The MCP now includes configurable outlier detection and robust statistical measu ```bash # Outlier Detection Strategy ANALYSIS_OUTLIER_METHOD=iqr # Options: iqr, percent, none (default: iqr) -ANALYSIS_IQR_MULTIPLIER=1.5 # 1.5=moderate, 3.0=conservative (default: 1.5) +ANALYSIS_IQR_MULTIPLIER=1.0 # 1.0=aggressive, 1.5=moderate, 3.0=conservative (default: 1.0) ANALYSIS_MIN_DEALS_FOR_OUTLIER_DETECTION=10 # Minimum deals needed (default: 10) # Hard Bounds (catches obvious data errors) @@ -273,10 +273,14 @@ ANALYSIS_INCLUDE_UNFILTERED_STATS=true # Report both filtered/unfiltered (defau from nadlan_mcp.govmap.statistics import calculate_deal_statistics from nadlan_mcp.config import GovmapConfig -# With outlier filtering (default behavior) +# With outlier filtering (default behavior, k=1.0) config = GovmapConfig() # Uses env vars or defaults stats = calculate_deal_statistics(deals, config) +# Override IQR multiplier for more/less aggressive filtering +stats = calculate_deal_statistics(deals, config, iqr_multiplier=1.5) # More conservative +stats = calculate_deal_statistics(deals, config, iqr_multiplier=0.5) # More aggressive + # Access filtered statistics (main results after outlier removal) filtered_mean = stats.filtered_price_per_sqm_statistics["mean"] filtered_median = stats.filtered_price_per_sqm_statistics["median"] @@ -292,10 +296,10 @@ original_mean = stats.price_per_sqm_statistics["mean"] ``` **Design Principles:** -- **Enabled by default**: Moderate filtering (k=1.5) to catch common errors +- **Enabled by default**: Aggressive filtering (k=1.0) to catch common errors & suspicious deals - **Transparent**: Both filtered and unfiltered statistics returned - **Conservative with real data**: Hard bounds + IQR preserve legitimate high-end properties -- **Configurable**: Adjust sensitivity via environment variables +- **Configurable**: Adjust sensitivity via environment variables or function parameters - **MCP provides data, LLM provides intelligence**: Outlier detection improves data quality; LLM interprets results ### Retry Logic diff --git a/nadlan_mcp/config.py b/nadlan_mcp/config.py index d3435c0..7602284 100644 --- a/nadlan_mcp/config.py +++ b/nadlan_mcp/config.py @@ -61,7 +61,7 @@ class GovmapConfig: default_factory=lambda: os.getenv("ANALYSIS_OUTLIER_METHOD", "iqr") ) analysis_iqr_multiplier: float = field( - default_factory=lambda: float(os.getenv("ANALYSIS_IQR_MULTIPLIER", "1.5")) + default_factory=lambda: float(os.getenv("ANALYSIS_IQR_MULTIPLIER", "1.0")) ) analysis_min_deals_for_outlier_detection: int = field( default_factory=lambda: int(os.getenv("ANALYSIS_MIN_DEALS_FOR_OUTLIER_DETECTION", "10")) diff --git a/nadlan_mcp/fastmcp_server.py b/nadlan_mcp/fastmcp_server.py index de2f7fb..af62cc6 100644 --- a/nadlan_mcp/fastmcp_server.py +++ b/nadlan_mcp/fastmcp_server.py @@ -772,11 +772,12 @@ def get_valuation_comparables( max_floor: Optional[int] = None, radius_meters: int = 100, max_comparables: int = 50, + iqr_multiplier: Optional[float] = None, ) -> str: """Get comparable properties for valuation analysis. This tool provides detailed comparable deals filtered by your criteria. - Automatically applies IQR outlier filtering (k=1.5) to remove statistical outliers + Automatically applies IQR outlier filtering (k=1.0 default) to remove statistical outliers and improve data quality. The response includes metadata about filtering so you can inform users about removed outliers. @@ -794,6 +795,7 @@ def get_valuation_comparables( max_floor: Maximum floor number radius_meters: Search radius in meters (default: 100, larger than find_recent_deals to get more comparables) max_comparables: Maximum number of deals to return (default: 50, optimized for MCP token limits) + iqr_multiplier: Override IQR multiplier for outlier detection (default: 1.0). Lower = more aggressive filtering Returns: JSON string containing: @@ -803,7 +805,7 @@ def get_valuation_comparables( - total_deals_before_filtering: Count before filtering - outliers_removed: Number of deals filtered out - filtering_method: Method used (e.g., "iqr") - - iqr_multiplier: IQR multiplier used (e.g., 1.5) + - iqr_multiplier: IQR multiplier used (e.g., 1.0) """ log_mcp_call( "get_valuation_comparables", @@ -820,6 +822,7 @@ def get_valuation_comparables( max_floor=max_floor, radius_meters=radius_meters, max_comparables=max_comparables, + iqr_multiplier=iqr_multiplier, ) try: # Get all deals for the address with higher limits for valuation @@ -876,10 +879,13 @@ def get_valuation_comparables( ): deals_before_outlier_filter = len(filtered_deals) filtered_deals, outlier_report = filter_deals_for_analysis( - filtered_deals, config, metric="price_per_sqm" + filtered_deals, config, metric="price_per_sqm", iqr_multiplier=iqr_multiplier + ) + effective_k = ( + iqr_multiplier if iqr_multiplier is not None else config.analysis_iqr_multiplier ) logger.info( - f"After outlier filtering ({config.analysis_outlier_method}, k={config.analysis_iqr_multiplier}): " + f"After outlier filtering ({config.analysis_outlier_method}, k={effective_k}): " f"{len(filtered_deals)} deals (removed {deals_before_outlier_filter - len(filtered_deals)} outliers)" ) else: @@ -946,6 +952,7 @@ def get_deal_statistics( property_type: Optional[str] = None, min_rooms: Optional[float] = None, max_rooms: Optional[float] = None, + iqr_multiplier: Optional[float] = None, ) -> str: """Calculate statistical aggregations on deal data for an address. @@ -958,6 +965,7 @@ def get_deal_statistics( property_type: Filter by property type (e.g., "דירה", "בית") min_rooms: Minimum number of rooms max_rooms: Maximum number of rooms + iqr_multiplier: Override IQR multiplier for outlier detection (default: 1.0). Lower = more aggressive filtering Returns: JSON string containing statistical metrics (mean, median, percentiles, etc.) @@ -969,6 +977,7 @@ def get_deal_statistics( property_type=property_type, min_rooms=min_rooms, max_rooms=max_rooms, + iqr_multiplier=iqr_multiplier, ) try: # Get all deals for the address @@ -998,7 +1007,7 @@ def get_deal_statistics( ) # Calculate statistics - stats = client.calculate_deal_statistics(deals) + stats = client.calculate_deal_statistics(deals, iqr_multiplier=iqr_multiplier) # Normalize response structure to match other tools return json.dumps( diff --git a/nadlan_mcp/govmap/client.py b/nadlan_mcp/govmap/client.py index 55dbc2c..d32e7ea 100644 --- a/nadlan_mcp/govmap/client.py +++ b/nadlan_mcp/govmap/client.py @@ -873,7 +873,9 @@ class GovmapClient: ) # Statistics methods (delegate to statistics module) - def calculate_deal_statistics(self, deals: List[Deal]) -> DealStatistics: + def calculate_deal_statistics( + self, deals: List[Deal], iqr_multiplier: Optional[float] = None + ) -> DealStatistics: """ Calculate statistical aggregations on deal data. @@ -881,11 +883,12 @@ class GovmapClient: Args: deals: List of Deal model instances + iqr_multiplier: Override IQR multiplier for outlier detection (optional) Returns: DealStatistics model with comprehensive metrics """ - return statistics.calculate_deal_statistics(deals) + return statistics.calculate_deal_statistics(deals, iqr_multiplier=iqr_multiplier) def _calculate_std_dev(self, values: List[float]) -> float: """ diff --git a/nadlan_mcp/govmap/outlier_detection.py b/nadlan_mcp/govmap/outlier_detection.py index 1f14708..518e3be 100644 --- a/nadlan_mcp/govmap/outlier_detection.py +++ b/nadlan_mcp/govmap/outlier_detection.py @@ -173,7 +173,10 @@ def apply_hard_bounds_deal_amount( def filter_deals_for_analysis( - deals: List[Deal], config: Optional[GovmapConfig] = None, metric: str = "price_per_sqm" + deals: List[Deal], + config: Optional[GovmapConfig] = None, + metric: str = "price_per_sqm", + iqr_multiplier: Optional[float] = None, ) -> Tuple[List[Deal], Dict[str, Any]]: """ Filter deals to remove outliers based on configuration. @@ -192,6 +195,7 @@ def filter_deals_for_analysis( config: Configuration object (optional, uses global if not provided) metric: Which metric to apply statistical outlier detection to Options: "price_per_sqm", "deal_amount" + iqr_multiplier: Override IQR multiplier (optional, uses config value if not provided) Returns: Tuple of: @@ -239,6 +243,11 @@ def filter_deals_for_analysis( filters_to_remove[i] = True # Step 3: Apply statistical outlier detection to specified metric + # Use override value if provided, otherwise use config + effective_iqr_multiplier = ( + iqr_multiplier if iqr_multiplier is not None else config.analysis_iqr_multiplier + ) + if config.analysis_outlier_method == "iqr": # Extract values for the specified metric if metric == "price_per_sqm": @@ -260,7 +269,7 @@ def filter_deals_for_analysis( value_indices = [] if values: - statistical_outliers = detect_outliers_iqr(values, config.analysis_iqr_multiplier) + statistical_outliers = detect_outliers_iqr(values, effective_iqr_multiplier) for i, is_outlier in enumerate(statistical_outliers): if is_outlier: filters_to_remove[value_indices[i]] = True @@ -302,7 +311,7 @@ def filter_deals_for_analysis( "outlier_indices": outlier_indices, "method_used": config.analysis_outlier_method, "parameters": { - "iqr_multiplier": config.analysis_iqr_multiplier + "iqr_multiplier": effective_iqr_multiplier if config.analysis_outlier_method == "iqr" else None, "metric": metric, diff --git a/nadlan_mcp/govmap/statistics.py b/nadlan_mcp/govmap/statistics.py index 4076161..3b0026c 100644 --- a/nadlan_mcp/govmap/statistics.py +++ b/nadlan_mcp/govmap/statistics.py @@ -152,7 +152,7 @@ def _calculate_basic_stats(deals: List[Deal]) -> Dict: def calculate_deal_statistics( - deals: List[Deal], config: Optional[GovmapConfig] = None + deals: List[Deal], config: Optional[GovmapConfig] = None, iqr_multiplier: Optional[float] = None ) -> DealStatistics: """ Calculate statistical aggregations on deal data with optional outlier filtering. @@ -164,6 +164,7 @@ def calculate_deal_statistics( Args: deals: List of Deal model instances config: Configuration object (optional, uses global config if not provided) + iqr_multiplier: Override IQR multiplier (optional, uses config value if not provided) Returns: DealStatistics model with comprehensive metrics, including: @@ -203,7 +204,7 @@ def calculate_deal_statistics( ): # Filter deals for analysis (primarily targeting price_per_sqm outliers) filtered_deals, report_dict = filter_deals_for_analysis( - deals, config, metric="price_per_sqm" + deals, config, metric="price_per_sqm", iqr_multiplier=iqr_multiplier ) # Create OutlierReport model