From 4de1a4822bac3b1d9c75c4ccacc2b6bfab4b46f9 Mon Sep 17 00:00:00 2001 From: Nitzan Pomerantz <9297302+nitzpo@users.noreply.github.com> Date: Sat, 6 Dec 2025 23:26:21 +0200 Subject: [PATCH] Add: Return outlier deals in analysis responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature: - New parameter `include_outlier_deals` (default=True) in: - filter_deals_for_analysis() - calculate_deal_statistics() - get_valuation_comparables() MCP tool - get_deal_statistics() MCP tool Behavior: - When ON: response includes `outlier_deals` field with removed deals - LLM can see what was filtered out, answer questions about it - Maintains full transparency on outlier removal Terminology fixed: - "outlier_deals" = deals removed as outliers (clearer than "filtered_deals") - "filtered deals" = deals that PASSED filtering - Added to OutlierReport model All 311 tests pass 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- nadlan_mcp/fastmcp_server.py | 135 ++++++++++++++----------- nadlan_mcp/govmap/client.py | 10 +- nadlan_mcp/govmap/models.py | 4 + nadlan_mcp/govmap/outlier_detection.py | 12 ++- nadlan_mcp/govmap/statistics.py | 15 ++- 5 files changed, 111 insertions(+), 65 deletions(-) diff --git a/nadlan_mcp/fastmcp_server.py b/nadlan_mcp/fastmcp_server.py index 25f6dde..0d8083a 100644 --- a/nadlan_mcp/fastmcp_server.py +++ b/nadlan_mcp/fastmcp_server.py @@ -799,6 +799,7 @@ def get_valuation_comparables( radius_meters: int = 100, max_comparables: int = 50, iqr_multiplier: Optional[float] = None, + include_outlier_deals: bool = True, ) -> str: """Get comparable properties for valuation analysis. @@ -822,16 +823,19 @@ def get_valuation_comparables( radius_meters: Search radius in meters (default: 100, larger than find_recent_deals to get more comparables) max_comparables: Maximum number of deals to return (default: 50, optimized for MCP token limits) iqr_multiplier: Override IQR multiplier for outlier detection (default: 1.0). Lower = more aggressive filtering + include_outlier_deals: If True (default), include the outlier deals separately in the response + This allows you to see what was filtered out and answer questions about it Returns: JSON string containing: - - Filtered comparable deals with full details + - deals: Comparable deals that passed filtering - deal_breakdown with outlier filtering metadata: - total_deals: Count after filtering - total_deals_before_filtering: Count before filtering - - outliers_removed: Number of deals filtered out + - outliers_removed: Number of deals removed as outliers - filtering_method: Method used (e.g., "iqr") - iqr_multiplier: IQR multiplier used (e.g., 1.0) + - outlier_deals: List of deals that were removed as outliers (if include_outlier_deals=True) """ log_mcp_call( "get_valuation_comparables", @@ -905,7 +909,11 @@ def get_valuation_comparables( ): deals_before_outlier_filter = len(filtered_deals) filtered_deals, outlier_report = filter_deals_for_analysis( - filtered_deals, config, metric="price_per_sqm", iqr_multiplier=iqr_multiplier + filtered_deals, + config, + metric="price_per_sqm", + iqr_multiplier=iqr_multiplier, + include_outlier_deals=include_outlier_deals, ) effective_k = ( iqr_multiplier if iqr_multiplier is not None else config.analysis_iqr_multiplier @@ -936,35 +944,37 @@ def get_valuation_comparables( if outlier_report["method_used"] == "iqr": deal_breakdown["iqr_multiplier"] = outlier_report["parameters"]["iqr_multiplier"] - # Normalize response structure to match other tools - return json.dumps( - { - "search_parameters": { - "address": address, - "years_back": years_back, - "radius_meters": radius_meters, - "max_comparables": max_comparables, - "filters_applied": { - "property_type": property_type, - "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None, - "price": f"{min_price}-{max_price}" if min_price or max_price else None, - "area": f"{min_area}-{max_area}" if min_area or max_area else None, - "floor": f"{min_floor}-{max_floor}" if min_floor or max_floor else None, - }, + # Build response with filtered deals + response_data = { + "search_parameters": { + "address": address, + "years_back": years_back, + "radius_meters": radius_meters, + "max_comparables": max_comparables, + "filters_applied": { + "property_type": property_type, + "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None, + "price": f"{min_price}-{max_price}" if min_price or max_price else None, + "area": f"{min_area}-{max_area}" if min_area or max_area else None, + "floor": f"{min_floor}-{max_floor}" if min_floor or max_floor else None, }, - "market_statistics": { - "deal_breakdown": deal_breakdown, - "price_statistics": stats.price_statistics, - "area_statistics": stats.area_statistics, - "price_per_sqm_statistics": stats.price_per_sqm_statistics, - "property_type_distribution": stats.property_type_distribution, - "date_range": stats.date_range, - }, - "deals": strip_bloat_fields(filtered_deals), }, - ensure_ascii=False, - indent=2, - ) + "market_statistics": { + "deal_breakdown": deal_breakdown, + "price_statistics": stats.price_statistics, + "area_statistics": stats.area_statistics, + "price_per_sqm_statistics": stats.price_per_sqm_statistics, + "property_type_distribution": stats.property_type_distribution, + "date_range": stats.date_range, + }, + "deals": strip_bloat_fields(filtered_deals), + } + + # Add outlier deals if present and requested + if include_outlier_deals and outlier_report and "outlier_deals" in outlier_report: + response_data["outlier_deals"] = strip_bloat_fields(outlier_report["outlier_deals"]) + + return json.dumps(response_data, ensure_ascii=False, indent=2) except Exception as e: logger.error(f"Error in get_valuation_comparables: {e}", exc_info=True) @@ -979,6 +989,7 @@ def get_deal_statistics( min_rooms: Optional[float] = None, max_rooms: Optional[float] = None, iqr_multiplier: Optional[float] = None, + include_outlier_deals: bool = True, ) -> str: """Calculate statistical aggregations on deal data for an address. @@ -992,9 +1003,13 @@ def get_deal_statistics( min_rooms: Minimum number of rooms max_rooms: Maximum number of rooms iqr_multiplier: Override IQR multiplier for outlier detection (default: 1.0). Lower = more aggressive filtering + include_outlier_deals: If True (default), include the outlier deals in the response + This allows you to see what was filtered out Returns: - JSON string containing statistical metrics (mean, median, percentiles, etc.) + JSON string containing: + - market_statistics: Statistical metrics (mean, median, percentiles, etc.) + - outlier_deals: Deals that were removed as outliers (if include_outlier_deals=True) """ log_mcp_call( "get_deal_statistics", @@ -1033,35 +1048,39 @@ def get_deal_statistics( ) # Calculate statistics - stats = client.calculate_deal_statistics(deals, iqr_multiplier=iqr_multiplier) - - # Normalize response structure to match other tools - return json.dumps( - { - "search_parameters": { - "address": address, - "years_back": years_back, - "filters_applied": { - "property_type": property_type, - "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None, - }, - }, - "market_statistics": { - "deal_breakdown": { - "total_deals": stats.total_deals, - }, - "price_statistics": stats.price_statistics, - "area_statistics": stats.area_statistics, - "price_per_sqm_statistics": stats.price_per_sqm_statistics, - "property_type_distribution": stats.property_type_distribution, - "date_range": stats.date_range, - }, - "deals": [], # Statistics-only query, no full deals returned - }, - ensure_ascii=False, - indent=2, + stats = client.calculate_deal_statistics( + deals, iqr_multiplier=iqr_multiplier, include_outlier_deals=include_outlier_deals ) + # Build response + response_data = { + "search_parameters": { + "address": address, + "years_back": years_back, + "filters_applied": { + "property_type": property_type, + "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None, + }, + }, + "market_statistics": { + "deal_breakdown": { + "total_deals": stats.total_deals, + }, + "price_statistics": stats.price_statistics, + "area_statistics": stats.area_statistics, + "price_per_sqm_statistics": stats.price_per_sqm_statistics, + "property_type_distribution": stats.property_type_distribution, + "date_range": stats.date_range, + }, + "deals": [], # Statistics-only query, no full deals returned + } + + # Add outlier deals if present and requested + if include_outlier_deals and stats.outlier_report and stats.outlier_report.outlier_deals: + response_data["outlier_deals"] = strip_bloat_fields(stats.outlier_report.outlier_deals) + + return json.dumps(response_data, ensure_ascii=False, indent=2) + except Exception as e: logger.error(f"Error in get_deal_statistics: {e}", exc_info=True) return f"Error calculating deal statistics: {str(e)}" diff --git a/nadlan_mcp/govmap/client.py b/nadlan_mcp/govmap/client.py index d32e7ea..3047eaa 100644 --- a/nadlan_mcp/govmap/client.py +++ b/nadlan_mcp/govmap/client.py @@ -874,7 +874,10 @@ class GovmapClient: # Statistics methods (delegate to statistics module) def calculate_deal_statistics( - self, deals: List[Deal], iqr_multiplier: Optional[float] = None + self, + deals: List[Deal], + iqr_multiplier: Optional[float] = None, + include_outlier_deals: bool = True, ) -> DealStatistics: """ Calculate statistical aggregations on deal data. @@ -884,11 +887,14 @@ class GovmapClient: Args: deals: List of Deal model instances iqr_multiplier: Override IQR multiplier for outlier detection (optional) + include_outlier_deals: If True (default), include removed outliers in report Returns: DealStatistics model with comprehensive metrics """ - return statistics.calculate_deal_statistics(deals, iqr_multiplier=iqr_multiplier) + return statistics.calculate_deal_statistics( + deals, iqr_multiplier=iqr_multiplier, include_outlier_deals=include_outlier_deals + ) def _calculate_std_dev(self, values: List[float]) -> float: """ diff --git a/nadlan_mcp/govmap/models.py b/nadlan_mcp/govmap/models.py index 6ba8c9f..688ade1 100644 --- a/nadlan_mcp/govmap/models.py +++ b/nadlan_mcp/govmap/models.py @@ -196,6 +196,7 @@ class OutlierReport(BaseModel): outlier_indices: Indices of removed deals in original list method_used: Outlier detection method ("iqr", "percent", "none") parameters: Configuration parameters used for detection + filtered_deals: The actual deals that were filtered out (optional) """ total_deals: int = Field(..., description="Total deals before filtering") @@ -205,6 +206,9 @@ class OutlierReport(BaseModel): parameters: Dict[str, Any] = Field( default_factory=dict, description="Detection parameters used" ) + outlier_deals: Optional[List["Deal"]] = Field( + None, description="Deals that were removed as outliers (if requested)" + ) class DealStatistics(BaseModel): diff --git a/nadlan_mcp/govmap/outlier_detection.py b/nadlan_mcp/govmap/outlier_detection.py index 9889830..b877b97 100644 --- a/nadlan_mcp/govmap/outlier_detection.py +++ b/nadlan_mcp/govmap/outlier_detection.py @@ -177,6 +177,7 @@ def filter_deals_for_analysis( config: Optional[GovmapConfig] = None, metric: str = "price_per_sqm", iqr_multiplier: Optional[float] = None, + include_outlier_deals: bool = True, ) -> Tuple[List[Deal], Dict[str, Any]]: """ Filter deals to remove outliers based on configuration. @@ -196,11 +197,13 @@ def filter_deals_for_analysis( metric: Which metric to apply statistical outlier detection to Options: "price_per_sqm", "deal_amount" iqr_multiplier: Override IQR multiplier (optional, uses config value if not provided) + include_outlier_deals: If True, include the removed outlier deals in the report + (default: True, allows LLM to see what was filtered out) Returns: Tuple of: - - Filtered list of deals - - Outlier report dict with details on what was filtered + - Filtered list of deals (deals that PASSED filtering) + - Outlier report dict with details on what was filtered out (includes outlier_deals if requested) """ if config is None: config = get_config() @@ -333,6 +336,7 @@ def filter_deals_for_analysis( # Filter deals filtered_deals = [deal for i, deal in enumerate(deals) if not filters_to_remove[i]] outlier_indices = [i for i, should_remove in enumerate(filters_to_remove) if should_remove] + outlier_deals = [deal for i, deal in enumerate(deals) if filters_to_remove[i]] # Create outlier report report = { @@ -357,4 +361,8 @@ def filter_deals_for_analysis( }, } + # Include outlier deals if requested + if include_outlier_deals and outlier_deals: + report["outlier_deals"] = outlier_deals + return filtered_deals, report diff --git a/nadlan_mcp/govmap/statistics.py b/nadlan_mcp/govmap/statistics.py index 3b0026c..2ce88e1 100644 --- a/nadlan_mcp/govmap/statistics.py +++ b/nadlan_mcp/govmap/statistics.py @@ -152,7 +152,10 @@ def _calculate_basic_stats(deals: List[Deal]) -> Dict: def calculate_deal_statistics( - deals: List[Deal], config: Optional[GovmapConfig] = None, iqr_multiplier: Optional[float] = None + deals: List[Deal], + config: Optional[GovmapConfig] = None, + iqr_multiplier: Optional[float] = None, + include_outlier_deals: bool = True, ) -> DealStatistics: """ Calculate statistical aggregations on deal data with optional outlier filtering. @@ -165,12 +168,14 @@ def calculate_deal_statistics( deals: List of Deal model instances config: Configuration object (optional, uses global config if not provided) iqr_multiplier: Override IQR multiplier (optional, uses config value if not provided) + include_outlier_deals: If True (default), include the removed outlier deals in the outlier report + This allows LLMs to see what was filtered out Returns: DealStatistics model with comprehensive metrics, including: - Original statistics (calculated on all deals) - Filtered statistics (calculated after outlier removal, if enabled) - - Outlier report (details on what was filtered, if enabled) + - Outlier report (details on what was filtered out, including outlier_deals if requested) Raises: ValueError: If deals is not a valid list @@ -204,7 +209,11 @@ def calculate_deal_statistics( ): # Filter deals for analysis (primarily targeting price_per_sqm outliers) filtered_deals, report_dict = filter_deals_for_analysis( - deals, config, metric="price_per_sqm", iqr_multiplier=iqr_multiplier + deals, + config, + metric="price_per_sqm", + iqr_multiplier=iqr_multiplier, + include_outlier_deals=include_outlier_deals, ) # Create OutlierReport model