Add: Return outlier deals in analysis responses

Feature:
- New parameter `include_outlier_deals` (default=True) in:  - filter_deals_for_analysis()
  - calculate_deal_statistics()
  - get_valuation_comparables() MCP tool
  - get_deal_statistics() MCP tool

Behavior:
- When ON: response includes `outlier_deals` field with removed deals
- LLM can see what was filtered out, answer questions about it
- Maintains full transparency on outlier removal

Terminology fixed:
- "outlier_deals" = deals removed as outliers (clearer than "filtered_deals")
- "filtered deals" = deals that PASSED filtering
- Added to OutlierReport model

All 311 tests pass

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

nadlan_mcp/fastmcp_server.py

@@ -799,6 +799,7 @@ def get_valuation_comparables(
     radius_meters: int = 100,
     max_comparables: int = 50,
     iqr_multiplier: Optional[float] = None,
+    include_outlier_deals: bool = True,
 ) -> str:
     """Get comparable properties for valuation analysis.
 
@@ -822,16 +823,19 @@ def get_valuation_comparables(
         radius_meters: Search radius in meters (default: 100, larger than find_recent_deals to get more comparables)
         max_comparables: Maximum number of deals to return (default: 50, optimized for MCP token limits)
         iqr_multiplier: Override IQR multiplier for outlier detection (default: 1.0). Lower = more aggressive filtering
+        include_outlier_deals: If True (default), include the outlier deals separately in the response
+                              This allows you to see what was filtered out and answer questions about it
 
     Returns:
         JSON string containing:
-        - Filtered comparable deals with full details
+        - deals: Comparable deals that passed filtering
         - deal_breakdown with outlier filtering metadata:
           - total_deals: Count after filtering
           - total_deals_before_filtering: Count before filtering
-          - outliers_removed: Number of deals filtered out
+          - outliers_removed: Number of deals removed as outliers
           - filtering_method: Method used (e.g., "iqr")
           - iqr_multiplier: IQR multiplier used (e.g., 1.0)
+        - outlier_deals: List of deals that were removed as outliers (if include_outlier_deals=True)
     """
     log_mcp_call(
         "get_valuation_comparables",
@@ -905,7 +909,11 @@ def get_valuation_comparables(
         ):
             deals_before_outlier_filter = len(filtered_deals)
             filtered_deals, outlier_report = filter_deals_for_analysis(
-                filtered_deals, config, metric="price_per_sqm", iqr_multiplier=iqr_multiplier
+                filtered_deals,
+                config,
+                metric="price_per_sqm",
+                iqr_multiplier=iqr_multiplier,
+                include_outlier_deals=include_outlier_deals,
             )
             effective_k = (
                 iqr_multiplier if iqr_multiplier is not None else config.analysis_iqr_multiplier
@@ -936,35 +944,37 @@ def get_valuation_comparables(
             if outlier_report["method_used"] == "iqr":
                 deal_breakdown["iqr_multiplier"] = outlier_report["parameters"]["iqr_multiplier"]
 
-        # Normalize response structure to match other tools
+        # Build response with filtered deals
-        return json.dumps(
+        response_data = {
-            {
+            "search_parameters": {
-                "search_parameters": {
+                "address": address,
-                    "address": address,
+                "years_back": years_back,
-                    "years_back": years_back,
+                "radius_meters": radius_meters,
-                    "radius_meters": radius_meters,
+                "max_comparables": max_comparables,
-                    "max_comparables": max_comparables,
+                "filters_applied": {
-                    "filters_applied": {
+                    "property_type": property_type,
-                        "property_type": property_type,
+                    "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None,
-                        "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None,
+                    "price": f"{min_price}-{max_price}" if min_price or max_price else None,
-                        "price": f"{min_price}-{max_price}" if min_price or max_price else None,
+                    "area": f"{min_area}-{max_area}" if min_area or max_area else None,
-                        "area": f"{min_area}-{max_area}" if min_area or max_area else None,
+                    "floor": f"{min_floor}-{max_floor}" if min_floor or max_floor else None,
-                        "floor": f"{min_floor}-{max_floor}" if min_floor or max_floor else None,
-                    },
                 },
-                "market_statistics": {
-                    "deal_breakdown": deal_breakdown,
-                    "price_statistics": stats.price_statistics,
-                    "area_statistics": stats.area_statistics,
-                    "price_per_sqm_statistics": stats.price_per_sqm_statistics,
-                    "property_type_distribution": stats.property_type_distribution,
-                    "date_range": stats.date_range,
-                },
-                "deals": strip_bloat_fields(filtered_deals),
             },
-            ensure_ascii=False,
+            "market_statistics": {
-            indent=2,
+                "deal_breakdown": deal_breakdown,
-        )
+                "price_statistics": stats.price_statistics,
+                "area_statistics": stats.area_statistics,
+                "price_per_sqm_statistics": stats.price_per_sqm_statistics,
+                "property_type_distribution": stats.property_type_distribution,
+                "date_range": stats.date_range,
+            },
+            "deals": strip_bloat_fields(filtered_deals),
+        }
+
+        # Add outlier deals if present and requested
+        if include_outlier_deals and outlier_report and "outlier_deals" in outlier_report:
+            response_data["outlier_deals"] = strip_bloat_fields(outlier_report["outlier_deals"])
+
+        return json.dumps(response_data, ensure_ascii=False, indent=2)
 
     except Exception as e:
         logger.error(f"Error in get_valuation_comparables: {e}", exc_info=True)
@@ -979,6 +989,7 @@ def get_deal_statistics(
     min_rooms: Optional[float] = None,
     max_rooms: Optional[float] = None,
     iqr_multiplier: Optional[float] = None,
+    include_outlier_deals: bool = True,
 ) -> str:
     """Calculate statistical aggregations on deal data for an address.
 
@@ -992,9 +1003,13 @@ def get_deal_statistics(
         min_rooms: Minimum number of rooms
         max_rooms: Maximum number of rooms
         iqr_multiplier: Override IQR multiplier for outlier detection (default: 1.0). Lower = more aggressive filtering
+        include_outlier_deals: If True (default), include the outlier deals in the response
+                              This allows you to see what was filtered out
 
     Returns:
-        JSON string containing statistical metrics (mean, median, percentiles, etc.)
+        JSON string containing:
+        - market_statistics: Statistical metrics (mean, median, percentiles, etc.)
+        - outlier_deals: Deals that were removed as outliers (if include_outlier_deals=True)
     """
     log_mcp_call(
         "get_deal_statistics",
@@ -1033,35 +1048,39 @@ def get_deal_statistics(
             )
 
         # Calculate statistics
-        stats = client.calculate_deal_statistics(deals, iqr_multiplier=iqr_multiplier)
+        stats = client.calculate_deal_statistics(
-
+            deals, iqr_multiplier=iqr_multiplier, include_outlier_deals=include_outlier_deals
-        # Normalize response structure to match other tools
-        return json.dumps(
-            {
-                "search_parameters": {
-                    "address": address,
-                    "years_back": years_back,
-                    "filters_applied": {
-                        "property_type": property_type,
-                        "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None,
-                    },
-                },
-                "market_statistics": {
-                    "deal_breakdown": {
-                        "total_deals": stats.total_deals,
-                    },
-                    "price_statistics": stats.price_statistics,
-                    "area_statistics": stats.area_statistics,
-                    "price_per_sqm_statistics": stats.price_per_sqm_statistics,
-                    "property_type_distribution": stats.property_type_distribution,
-                    "date_range": stats.date_range,
-                },
-                "deals": [],  # Statistics-only query, no full deals returned
-            },
-            ensure_ascii=False,
-            indent=2,
         )
 
+        # Build response
+        response_data = {
+            "search_parameters": {
+                "address": address,
+                "years_back": years_back,
+                "filters_applied": {
+                    "property_type": property_type,
+                    "rooms": f"{min_rooms}-{max_rooms}" if min_rooms or max_rooms else None,
+                },
+            },
+            "market_statistics": {
+                "deal_breakdown": {
+                    "total_deals": stats.total_deals,
+                },
+                "price_statistics": stats.price_statistics,
+                "area_statistics": stats.area_statistics,
+                "price_per_sqm_statistics": stats.price_per_sqm_statistics,
+                "property_type_distribution": stats.property_type_distribution,
+                "date_range": stats.date_range,
+            },
+            "deals": [],  # Statistics-only query, no full deals returned
+        }
+
+        # Add outlier deals if present and requested
+        if include_outlier_deals and stats.outlier_report and stats.outlier_report.outlier_deals:
+            response_data["outlier_deals"] = strip_bloat_fields(stats.outlier_report.outlier_deals)
+
+        return json.dumps(response_data, ensure_ascii=False, indent=2)
+
     except Exception as e:
         logger.error(f"Error in get_deal_statistics: {e}", exc_info=True)
         return f"Error calculating deal statistics: {str(e)}"

nadlan_mcp/govmap/client.py

@@ -874,7 +874,10 @@ class GovmapClient:
 
     # Statistics methods (delegate to statistics module)
     def calculate_deal_statistics(
-        self, deals: List[Deal], iqr_multiplier: Optional[float] = None
+        self,
+        deals: List[Deal],
+        iqr_multiplier: Optional[float] = None,
+        include_outlier_deals: bool = True,
     ) -> DealStatistics:
         """
         Calculate statistical aggregations on deal data.
@@ -884,11 +887,14 @@ class GovmapClient:
         Args:
             deals: List of Deal model instances
             iqr_multiplier: Override IQR multiplier for outlier detection (optional)
+            include_outlier_deals: If True (default), include removed outliers in report
 
         Returns:
             DealStatistics model with comprehensive metrics
         """
-        return statistics.calculate_deal_statistics(deals, iqr_multiplier=iqr_multiplier)
+        return statistics.calculate_deal_statistics(
+            deals, iqr_multiplier=iqr_multiplier, include_outlier_deals=include_outlier_deals
+        )
 
     def _calculate_std_dev(self, values: List[float]) -> float:
         """

nadlan_mcp/govmap/models.py

@@ -196,6 +196,7 @@ class OutlierReport(BaseModel):
         outlier_indices: Indices of removed deals in original list
         method_used: Outlier detection method ("iqr", "percent", "none")
         parameters: Configuration parameters used for detection
+        filtered_deals: The actual deals that were filtered out (optional)
     """
 
     total_deals: int = Field(..., description="Total deals before filtering")
@@ -205,6 +206,9 @@ class OutlierReport(BaseModel):
     parameters: Dict[str, Any] = Field(
         default_factory=dict, description="Detection parameters used"
     )
+    outlier_deals: Optional[List["Deal"]] = Field(
+        None, description="Deals that were removed as outliers (if requested)"
+    )
 
 
 class DealStatistics(BaseModel):

nadlan_mcp/govmap/outlier_detection.py

@@ -177,6 +177,7 @@ def filter_deals_for_analysis(
     config: Optional[GovmapConfig] = None,
     metric: str = "price_per_sqm",
     iqr_multiplier: Optional[float] = None,
+    include_outlier_deals: bool = True,
 ) -> Tuple[List[Deal], Dict[str, Any]]:
     """
     Filter deals to remove outliers based on configuration.
@@ -196,11 +197,13 @@ def filter_deals_for_analysis(
         metric: Which metric to apply statistical outlier detection to
                 Options: "price_per_sqm", "deal_amount"
         iqr_multiplier: Override IQR multiplier (optional, uses config value if not provided)
+        include_outlier_deals: If True, include the removed outlier deals in the report
+                              (default: True, allows LLM to see what was filtered out)
 
     Returns:
         Tuple of:
-        - Filtered list of deals
+        - Filtered list of deals (deals that PASSED filtering)
-        - Outlier report dict with details on what was filtered
+        - Outlier report dict with details on what was filtered out (includes outlier_deals if requested)
     """
     if config is None:
         config = get_config()
@@ -333,6 +336,7 @@ def filter_deals_for_analysis(
     # Filter deals
     filtered_deals = [deal for i, deal in enumerate(deals) if not filters_to_remove[i]]
     outlier_indices = [i for i, should_remove in enumerate(filters_to_remove) if should_remove]
+    outlier_deals = [deal for i, deal in enumerate(deals) if filters_to_remove[i]]
 
     # Create outlier report
     report = {
@@ -357,4 +361,8 @@ def filter_deals_for_analysis(
         },
     }
 
+    # Include outlier deals if requested
+    if include_outlier_deals and outlier_deals:
+        report["outlier_deals"] = outlier_deals
+
     return filtered_deals, report

nadlan_mcp/govmap/statistics.py

@@ -152,7 +152,10 @@ def _calculate_basic_stats(deals: List[Deal]) -> Dict:
 
 
 def calculate_deal_statistics(
-    deals: List[Deal], config: Optional[GovmapConfig] = None, iqr_multiplier: Optional[float] = None
+    deals: List[Deal],
+    config: Optional[GovmapConfig] = None,
+    iqr_multiplier: Optional[float] = None,
+    include_outlier_deals: bool = True,
 ) -> DealStatistics:
     """
     Calculate statistical aggregations on deal data with optional outlier filtering.
@@ -165,12 +168,14 @@ def calculate_deal_statistics(
         deals: List of Deal model instances
         config: Configuration object (optional, uses global config if not provided)
         iqr_multiplier: Override IQR multiplier (optional, uses config value if not provided)
+        include_outlier_deals: If True (default), include the removed outlier deals in the outlier report
+                              This allows LLMs to see what was filtered out
 
     Returns:
         DealStatistics model with comprehensive metrics, including:
         - Original statistics (calculated on all deals)
         - Filtered statistics (calculated after outlier removal, if enabled)
-        - Outlier report (details on what was filtered, if enabled)
+        - Outlier report (details on what was filtered out, including outlier_deals if requested)
 
     Raises:
         ValueError: If deals is not a valid list
@@ -204,7 +209,11 @@ def calculate_deal_statistics(
     ):
         # Filter deals for analysis (primarily targeting price_per_sqm outliers)
         filtered_deals, report_dict = filter_deals_for_analysis(
-            deals, config, metric="price_per_sqm", iqr_multiplier=iqr_multiplier
+            deals,
+            config,
+            metric="price_per_sqm",
+            iqr_multiplier=iqr_multiplier,
+            include_outlier_deals=include_outlier_deals,
         )
 
         # Create OutlierReport model