Add statistical refinement and outlier detection system
Implement configurable outlier detection and robust statistical measures to improve analysis accuracy for real estate data. Addresses issues with data entry errors, partial deals, and other anomalies that skew statistics. Key Features: - IQR-based outlier detection (moderate filtering by default, k=1.5) - Hard bounds filtering for obvious errors (price_per_sqm, deal_amount) - Robust volatility using IQR instead of std_dev for investment analysis - Transparent reporting with both filtered and unfiltered statistics Implementation: - Add outlier_detection.py module with IQR/percent/hard bounds methods - Add OutlierReport model and enhance DealStatistics with filtered fields - Update calculate_deal_statistics() to support optional outlier filtering - Update analyze_investment_potential() to use robust volatility - Add 9 new configuration parameters for customization - Add comprehensive test suite (24 tests) for outlier detection - Update CLAUDE.md with usage documentation Configuration (all via env vars): - ANALYSIS_OUTLIER_METHOD=iqr (default, or percent/none) - ANALYSIS_IQR_MULTIPLIER=1.5 (moderate, 3.0=conservative) - ANALYSIS_PRICE_PER_SQM_MIN/MAX=1000/100000 (bounds in NIS/sqm) - ANALYSIS_MIN_DEAL_AMOUNT=100000 (catches partial deals) - ANALYSIS_USE_ROBUST_VOLATILITY=true (IQR-based CV) - ANALYSIS_USE_ROBUST_TRENDS=true (filter before regression) Testing: - All existing tests pass (326 passed) - 24 new comprehensive outlier detection tests - Real-world scenario tests (partial deals, data errors) Backward Compatible: - Default behavior improves accuracy without breaking changes - All new fields in models are optional - Config parameters have sensible defaults 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Nitzan P
@@ -10,7 +10,10 @@ from datetime import date, datetime, timedelta
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from nadlan_mcp.config import GovmapConfig, get_config
|
||||
|
||||
from .models import Deal, InvestmentAnalysis, LiquidityMetrics, MarketActivityScore
|
||||
from .outlier_detection import calculate_iqr, filter_deals_for_analysis
|
||||
from .statistics import calculate_std_dev
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -201,7 +204,9 @@ def calculate_market_activity_score(
|
||||
)
|
||||
|
||||
|
||||
def analyze_investment_potential(deals: List[Deal]) -> InvestmentAnalysis:
|
||||
def analyze_investment_potential(
|
||||
deals: List[Deal], config: Optional[GovmapConfig] = None
|
||||
) -> InvestmentAnalysis:
|
||||
"""
|
||||
Analyze investment potential based on price trends and market stability.
|
||||
|
||||
@@ -209,8 +214,12 @@ def analyze_investment_potential(deals: List[Deal]) -> InvestmentAnalysis:
|
||||
and provides investment metrics for decision-making. The MCP provides
|
||||
data metrics; the LLM interprets them for investment advice.
|
||||
|
||||
With outlier filtering enabled, this function removes statistical outliers
|
||||
before calculating volatility and trends, resulting in more accurate assessments.
|
||||
|
||||
Args:
|
||||
deals: List of Deal model instances with price and date information
|
||||
config: Configuration object (optional, uses global config if not provided)
|
||||
|
||||
Returns:
|
||||
InvestmentAnalysis model containing:
|
||||
@@ -229,9 +238,27 @@ def analyze_investment_potential(deals: List[Deal]) -> InvestmentAnalysis:
|
||||
if not deals:
|
||||
raise ValueError("Cannot analyze investment potential from empty deals list")
|
||||
|
||||
# Extract price per sqm and dates
|
||||
if config is None:
|
||||
config = get_config()
|
||||
|
||||
# Step 1: Apply outlier filtering if configured
|
||||
deals_for_analysis = deals
|
||||
if (
|
||||
config.analysis_use_robust_trends
|
||||
and config.analysis_outlier_method != "none"
|
||||
and len(deals) >= config.analysis_min_deals_for_outlier_detection
|
||||
):
|
||||
filtered_deals, _ = filter_deals_for_analysis(deals, config, metric="price_per_sqm")
|
||||
if filtered_deals and len(filtered_deals) >= 3: # Need at least 3 deals for analysis
|
||||
deals_for_analysis = filtered_deals
|
||||
logger.info(
|
||||
f"Filtered {len(deals) - len(filtered_deals)} outliers "
|
||||
f"from investment analysis ({len(deals)} → {len(filtered_deals)} deals)"
|
||||
)
|
||||
|
||||
# Step 2: Extract price per sqm and dates from deals_for_analysis
|
||||
price_data = []
|
||||
for deal in deals:
|
||||
for deal in deals_for_analysis:
|
||||
price_per_sqm = deal.price_per_sqm # Use computed field from Deal model
|
||||
|
||||
if price_per_sqm and price_per_sqm > 0 and deal.deal_date:
|
||||
@@ -292,12 +319,26 @@ def analyze_investment_potential(deals: List[Deal]) -> InvestmentAnalysis:
|
||||
else:
|
||||
price_trend = "stable"
|
||||
|
||||
# Calculate price volatility (coefficient of variation)
|
||||
std_dev = calculate_std_dev(prices)
|
||||
if avg_price_per_sqm > 0:
|
||||
coefficient_of_variation = (std_dev / avg_price_per_sqm) * 100
|
||||
# Step 3: Calculate price volatility
|
||||
# Use robust volatility (IQR-based) if configured, otherwise use traditional CV
|
||||
if config.analysis_use_robust_volatility:
|
||||
# Robust volatility using IQR (less sensitive to outliers)
|
||||
iqr = calculate_iqr(prices)
|
||||
# Calculate median for robust CV
|
||||
sorted_prices = sorted(prices)
|
||||
median_price = sorted_prices[len(sorted_prices) // 2]
|
||||
if median_price > 0:
|
||||
# Robust coefficient of variation: IQR / median × 100
|
||||
coefficient_of_variation = (iqr / median_price) * 100
|
||||
else:
|
||||
coefficient_of_variation = 0
|
||||
else:
|
||||
coefficient_of_variation = 0
|
||||
# Traditional volatility (coefficient of variation using std_dev)
|
||||
std_dev = calculate_std_dev(prices)
|
||||
if avg_price_per_sqm > 0:
|
||||
coefficient_of_variation = (std_dev / avg_price_per_sqm) * 100
|
||||
else:
|
||||
coefficient_of_variation = 0
|
||||
|
||||
# Convert CV to volatility score (0-100, lower is better)
|
||||
# Using defined volatility thresholds
|
||||
|
||||
@@ -183,6 +183,30 @@ class Deal(BaseModel):
|
||||
raise TypeError(f"Unsupported type for date parsing: {type(v)}")
|
||||
|
||||
|
||||
class OutlierReport(BaseModel):
|
||||
"""
|
||||
Report on outliers detected and removed from analysis.
|
||||
|
||||
Provides transparency about what data was filtered and why,
|
||||
allowing LLMs to understand data quality issues.
|
||||
|
||||
Attributes:
|
||||
total_deals: Total number of deals before filtering
|
||||
outliers_removed: Number of outliers removed
|
||||
outlier_indices: Indices of removed deals in original list
|
||||
method_used: Outlier detection method ("iqr", "percent", "none")
|
||||
parameters: Configuration parameters used for detection
|
||||
"""
|
||||
|
||||
total_deals: int = Field(..., description="Total deals before filtering")
|
||||
outliers_removed: int = Field(..., description="Number of outliers removed")
|
||||
outlier_indices: List[int] = Field(default_factory=list, description="Indices of removed deals")
|
||||
method_used: str = Field(..., description="Detection method (iqr, percent, none)")
|
||||
parameters: Dict[str, Any] = Field(
|
||||
default_factory=dict, description="Detection parameters used"
|
||||
)
|
||||
|
||||
|
||||
class DealStatistics(BaseModel):
|
||||
"""
|
||||
Statistical analysis of real estate deals.
|
||||
@@ -222,6 +246,25 @@ class DealStatistics(BaseModel):
|
||||
# Date range
|
||||
date_range: Optional[Dict[str, str]] = Field(None, description="Earliest and latest deal dates")
|
||||
|
||||
# Outlier detection and filtered statistics
|
||||
outlier_report: Optional[OutlierReport] = Field(
|
||||
None, description="Outlier detection report (if filtering was applied)"
|
||||
)
|
||||
|
||||
# Filtered statistics (after outlier removal)
|
||||
filtered_deal_count: Optional[int] = Field(
|
||||
None, description="Number of deals after outlier removal"
|
||||
)
|
||||
filtered_price_statistics: Optional[Dict[str, float]] = Field(
|
||||
None, description="Price statistics after outlier filtering"
|
||||
)
|
||||
filtered_area_statistics: Optional[Dict[str, float]] = Field(
|
||||
None, description="Area statistics after outlier filtering"
|
||||
)
|
||||
filtered_price_per_sqm_statistics: Optional[Dict[str, float]] = Field(
|
||||
None, description="Price/sqm statistics after outlier filtering"
|
||||
)
|
||||
|
||||
|
||||
class MarketActivityScore(BaseModel):
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
Outlier detection and robust statistical analysis for real estate data.
|
||||
|
||||
This module provides functions to detect and filter outliers from deal data,
|
||||
improving the accuracy of statistical analyses and market assessments.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from nadlan_mcp.config import GovmapConfig, get_config
|
||||
from nadlan_mcp.govmap.models import Deal
|
||||
|
||||
|
||||
def calculate_iqr(values: List[float]) -> float:
|
||||
"""
|
||||
Calculate the Interquartile Range (IQR) of a dataset.
|
||||
|
||||
Args:
|
||||
values: List of numeric values
|
||||
|
||||
Returns:
|
||||
IQR (Q3 - Q1)
|
||||
"""
|
||||
if not values:
|
||||
return 0.0
|
||||
|
||||
sorted_values = sorted(values)
|
||||
n = len(sorted_values)
|
||||
|
||||
# Calculate Q1 (25th percentile)
|
||||
q1_index = n // 4
|
||||
q1 = sorted_values[q1_index]
|
||||
|
||||
# Calculate Q3 (75th percentile)
|
||||
q3_index = (3 * n) // 4
|
||||
q3 = sorted_values[q3_index]
|
||||
|
||||
return q3 - q1
|
||||
|
||||
|
||||
def detect_outliers_iqr(values: List[float], multiplier: float = 1.5) -> List[bool]:
|
||||
"""
|
||||
Detect outliers using the IQR (Interquartile Range) method.
|
||||
|
||||
This is the most robust method for real estate data, as it doesn't assume
|
||||
a normal distribution and handles skewed data well.
|
||||
|
||||
Outliers are defined as values outside [Q1 - k*IQR, Q3 + k*IQR] where:
|
||||
- k=1.5 (mild outliers, more aggressive filtering)
|
||||
- k=3.0 (extreme outliers, conservative filtering)
|
||||
|
||||
Args:
|
||||
values: List of numeric values to check
|
||||
multiplier: IQR multiplier (default 1.5)
|
||||
|
||||
Returns:
|
||||
List of booleans, True if value is an outlier
|
||||
"""
|
||||
if not values or len(values) < 4:
|
||||
return [False] * len(values)
|
||||
|
||||
sorted_values = sorted(values)
|
||||
n = len(sorted_values)
|
||||
|
||||
# Calculate Q1 and Q3
|
||||
q1_index = n // 4
|
||||
q1 = sorted_values[q1_index]
|
||||
|
||||
q3_index = (3 * n) // 4
|
||||
q3 = sorted_values[q3_index]
|
||||
|
||||
iqr = q3 - q1
|
||||
|
||||
# Calculate bounds
|
||||
lower_bound = q1 - (multiplier * iqr)
|
||||
upper_bound = q3 + (multiplier * iqr)
|
||||
|
||||
# Mark outliers
|
||||
return [value < lower_bound or value > upper_bound for value in values]
|
||||
|
||||
|
||||
def detect_outliers_percent(values: List[float], threshold: float = 0.5) -> List[bool]:
|
||||
"""
|
||||
Detect outliers using percentage-based thresholds from the median.
|
||||
|
||||
Outliers are defined as values outside [median * (1-threshold), median * (1+threshold)]
|
||||
|
||||
Example: threshold=0.5 means values >150% or <50% of median are outliers
|
||||
|
||||
Args:
|
||||
values: List of numeric values to check
|
||||
threshold: Percentage threshold (0.5 = 50%)
|
||||
|
||||
Returns:
|
||||
List of booleans, True if value is an outlier
|
||||
"""
|
||||
if not values:
|
||||
return []
|
||||
|
||||
sorted_values = sorted(values)
|
||||
n = len(sorted_values)
|
||||
median = sorted_values[n // 2]
|
||||
|
||||
lower_bound = median * (1 - threshold)
|
||||
upper_bound = median * (1 + threshold)
|
||||
|
||||
return [value < lower_bound or value > upper_bound for value in values]
|
||||
|
||||
|
||||
def apply_hard_bounds_price_per_sqm(
|
||||
deals: List[Deal], config: Optional[GovmapConfig] = None
|
||||
) -> List[bool]:
|
||||
"""
|
||||
Apply hard bounds filtering to price per sqm values.
|
||||
|
||||
This catches obvious data entry errors (e.g., wrong area leading to
|
||||
extreme price per sqm values).
|
||||
|
||||
Args:
|
||||
deals: List of deals to check
|
||||
config: Configuration object (optional, uses global if not provided)
|
||||
|
||||
Returns:
|
||||
List of booleans, True if deal should be filtered out
|
||||
"""
|
||||
if config is None:
|
||||
config = get_config()
|
||||
|
||||
filters = []
|
||||
for deal in deals:
|
||||
price_per_sqm = deal.price_per_sqm
|
||||
|
||||
# Skip deals without valid price per sqm
|
||||
if price_per_sqm is None:
|
||||
filters.append(False)
|
||||
continue
|
||||
|
||||
# Check bounds
|
||||
is_outlier = (
|
||||
price_per_sqm < config.analysis_price_per_sqm_min
|
||||
or price_per_sqm > config.analysis_price_per_sqm_max
|
||||
)
|
||||
filters.append(is_outlier)
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def apply_hard_bounds_deal_amount(
|
||||
deals: List[Deal], config: Optional[GovmapConfig] = None
|
||||
) -> List[bool]:
|
||||
"""
|
||||
Apply hard bounds filtering to deal amounts.
|
||||
|
||||
This catches partial deals and obvious data errors (e.g., 400K apartment
|
||||
when typical apartments are 1.6M).
|
||||
|
||||
Args:
|
||||
deals: List of deals to check
|
||||
config: Configuration object (optional, uses global if not provided)
|
||||
|
||||
Returns:
|
||||
List of booleans, True if deal should be filtered out
|
||||
"""
|
||||
if config is None:
|
||||
config = get_config()
|
||||
|
||||
filters = []
|
||||
for deal in deals:
|
||||
is_outlier = deal.deal_amount < config.analysis_min_deal_amount
|
||||
filters.append(is_outlier)
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def filter_deals_for_analysis(
|
||||
deals: List[Deal], config: Optional[GovmapConfig] = None, metric: str = "price_per_sqm"
|
||||
) -> Tuple[List[Deal], Dict[str, Any]]:
|
||||
"""
|
||||
Filter deals to remove outliers based on configuration.
|
||||
|
||||
This is the main entry point for outlier filtering. It applies a combination
|
||||
of hard bounds and statistical outlier detection.
|
||||
|
||||
Process:
|
||||
1. Apply hard bounds to price per sqm (catches data errors)
|
||||
2. Apply hard bounds to deal amount (catches partial deals)
|
||||
3. Apply statistical outlier detection (IQR or percent method) to specified metric
|
||||
4. Combine all filters and remove outliers
|
||||
|
||||
Args:
|
||||
deals: List of deals to filter
|
||||
config: Configuration object (optional, uses global if not provided)
|
||||
metric: Which metric to apply statistical outlier detection to
|
||||
Options: "price_per_sqm", "deal_amount"
|
||||
|
||||
Returns:
|
||||
Tuple of:
|
||||
- Filtered list of deals
|
||||
- Outlier report dict with details on what was filtered
|
||||
"""
|
||||
if config is None:
|
||||
config = get_config()
|
||||
|
||||
if not deals:
|
||||
return [], {
|
||||
"total_deals": 0,
|
||||
"outliers_removed": 0,
|
||||
"outlier_indices": [],
|
||||
"method_used": config.analysis_outlier_method,
|
||||
"parameters": {},
|
||||
}
|
||||
|
||||
# Skip outlier detection if disabled or insufficient data
|
||||
if (
|
||||
config.analysis_outlier_method == "none"
|
||||
or len(deals) < config.analysis_min_deals_for_outlier_detection
|
||||
):
|
||||
return deals, {
|
||||
"total_deals": len(deals),
|
||||
"outliers_removed": 0,
|
||||
"outlier_indices": [],
|
||||
"method_used": "none",
|
||||
"parameters": {"reason": "disabled or insufficient data"},
|
||||
}
|
||||
|
||||
# Initialize filter masks (True = keep, False = remove)
|
||||
filters_to_remove = [False] * len(deals)
|
||||
|
||||
# Step 1: Apply hard bounds to price per sqm
|
||||
price_per_sqm_outliers = apply_hard_bounds_price_per_sqm(deals, config)
|
||||
for i, is_outlier in enumerate(price_per_sqm_outliers):
|
||||
if is_outlier:
|
||||
filters_to_remove[i] = True
|
||||
|
||||
# Step 2: Apply hard bounds to deal amount
|
||||
deal_amount_outliers = apply_hard_bounds_deal_amount(deals, config)
|
||||
for i, is_outlier in enumerate(deal_amount_outliers):
|
||||
if is_outlier:
|
||||
filters_to_remove[i] = True
|
||||
|
||||
# Step 3: Apply statistical outlier detection to specified metric
|
||||
if config.analysis_outlier_method == "iqr":
|
||||
# Extract values for the specified metric
|
||||
if metric == "price_per_sqm":
|
||||
values = [
|
||||
deal.price_per_sqm
|
||||
for deal in deals
|
||||
if deal.price_per_sqm is not None and not filters_to_remove[i]
|
||||
]
|
||||
value_indices = [
|
||||
i
|
||||
for i, deal in enumerate(deals)
|
||||
if deal.price_per_sqm is not None and not filters_to_remove[i]
|
||||
]
|
||||
elif metric == "deal_amount":
|
||||
values = [deal.deal_amount for i, deal in enumerate(deals) if not filters_to_remove[i]]
|
||||
value_indices = [i for i in range(len(deals)) if not filters_to_remove[i]]
|
||||
else:
|
||||
values = []
|
||||
value_indices = []
|
||||
|
||||
if values:
|
||||
statistical_outliers = detect_outliers_iqr(values, config.analysis_iqr_multiplier)
|
||||
for i, is_outlier in enumerate(statistical_outliers):
|
||||
if is_outlier:
|
||||
filters_to_remove[value_indices[i]] = True
|
||||
|
||||
elif config.analysis_outlier_method == "percent":
|
||||
# Extract values for the specified metric
|
||||
if metric == "price_per_sqm":
|
||||
values = [
|
||||
deal.price_per_sqm
|
||||
for i, deal in enumerate(deals)
|
||||
if deal.price_per_sqm is not None and not filters_to_remove[i]
|
||||
]
|
||||
value_indices = [
|
||||
i
|
||||
for i, deal in enumerate(deals)
|
||||
if deal.price_per_sqm is not None and not filters_to_remove[i]
|
||||
]
|
||||
elif metric == "deal_amount":
|
||||
values = [deal.deal_amount for i, deal in enumerate(deals) if not filters_to_remove[i]]
|
||||
value_indices = [i for i in range(len(deals)) if not filters_to_remove[i]]
|
||||
else:
|
||||
values = []
|
||||
value_indices = []
|
||||
|
||||
if values:
|
||||
statistical_outliers = detect_outliers_percent(values, 0.5)
|
||||
for i, is_outlier in enumerate(statistical_outliers):
|
||||
if is_outlier:
|
||||
filters_to_remove[value_indices[i]] = True
|
||||
|
||||
# Filter deals
|
||||
filtered_deals = [deal for i, deal in enumerate(deals) if not filters_to_remove[i]]
|
||||
outlier_indices = [i for i, should_remove in enumerate(filters_to_remove) if should_remove]
|
||||
|
||||
# Create outlier report
|
||||
report = {
|
||||
"total_deals": len(deals),
|
||||
"outliers_removed": len(outlier_indices),
|
||||
"outlier_indices": outlier_indices,
|
||||
"method_used": config.analysis_outlier_method,
|
||||
"parameters": {
|
||||
"iqr_multiplier": config.analysis_iqr_multiplier
|
||||
if config.analysis_outlier_method == "iqr"
|
||||
else None,
|
||||
"metric": metric,
|
||||
"price_per_sqm_min": config.analysis_price_per_sqm_min,
|
||||
"price_per_sqm_max": config.analysis_price_per_sqm_max,
|
||||
"min_deal_amount": config.analysis_min_deal_amount,
|
||||
},
|
||||
}
|
||||
|
||||
return filtered_deals, report
|
||||
@@ -6,39 +6,27 @@ This module provides pure mathematical functions for analyzing real estate deal
|
||||
|
||||
from collections import Counter
|
||||
import logging
|
||||
from typing import List
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .models import Deal, DealStatistics
|
||||
from nadlan_mcp.config import GovmapConfig, get_config
|
||||
|
||||
from .models import Deal, DealStatistics, OutlierReport
|
||||
from .outlier_detection import filter_deals_for_analysis
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_deal_statistics(deals: List[Deal]) -> DealStatistics:
|
||||
def _calculate_basic_stats(deals: List[Deal]) -> Dict:
|
||||
"""
|
||||
Calculate statistical aggregations on deal data.
|
||||
Internal helper to calculate basic statistics from a deal list.
|
||||
|
||||
Args:
|
||||
deals: List of Deal model instances
|
||||
|
||||
Returns:
|
||||
DealStatistics model with comprehensive metrics
|
||||
|
||||
Raises:
|
||||
ValueError: If deals is not a valid list
|
||||
Dictionary with price_stats, area_stats, price_per_sqm_stats,
|
||||
property_type_dist, and date_range
|
||||
"""
|
||||
if not isinstance(deals, list):
|
||||
raise ValueError("deals must be a list")
|
||||
|
||||
if not deals:
|
||||
return DealStatistics(
|
||||
total_deals=0,
|
||||
price_statistics={},
|
||||
area_statistics={},
|
||||
price_per_sqm_statistics={},
|
||||
property_type_distribution={},
|
||||
date_range=None,
|
||||
)
|
||||
|
||||
# Extract numeric values
|
||||
prices = []
|
||||
areas = []
|
||||
@@ -154,13 +142,91 @@ def calculate_deal_statistics(deals: List[Deal]) -> DealStatistics:
|
||||
logger.warning("Invalid date format in date range calculation")
|
||||
pass
|
||||
|
||||
return {
|
||||
"price_statistics": price_stats,
|
||||
"area_statistics": area_stats,
|
||||
"price_per_sqm_statistics": price_per_sqm_stats,
|
||||
"property_type_distribution": property_type_dist,
|
||||
"date_range": date_range_dict,
|
||||
}
|
||||
|
||||
|
||||
def calculate_deal_statistics(
|
||||
deals: List[Deal], config: Optional[GovmapConfig] = None
|
||||
) -> DealStatistics:
|
||||
"""
|
||||
Calculate statistical aggregations on deal data with optional outlier filtering.
|
||||
|
||||
This function calculates comprehensive statistics on real estate deals, optionally
|
||||
filtering outliers based on configuration. When outlier filtering is enabled, it
|
||||
returns both original (unfiltered) and filtered statistics for transparency.
|
||||
|
||||
Args:
|
||||
deals: List of Deal model instances
|
||||
config: Configuration object (optional, uses global config if not provided)
|
||||
|
||||
Returns:
|
||||
DealStatistics model with comprehensive metrics, including:
|
||||
- Original statistics (calculated on all deals)
|
||||
- Filtered statistics (calculated after outlier removal, if enabled)
|
||||
- Outlier report (details on what was filtered, if enabled)
|
||||
|
||||
Raises:
|
||||
ValueError: If deals is not a valid list
|
||||
"""
|
||||
if not isinstance(deals, list):
|
||||
raise ValueError("deals must be a list")
|
||||
|
||||
if config is None:
|
||||
config = get_config()
|
||||
|
||||
if not deals:
|
||||
return DealStatistics(
|
||||
total_deals=0,
|
||||
price_statistics={},
|
||||
area_statistics={},
|
||||
price_per_sqm_statistics={},
|
||||
property_type_distribution={},
|
||||
date_range=None,
|
||||
)
|
||||
|
||||
# Step 1: Calculate statistics on original data
|
||||
original_stats = _calculate_basic_stats(deals)
|
||||
|
||||
# Step 2: Apply outlier filtering if enabled
|
||||
outlier_report_data = None
|
||||
filtered_stats = None
|
||||
|
||||
if (
|
||||
config.analysis_outlier_method != "none"
|
||||
and len(deals) >= config.analysis_min_deals_for_outlier_detection
|
||||
):
|
||||
# Filter deals for analysis (primarily targeting price_per_sqm outliers)
|
||||
filtered_deals, report_dict = filter_deals_for_analysis(
|
||||
deals, config, metric="price_per_sqm"
|
||||
)
|
||||
|
||||
# Create OutlierReport model
|
||||
outlier_report_data = OutlierReport(**report_dict)
|
||||
|
||||
# Calculate statistics on filtered data
|
||||
if filtered_deals and len(filtered_deals) > 0:
|
||||
filtered_basic_stats = _calculate_basic_stats(filtered_deals)
|
||||
filtered_stats = {
|
||||
"filtered_deal_count": len(filtered_deals),
|
||||
"filtered_price_statistics": filtered_basic_stats["price_statistics"],
|
||||
"filtered_area_statistics": filtered_basic_stats["area_statistics"],
|
||||
"filtered_price_per_sqm_statistics": filtered_basic_stats[
|
||||
"price_per_sqm_statistics"
|
||||
],
|
||||
}
|
||||
|
||||
# Step 3: Return comprehensive DealStatistics with both original and filtered data
|
||||
return DealStatistics(
|
||||
total_deals=len(deals),
|
||||
price_statistics=price_stats,
|
||||
area_statistics=area_stats,
|
||||
price_per_sqm_statistics=price_per_sqm_stats,
|
||||
property_type_distribution=property_type_dist,
|
||||
date_range=date_range_dict,
|
||||
**original_stats,
|
||||
outlier_report=outlier_report_data,
|
||||
**(filtered_stats if filtered_stats else {}),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user