Fix too many polygons issue
This commit is contained in:
@@ -366,6 +366,9 @@ GOVMAP_REQUESTS_PER_SECOND=5.0
|
|||||||
GOVMAP_DEFAULT_RADIUS=50
|
GOVMAP_DEFAULT_RADIUS=50
|
||||||
GOVMAP_DEFAULT_YEARS_BACK=2
|
GOVMAP_DEFAULT_YEARS_BACK=2
|
||||||
GOVMAP_DEFAULT_DEAL_LIMIT=100
|
GOVMAP_DEFAULT_DEAL_LIMIT=100
|
||||||
|
|
||||||
|
# Performance
|
||||||
|
GOVMAP_MAX_POLYGONS=10 # Max polygons to query per search (limits API calls)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Tuning Guidelines
|
### Tuning Guidelines
|
||||||
@@ -379,6 +382,7 @@ GOVMAP_DEFAULT_DEAL_LIMIT=100
|
|||||||
- `MAX_RETRIES=2`
|
- `MAX_RETRIES=2`
|
||||||
- `RETRY_MAX_WAIT=5`
|
- `RETRY_MAX_WAIT=5`
|
||||||
- `REQUESTS_PER_SECOND=10.0`
|
- `REQUESTS_PER_SECOND=10.0`
|
||||||
|
- `MAX_POLYGONS=5` (faster, fewer results)
|
||||||
|
|
||||||
**Conservative (Shared API):**
|
**Conservative (Shared API):**
|
||||||
- `MAX_RETRIES=3`
|
- `MAX_RETRIES=3`
|
||||||
|
|||||||
@@ -272,6 +272,9 @@ GOVMAP_REQUESTS_PER_SECOND=5.0
|
|||||||
GOVMAP_DEFAULT_RADIUS=50
|
GOVMAP_DEFAULT_RADIUS=50
|
||||||
GOVMAP_DEFAULT_YEARS_BACK=2
|
GOVMAP_DEFAULT_YEARS_BACK=2
|
||||||
GOVMAP_DEFAULT_DEAL_LIMIT=100
|
GOVMAP_DEFAULT_DEAL_LIMIT=100
|
||||||
|
|
||||||
|
# Performance
|
||||||
|
GOVMAP_MAX_POLYGONS=10 # Limit polygons per search (reduces API calls, improves speed)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Development Roadmap
|
## Development Roadmap
|
||||||
|
|||||||
@@ -58,6 +58,11 @@ class GovmapConfig:
|
|||||||
default_factory=lambda: int(os.getenv("GOVMAP_DEFAULT_DEAL_LIMIT", "100"))
|
default_factory=lambda: int(os.getenv("GOVMAP_DEFAULT_DEAL_LIMIT", "100"))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Performance optimization
|
||||||
|
max_polygons_to_query: int = field(
|
||||||
|
default_factory=lambda: int(os.getenv("GOVMAP_MAX_POLYGONS", "10"))
|
||||||
|
)
|
||||||
|
|
||||||
# User agent
|
# User agent
|
||||||
user_agent: str = field(
|
user_agent: str = field(
|
||||||
default_factory=lambda: os.getenv(
|
default_factory=lambda: os.getenv(
|
||||||
@@ -90,6 +95,8 @@ class GovmapConfig:
|
|||||||
raise ValueError("default_years_back must be positive")
|
raise ValueError("default_years_back must be positive")
|
||||||
if self.default_deal_limit <= 0:
|
if self.default_deal_limit <= 0:
|
||||||
raise ValueError("default_deal_limit must be positive")
|
raise ValueError("default_deal_limit must be positive")
|
||||||
|
if self.max_polygons_to_query <= 0:
|
||||||
|
raise ValueError("max_polygons_to_query must be positive")
|
||||||
if not self.base_url:
|
if not self.base_url:
|
||||||
raise ValueError("base_url cannot be empty")
|
raise ValueError("base_url cannot be empty")
|
||||||
if not self.user_agent:
|
if not self.user_agent:
|
||||||
|
|||||||
@@ -604,6 +604,12 @@ class GovmapClient:
|
|||||||
|
|
||||||
logger.info(f"Found {len(polygon_ids)} unique polygon IDs")
|
logger.info(f"Found {len(polygon_ids)} unique polygon IDs")
|
||||||
|
|
||||||
|
# Limit polygons to query (performance optimization)
|
||||||
|
max_polygons = self.config.max_polygons_to_query
|
||||||
|
if len(polygon_ids) > max_polygons:
|
||||||
|
polygon_ids = list(polygon_ids)[:max_polygons]
|
||||||
|
logger.info(f"Limited to {max_polygons} polygons for performance")
|
||||||
|
|
||||||
# Step 3: Calculate date range
|
# Step 3: Calculate date range
|
||||||
end_date = datetime.now()
|
end_date = datetime.now()
|
||||||
start_date = end_date - timedelta(days=years_back * 365)
|
start_date = end_date - timedelta(days=years_back * 365)
|
||||||
@@ -618,6 +624,12 @@ class GovmapClient:
|
|||||||
seen_deals = set() # For deduplication
|
seen_deals = set() # For deduplication
|
||||||
|
|
||||||
for polygon_id in polygon_ids:
|
for polygon_id in polygon_ids:
|
||||||
|
# Early termination if we have enough deals
|
||||||
|
total_collected = len(building_deals) + len(street_deals) + len(neighborhood_deals)
|
||||||
|
if total_collected >= max_deals:
|
||||||
|
logger.info(f"Collected {total_collected} deals, stopping polygon queries")
|
||||||
|
break
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Get street deals first (higher priority)
|
# Get street deals first (higher priority)
|
||||||
current_street_deals = self.get_street_deals(
|
current_street_deals = self.get_street_deals(
|
||||||
@@ -628,14 +640,17 @@ class GovmapClient:
|
|||||||
deal_type=deal_type,
|
deal_type=deal_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Get neighborhood deals (lower priority)
|
# Get neighborhood deals (lower priority) - optional for performance
|
||||||
current_neighborhood_deals = self.get_neighborhood_deals(
|
current_neighborhood_deals = []
|
||||||
polygon_id,
|
# Skip neighborhood deals if we have enough street deals
|
||||||
limit=max_deals // 4, # Allocate less to neighborhood deals
|
if len(street_deals) < max_deals // 2:
|
||||||
start_date=start_date_str,
|
current_neighborhood_deals = self.get_neighborhood_deals(
|
||||||
end_date=end_date_str,
|
polygon_id,
|
||||||
deal_type=deal_type,
|
limit=max_deals // 4, # Allocate less to neighborhood deals
|
||||||
)
|
start_date=start_date_str,
|
||||||
|
end_date=end_date_str,
|
||||||
|
deal_type=deal_type,
|
||||||
|
)
|
||||||
|
|
||||||
# Process street deals and separate building deals
|
# Process street deals and separate building deals
|
||||||
for deal in current_street_deals:
|
for deal in current_street_deals:
|
||||||
|
|||||||
@@ -55,10 +55,11 @@ def validate_coordinates(point: Tuple[float, float]) -> Tuple[float, float]:
|
|||||||
raise ValueError("Coordinates must be numeric values")
|
raise ValueError("Coordinates must be numeric values")
|
||||||
|
|
||||||
# Basic validation for Israeli coordinates (ITM projection)
|
# Basic validation for Israeli coordinates (ITM projection)
|
||||||
if not (0 < lon < 400000): # Rough bounds for Israeli ITM longitude
|
# ITM bounds for Israel: X (longitude) ~150,000-300,000, Y (latitude) ~3,500,000-4,000,000
|
||||||
logger.warning(f"Longitude {lon} may be outside Israeli bounds")
|
if not (150000 <= lon <= 300000): # ITM longitude bounds for Israel
|
||||||
if not (0 < lat < 1400000): # Rough bounds for Israeli ITM latitude
|
logger.warning(f"Longitude {lon} appears to be outside Israeli ITM bounds (150,000-300,000)")
|
||||||
logger.warning(f"Latitude {lat} may be outside Israeli bounds")
|
if not (3500000 <= lat <= 4000000): # ITM latitude bounds for Israel
|
||||||
|
logger.warning(f"Latitude {lat} appears to be outside Israeli ITM bounds (3,500,000-4,000,000)")
|
||||||
|
|
||||||
return (lon, lat)
|
return (lon, lat)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user