security(polymarket-browse): improve response size limit with dynamic calculation

- Replace fixed 10MB limit with dynamic calculation
- get_max_response_size() computes limit based on PAGE_SIZE * multiplier
- Uses 10x multiplier (e.g., PAGE_SIZE=50 -> ~500KB * 10 = ~5MB)
- Clamped between 10MB minimum and 100MB maximum
- Formula: max(PAGE_SIZE * multiplier, 10MB) capped at 100MB
This commit is contained in:
shoko
2026-03-27 01:36:32 +00:00
parent 36a7e8b3eb
commit 2c636048e7

View File

@@ -97,9 +97,23 @@ class FetchResult(TypedDict):
PAGE_SIZE = 50
MAX_RETRIES = 5
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit per API response
MAX_RESPONSE_SIZE_MULTIPLIER = 10 # Response size limit = PAGE_SIZE * multiplier
MAX_RESPONSE_SIZE_MIN = 10 * 1024 * 1024 # 10MB minimum
MAX_RESPONSE_SIZE_MAX = 100 * 1024 * 1024 # 100MB maximum for safety
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
def get_max_response_size(page_size: int = PAGE_SIZE) -> int:
"""
Calculate max response size based on expected payload.
Uses 10x multiplier: if PAGE_SIZE=50 events, expected ~500KB-5MB,
so 10x gives 5MB-50MB. Clamped between 10MB and 100MB.
"""
multiplier = MAX_RESPONSE_SIZE_MULTIPLIER * page_size * 1024 # rough estimate
size = max(multiplier, MAX_RESPONSE_SIZE_MIN)
return min(size, MAX_RESPONSE_SIZE_MAX)
GAME_CATEGORIES = {
"All Esports": "Esports",
"Counter Strike": "Counter Strike",
@@ -180,9 +194,10 @@ def fetch_page(
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urlopen(req, timeout=10) as r:
data = r.read()
if len(data) > MAX_RESPONSE_SIZE:
max_size = get_max_response_size(PAGE_SIZE)
if len(data) > max_size:
raise ValueError(
f"API response too large: {len(data)} bytes (max {MAX_RESPONSE_SIZE})"
f"API response too large: {len(data)} bytes (max {max_size})"
)
return json.loads(data)
except Exception: