security(polymarket-browse): improve response size limit with dynamic calculation
- Replace fixed 10MB limit with dynamic calculation - get_max_response_size() computes limit based on PAGE_SIZE * multiplier - Uses 10x multiplier (e.g., PAGE_SIZE=50 -> ~500KB * 10 = ~5MB) - Clamped between 10MB minimum and 100MB maximum - Formula: max(PAGE_SIZE * multiplier, 10MB) capped at 100MB
This commit is contained in:
@@ -97,9 +97,23 @@ class FetchResult(TypedDict):
|
||||
PAGE_SIZE = 50
|
||||
MAX_RETRIES = 5
|
||||
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
|
||||
MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit per API response
|
||||
MAX_RESPONSE_SIZE_MULTIPLIER = 10 # Response size limit = PAGE_SIZE * multiplier
|
||||
MAX_RESPONSE_SIZE_MIN = 10 * 1024 * 1024 # 10MB minimum
|
||||
MAX_RESPONSE_SIZE_MAX = 100 * 1024 * 1024 # 100MB maximum for safety
|
||||
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
|
||||
|
||||
|
||||
def get_max_response_size(page_size: int = PAGE_SIZE) -> int:
|
||||
"""
|
||||
Calculate max response size based on expected payload.
|
||||
Uses 10x multiplier: if PAGE_SIZE=50 events, expected ~500KB-5MB,
|
||||
so 10x gives 5MB-50MB. Clamped between 10MB and 100MB.
|
||||
"""
|
||||
multiplier = MAX_RESPONSE_SIZE_MULTIPLIER * page_size * 1024 # rough estimate
|
||||
size = max(multiplier, MAX_RESPONSE_SIZE_MIN)
|
||||
return min(size, MAX_RESPONSE_SIZE_MAX)
|
||||
|
||||
|
||||
GAME_CATEGORIES = {
|
||||
"All Esports": "Esports",
|
||||
"Counter Strike": "Counter Strike",
|
||||
@@ -180,9 +194,10 @@ def fetch_page(
|
||||
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urlopen(req, timeout=10) as r:
|
||||
data = r.read()
|
||||
if len(data) > MAX_RESPONSE_SIZE:
|
||||
max_size = get_max_response_size(PAGE_SIZE)
|
||||
if len(data) > max_size:
|
||||
raise ValueError(
|
||||
f"API response too large: {len(data)} bytes (max {MAX_RESPONSE_SIZE})"
|
||||
f"API response too large: {len(data)} bytes (max {max_size})"
|
||||
)
|
||||
return json.loads(data)
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user