security(polymarket-browse): improve response size limit with dynamic calculation
- Replace fixed 10MB limit with dynamic calculation - get_max_response_size() computes limit based on PAGE_SIZE * multiplier - Uses 10x multiplier (e.g., PAGE_SIZE=50 -> ~500KB * 10 = ~5MB) - Clamped between 10MB minimum and 100MB maximum - Formula: max(PAGE_SIZE * multiplier, 10MB) capped at 100MB
This commit is contained in:
@@ -97,9 +97,23 @@ class FetchResult(TypedDict):
|
|||||||
PAGE_SIZE = 50
|
PAGE_SIZE = 50
|
||||||
MAX_RETRIES = 5
|
MAX_RETRIES = 5
|
||||||
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
|
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
|
||||||
MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit per API response
|
MAX_RESPONSE_SIZE_MULTIPLIER = 10 # Response size limit = PAGE_SIZE * multiplier
|
||||||
|
MAX_RESPONSE_SIZE_MIN = 10 * 1024 * 1024 # 10MB minimum
|
||||||
|
MAX_RESPONSE_SIZE_MAX = 100 * 1024 * 1024 # 100MB maximum for safety
|
||||||
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
|
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
|
||||||
|
|
||||||
|
|
||||||
|
def get_max_response_size(page_size: int = PAGE_SIZE) -> int:
|
||||||
|
"""
|
||||||
|
Calculate max response size based on expected payload.
|
||||||
|
Uses 10x multiplier: if PAGE_SIZE=50 events, expected ~500KB-5MB,
|
||||||
|
so 10x gives 5MB-50MB. Clamped between 10MB and 100MB.
|
||||||
|
"""
|
||||||
|
multiplier = MAX_RESPONSE_SIZE_MULTIPLIER * page_size * 1024 # rough estimate
|
||||||
|
size = max(multiplier, MAX_RESPONSE_SIZE_MIN)
|
||||||
|
return min(size, MAX_RESPONSE_SIZE_MAX)
|
||||||
|
|
||||||
|
|
||||||
GAME_CATEGORIES = {
|
GAME_CATEGORIES = {
|
||||||
"All Esports": "Esports",
|
"All Esports": "Esports",
|
||||||
"Counter Strike": "Counter Strike",
|
"Counter Strike": "Counter Strike",
|
||||||
@@ -180,9 +194,10 @@ def fetch_page(
|
|||||||
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||||
with urlopen(req, timeout=10) as r:
|
with urlopen(req, timeout=10) as r:
|
||||||
data = r.read()
|
data = r.read()
|
||||||
if len(data) > MAX_RESPONSE_SIZE:
|
max_size = get_max_response_size(PAGE_SIZE)
|
||||||
|
if len(data) > max_size:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"API response too large: {len(data)} bytes (max {MAX_RESPONSE_SIZE})"
|
f"API response too large: {len(data)} bytes (max {max_size})"
|
||||||
)
|
)
|
||||||
return json.loads(data)
|
return json.loads(data)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
Reference in New Issue
Block a user