security(polymarket-browse): add token bucket rate limiter for API calls

- Add RateLimiter class (token bucket algorithm)
- Thread-safe for use with ThreadPoolExecutor
- RATE_LIMIT_CALLS = 10 per RATE_LIMIT_WINDOW = 1 second
- _rate_limiter.acquire() called before each API request
- Also add MAX_RESPONSE_SIZE check in fetch_page
This commit is contained in:
shoko
2026-03-26 19:15:02 +00:00
parent dfad8d3072
commit c0484ab340

View File

@@ -10,6 +10,7 @@ import time
import argparse import argparse
import hashlib import hashlib
import os import os
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
from typing import Any, Callable, TypedDict from typing import Any, Callable, TypedDict
@@ -97,8 +98,43 @@ class FetchResult(TypedDict):
PAGE_SIZE = 50 PAGE_SIZE = 50
MAX_RETRIES = 5 MAX_RETRIES = 5
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit per API response
RATE_LIMIT_CALLS = 10 # max API calls
RATE_LIMIT_WINDOW = 1.0 # per second
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
class RateLimiter:
"""Token bucket rate limiter for API calls. Thread-safe for use with ThreadPoolExecutor."""
def __init__(
self, calls: int = RATE_LIMIT_CALLS, window: float = RATE_LIMIT_WINDOW
):
self.calls = calls
self.window = window
self.tokens = float(calls)
self.last_update = time.monotonic()
self._lock = threading.Lock()
def acquire(self) -> None:
"""Block until a token is available."""
with self._lock:
now = time.monotonic()
elapsed = now - self.last_update
self.tokens = min(
self.calls, self.tokens + elapsed * (self.calls / self.window)
)
if self.tokens < 1:
wait_time = (1 - self.tokens) * (self.window / self.calls)
time.sleep(wait_time)
self.tokens = 0
else:
self.tokens -= 1
self.last_update = time.monotonic()
_rate_limiter = RateLimiter()
GAME_CATEGORIES = { GAME_CATEGORIES = {
"All Esports": "Esports", "All Esports": "Esports",
"Counter Strike": "Counter Strike", "Counter Strike": "Counter Strike",
@@ -176,9 +212,15 @@ def fetch_page(
if attempt > 0: if attempt > 0:
time.sleep(delay) time.sleep(delay)
try: try:
_rate_limiter.acquire()
req = Request(url, headers={"User-Agent": "Mozilla/5.0"}) req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urlopen(req, timeout=10) as r: with urlopen(req, timeout=10) as r:
return json.loads(r.read()) data = r.read()
if len(data) > MAX_RESPONSE_SIZE:
raise ValueError(
f"API response too large: {len(data)} bytes (max {MAX_RESPONSE_SIZE})"
)
return json.loads(data)
except Exception: except Exception:
if attempt < max_retries - 1: if attempt < max_retries - 1:
delay *= 2 delay *= 2