security(polymarket-browse): add token bucket rate limiter for API calls
- Add RateLimiter class (token bucket algorithm) - Thread-safe for use with ThreadPoolExecutor - RATE_LIMIT_CALLS = 10 per RATE_LIMIT_WINDOW = 1 second - _rate_limiter.acquire() called before each API request - Also add MAX_RESPONSE_SIZE check in fetch_page
This commit is contained in:
@@ -10,6 +10,7 @@ import time
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any, Callable, TypedDict
|
||||
@@ -97,8 +98,43 @@ class FetchResult(TypedDict):
|
||||
PAGE_SIZE = 50
|
||||
MAX_RETRIES = 5
|
||||
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
|
||||
MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit per API response
|
||||
RATE_LIMIT_CALLS = 10 # max API calls
|
||||
RATE_LIMIT_WINDOW = 1.0 # per second
|
||||
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Token bucket rate limiter for API calls. Thread-safe for use with ThreadPoolExecutor."""
|
||||
|
||||
def __init__(
|
||||
self, calls: int = RATE_LIMIT_CALLS, window: float = RATE_LIMIT_WINDOW
|
||||
):
|
||||
self.calls = calls
|
||||
self.window = window
|
||||
self.tokens = float(calls)
|
||||
self.last_update = time.monotonic()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def acquire(self) -> None:
|
||||
"""Block until a token is available."""
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self.last_update
|
||||
self.tokens = min(
|
||||
self.calls, self.tokens + elapsed * (self.calls / self.window)
|
||||
)
|
||||
if self.tokens < 1:
|
||||
wait_time = (1 - self.tokens) * (self.window / self.calls)
|
||||
time.sleep(wait_time)
|
||||
self.tokens = 0
|
||||
else:
|
||||
self.tokens -= 1
|
||||
self.last_update = time.monotonic()
|
||||
|
||||
|
||||
_rate_limiter = RateLimiter()
|
||||
|
||||
GAME_CATEGORIES = {
|
||||
"All Esports": "Esports",
|
||||
"Counter Strike": "Counter Strike",
|
||||
@@ -176,9 +212,15 @@ def fetch_page(
|
||||
if attempt > 0:
|
||||
time.sleep(delay)
|
||||
try:
|
||||
_rate_limiter.acquire()
|
||||
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urlopen(req, timeout=10) as r:
|
||||
return json.loads(r.read())
|
||||
data = r.read()
|
||||
if len(data) > MAX_RESPONSE_SIZE:
|
||||
raise ValueError(
|
||||
f"API response too large: {len(data)} bytes (max {MAX_RESPONSE_SIZE})"
|
||||
)
|
||||
return json.loads(data)
|
||||
except Exception:
|
||||
if attempt < max_retries - 1:
|
||||
delay *= 2
|
||||
|
||||
Reference in New Issue
Block a user