security(polymarket-browse): add token bucket rate limiter for API calls
- Add RateLimiter class (token bucket algorithm) - Thread-safe for use with ThreadPoolExecutor - RATE_LIMIT_CALLS = 10 per RATE_LIMIT_WINDOW = 1 second - _rate_limiter.acquire() called before each API request - Also add MAX_RESPONSE_SIZE check in fetch_page
This commit is contained in:
@@ -10,6 +10,7 @@ import time
|
|||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timezone, timedelta
|
from datetime import datetime, timezone, timedelta
|
||||||
from typing import Any, Callable, TypedDict
|
from typing import Any, Callable, TypedDict
|
||||||
@@ -97,8 +98,43 @@ class FetchResult(TypedDict):
|
|||||||
PAGE_SIZE = 50
|
PAGE_SIZE = 50
|
||||||
MAX_RETRIES = 5
|
MAX_RETRIES = 5
|
||||||
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
|
INITIAL_RETRY_DELAY = 2 # exponential backoff starts at 2s
|
||||||
|
MAX_RESPONSE_SIZE = 10 * 1024 * 1024 # 10MB limit per API response
|
||||||
|
RATE_LIMIT_CALLS = 10 # max API calls
|
||||||
|
RATE_LIMIT_WINDOW = 1.0 # per second
|
||||||
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
|
WIB = timezone(timedelta(hours=7)) # UTC+7 for Indonesian users
|
||||||
|
|
||||||
|
|
||||||
|
class RateLimiter:
|
||||||
|
"""Token bucket rate limiter for API calls. Thread-safe for use with ThreadPoolExecutor."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, calls: int = RATE_LIMIT_CALLS, window: float = RATE_LIMIT_WINDOW
|
||||||
|
):
|
||||||
|
self.calls = calls
|
||||||
|
self.window = window
|
||||||
|
self.tokens = float(calls)
|
||||||
|
self.last_update = time.monotonic()
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def acquire(self) -> None:
|
||||||
|
"""Block until a token is available."""
|
||||||
|
with self._lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
elapsed = now - self.last_update
|
||||||
|
self.tokens = min(
|
||||||
|
self.calls, self.tokens + elapsed * (self.calls / self.window)
|
||||||
|
)
|
||||||
|
if self.tokens < 1:
|
||||||
|
wait_time = (1 - self.tokens) * (self.window / self.calls)
|
||||||
|
time.sleep(wait_time)
|
||||||
|
self.tokens = 0
|
||||||
|
else:
|
||||||
|
self.tokens -= 1
|
||||||
|
self.last_update = time.monotonic()
|
||||||
|
|
||||||
|
|
||||||
|
_rate_limiter = RateLimiter()
|
||||||
|
|
||||||
GAME_CATEGORIES = {
|
GAME_CATEGORIES = {
|
||||||
"All Esports": "Esports",
|
"All Esports": "Esports",
|
||||||
"Counter Strike": "Counter Strike",
|
"Counter Strike": "Counter Strike",
|
||||||
@@ -176,9 +212,15 @@ def fetch_page(
|
|||||||
if attempt > 0:
|
if attempt > 0:
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
try:
|
try:
|
||||||
|
_rate_limiter.acquire()
|
||||||
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||||
with urlopen(req, timeout=10) as r:
|
with urlopen(req, timeout=10) as r:
|
||||||
return json.loads(r.read())
|
data = r.read()
|
||||||
|
if len(data) > MAX_RESPONSE_SIZE:
|
||||||
|
raise ValueError(
|
||||||
|
f"API response too large: {len(data)} bytes (max {MAX_RESPONSE_SIZE})"
|
||||||
|
)
|
||||||
|
return json.loads(data)
|
||||||
except Exception:
|
except Exception:
|
||||||
if attempt < max_retries - 1:
|
if attempt < max_retries - 1:
|
||||||
delay *= 2
|
delay *= 2
|
||||||
|
|||||||
Reference in New Issue
Block a user