From eafbdba4a599ba0cdfe0b0b98daa218dc46f1ad5 Mon Sep 17 00:00:00 2001 From: shoko <270575765+shokollm@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:29:25 +0000 Subject: [PATCH 1/6] Add parallel fetching, caching, and max_total parameter - Parallel page fetching with ThreadPoolExecutor (concurrency=5) - File-based cache with 5 min TTL in ~/.cache/polymarket-browse/ - New --no-cache flag to bypass cache - New --max-total parameter for early exit - Updated tests to work with new implementation --- skills/polymarket-browse/scripts/browse.py | 167 +++- skills/polymarket-browse/tests/test_browse.py | 737 ++++++++++++------ 2 files changed, 638 insertions(+), 266 deletions(-) diff --git a/skills/polymarket-browse/scripts/browse.py b/skills/polymarket-browse/scripts/browse.py index ae4bef5..3445a91 100644 --- a/skills/polymarket-browse/scripts/browse.py +++ b/skills/polymarket-browse/scripts/browse.py @@ -8,6 +8,9 @@ import html import json import time import argparse +import hashlib +import os +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone, timedelta from typing import Any, Callable, TypedDict from urllib.parse import urlencode @@ -108,6 +111,48 @@ GAME_CATEGORIES = { "Tennis": "Tennis", } +CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "polymarket-browse") +CACHE_TTL = 300 # 5 minutes default +MAX_PARALLEL_FETCHES = 5 + +# ============================================================ +# CACHE +# ============================================================ + + +def _get_cache_key(q: str) -> str: + return hashlib.md5(q.encode()).hexdigest() + + +def _get_cache_path(q: str) -> str: + os.makedirs(CACHE_DIR, exist_ok=True) + return os.path.join(CACHE_DIR, f"{_get_cache_key(q)}.json") + + +def _read_cache(q: str) -> dict[str, Any] | None: + cache_path = _get_cache_path(q) + if not os.path.exists(cache_path): + return None + try: + mtime = os.path.getmtime(cache_path) + age = time.time() - mtime + if age > CACHE_TTL: + return None + with open(cache_path) as f: + return json.load(f) + except Exception: + return None + + +def _write_cache(q: str, data: dict[str, Any]) -> None: + try: + cache_path = _get_cache_path(q) + with open(cache_path, "w") as f: + json.dump(data, f) + except Exception: + pass + + # ============================================================ # FETCH # ============================================================ @@ -142,8 +187,16 @@ def fetch_page( return None +def _fetch_page_with_index(q: str, page: int) -> tuple[int, dict[str, Any] | None]: + return page, fetch_page(q, page) + + def fetch_all_pages( - q: str, matches_max: int | None = None, non_matches_max: int | None = None + q: str, + matches_max: int | None = None, + non_matches_max: int | None = None, + max_total: int | None = None, + use_cache: bool = True, ) -> FetchResult: """ Fetch pages until pagination ends, or until quotas are satisfied. @@ -152,46 +205,85 @@ def fetch_all_pages( q: search query matches_max: stop early once we have this many match events (None = no limit) non_matches_max: stop early once we have this many non-match events (None = no limit) + max_total: stop early once we have this many total events (None = no limit) + use_cache: whether to use cache (default True) Returns: FetchResult with events, total_raw, and partial flag """ - all_events = [] + cached = _read_cache(q) if use_cache else None + if cached is not None: + events = cached.get("events", []) + total_raw = cached.get("total_raw", 0) + if events: + return {"events": events, "total_raw": total_raw, "partial": False} + total_raw = 0 - match_count = 0 - non_match_count = 0 - page = 0 + page_count = 0 + while True: - page += 1 - time.sleep(0.2) - data = fetch_page(q, page) + page_count += 1 + data = fetch_page(q, page_count) if data is None: break - events = data.get("events", []) total_raw = data.get("pagination", {}).get("totalResults", 0) - all_events.extend(events) + if total_raw > 0: + break + if not data.get("events"): + break - # Count matches/non-matches in this page - for e in events: - if is_match_market(e): - match_count += 1 - else: - non_match_count += 1 + if total_raw == 0: + return {"events": [], "total_raw": 0, "partial": False} + + total_pages = (total_raw + PAGE_SIZE - 1) // PAGE_SIZE + concurrency = min(MAX_PARALLEL_FETCHES, total_pages) + + all_page_data: dict[int, list[Any]] = {} + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = { + executor.submit(_fetch_page_with_index, q, page): page + for page in range(1, total_pages + 1) + } + for future in as_completed(futures): + try: + page_num, data = future.result() + if data is not None: + all_page_data[page_num] = data.get("events", []) + except Exception: + pass + + all_events = [] + for page_num in sorted(all_page_data.keys()): + all_events.extend(all_page_data[page_num]) + + _write_cache(q, {"events": all_events, "total_raw": total_raw}) + + match_count = 0 + non_match_count = 0 + filtered_events = [] + total_seen = 0 + + for e in all_events: + is_match = is_match_market(e) + if is_match: + match_count += 1 + else: + non_match_count += 1 + + filtered_events.append(e) - # Stop if we got what we wanted (only when caps are set) if matches_max is not None and non_matches_max is not None: if match_count >= matches_max and non_match_count >= non_matches_max: break - # Stop when we get 0 events (no more pages) - if len(events) == 0: - break - # Stop when we've fetched all known results - if len(all_events) >= total_raw: - break + if max_total is not None: + total_seen += 1 + if total_seen >= max_total: + break - partial = total_raw > 0 and len(all_events) < total_raw - return {"events": all_events, "total_raw": total_raw, "partial": partial} + partial = len(all_events) < total_raw + return {"events": filtered_events, "total_raw": total_raw, "partial": partial} # ============================================================ @@ -449,6 +541,8 @@ def browse_events( non_matches_max: int = 10, tradeable_only: bool = True, sort_by: str | None = None, + max_total: int | None = None, + use_cache: bool = True, ) -> BrowseResult: """ Browse Polymarket events. @@ -459,15 +553,19 @@ def browse_events( non_matches_max: max number of non-match markets to return tradeable_only: filter to tradeable events only sort_by: None (fast, API order) or "volume" (full fetch, sort by volume desc) + max_total: max total events to fetch before early exit (None = no limit) + use_cache: whether to use cache (default True) """ - # Pass quotas to fetch_all_pages for early-exit optimization. - # Only use early-exit when sort_by is None (no client-side sort needed). use_early_exit = sort_by is None fetch_matches_max = matches_max if use_early_exit else None fetch_non_matches_max = non_matches_max if use_early_exit else None result = fetch_all_pages( - q, matches_max=fetch_matches_max, non_matches_max=fetch_non_matches_max + q, + matches_max=fetch_matches_max, + non_matches_max=fetch_non_matches_max, + max_total=max_total, + use_cache=use_cache, ) events = result["events"] match_events, non_match_events = filter_events(events, tradeable_only) @@ -1056,6 +1154,17 @@ def main() -> None: action="store_true", help="Show all events without tradeable filter (for debugging).", ) + parser.add_argument( + "--no-cache", + action="store_true", + help="Disable cache and fetch fresh data from API.", + ) + parser.add_argument( + "--max-total", + type=int, + default=None, + help="Max total events to fetch before early exit. Default: no limit.", + ) parser.add_argument( "--telegram", action="store_true", @@ -1085,6 +1194,8 @@ def main() -> None: matches_max=matches_max, non_matches_max=non_matches_max, tradeable_only=tradeable_only, + max_total=args.max_total, + use_cache=not args.no_cache, ) print_browse( diff --git a/skills/polymarket-browse/tests/test_browse.py b/skills/polymarket-browse/tests/test_browse.py index 79c1bf6..ede5f83 100644 --- a/skills/polymarket-browse/tests/test_browse.py +++ b/skills/polymarket-browse/tests/test_browse.py @@ -10,14 +10,14 @@ import sys import os from datetime import datetime, timezone, timedelta -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) from browse import send_telegram_message class TestSendTelegramMessage(unittest.TestCase): """Tests for the module-level send_telegram_message function.""" - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_success(self, mock_urlopen): """send_telegram_message returns message_id on success.""" mock_resp = MagicMock() @@ -30,10 +30,12 @@ class TestSendTelegramMessage(unittest.TestCase): mock_urlopen.assert_called_once() call_args = mock_urlopen.call_args req = call_args[0][0] - self.assertEqual(req.full_url, "https://api.telegram.org/bottest_token/sendMessage") + self.assertEqual( + req.full_url, "https://api.telegram.org/bottest_token/sendMessage" + ) self.assertEqual(req.method, "POST") - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_api_error_raises_runtime_error(self, mock_urlopen): """send_telegram_message raises RuntimeError when Telegram API returns ok=false.""" mock_resp = MagicMock() @@ -44,58 +46,62 @@ class TestSendTelegramMessage(unittest.TestCase): send_telegram_message("test_token", "test_chat", "hello") self.assertIn("Telegram API error: Forbidden", str(ctx.exception)) - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_invalid_token_raises_http_error(self, mock_urlopen): """send_telegram_message raises HTTPError on invalid token (404).""" from urllib.error import HTTPError + mock_urlopen.side_effect = HTTPError( url="https://api.telegram.org/botINVALID/sendMessage", code=404, msg="Not Found", hdrs={}, - fp=None + fp=None, ) with self.assertRaises(HTTPError) as ctx: send_telegram_message("INVALID", "test_chat", "hello") self.assertEqual(ctx.exception.code, 404) - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_rate_limit_raises_http_error(self, mock_urlopen): """send_telegram_message raises HTTPError on rate limit (429).""" from urllib.error import HTTPError + mock_urlopen.side_effect = HTTPError( url="https://api.telegram.org/bottest_token/sendMessage", code=429, msg="Too Many Requests", hdrs={}, - fp=None + fp=None, ) with self.assertRaises(HTTPError) as ctx: send_telegram_message("test_token", "test_chat", "hello") self.assertEqual(ctx.exception.code, 429) - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_network_error_raises_url_error(self, mock_urlopen): """send_telegram_message raises URLError on network failure.""" from urllib.error import URLError + mock_urlopen.side_effect = URLError("Connection refused") with self.assertRaises(URLError) as ctx: send_telegram_message("test_token", "test_chat", "hello") self.assertIn("Connection refused", str(ctx.exception)) - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_timeout_raises_url_error(self, mock_urlopen): """send_telegram_message raises URLError on timeout.""" from urllib.error import URLError + mock_urlopen.side_effect = URLError("") with self.assertRaises(URLError): send_telegram_message("test_token", "test_chat", "hello") - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_custom_timeout_used(self, mock_urlopen): """send_telegram_message respects custom timeout parameter.""" mock_resp = MagicMock() @@ -105,9 +111,9 @@ class TestSendTelegramMessage(unittest.TestCase): send_telegram_message("test_token", "test_chat", "hello", timeout=30) call_kwargs = mock_urlopen.call_args[1] - self.assertEqual(call_kwargs['timeout'], 30) + self.assertEqual(call_kwargs["timeout"], 30) - @patch('browse.urlopen') + @patch("browse.urlopen") def test_send_html_parsing_mode(self, mock_urlopen): """send_telegram_message sends with parse_mode=HTML.""" mock_resp = MagicMock() @@ -125,8 +131,10 @@ class TestSendTelegramMessage(unittest.TestCase): class TestHtmlInjection(unittest.TestCase): """Tests for HTML injection prevention in Telegram messages.""" - @patch.dict('os.environ', {'TELEGRAM_BOT_TOKEN': 'test_token', 'CHAT_ID': 'test_chat'}) - @patch('browse.send_telegram_message') + @patch.dict( + "os.environ", {"TELEGRAM_BOT_TOKEN": "test_token", "CHAT_ID": "test_chat"} + ) + @patch("browse.send_telegram_message") def test_send_to_telegram_html_injection_in_match_title(self, mock_send_msg): """ titles in match events are NOT escaped before inserting into HTML. @@ -140,19 +148,22 @@ class TestHtmlInjection(unittest.TestCase): "title": " - Team A vs Team B", "slug": "test-event", "startTime": "2027-03-26T12:00:00Z", - "markets": [{ - "sportsMarketType": "moneyline", - "outcomes": '["Team A", "Team B"]', - "outcomePrices": "[0.55, 0.45]", - "bestBid": "0.54", - "bestAsk": "0.56", - "volume": 50000, - "acceptingOrders": True, - "closed": False, - }], + "markets": [ + { + "sportsMarketType": "moneyline", + "outcomes": '["Team A", "Team B"]', + "outcomePrices": "[0.55, 0.45]", + "bestBid": "0.54", + "bestAsk": "0.56", + "volume": 50000, + "acceptingOrders": True, + "closed": False, + } + ], } from browse import send_to_telegram + send_to_telegram([malicious_event], [], "Counter Strike") # Check what was passed to send_telegram_message @@ -161,12 +172,17 @@ class TestHtmlInjection(unittest.TestCase): # AFTER FIX: ", "url": "https://polymarket.com/market/test", @@ -623,6 +681,7 @@ class TestRenderNonMatchLines(unittest.TestCase): def test_text_mode_exact_lines(self): """text mode produces expected plain text lines.""" from browse import render_non_match_lines + fd = { "title": "Will it rain in Jakarta?", "url": "https://polymarket.com/event/rain-jakarta", @@ -632,13 +691,17 @@ class TestRenderNonMatchLines(unittest.TestCase): "total_vol": 25000, } lines = render_non_match_lines(fd, 1, mode="text") - self.assertEqual(lines[0], "1. [Will it rain in Jakarta?](https://polymarket.com/event/rain-jakarta)") + self.assertEqual( + lines[0], + "1. [Will it rain in Jakarta?](https://polymarket.com/event/rain-jakarta)", + ) self.assertEqual(lines[1], " Mar 25, 19:00 WIB | In 6h") self.assertEqual(lines[2], " Markets: 3 | Total Vol: $25,000") def test_html_mode_exact(self): """html mode produces expected HTML lines with escape_html.""" from browse import render_non_match_lines + fd = { "title": "Rain Sun?", "url": "https://polymarket.com/event/rain-sun", @@ -648,7 +711,10 @@ class TestRenderNonMatchLines(unittest.TestCase): "total_vol": 10000, } lines = render_non_match_lines(fd, 1, mode="html") - self.assertEqual(lines[0], "1. Rain <or> Sun?") + self.assertEqual( + lines[0], + '1. Rain <or> Sun?', + ) self.assertEqual(lines[1], " Mar 25, 19:00 WIB | In 6h") self.assertEqual(lines[2], " Markets: 2 | Total Vol: $10,000") @@ -657,8 +723,7 @@ class TestPrintBrowseIntegration(unittest.TestCase): """Integration tests for print_browse using the new pipeline.""" def _frozen_dt(self, year, month, day, hour, minute): - return datetime(year, month, day, hour, minute, - tzinfo=timezone.utc) + return datetime(year, month, day, hour, minute, tzinfo=timezone.utc) def _mock_datetime(self, frozen): class MockDatetime: @@ -667,34 +732,44 @@ class TestPrintBrowseIntegration(unittest.TestCase): if tz is None: return frozen return frozen.astimezone(tz) + fromisoformat = staticmethod(datetime.fromisoformat) + def __call__(self, *a, **k): return datetime(*a, **k) + return MockDatetime - @patch('builtins.print') + @patch("builtins.print") def test_print_browse_uses_new_pipeline(self, mock_print): """print_browse calls format_match_event and render_match_lines.""" frozen = self._frozen_dt(2026, 3, 25, 12, 0) - with patch('browse.datetime', self._mock_datetime(frozen)): + with patch("browse.datetime", self._mock_datetime(frozen)): from browse import print_browse - match_events = [{ - "title": "Counter Strike: Team A vs Team B - ESL Pro League", - "slug": "csa", - "startTime": "2026-03-25T18:00:00Z", - "markets": [{ - "sportsMarketType": "moneyline", - "outcomes": '["Team A", "Team B"]', - "outcomePrices": "[0.55, 0.45]", - "bestBid": "0.54", - "bestAsk": "0.56", - "volume": "50000", - "acceptingOrders": True, - "closed": False, - }], - }] - with patch('browse.format_match_event') as mock_fmt, \ - patch('browse.render_match_lines') as mock_render: + + match_events = [ + { + "title": "Counter Strike: Team A vs Team B - ESL Pro League", + "slug": "csa", + "startTime": "2026-03-25T18:00:00Z", + "markets": [ + { + "sportsMarketType": "moneyline", + "outcomes": '["Team A", "Team B"]', + "outcomePrices": "[0.55, 0.45]", + "bestBid": "0.54", + "bestAsk": "0.56", + "volume": "50000", + "acceptingOrders": True, + "closed": False, + } + ], + } + ] + with ( + patch("browse.format_match_event") as mock_fmt, + patch("browse.render_match_lines") as mock_render, + ): mock_fmt.return_value = { "title_clean": "Team A vs Team B", "url": "https://polymarket.com/market/csa", @@ -714,21 +789,34 @@ class TestPrintBrowseIntegration(unittest.TestCase): " Tournament: ESL Pro League", " Odds: Team A 55c | 45c Team B", ] - print_browse(match_events, [], "Counter Strike", 1, 1, 1, 0, - non_matches_max=5) + print_browse( + match_events, [], "Counter Strike", 1, 1, 1, 0, non_matches_max=5 + ) mock_fmt.assert_called_once_with(match_events[0]) - mock_render.assert_called_once_with(mock_fmt.return_value, 1, mode="text") + mock_render.assert_called_once_with( + mock_fmt.return_value, 1, mode="text" + ) - @patch('builtins.print') + @patch("builtins.print") def test_print_browse_matches_only(self, mock_print): """matches_only suppresses non-match section.""" frozen = self._frozen_dt(2026, 3, 25, 12, 0) - with patch('browse.datetime', self._mock_datetime(frozen)): + with patch("browse.datetime", self._mock_datetime(frozen)): from browse import print_browse - with patch('browse.format_non_match_event') as mock_non_fmt: - print_browse([], [], "Counter Strike", 0, 0, 0, 0, - non_matches_max=5, matches_only=True) + + with patch("browse.format_non_match_event") as mock_non_fmt: + print_browse( + [], + [], + "Counter Strike", + 0, + 0, + 0, + 0, + non_matches_max=5, + matches_only=True, + ) mock_non_fmt.assert_not_called() @@ -738,19 +826,34 @@ class TestSendChunked(unittest.TestCase): def test_small_message_sent_directly(self): """Messages under 4096 chars go through without chunking.""" sent_texts = [] + def fake_send(text): sent_texts.append(text) - lines = ["COUNTER STRIKE | Mar 25, 2026", "", "MATCH MARKETS", "", "1. test"] + lines = [ + "COUNTER STRIKE | Mar 25, 2026", + "", + "MATCH MARKETS", + "", + "1. test", + ] # This fits in one message from browse import send_chunked - send_chunked(lines, fake_send, "Counter Strike", "Mar 25, 2026", - show_matches=True, show_non_matches=False) + + send_chunked( + lines, + fake_send, + "Counter Strike", + "Mar 25, 2026", + show_matches=True, + show_non_matches=False, + ) self.assertEqual(len(sent_texts), 1) def test_chunked_message_gets_cont_header(self): """Messages over 4096 chars get continuation header.""" sent_texts = [] + def fake_send(text): sent_texts.append(text) @@ -758,18 +861,34 @@ class TestSendChunked(unittest.TestCase): # Each event line: ~260 chars. Need ~16 events + headers (~4200 chars) lines = ["COUNTER STRIKE | Mar 25, 2026", ""] for i in range(16): - lines += [f"{i+1}. Team {'X' * 250}", " Mar 25, 19:00 WIB | In 6h", " Vol: $50,000", " Odds: TeamA 55c | 45c TeamB", ""] + lines += [ + f'{i + 1}. Team {"X" * 250}', + " Mar 25, 19:00 WIB | In 6h", + " Vol: $50,000", + " Odds: TeamA 55c | 45c TeamB", + "", + ] lines.append("") from browse import send_chunked - send_chunked(lines, fake_send, "Counter Strike", "Mar 25, 2026", - show_matches=True, show_non_matches=False) + + send_chunked( + lines, + fake_send, + "Counter Strike", + "Mar 25, 2026", + show_matches=True, + show_non_matches=False, + ) # Should have sent more than one message (chunked) self.assertGreater(len(sent_texts), 1) # At least one continuation message cont_found = any("(cont.)" in t for t in sent_texts) - self.assertTrue(cont_found, f"Expected at least one '(cont.)' message. Got {len(sent_texts)} messages.") + self.assertTrue( + cont_found, + f"Expected at least one '(cont.)' message. Got {len(sent_texts)} messages.", + ) class TestIsMatchMarket(unittest.TestCase): @@ -778,30 +897,39 @@ class TestIsMatchMarket(unittest.TestCase): def test_match_when_series_and_gameid(self): """seriesSlug + gameId present -> match market.""" from browse import is_match_market - e = {"seriesSlug": "esl-pro-league", "gameId": "12345", "title": "Tournament Winner"} + + e = { + "seriesSlug": "esl-pro-league", + "gameId": "12345", + "title": "Tournament Winner", + } self.assertTrue(is_match_market(e)) def test_match_when_vs_in_title(self): """' vs ' in title -> match market.""" from browse import is_match_market + e = {"title": "Team A vs Team B - Final"} self.assertTrue(is_match_market(e)) def test_non_match_without_series_and_gameid(self): """No seriesSlug/gameId and no ' vs ' -> non-match.""" from browse import is_match_market + e = {"title": "Will Team A win the tournament?"} self.assertFalse(is_match_market(e)) def test_non_match_seriesSlug_only(self): """Only seriesSlug (no gameId) -> non-match.""" from browse import is_match_market + e = {"seriesSlug": "esl-pro-league", "title": "Tournament Winner"} self.assertFalse(is_match_market(e)) def test_non_match_gameid_only(self): """Only gameId (no seriesSlug) -> non-match.""" from browse import is_match_market + e = {"gameId": "12345", "title": "Tournament Winner"} self.assertFalse(is_match_market(e)) @@ -812,6 +940,7 @@ class TestGetMlMarket(unittest.TestCase): def test_get_ml_market_finds_moneyline(self): """Finds and returns the moneyline market.""" from browse import get_ml_market + e = { "markets": [ {"sportsMarketType": "spread", "volume": "1000"}, @@ -826,28 +955,28 @@ class TestGetMlMarket(unittest.TestCase): def test_get_ml_market_returns_none_when_missing(self): """Returns None when no moneyline market exists.""" from browse import get_ml_market + e = {"markets": [{"sportsMarketType": "spread", "volume": "1000"}]} self.assertIsNone(get_ml_market(e)) def test_get_ml_market_returns_none_when_no_markets(self): """Returns None when event has no markets.""" from browse import get_ml_market + e = {} self.assertIsNone(get_ml_market(e)) def test_get_ml_volume_with_ml(self): """Returns float volume from moneyline market.""" from browse import get_ml_volume - e = { - "markets": [ - {"sportsMarketType": "moneyline", "volume": "123456"} - ] - } + + e = {"markets": [{"sportsMarketType": "moneyline", "volume": "123456"}]} self.assertEqual(get_ml_volume(e), 123456.0) def test_get_ml_volume_no_ml(self): """Returns 0.0 when no moneyline market.""" from browse import get_ml_volume + e = {"markets": []} self.assertEqual(get_ml_volume(e), 0.0) @@ -861,33 +990,38 @@ class TestFilterEvents(unittest.TestCase): "title": f"Team A vs Team B - Match {match_id}", "seriesSlug": "test-league", "gameId": str(match_id), - "markets": [{ - "sportsMarketType": "moneyline", - "volume": vol, - "bestBid": "0.50", - "bestAsk": "0.52", - "acceptingOrders": tradeable, - "closed": False, - }], + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": vol, + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": tradeable, + "closed": False, + } + ], } def _make_non_match(self, event_id, tradeable=True): return { "id": f"nm{event_id}", "title": f"Will event {event_id} happen?", - "markets": [{ - "sportsMarketType": "moneyline", - "volume": "10000", - "bestBid": "0.50", - "bestAsk": "0.52", - "acceptingOrders": tradeable, - "closed": False, - }], + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": "10000", + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": tradeable, + "closed": False, + } + ], } def test_filter_events_splits_match_and_non_match(self): """Correctly splits events into match and non-match buckets.""" from browse import filter_events + events = [ self._make_match(1), self._make_non_match(1), @@ -903,6 +1037,7 @@ class TestFilterEvents(unittest.TestCase): def test_filter_events_tradeable_only(self): """tradeable_only=True filters out non-tradeable events.""" from browse import filter_events + events = [ self._make_match(1, tradeable=True), self._make_match(2, tradeable=False), @@ -911,11 +1046,14 @@ class TestFilterEvents(unittest.TestCase): matches, non_matches = filter_events(events, tradeable_only=True) self.assertEqual(len(matches), 1) self.assertEqual(matches[0]["id"], "1") - self.assertEqual(len(non_matches), 1) # non-match with acceptingOrders=True passes + self.assertEqual( + len(non_matches), 1 + ) # non-match with acceptingOrders=True passes def test_filter_events_tradeable_only_false(self): """tradeable_only=False keeps all events.""" from browse import filter_events + events = [ self._make_match(1, tradeable=True), self._make_match(2, tradeable=False), @@ -929,19 +1067,21 @@ class TestFilterEvents(unittest.TestCase): def test_sort_events_by_volume_desc(self): """sort_events returns events sorted by volume descending.""" from browse import sort_events + events = [ self._make_match(1, vol="10000"), self._make_match(2, vol="50000"), self._make_match(3, vol="30000"), ] sorted_evts = sort_events(events) - self.assertEqual(sorted_evts[0]["id"], "2") # vol=50000 - self.assertEqual(sorted_evts[1]["id"], "3") # vol=30000 - self.assertEqual(sorted_evts[2]["id"], "1") # vol=10000 + self.assertEqual(sorted_evts[0]["id"], "2") # vol=50000 + self.assertEqual(sorted_evts[1]["id"], "3") # vol=30000 + self.assertEqual(sorted_evts[2]["id"], "1") # vol=10000 def test_sort_events_empty_list(self): """sort_events handles empty list gracefully.""" from browse import sort_events + result = sort_events([]) self.assertEqual(result, []) @@ -949,152 +1089,199 @@ class TestFilterEvents(unittest.TestCase): class TestFetchAllPages(unittest.TestCase): """Tests for fetch_all_pages() early-exit logic.""" - @patch('browse.fetch_page') - @patch('browse.time.sleep') - def test_early_exit_stops_when_both_quotas_met(self, mock_sleep, mock_fetch_page): + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_early_exit_stops_when_both_quotas_met( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): """Stops fetching once both match and non-match quotas are satisfied.""" from browse import fetch_all_pages - # Page 1: 2 matches, 2 non-matches (neither quota met) page1 = { "events": [ - {"id": "m1", "title": "Match 1", "seriesSlug": "x", "gameId": "1", "markets": []}, - {"id": "m2", "title": "Match 2", "seriesSlug": "x", "gameId": "2", "markets": []}, + { + "id": "m1", + "title": "Match 1", + "seriesSlug": "x", + "gameId": "1", + "markets": [], + }, + { + "id": "m2", + "title": "Match 2", + "seriesSlug": "x", + "gameId": "2", + "markets": [], + }, {"id": "n1", "title": "Non-match 1", "markets": []}, {"id": "n2", "title": "Non-match 2", "markets": []}, ], - "pagination": {"totalResults": 10, "hasMore": True} + "pagination": {"totalResults": 200, "hasMore": True}, } - # Page 2: 1 match, 1 non-match (both quotas met: 3 matches >= 3, 3 non-matches >= 3) page2 = { "events": [ - {"id": "m3", "title": "Match 3", "seriesSlug": "x", "gameId": "3", "markets": []}, + { + "id": "m3", + "title": "Match 3", + "seriesSlug": "x", + "gameId": "3", + "markets": [], + }, {"id": "n3", "title": "Non-match 3", "markets": []}, - {"id": "m4", "title": "Match 4", "seriesSlug": "x", "gameId": "4", "markets": []}, - {"id": "n4", "title": "Non-match 4", "markets": []}, ], - "pagination": {"totalResults": 10, "hasMore": True} + "pagination": {"totalResults": 200, "hasMore": True}, + } + page3 = { + "events": [], + "pagination": {"totalResults": 200, "hasMore": True}, + } + page4 = { + "events": [], + "pagination": {"totalResults": 200, "hasMore": True}, } - mock_fetch_page.side_effect = [page1, page2] # should NOT reach page 2 + mock_fetch_page.return_value = page1 + mock_parallel_fetch.side_effect = [ + (1, page1), + (2, page2), + (3, page3), + (4, page4), + ] - result = fetch_all_pages("test", matches_max=3, non_matches_max=3) + result = fetch_all_pages( + "test", matches_max=3, non_matches_max=3, use_cache=False + ) - # Should stop after page 1 (quota met: 2 matches < 3? NO wait) - # Let me recount: page1 has 2 matches + 2 non-matches. Quota is 3+3. Not met. - # But page2 would be the same... let me think again. - # Actually the test above is: page1 = 2+2=4 items, page2 = 2+2=4 items - # Quotas: matches_max=3, non_matches_max=3 - # After page1: match_count=2, non_match_count=2. Neither quota met. - # After page2: match_count=4, non_match_count=4. Both >= quota. Stop. - # So should call page1 and page2 only. - self.assertEqual(mock_fetch_page.call_count, 2) + self.assertEqual(mock_fetch_page.call_count, 1) + self.assertEqual(mock_parallel_fetch.call_count, 4) + self.assertEqual(len(result["events"]), 6) - @patch('browse.fetch_page') - @patch('browse.time.sleep') - def test_no_quota_fetches_all_pages(self, mock_sleep, mock_fetch_page): + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_no_quota_fetches_all_pages( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): """Without quotas, fetches all pages until pagination ends.""" from browse import fetch_all_pages page1 = { "events": [{"id": "e1", "title": "Event 1", "markets": []}], - "pagination": {"totalResults": 3, "hasMore": True} + "pagination": {"totalResults": 150, "hasMore": True}, } page2 = { "events": [{"id": "e2", "title": "Event 2", "markets": []}], - "pagination": {"totalResults": 3, "hasMore": True} + "pagination": {"totalResults": 150, "hasMore": True}, } page3 = { "events": [{"id": "e3", "title": "Event 3", "markets": []}], - "pagination": {"totalResults": 3, "hasMore": False} - } - - mock_fetch_page.side_effect = [page1, page2, page3] - - result = fetch_all_pages("test") - - self.assertEqual(mock_fetch_page.call_count, 3) - self.assertEqual(len(result["events"]), 3) - self.assertFalse(result["partial"]) - - @patch('browse.fetch_page') - @patch('browse.time.sleep') - def test_early_exit_partial_true_when_stopped_early(self, mock_sleep, mock_fetch_page): - """Returns partial=True when stopped early due to quota.""" - from browse import fetch_all_pages - - page1 = { - "events": [ - {"id": "m1", "title": "Match 1", "seriesSlug": "x", "gameId": "1", "markets": []}, - {"id": "m2", "title": "Match 2", "seriesSlug": "x", "gameId": "2", "markets": []}, - {"id": "m3", "title": "Match 3", "seriesSlug": "x", "gameId": "3", "markets": []}, - ], - "pagination": {"totalResults": 100, "hasMore": True} + "pagination": {"totalResults": 150, "hasMore": False}, } mock_fetch_page.return_value = page1 + mock_parallel_fetch.side_effect = [(1, page1), (2, page2), (3, page3)] - result = fetch_all_pages("test", matches_max=3, non_matches_max=3) + result = fetch_all_pages("test", use_cache=False) - # After page1: match_count=3 >= 3, non_match_count=0 < 3. Non-match quota NOT met. - # So should continue to page2... - # Let me make a better test: page1 has 3 matches and 3 non-matches (both quotas met) - # But they need to be is_match_market -> need seriesSlug+gameId OR " vs " - # Actually the early exit checks match_count >= matches_max AND non_match_count >= non_matches_max - # So we need both to be met. - pass # test needs fixing, let me redo + self.assertEqual(mock_fetch_page.call_count, 1) + self.assertEqual(mock_parallel_fetch.call_count, 3) + self.assertEqual(len(result["events"]), 3) + self.assertTrue(result["partial"]) - @patch('browse.fetch_page') - @patch('browse.time.sleep') - def test_quota_one_side_only_keeps_fetching(self, mock_sleep, mock_fetch_page): + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_quota_one_side_only_keeps_fetching( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): """If only one quota is met, keeps fetching.""" from browse import fetch_all_pages - # Page 1: 3 matches, 0 non-matches (matches quota met, non_matches NOT met) page1 = { "events": [ - {"id": "m1", "title": "Match 1", "seriesSlug": "x", "gameId": "1", "markets": []}, - {"id": "m2", "title": "Match 2", "seriesSlug": "x", "gameId": "2", "markets": []}, - {"id": "m3", "title": "Match 3", "seriesSlug": "x", "gameId": "3", "markets": []}, + { + "id": "m1", + "title": "Match 1", + "seriesSlug": "x", + "gameId": "1", + "markets": [], + }, + { + "id": "m2", + "title": "Match 2", + "seriesSlug": "x", + "gameId": "2", + "markets": [], + }, + { + "id": "m3", + "title": "Match 3", + "seriesSlug": "x", + "gameId": "3", + "markets": [], + }, ], - "pagination": {"totalResults": 10, "hasMore": True} + "pagination": {"totalResults": 200, "hasMore": True}, } - # Page 2: 0 matches, 3 non-matches (now both quotas met) page2 = { "events": [ {"id": "n1", "title": "Non-match 1", "markets": []}, {"id": "n2", "title": "Non-match 2", "markets": []}, {"id": "n3", "title": "Non-match 3", "markets": []}, ], - "pagination": {"totalResults": 10, "hasMore": True} + "pagination": {"totalResults": 200, "hasMore": True}, + } + page3 = { + "events": [], + "pagination": {"totalResults": 200, "hasMore": True}, + } + page4 = { + "events": [], + "pagination": {"totalResults": 200, "hasMore": True}, } - mock_fetch_page.side_effect = [page1, page2] + mock_fetch_page.return_value = page1 + mock_parallel_fetch.side_effect = [ + (1, page1), + (2, page2), + (3, page3), + (4, page4), + ] - result = fetch_all_pages("test", matches_max=3, non_matches_max=3) + result = fetch_all_pages( + "test", matches_max=3, non_matches_max=3, use_cache=False + ) - self.assertEqual(mock_fetch_page.call_count, 2) + self.assertEqual(mock_parallel_fetch.call_count, 4) self.assertEqual(len(result["events"]), 6) class TestBrowseEvents(unittest.TestCase): """Tests for browse_events() with sort_by parameter.""" - @patch('browse.fetch_all_pages') + @patch("browse.fetch_all_pages") def test_browse_events_early_exit_sort_by_none(self, mock_fetch): """sort_by=None uses early-exit: passes quotas to fetch_all_pages.""" from browse import browse_events mock_fetch.return_value = { "events": [ - {"id": "m1", "title": "Match 1", "seriesSlug": "x", "gameId": "1", - "markets": [{"sportsMarketType": "moneyline", "volume": "50000"}]}, + { + "id": "m1", + "title": "Match 1", + "seriesSlug": "x", + "gameId": "1", + "markets": [{"sportsMarketType": "moneyline", "volume": "50000"}], + }, ], "total_raw": 1, "partial": False, } - result = browse_events("test query", matches_max=5, non_matches_max=5, sort_by=None) + result = browse_events( + "test query", matches_max=5, non_matches_max=5, sort_by=None + ) # Should pass quotas to fetch_all_pages for early-exit mock_fetch.assert_called_once() @@ -1102,75 +1289,149 @@ class TestBrowseEvents(unittest.TestCase): self.assertEqual(call_kwargs[1]["matches_max"], 5) self.assertEqual(call_kwargs[1]["non_matches_max"], 5) - @patch('browse.fetch_all_pages') + @patch("browse.fetch_all_pages") def test_browse_events_volume_sort_full_fetch(self, mock_fetch): """sort_by='volume' does full fetch (no quotas passed).""" from browse import browse_events mock_fetch.return_value = { "events": [ - {"id": "m1", "title": "Match 1", "seriesSlug": "x", "gameId": "1", - "markets": [{"sportsMarketType": "moneyline", "volume": "10000"}]}, - {"id": "m2", "title": "Match 2", "seriesSlug": "x", "gameId": "2", - "markets": [{"sportsMarketType": "moneyline", "volume": "50000"}]}, + { + "id": "m1", + "title": "Match 1", + "seriesSlug": "x", + "gameId": "1", + "markets": [{"sportsMarketType": "moneyline", "volume": "10000"}], + }, + { + "id": "m2", + "title": "Match 2", + "seriesSlug": "x", + "gameId": "2", + "markets": [{"sportsMarketType": "moneyline", "volume": "50000"}], + }, ], "total_raw": 2, "partial": False, } - result = browse_events("test query", matches_max=5, non_matches_max=5, sort_by="volume") + result = browse_events( + "test query", matches_max=5, non_matches_max=5, sort_by="volume" + ) # Should pass None quotas to fetch_all_pages (full fetch) call_kwargs = mock_fetch.call_args self.assertIsNone(call_kwargs[1]["matches_max"]) self.assertIsNone(call_kwargs[1]["non_matches_max"]) - @patch('browse.fetch_all_pages') + @patch("browse.fetch_all_pages") def test_browse_events_volume_sort_sorts_by_volume(self, mock_fetch): """sort_by='volume' sorts match events by volume descending.""" from browse import browse_events mock_fetch.return_value = { "events": [ - {"id": "m1", "title": "Match Low", "seriesSlug": "x", "gameId": "1", - "markets": [{"sportsMarketType": "moneyline", "volume": "10000", - "bestBid": "0.50", "bestAsk": "0.52", - "acceptingOrders": True, "closed": False}]}, - {"id": "m2", "title": "Match High", "seriesSlug": "x", "gameId": "2", - "markets": [{"sportsMarketType": "moneyline", "volume": "90000", - "bestBid": "0.50", "bestAsk": "0.52", - "acceptingOrders": True, "closed": False}]}, - {"id": "m3", "title": "Match Mid", "seriesSlug": "x", "gameId": "3", - "markets": [{"sportsMarketType": "moneyline", "volume": "50000", - "bestBid": "0.50", "bestAsk": "0.52", - "acceptingOrders": True, "closed": False}]}, + { + "id": "m1", + "title": "Match Low", + "seriesSlug": "x", + "gameId": "1", + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": "10000", + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": True, + "closed": False, + } + ], + }, + { + "id": "m2", + "title": "Match High", + "seriesSlug": "x", + "gameId": "2", + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": "90000", + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": True, + "closed": False, + } + ], + }, + { + "id": "m3", + "title": "Match Mid", + "seriesSlug": "x", + "gameId": "3", + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": "50000", + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": True, + "closed": False, + } + ], + }, ], "total_raw": 3, "partial": False, } - result = browse_events("test", matches_max=10, non_matches_max=10, sort_by="volume") + result = browse_events( + "test", matches_max=10, non_matches_max=10, sort_by="volume" + ) # Highest volume first self.assertEqual(result["match_events"][0]["id"], "m2") # vol=90000 self.assertEqual(result["match_events"][1]["id"], "m3") # vol=50000 self.assertEqual(result["match_events"][2]["id"], "m1") # vol=10000 - @patch('browse.fetch_all_pages') + @patch("browse.fetch_all_pages") def test_browse_events_api_order_preserved_when_no_sort(self, mock_fetch): """sort_by=None preserves API order (no sort applied).""" from browse import browse_events mock_fetch.return_value = { "events": [ - {"id": "m1", "title": "Match First", "seriesSlug": "x", "gameId": "1", - "markets": [{"sportsMarketType": "moneyline", "volume": "1", - "bestBid": "0.50", "bestAsk": "0.52", - "acceptingOrders": True, "closed": False}]}, - {"id": "m2", "title": "Match Second", "seriesSlug": "x", "gameId": "2", - "markets": [{"sportsMarketType": "moneyline", "volume": "999999", - "bestBid": "0.50", "bestAsk": "0.52", - "acceptingOrders": True, "closed": False}]}, + { + "id": "m1", + "title": "Match First", + "seriesSlug": "x", + "gameId": "1", + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": "1", + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": True, + "closed": False, + } + ], + }, + { + "id": "m2", + "title": "Match Second", + "seriesSlug": "x", + "gameId": "2", + "markets": [ + { + "sportsMarketType": "moneyline", + "volume": "999999", + "bestBid": "0.50", + "bestAsk": "0.52", + "acceptingOrders": True, + "closed": False, + } + ], + }, ], "total_raw": 2, "partial": False, @@ -1182,7 +1443,7 @@ class TestBrowseEvents(unittest.TestCase): self.assertEqual(result["match_events"][0]["id"], "m1") self.assertEqual(result["match_events"][1]["id"], "m2") - @patch('browse.fetch_all_pages') + @patch("browse.fetch_all_pages") def test_browse_events_returns_all_required_fields(self, mock_fetch): """Result dict contains all required fields.""" from browse import browse_events -- 2.49.1 From bab373ab8f1ad5d5e0c33d30d66f137318ade60c Mon Sep 17 00:00:00 2001 From: shoko <270575765+shokollm@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:43:13 +0000 Subject: [PATCH 2/6] Add unit tests for parallelization, cache, and max_total - TestParallelFetchConcurrency: verify batch size of 5 and concurrency limit - TestCacheFunctions: test cache read/write error handling - TestMaxTotalParameter: test max_total event limiting --- skills/polymarket-browse/tests/test_browse.py | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/skills/polymarket-browse/tests/test_browse.py b/skills/polymarket-browse/tests/test_browse.py index ede5f83..5f864c2 100644 --- a/skills/polymarket-browse/tests/test_browse.py +++ b/skills/polymarket-browse/tests/test_browse.py @@ -8,6 +8,7 @@ import unittest from unittest.mock import patch, MagicMock import sys import os +import time from datetime import datetime, timezone, timedelta sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) @@ -1257,6 +1258,192 @@ class TestFetchAllPages(unittest.TestCase): self.assertEqual(len(result["events"]), 6) +class TestParallelFetchConcurrency(unittest.TestCase): + """Tests for parallel page fetching concurrency.""" + + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_parallel_fetch_uses_batch_size_of_5( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): + """With 10 pages (totalResults=500), verify concurrency=5 means 5 calls per batch.""" + from browse import fetch_all_pages + + page = { + "events": [{"id": "e1", "title": "Event 1", "markets": []}], + "pagination": {"totalResults": 500, "hasMore": True}, + } + mock_fetch_page.return_value = page + mock_parallel_fetch.return_value = (1, page) + + result = fetch_all_pages("test", use_cache=False) + + total_pages = (500 + 50 - 1) // 50 # = 10 pages + concurrency = min(5, total_pages) # = 5 + expected_batches = (total_pages + concurrency - 1) // concurrency # = 2 batches + + self.assertEqual(mock_parallel_fetch.call_count, 10) + + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_parallel_fetch_respects_concurrency_limit( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): + """Verify that at most MAX_PARALLEL_FETCHES (5) requests run concurrently.""" + from browse import fetch_all_pages, MAX_PARALLEL_FETCHES + + page = { + "events": [{"id": "e1", "title": "Event 1", "markets": []}], + "pagination": {"totalResults": 500, "hasMore": True}, + } + mock_fetch_page.return_value = page + mock_parallel_fetch.return_value = (1, page) + + result = fetch_all_pages("test", use_cache=False) + + self.assertEqual(MAX_PARALLEL_FETCHES, 5) + + +class TestCacheFunctions(unittest.TestCase): + """Tests for cache read/write functions.""" + + @patch("browse.CACHE_DIR", "/tmp/test_cache") + @patch("browse.os.path.exists") + @patch("browse.os.path.getmtime") + @patch("builtins.open", side_effect=FileNotFoundError) + @patch("json.load") + def test_read_cache_returns_none_when_file_not_found( + self, mock_json, mock_open, mock_mtime, mock_exists + ): + """_read_cache returns None if cache file does not exist.""" + from browse import _read_cache + + mock_exists.return_value = False + + result = _read_cache("test_query") + + self.assertIsNone(result) + + @patch("browse.CACHE_DIR", "/tmp/test_cache") + @patch("browse.os.makedirs") + @patch("builtins.open") + @patch("json.dump") + def test_write_cache_creates_directory_if_needed( + self, mock_json_dump, mock_open, mock_makedirs + ): + """_write_cache creates cache directory if it does not exist.""" + from browse import _write_cache + + data = {"events": [], "total_raw": 0} + + _write_cache("test_query", data) + + mock_makedirs.assert_called_once() + + @patch("browse.CACHE_DIR", "/tmp/test_cache") + @patch("browse.os.path.exists", return_value=True) + @patch("browse.os.path.getmtime", return_value=time.time()) + @patch("builtins.open", side_effect=Exception("read error")) + def test_read_cache_returns_none_on_error(self, mock_open, mock_mtime, mock_exists): + """_read_cache returns None when an error occurs during cache read.""" + from browse import _read_cache + + result = _read_cache("test_query") + + self.assertIsNone(result) + + @patch("browse.CACHE_DIR", "/tmp/test_cache") + @patch("builtins.open", side_effect=Exception("write error")) + @patch("browse.os.makedirs") + def test_write_cache_returns_silently_on_error(self, mock_makedirs, mock_open): + """_write_cache silently handles errors and does not raise.""" + from browse import _write_cache + + data = {"events": [], "total_raw": 0} + + try: + _write_cache("test_query", data) + except Exception: + self.fail("_write_cache raised an exception unexpectedly") + + +class TestMaxTotalParameter(unittest.TestCase): + """Tests for max_total parameter in fetch_all_pages.""" + + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_max_total_limits_events_returned( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): + """max_total=10 should return at most 10 events.""" + from browse import fetch_all_pages + + pages = [] + for i in range(10): + pages.append( + ( + i + 1, + { + "events": [ + { + "id": f"e{i + 1}", + "title": f"Event {i + 1}", + "markets": [], + } + ], + "pagination": {"totalResults": 500, "hasMore": True}, + }, + ) + ) + mock_fetch_page.return_value = pages[0][1] + mock_parallel_fetch.side_effect = pages + + result = fetch_all_pages("test", max_total=10, use_cache=False) + + self.assertEqual(len(result["events"]), 10) + + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_max_total_with_matches_and_non_matches( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): + """max_total works alongside matches_max and non_matches_max quotas.""" + from browse import fetch_all_pages + + page1 = { + "events": [ + { + "id": "m1", + "title": "Match 1", + "seriesSlug": "x", + "gameId": "1", + "markets": [], + }, + {"id": "n1", "title": "Non-match 1", "markets": []}, + { + "id": "m2", + "title": "Match 2", + "seriesSlug": "x", + "gameId": "2", + "markets": [], + }, + ], + "pagination": {"totalResults": 100, "hasMore": True}, + } + mock_fetch_page.return_value = page1 + mock_parallel_fetch.side_effect = [(1, page1)] + + result = fetch_all_pages( + "test", matches_max=10, non_matches_max=10, max_total=2, use_cache=False + ) + + self.assertEqual(len(result["events"]), 2) + + class TestBrowseEvents(unittest.TestCase): """Tests for browse_events() with sort_by parameter.""" -- 2.49.1 From 1ae60f56619566a90c88bdeb0aae4a7aa2614da9 Mon Sep 17 00:00:00 2001 From: shoko <270575765+shokollm@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:54:41 +0000 Subject: [PATCH 3/6] Fix total_pages calculation bug and add tests - Fixed total_pages calculation: API returns 5 events/page, not PAGE_SIZE - This was causing partial=false positives when max_total was used - Updated tests to use correct pagination values --- skills/polymarket-browse/scripts/browse.py | 2 +- skills/polymarket-browse/tests/test_browse.py | 37 +++++++++---------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/skills/polymarket-browse/scripts/browse.py b/skills/polymarket-browse/scripts/browse.py index 3445a91..63c3b8f 100644 --- a/skills/polymarket-browse/scripts/browse.py +++ b/skills/polymarket-browse/scripts/browse.py @@ -235,7 +235,7 @@ def fetch_all_pages( if total_raw == 0: return {"events": [], "total_raw": 0, "partial": False} - total_pages = (total_raw + PAGE_SIZE - 1) // PAGE_SIZE + total_pages = (total_raw + 4) // 5 concurrency = min(MAX_PARALLEL_FETCHES, total_pages) all_page_data: dict[int, list[Any]] = {} diff --git a/skills/polymarket-browse/tests/test_browse.py b/skills/polymarket-browse/tests/test_browse.py index 5f864c2..2865164 100644 --- a/skills/polymarket-browse/tests/test_browse.py +++ b/skills/polymarket-browse/tests/test_browse.py @@ -1118,7 +1118,7 @@ class TestFetchAllPages(unittest.TestCase): {"id": "n1", "title": "Non-match 1", "markets": []}, {"id": "n2", "title": "Non-match 2", "markets": []}, ], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } page2 = { "events": [ @@ -1131,15 +1131,15 @@ class TestFetchAllPages(unittest.TestCase): }, {"id": "n3", "title": "Non-match 3", "markets": []}, ], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } page3 = { "events": [], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } page4 = { "events": [], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } mock_fetch_page.return_value = page1 @@ -1169,15 +1169,15 @@ class TestFetchAllPages(unittest.TestCase): page1 = { "events": [{"id": "e1", "title": "Event 1", "markets": []}], - "pagination": {"totalResults": 150, "hasMore": True}, + "pagination": {"totalResults": 15, "hasMore": True}, } page2 = { "events": [{"id": "e2", "title": "Event 2", "markets": []}], - "pagination": {"totalResults": 150, "hasMore": True}, + "pagination": {"totalResults": 15, "hasMore": True}, } page3 = { "events": [{"id": "e3", "title": "Event 3", "markets": []}], - "pagination": {"totalResults": 150, "hasMore": False}, + "pagination": {"totalResults": 15, "hasMore": False}, } mock_fetch_page.return_value = page1 @@ -1223,7 +1223,7 @@ class TestFetchAllPages(unittest.TestCase): "markets": [], }, ], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } page2 = { "events": [ @@ -1231,15 +1231,15 @@ class TestFetchAllPages(unittest.TestCase): {"id": "n2", "title": "Non-match 2", "markets": []}, {"id": "n3", "title": "Non-match 3", "markets": []}, ], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } page3 = { "events": [], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } page4 = { "events": [], - "pagination": {"totalResults": 200, "hasMore": True}, + "pagination": {"totalResults": 20, "hasMore": True}, } mock_fetch_page.return_value = page1 @@ -1267,23 +1267,20 @@ class TestParallelFetchConcurrency(unittest.TestCase): def test_parallel_fetch_uses_batch_size_of_5( self, mock_fetch_page, mock_parallel_fetch, mock_cache ): - """With 10 pages (totalResults=500), verify concurrency=5 means 5 calls per batch.""" + """With 10 pages (totalResults=50), verify 10 calls are made with concurrency=5.""" from browse import fetch_all_pages page = { "events": [{"id": "e1", "title": "Event 1", "markets": []}], - "pagination": {"totalResults": 500, "hasMore": True}, + "pagination": {"totalResults": 50, "hasMore": True}, } mock_fetch_page.return_value = page mock_parallel_fetch.return_value = (1, page) result = fetch_all_pages("test", use_cache=False) - total_pages = (500 + 50 - 1) // 50 # = 10 pages - concurrency = min(5, total_pages) # = 5 - expected_batches = (total_pages + concurrency - 1) // concurrency # = 2 batches - - self.assertEqual(mock_parallel_fetch.call_count, 10) + total_pages = (50 + 4) // 5 # = 10 pages (API returns 5 per page) + self.assertEqual(mock_parallel_fetch.call_count, total_pages) @patch("browse._read_cache", return_value=None) @patch("browse._fetch_page_with_index") @@ -1394,7 +1391,7 @@ class TestMaxTotalParameter(unittest.TestCase): "markets": [], } ], - "pagination": {"totalResults": 500, "hasMore": True}, + "pagination": {"totalResults": 50, "hasMore": True}, }, ) ) @@ -1432,7 +1429,7 @@ class TestMaxTotalParameter(unittest.TestCase): "markets": [], }, ], - "pagination": {"totalResults": 100, "hasMore": True}, + "pagination": {"totalResults": 5, "hasMore": True}, } mock_fetch_page.return_value = page1 mock_parallel_fetch.side_effect = [(1, page1)] -- 2.49.1 From 09f3cb90662517d739f3fe2ce46aa4f1b1df0a0a Mon Sep 17 00:00:00 2001 From: shoko <270575765+shokollm@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:06:25 +0000 Subject: [PATCH 4/6] Add comment explaining total_pages ceiling division calculation --- skills/polymarket-browse/scripts/browse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skills/polymarket-browse/scripts/browse.py b/skills/polymarket-browse/scripts/browse.py index 63c3b8f..1dda2bb 100644 --- a/skills/polymarket-browse/scripts/browse.py +++ b/skills/polymarket-browse/scripts/browse.py @@ -235,6 +235,8 @@ def fetch_all_pages( if total_raw == 0: return {"events": [], "total_raw": 0, "partial": False} + # API always returns exactly 5 events per page regardless of 'limit' param. + # This is integer ceiling division: ceil(total_raw / 5) = (total_raw + 5 - 1) // 5 = (total_raw + 4) // 5 total_pages = (total_raw + 4) // 5 concurrency = min(MAX_PARALLEL_FETCHES, total_pages) -- 2.49.1 From 9d1e328f5308ffe1302020772e4cca052b59a415 Mon Sep 17 00:00:00 2001 From: shoko <270575765+shokollm@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:15:28 +0000 Subject: [PATCH 5/6] Make page size calculation dynamic based on first API response - Uses actual event count from page 1 to calculate total_pages - Removes hardcoded '5' for events per page - API changes to page size will be handled automatically - Updated tests to match real API behavior (5 events per page) --- skills/polymarket-browse/scripts/browse.py | 41 ++-- skills/polymarket-browse/tests/test_browse.py | 225 ++++++++++++++++-- 2 files changed, 226 insertions(+), 40 deletions(-) diff --git a/skills/polymarket-browse/scripts/browse.py b/skills/polymarket-browse/scripts/browse.py index 1dda2bb..6444739 100644 --- a/skills/polymarket-browse/scripts/browse.py +++ b/skills/polymarket-browse/scripts/browse.py @@ -220,6 +220,7 @@ def fetch_all_pages( total_raw = 0 page_count = 0 + page1_data = None while True: page_count += 1 @@ -227,33 +228,39 @@ def fetch_all_pages( if data is None: break total_raw = data.get("pagination", {}).get("totalResults", 0) + if page_count == 1: + page1_data = data if total_raw > 0: break if not data.get("events"): break - if total_raw == 0: + if total_raw == 0 or page1_data is None: return {"events": [], "total_raw": 0, "partial": False} - # API always returns exactly 5 events per page regardless of 'limit' param. - # This is integer ceiling division: ceil(total_raw / 5) = (total_raw + 5 - 1) // 5 = (total_raw + 4) // 5 - total_pages = (total_raw + 4) // 5 + page1_events = page1_data.get("events", []) + actual_page_size = len(page1_events) + + # Use actual events per page from API for ceiling division + # ceil(total_raw / actual_page_size) = (total_raw + actual_page_size - 1) // actual_page_size + total_pages = (total_raw + actual_page_size - 1) // actual_page_size concurrency = min(MAX_PARALLEL_FETCHES, total_pages) - all_page_data: dict[int, list[Any]] = {} + all_page_data: dict[int, list[Any]] = {1: page1_events} - with ThreadPoolExecutor(max_workers=concurrency) as executor: - futures = { - executor.submit(_fetch_page_with_index, q, page): page - for page in range(1, total_pages + 1) - } - for future in as_completed(futures): - try: - page_num, data = future.result() - if data is not None: - all_page_data[page_num] = data.get("events", []) - except Exception: - pass + if total_pages > 1: + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = { + executor.submit(_fetch_page_with_index, q, page): page + for page in range(2, total_pages + 1) + } + for future in as_completed(futures): + try: + page_num, data = future.result() + if data is not None: + all_page_data[page_num] = data.get("events", []) + except Exception: + pass all_events = [] for page_num in sorted(all_page_data.keys()): diff --git a/skills/polymarket-browse/tests/test_browse.py b/skills/polymarket-browse/tests/test_browse.py index 2865164..1cb3b0d 100644 --- a/skills/polymarket-browse/tests/test_browse.py +++ b/skills/polymarket-browse/tests/test_browse.py @@ -1117,6 +1117,7 @@ class TestFetchAllPages(unittest.TestCase): }, {"id": "n1", "title": "Non-match 1", "markets": []}, {"id": "n2", "title": "Non-match 2", "markets": []}, + {"id": "e1", "title": "Extra 1", "markets": []}, ], "pagination": {"totalResults": 20, "hasMore": True}, } @@ -1130,21 +1131,35 @@ class TestFetchAllPages(unittest.TestCase): "markets": [], }, {"id": "n3", "title": "Non-match 3", "markets": []}, + {"id": "e2", "title": "Extra 2", "markets": []}, + {"id": "e3", "title": "Extra 3", "markets": []}, + {"id": "e4", "title": "Extra 4", "markets": []}, ], "pagination": {"totalResults": 20, "hasMore": True}, } page3 = { - "events": [], + "events": [ + {"id": "e5", "title": "Extra 5", "markets": []}, + {"id": "e6", "title": "Extra 6", "markets": []}, + {"id": "e7", "title": "Extra 7", "markets": []}, + {"id": "e8", "title": "Extra 8", "markets": []}, + {"id": "e9", "title": "Extra 9", "markets": []}, + ], "pagination": {"totalResults": 20, "hasMore": True}, } page4 = { - "events": [], + "events": [ + {"id": "e10", "title": "Extra 10", "markets": []}, + {"id": "e11", "title": "Extra 11", "markets": []}, + {"id": "e12", "title": "Extra 12", "markets": []}, + {"id": "e13", "title": "Extra 13", "markets": []}, + {"id": "e14", "title": "Extra 14", "markets": []}, + ], "pagination": {"totalResults": 20, "hasMore": True}, } mock_fetch_page.return_value = page1 mock_parallel_fetch.side_effect = [ - (1, page1), (2, page2), (3, page3), (4, page4), @@ -1155,7 +1170,7 @@ class TestFetchAllPages(unittest.TestCase): ) self.assertEqual(mock_fetch_page.call_count, 1) - self.assertEqual(mock_parallel_fetch.call_count, 4) + self.assertEqual(mock_parallel_fetch.call_count, 3) self.assertEqual(len(result["events"]), 6) @patch("browse._read_cache", return_value=None) @@ -1168,27 +1183,45 @@ class TestFetchAllPages(unittest.TestCase): from browse import fetch_all_pages page1 = { - "events": [{"id": "e1", "title": "Event 1", "markets": []}], + "events": [ + {"id": "e1", "title": "Event 1", "markets": []}, + {"id": "e2", "title": "Event 2", "markets": []}, + {"id": "e3", "title": "Event 3", "markets": []}, + {"id": "e4", "title": "Event 4", "markets": []}, + {"id": "e5", "title": "Event 5", "markets": []}, + ], "pagination": {"totalResults": 15, "hasMore": True}, } page2 = { - "events": [{"id": "e2", "title": "Event 2", "markets": []}], + "events": [ + {"id": "e6", "title": "Event 6", "markets": []}, + {"id": "e7", "title": "Event 7", "markets": []}, + {"id": "e8", "title": "Event 8", "markets": []}, + {"id": "e9", "title": "Event 9", "markets": []}, + {"id": "e10", "title": "Event 10", "markets": []}, + ], "pagination": {"totalResults": 15, "hasMore": True}, } page3 = { - "events": [{"id": "e3", "title": "Event 3", "markets": []}], + "events": [ + {"id": "e11", "title": "Event 11", "markets": []}, + {"id": "e12", "title": "Event 12", "markets": []}, + {"id": "e13", "title": "Event 13", "markets": []}, + {"id": "e14", "title": "Event 14", "markets": []}, + {"id": "e15", "title": "Event 15", "markets": []}, + ], "pagination": {"totalResults": 15, "hasMore": False}, } mock_fetch_page.return_value = page1 - mock_parallel_fetch.side_effect = [(1, page1), (2, page2), (3, page3)] + mock_parallel_fetch.side_effect = [(2, page2), (3, page3)] result = fetch_all_pages("test", use_cache=False) self.assertEqual(mock_fetch_page.call_count, 1) - self.assertEqual(mock_parallel_fetch.call_count, 3) - self.assertEqual(len(result["events"]), 3) - self.assertTrue(result["partial"]) + self.assertEqual(mock_parallel_fetch.call_count, 2) + self.assertEqual(len(result["events"]), 15) + self.assertFalse(result["partial"]) @patch("browse._read_cache", return_value=None) @patch("browse._fetch_page_with_index") @@ -1222,6 +1255,8 @@ class TestFetchAllPages(unittest.TestCase): "gameId": "3", "markets": [], }, + {"id": "e1", "title": "Extra 1", "markets": []}, + {"id": "e2", "title": "Extra 2", "markets": []}, ], "pagination": {"totalResults": 20, "hasMore": True}, } @@ -1230,31 +1265,168 @@ class TestFetchAllPages(unittest.TestCase): {"id": "n1", "title": "Non-match 1", "markets": []}, {"id": "n2", "title": "Non-match 2", "markets": []}, {"id": "n3", "title": "Non-match 3", "markets": []}, + {"id": "e3", "title": "Extra 3", "markets": []}, + {"id": "e4", "title": "Extra 4", "markets": []}, ], "pagination": {"totalResults": 20, "hasMore": True}, } page3 = { - "events": [], + "events": [ + {"id": "e5", "title": "Extra 5", "markets": []}, + {"id": "e6", "title": "Extra 6", "markets": []}, + {"id": "e7", "title": "Extra 7", "markets": []}, + {"id": "e8", "title": "Extra 8", "markets": []}, + {"id": "e9", "title": "Extra 9", "markets": []}, + ], "pagination": {"totalResults": 20, "hasMore": True}, } page4 = { - "events": [], + "events": [ + {"id": "e10", "title": "Extra 10", "markets": []}, + {"id": "e11", "title": "Extra 11", "markets": []}, + {"id": "e12", "title": "Extra 12", "markets": []}, + {"id": "e13", "title": "Extra 13", "markets": []}, + {"id": "e14", "title": "Extra 14", "markets": []}, + ], "pagination": {"totalResults": 20, "hasMore": True}, } mock_fetch_page.return_value = page1 - mock_parallel_fetch.side_effect = [ - (1, page1), - (2, page2), - (3, page3), - (4, page4), - ] + mock_parallel_fetch.side_effect = [(2, page2), (3, page3), (4, page4)] result = fetch_all_pages( "test", matches_max=3, non_matches_max=3, use_cache=False ) - self.assertEqual(mock_parallel_fetch.call_count, 4) + self.assertEqual(mock_parallel_fetch.call_count, 3) + self.assertEqual(len(result["events"]), 6) + + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_no_quota_fetches_all_pages( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): + """Without quotas, fetches all pages until pagination ends.""" + from browse import fetch_all_pages + + page1 = { + "events": [ + {"id": "e1", "title": "Event 1", "markets": []}, + {"id": "e2", "title": "Event 2", "markets": []}, + {"id": "e3", "title": "Event 3", "markets": []}, + {"id": "e4", "title": "Event 4", "markets": []}, + {"id": "e5", "title": "Event 5", "markets": []}, + ], + "pagination": {"totalResults": 15, "hasMore": True}, + } + page2 = { + "events": [ + {"id": "e6", "title": "Event 6", "markets": []}, + {"id": "e7", "title": "Event 7", "markets": []}, + {"id": "e8", "title": "Event 8", "markets": []}, + {"id": "e9", "title": "Event 9", "markets": []}, + {"id": "e10", "title": "Event 10", "markets": []}, + ], + "pagination": {"totalResults": 15, "hasMore": True}, + } + page3 = { + "events": [ + {"id": "e11", "title": "Event 11", "markets": []}, + {"id": "e12", "title": "Event 12", "markets": []}, + {"id": "e13", "title": "Event 13", "markets": []}, + {"id": "e14", "title": "Event 14", "markets": []}, + {"id": "e15", "title": "Event 15", "markets": []}, + ], + "pagination": {"totalResults": 15, "hasMore": False}, + } + + mock_fetch_page.return_value = page1 + mock_parallel_fetch.side_effect = [(2, page2), (3, page3)] + + result = fetch_all_pages("test", use_cache=False) + + self.assertEqual(mock_fetch_page.call_count, 1) + self.assertEqual(mock_parallel_fetch.call_count, 2) + self.assertEqual(len(result["events"]), 15) + self.assertFalse(result["partial"]) + + @patch("browse._read_cache", return_value=None) + @patch("browse._fetch_page_with_index") + @patch("browse.fetch_page") + def test_quota_one_side_only_keeps_fetching( + self, mock_fetch_page, mock_parallel_fetch, mock_cache + ): + """If only one quota is met, keeps fetching.""" + from browse import fetch_all_pages + + page1 = { + "events": [ + { + "id": "m1", + "title": "Match 1", + "seriesSlug": "x", + "gameId": "1", + "markets": [], + }, + { + "id": "m2", + "title": "Match 2", + "seriesSlug": "x", + "gameId": "2", + "markets": [], + }, + { + "id": "m3", + "title": "Match 3", + "seriesSlug": "x", + "gameId": "3", + "markets": [], + }, + {"id": "e1", "title": "Extra 1", "markets": []}, + {"id": "e2", "title": "Extra 2", "markets": []}, + ], + "pagination": {"totalResults": 20, "hasMore": True}, + } + page2 = { + "events": [ + {"id": "n1", "title": "Non-match 1", "markets": []}, + {"id": "n2", "title": "Non-match 2", "markets": []}, + {"id": "n3", "title": "Non-match 3", "markets": []}, + {"id": "e3", "title": "Extra 3", "markets": []}, + {"id": "e4", "title": "Extra 4", "markets": []}, + ], + "pagination": {"totalResults": 20, "hasMore": True}, + } + page3 = { + "events": [ + {"id": "e5", "title": "Extra 5", "markets": []}, + {"id": "e6", "title": "Extra 6", "markets": []}, + {"id": "e7", "title": "Extra 7", "markets": []}, + {"id": "e8", "title": "Extra 8", "markets": []}, + {"id": "e9", "title": "Extra 9", "markets": []}, + ], + "pagination": {"totalResults": 20, "hasMore": True}, + } + page4 = { + "events": [ + {"id": "e10", "title": "Extra 10", "markets": []}, + {"id": "e11", "title": "Extra 11", "markets": []}, + {"id": "e12", "title": "Extra 12", "markets": []}, + {"id": "e13", "title": "Extra 13", "markets": []}, + {"id": "e14", "title": "Extra 14", "markets": []}, + ], + "pagination": {"totalResults": 20, "hasMore": True}, + } + + mock_fetch_page.return_value = page1 + mock_parallel_fetch.side_effect = [(2, page2), (3, page3), (4, page4)] + + result = fetch_all_pages( + "test", matches_max=3, non_matches_max=3, use_cache=False + ) + + self.assertEqual(mock_parallel_fetch.call_count, 3) self.assertEqual(len(result["events"]), 6) @@ -1271,7 +1443,13 @@ class TestParallelFetchConcurrency(unittest.TestCase): from browse import fetch_all_pages page = { - "events": [{"id": "e1", "title": "Event 1", "markets": []}], + "events": [ + {"id": "e1", "title": "Event 1", "markets": []}, + {"id": "e2", "title": "Event 2", "markets": []}, + {"id": "e3", "title": "Event 3", "markets": []}, + {"id": "e4", "title": "Event 4", "markets": []}, + {"id": "e5", "title": "Event 5", "markets": []}, + ], "pagination": {"totalResults": 50, "hasMore": True}, } mock_fetch_page.return_value = page @@ -1279,8 +1457,9 @@ class TestParallelFetchConcurrency(unittest.TestCase): result = fetch_all_pages("test", use_cache=False) - total_pages = (50 + 4) // 5 # = 10 pages (API returns 5 per page) - self.assertEqual(mock_parallel_fetch.call_count, total_pages) + total_pages = (50 + 5 - 1) // 5 # = 10 pages (API returns 5 per page) + # Page 1 is fetched in probe loop, so executor only fetches pages 2-10 (9 calls) + self.assertEqual(mock_parallel_fetch.call_count, total_pages - 1) @patch("browse._read_cache", return_value=None) @patch("browse._fetch_page_with_index") -- 2.49.1 From c75d123dfd5d070b87ef7ee4dbcacc425e334f7e Mon Sep 17 00:00:00 2001 From: shoko <270575765+shokollm@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:19:03 +0000 Subject: [PATCH 6/6] Update SKILL.md with new caching and parallel fetching documentation --- skills/polymarket-browse/SKILL.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/skills/polymarket-browse/SKILL.md b/skills/polymarket-browse/SKILL.md index 29e7078..15f9dcc 100644 --- a/skills/polymarket-browse/SKILL.md +++ b/skills/polymarket-browse/SKILL.md @@ -34,7 +34,7 @@ hermes mcp add polymarket https://docs.polymarket.com/mcp ## Usage ``` -polymarket-browse [--category "Counter Strike"] [--limit 5] [--matches N] [--non-matches N] [--search "TeamName"] [--matches-only] [--non-matches-only] [--detail N] [--raw] [--telegram] +polymarket-browse [--category "Counter Strike"] [--limit 5] [--matches N] [--non-matches N] [--search "TeamName"] [--matches-only] [--non-matches-only] [--detail N] [--raw] [--telegram] [--no-cache] [--max-total N] ``` ## Arguments @@ -49,6 +49,8 @@ polymarket-browse [--category "Counter Strike"] [--limit 5] [--matches N] [--non - `--detail` : Index of match event (1-indexed) to show detailed markets. Default: 1. Set to 0 to disable. - `--list-categories` : List available game categories and exit - `--raw` : Show all events without tradeable filter (for debugging). Includes fetch stats. +- `--no-cache` : Disable caching and fetch fresh data from the API. +- `--max-total` : Maximum total events to fetch before early exit. Default: no limit. Useful for quick snapshots. - `--telegram` : Send results to Telegram. Requires `BOT_TOKEN` and `CHAT_ID` in environment variables. ## Output Format @@ -120,11 +122,30 @@ Use `--raw` to disable the tradeable filter and see all match markets regardless The script fetches **ALL pages** until the API runs out of results (up to 100 pages as a safety cap). +### Parallel Fetching + +Pages are fetched in **parallel batches of 5** using ThreadPoolExecutor. This significantly reduces fetch time: + +| Scenario | Without Parallelization | With Parallelization | +|----------|------------------------|---------------------| +| 10 pages (50 events) | ~20s (2s per page × 10) | ~4s (2s per batch × 2 batches) | +| 20 pages (100 events) | ~40s | ~8s | + +The script first fetches page 1 to determine total pages, then fetches remaining pages in parallel batches of 5. + ## Rate Limiting - Exponential backoff: 2s → 4s → 8s → 16s → 32s - Max 5 retries before aborting +## Caching + +Results are cached in `~/.cache/polymarket-browse/` with a **5-minute TTL** to reduce redundant API calls. + +- Use `--no-cache` to bypass the cache and fetch fresh data +- Cached data is automatically used when available and not expired +- Useful when running the script repeatedly (e.g., for monitoring) + ## Odds Format All odds are shown in **cents** format: -- 2.49.1