```html Browser (vanilla JS, save as index.html) theme={"theme":{"light":"github-light","dark":"vesper"}}
Simkl Trending
Trending on Simkl
Loads simkl-trending-urls.json, wires up SimklTrendingClient, subscribes to all configured catalogs, renders one carousel per row.
```
```js JavaScript (save as simkl-trending-client.js) theme={"theme":{"light":"github-light","dark":"vesper"}}
// === Universal ES module — works in any modern browser AND Node 22+ ===
// This is the single SDK file. Save the entire block below as `simkl-trending-client.js`.
// The "Browser (vanilla JS)" tab above imports from this exact file.
// For Node: also add `"type": "module"` to your package.json (or save as `.mjs`).
//
// Then pass it the URL catalog object — three ways to load that:
// Browser / Deno / CF Workers: const data = await (await fetch('./simkl-trending-urls.json')).json();
// Node 22+ (ESM): import data from './simkl-trending-urls.json' with { type: 'json' };
// Node (CommonJS .cjs) const data = require('./simkl-trending-urls.json');
//
// Then: `new SimklTrendingClient(config, data)` or set `globalThis.SIMKL_TRENDING_URLS = data` first.
// === SIMKL_TRENDING_DEFAULTS — tune to match your app ===
export const SIMKL_TRENDING_DEFAULTS = {
// What to load
supportedTypes: ['movies', 'tv', 'anime', 'dvd'], // movies / tv / anime / dvd — drop ones your app doesn't surface
timeframes: ['today'], // today / week / month (ignored for dvd)
size: 100, // 100 or 500 — use 500 if you render derived catalogs (see "Build your own catalogs" below)
// HTTP — replace with YOUR values
clientId: 'YOUR_CLIENT_ID',
appName: 'my-app-name',
appVersion: '1.0',
userAgent: 'my-app-name/1.0',
requestTimeoutMs: 8_000,
minIntervalMs: 250, // process-wide floor between consecutive requests
fetchRetry: {
maxAttempts: 3,
baseBackoffMs: 1_000,
maxBackoffMs: 16_000,
retryStatuses: [408, 425, 429, 500, 502, 503, 504], // never retry other 4xx — they're permanent
},
// Refresh
regenRetrySeconds: [300, 900, 1_800], // 5 / 15 / 30 min after a 304 (server not regenerated yet)
jitterPercent: 0.05, // ±5% so all clients don't hit the CDN at exactly :00
// Persistent cache. Browser: localStorage[storageKey]. Node: file at persistPath.
// Set BOTH to null to disable. For server-side runtimes with many processes, swap
// this for a Redis / KV adapter (see _persist / _restorePersisted).
storageKey: 'simkl-trending-cache',
persistPath: './.simkl-trending-cache.json',
persistMaxAgeSeconds: 7 * 86_400, // discard cache older than 7d
};
export class SimklTrendingError extends Error {
constructor(message, { status, retryAfterMs, permanent } = {}) {
super(message);
this.name = 'SimklTrendingError';
this.status = status ?? null;
this.retryAfterMs = retryAfterMs ?? null;
this.permanent = Boolean(permanent);
}
}
export class SimklTrendingClient {
constructor(config = {}, data = globalThis.SIMKL_TRENDING_URLS) {
if (!data) throw new SimklTrendingError('Pass the URL catalog (SIMKL_TRENDING_URLS) as the second arg or set it globally.');
this.cfg = { ...SIMKL_TRENDING_DEFAULTS, ...config };
this.urls = data.urls ?? data; // unwrap if { urls: {...}, catalogs: [...] } shape
this.defaultCatalogs = data.catalogs ?? null; // pre-defined rows from the JSON
this._cache = new Map();
this._inflight = new Map();
this._subs = new Map();
this._timers = new Set();
this._abort = new AbortController();
this._stopped = false;
this._lastRequestAt = 0;
}
// === Public API ===
async start() {
await this._restorePersisted();
for (const t of new Set(this._subs.keys())) this._notify(t); // paint immediately from restored cache
for (const entry of this._plan()) this._loop(entry);
}
stop() {
this._stopped = true;
this._abort.abort();
for (const t of this._timers) clearTimeout(t);
this._timers.clear();
}
get(type) {
for (const v of this._cache.values()) if (v.itemsByType[type]) return v.itemsByType[type];
return [];
}
subscribe(type, cb) {
if (!this._subs.has(type)) this._subs.set(type, new Set());
this._subs.get(type).add(cb);
const items = this.get(type);
if (items.length) cb(items); // immediate paint from cache
return () => this._subs.get(type)?.delete(cb);
}
// === Catalogs — rows on the home screen ===
catalogs() { return this._catalogConfigs().map(c => this._buildCatalog(c)); }
subscribeCatalogs(cb) {
const sources = new Set(this._catalogConfigs().map(c => c.source));
const unsubs = [];
let primed = false;
// Coalesce: a combined fetch notifies 3 types (movies/tv/anime) in one tick;
// cache-restore fires per subscribed source. queueMicrotask flushes once per tick.
let queued = false;
const fire = () => {
if (queued) return;
queued = true;
queueMicrotask(() => { queued = false; cb(this.catalogs()); });
};
for (const s of sources) unsubs.push(this.subscribe(s, () => { primed = true; fire(); }));
if (!primed) fire(); // empty cache — skeleton paint
return () => unsubs.forEach(u => u());
}
_catalogConfigs() {
if (Array.isArray(this.cfg.catalogs)) return this.cfg.catalogs; // explicit override wins
if (Array.isArray(this.defaultCatalogs)) return this.defaultCatalogs; // pulled from JSON's catalogs[] (recommended path)
const tf0 = this.cfg.timeframes?.[0] ?? 'today'; // last resort: auto-build
return this.cfg.supportedTypes.map(t => t === 'dvd'
? { id: 'dvd', source: 'dvd' }
: { id: `${t}_${tf0}`, source: t, timeframe: tf0 });
}
_fileTitleFor(source, timeframe) {
const sizeKey = `top_${this.cfg.size}`;
const file = source === 'dvd' ? this.urls.dvd?.[sizeKey] : this.urls?.[source]?.[timeframe]?.[sizeKey];
return file?.title ?? `Trending ${source} on Simkl`;
}
_buildCatalog(c) {
if (!c.id || !c.source) throw new SimklTrendingError('catalogs[] entries need `id` and `source`');
const items = this.get(c.source);
let out = items;
if (typeof c.transform === 'function') { // inline closure (per-app overrides)
out = c.transform(items);
} else if (typeof c.recipe === 'string') { // JSON catalogs reference by name
const fn = SimklTrendingRecipes[c.recipe];
if (typeof fn === 'function') out = fn(items, c.recipe_opts || {});
else console.warn(`SimklTrendingClient: unknown recipe "${c.recipe}" for catalog "${c.id}" — rendering raw items.`);
}
return { id: c.id, title: c.title ?? this._fileTitleFor(c.source, c.timeframe ?? this.cfg.timeframes?.[0] ?? 'today'), items: out };
}
// Call from a background scheduler (node-cron, Cloudflare Cron, AWS EventBridge) for a single pass.
async tick() {
await this._restorePersisted();
for (const entry of this._plan()) {
const cached = this._cache.get(entry.url);
const r = await this._fetch(entry, cached?.lastModified);
if (!r.notModified) this._storeAndNotify(entry, r);
}
await this._persist();
}
// === Internals ===
_requiredQuery() {
const { clientId, appName, appVersion } = this.cfg;
return `client_id=${clientId}&app-name=${encodeURIComponent(appName)}&app-version=${encodeURIComponent(appVersion)}`;
}
_plan() {
const { supportedTypes, timeframes, size } = this.cfg;
if (size !== 100 && size !== 500) throw new SimklTrendingError('size must be 100 or 500');
const key = `top_${size}`;
const tvma = supportedTypes.filter(t => ['movies','tv','anime'].includes(t));
const useCombined = tvma.length >= 2; // 1 round-trip < 2-3 per timeframe
const plan = [];
if (tvma.length) {
for (const tf of timeframes) {
const file = useCombined ? this.urls.combined[tf][key]
: this.urls[tvma[0]][tf][key];
plan.push({ ...file, types: tvma });
}
}
if (supportedTypes.includes('dvd')) plan.push({ ...this.urls.dvd[key], types: ['dvd'] });
return plan;
}
// Process-wide rate-limit floor (so N entries refreshing at once don't fire N simultaneous requests).
async _throttle() {
const gap = Date.now() - this._lastRequestAt;
if (gap < this.cfg.minIntervalMs) await this._sleep((this.cfg.minIntervalMs - gap) / 1_000);
this._lastRequestAt = Date.now();
}
static _parseRetryAfter(value) {
if (!value) return null;
const secs = Number(value);
if (Number.isFinite(secs)) return Math.max(0, secs * 1_000);
const dateMs = Date.parse(value);
return Number.isFinite(dateMs) ? Math.max(0, dateMs - Date.now()) : null;
}
// _fetch — dedup + retry + timeout + conditional revalidation + Retry-After. Normalizes array vs object response into itemsByType.
async _fetch(entry, lastModified) {
const sep = entry.url.includes('?') ? '&' : '?';
const url = entry.url + sep + this._requiredQuery();
if (this._inflight.has(url)) return this._inflight.get(url);
// In Node/Deno/Bun/CF Workers, set If-Modified-Since manually for conditional revalidation.
// In BROWSERS, the manual header would trigger a CORS preflight (If-Modified-Since isn't
// CORS-safelisted), and most CDNs respond to OPTIONS with 405 → preflight fails → fetch
// fails. Instead we use `cache: 'no-cache'` so the browser handles revalidation via its
// own HTTP cache (browser-added If-None-Match / If-Modified-Since don't count as author
// headers and skip preflight). User-Agent is dropped in browsers too — it's a forbidden
// header per Fetch spec and silently ignored.
const isBrowser = typeof window !== 'undefined';
const headers = { 'Accept': 'application/json' };
if (!isBrowser) {
headers['User-Agent'] = this.cfg.userAgent;
if (lastModified) headers['If-Modified-Since'] = lastModified;
}
const { fetchRetry, requestTimeoutMs } = this.cfg;
const run = async () => {
let lastErr;
for (let attempt = 0; attempt < fetchRetry.maxAttempts; attempt++) {
await this._throttle();
const ctl = new AbortController();
const time = setTimeout(() => ctl.abort(), requestTimeoutMs);
const off = () => ctl.abort();
this._abort.signal.addEventListener('abort', off);
try {
const fetchOpts = { headers, signal: ctl.signal };
if (isBrowser) fetchOpts.cache = 'no-cache';
const res = await fetch(url, fetchOpts);
if (res.status === 304) return { itemsByType: null, lastModified, notModified: true };
if (!res.ok) {
const retryAfterMs = SimklTrendingClient._parseRetryAfter(res.headers.get('Retry-After'));
const permanent = !fetchRetry.retryStatuses.includes(res.status);
throw new SimklTrendingError(`${res.status} ${res.statusText} — ${entry.title}`,
{ status: res.status, retryAfterMs, permanent });
}
// In browsers with cache:'no-cache', fetch resolves to 200 + cached body when the
// server returned 304 to the browser's conditional request. Detect that via
// Last-Modified equality so we skip a no-op re-render.
const newLastModified = res.headers.get('Last-Modified');
if (isBrowser && lastModified && newLastModified === lastModified) {
return { itemsByType: null, lastModified, notModified: true };
}
const raw = await res.json();
// Combined endpoint returns {movies, tv, anime}. Keep ALL keys (not just wanted types) so
// a dev who later adds 'anime' to supportedTypes can serve it from the same cached response.
const itemsByType = Array.isArray(raw)
? { [entry.types[0]]: raw } // per-type / dvd → array
: raw; // combined → store full object
return { itemsByType, lastModified: newLastModified, notModified: false };
} catch (e) {
if (this._stopped) throw e;
if (e instanceof SimklTrendingError && e.permanent) throw e; // don't waste backoff on 404/400/etc
lastErr = e;
const honored = (e instanceof SimklTrendingError && e.retryAfterMs != null) ? e.retryAfterMs : null;
const backoff = honored ?? Math.min(fetchRetry.baseBackoffMs * (2 ** attempt), fetchRetry.maxBackoffMs);
const jitter = backoff * (Math.random() * 0.2 - 0.1); // ±10% jitter on retry delay
await this._sleep(Math.max(0, backoff + jitter) / 1_000);
} finally {
clearTimeout(time);
this._abort.signal.removeEventListener('abort', off);
}
}
throw lastErr;
};
const p = run().finally(() => this._inflight.delete(url));
this._inflight.set(url, p);
return p;
}
async _loop(entry) {
let lastModified = this._cache.get(entry.url)?.lastModified ?? null;
let retry = 0;
while (!this._stopped) {
let r;
try { r = await this._fetch(entry, lastModified); }
catch (e) {
if (this._stopped) return;
if (e instanceof SimklTrendingError && e.permanent) { // 404 / 401 / etc — halt this entry's loop
console.error(`${entry.title}: ${e.message} (permanent, halting loop)`);
return;
}
console.error(`${entry.title}:`, e.message);
await this._sleep(30); continue;
}
let wait;
if (r.notModified) {
wait = this.cfg.regenRetrySeconds[Math.min(retry++, this.cfg.regenRetrySeconds.length - 1)];
} else {
this._storeAndNotify(entry, r);
lastModified = r.lastModified;
retry = 0;
await this._persist();
const j = this.cfg.jitterPercent;
wait = Math.round(entry.refresh_seconds * (1 + (Math.random() - 0.5) * 2 * j));
}
await this._sleep(wait);
}
}
_storeAndNotify(entry, r) {
this._cache.set(entry.url, { ...r, types: entry.types, fetchedAt: Date.now() });
for (const t of entry.types) this._notify(t);
}
_notify(type) { const items = this.get(type); for (const cb of (this._subs.get(type) ?? [])) cb(items); }
_sleep(seconds) {
return new Promise(resolve => {
const t = setTimeout(() => { this._timers.delete(t); resolve(); }, seconds * 1_000);
this._timers.add(t);
});
}
// Wipe in-memory + persistent cache and notify subscribers (so UI repaints empty).
// Call client.tick() after this if you want to immediately re-fetch.
async clearCache() {
this._cache.clear();
const ls = this._localStorage();
if (ls && this.cfg.storageKey) { try { ls.removeItem(this.cfg.storageKey); } catch { /* ignore */ } }
if (this.cfg.persistPath && !ls) {
try { const fs = await import('node:fs/promises'); await fs.unlink(this.cfg.persistPath); }
catch { /* no file — fine */ }
}
for (const t of this._subs.keys()) this._notify(t);
}
// Persistence — auto-detects environment. Browser → localStorage. Node → file.
// Swap these two methods (plus clearCache) for a Redis / KV adapter in server-side runtimes.
_localStorage() { try { return globalThis.localStorage ?? null; } catch { return null; } }
async _persist() {
const data = JSON.stringify(Object.fromEntries(this._cache));
const ls = this._localStorage();
if (ls && this.cfg.storageKey) { try { ls.setItem(this.cfg.storageKey, data); } catch { /* quota etc */ } return; }
if (!this.cfg.persistPath) return;
try {
const fs = await import('node:fs/promises');
await fs.writeFile(this.cfg.persistPath, data);
} catch { /* best-effort */ }
}
async _restorePersisted() {
const cutoff = Date.now() - this.cfg.persistMaxAgeSeconds * 1_000;
let raw = null;
const ls = this._localStorage();
if (ls && this.cfg.storageKey) {
try { const s = ls.getItem(this.cfg.storageKey); if (s) raw = JSON.parse(s); } catch { /* corrupt — ignore */ }
}
if (!raw && this.cfg.persistPath && !ls) {
try {
const fs = await import('node:fs/promises');
raw = JSON.parse(await fs.readFile(this.cfg.persistPath, 'utf8'));
} catch { /* no cache yet — fine */ }
}
if (!raw) return;
for (const [url, val] of Object.entries(raw)) {
if (val.fetchedAt && val.fetchedAt >= cutoff) this._cache.set(url, val);
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// SimklTrendingRecipes — the catalog transforms referenced by name from the
// JSON's `catalogs[]` array. Pure functions: `(items, opts?) => items`.
// ─────────────────────────────────────────────────────────────────────────────
export const SimklTrendingRecipes = {
// Sorts
byRank(items) { return [...items].sort((a, b) => (a.rank ?? Infinity) - (b.rank ?? Infinity)); },
// Sorts by plan_to_watch desc — i.e. how many users have added the item to their watchlist.
// Despite the historical "mostAnticipated" name, this fires for already-released titles too
// (Game of Thrones, Avatar, One Piece — all rank high because they have huge backlog appeal).
// `mostWatchlisted` is the accurate name; `mostAnticipated` is kept as an alias for back-compat.
mostWatchlisted(items) { return [...items].sort((a, b) => (b.plan_to_watch ?? 0) - (a.plan_to_watch ?? 0)); },
mostAnticipated(items) { return [...items].sort((a, b) => (b.plan_to_watch ?? 0) - (a.plan_to_watch ?? 0)); }, // alias of mostWatchlisted
newest(items) { return [...items].filter(i => i.release_date).sort((a, b) => +new Date(b.release_date) - +new Date(a.release_date)); },
stickiest(items) { return [...items].sort((a, b) => parseFloat(a.drop_rate ?? 'Infinity') - parseFloat(b.drop_rate ?? 'Infinity')); },
highestRated(items, { source = 'simkl', minVotes = 500 } = {}) {
return items.filter(i => (i.ratings?.[source]?.votes ?? 0) >= minVotes)
.sort((a, b) => (b.ratings?.[source]?.rating ?? 0) - (a.ratings?.[source]?.rating ?? 0));
},
bestBoxOffice(items) { // movies only
const bo = i => { const m = (i.metadata || '').match(/Box office \$([\d.]+)([MBK])/);
return m ? parseFloat(m[1]) * { K: 1e3, M: 1e6, B: 1e9 }[m[2]] : null; };
return items.map(i => ({ i, v: bo(i) })).filter(x => x.v != null).sort((a, b) => b.v - a.v).map(x => x.i);
},
// Composed (multiple signals)
// Hidden gems = well-rated titles OUTSIDE Simkl's top N by all-time popularity. The
// `watched` field on trending feeds is last-24h viewers (not lifetime), so filtering on
// it surfaces mainstream titles (The Office, Interstellar) instead of actual gems.
// `rank` is the right gate: mainstream shows rank low (~50-500), obscure-but-loved
// ones rank in the thousands. Sort by rating so the best gems surface first.
hiddenGems(items, { minRating = 7.5, minVotes = 500, minRank = 2000, source = 'simkl' } = {}) {
return items
.filter(i => (i.rank ?? 0) > minRank &&
(i.ratings?.[source]?.rating ?? 0) >= minRating &&
(i.ratings?.[source]?.votes ?? 0) >= minVotes)
.sort((a, b) => (b.ratings?.[source]?.rating ?? 0) -
(a.ratings?.[source]?.rating ?? 0));
},
criticallyAcclaimed(items, { minRating = 8.5, minVotes = 1000, source = 'simkl' } = {}) {
return items.filter(i => (i.ratings?.[source]?.rating ?? 0) >= minRating &&
(i.ratings?.[source]?.votes ?? 0) >= minVotes)
.sort((a, b) => (b.ratings?.[source]?.rating ?? 0) - (a.ratings?.[source]?.rating ?? 0));
},
marathonWorthy(items, { minEpisodes = 20, maxEpisodes = 100, minRating = 8, source = 'simkl' } = {}) {
return items.filter(i => i.status === 'ended' &&
(i.total_episodes ?? 0) >= minEpisodes && (i.total_episodes ?? 0) <= maxEpisodes &&
(i.ratings?.[source]?.rating ?? 0) >= minRating);
},
crossPlatformConsensus(items, { simklMin = 8, otherMin = 8 } = {}) {
return items.filter(i => (i.ratings?.simkl?.rating ?? 0) >= simklMin &&
((i.ratings?.imdb?.rating ?? i.ratings?.mal?.rating) ?? 0) >= otherMin);
},
massiveAnticipation(items, { minRatio = 300 } = {}) {
return items.filter(i => (i.plan_to_watch ?? 0) / ((i.watched ?? 0) + 1) >= minRatio)
.sort((a, b) => (b.plan_to_watch / (b.watched + 1)) - (a.plan_to_watch / (a.watched + 1)));
},
bestOfNetwork(items, { network, minRating = 7.5, requireEnded = false, source = 'simkl' } = {}) {
return items.filter(i => i.network === network &&
(!requireEnded || i.status === 'ended') &&
(i.ratings?.[source]?.rating ?? 0) >= minRating)
.sort((a, b) => (b.ratings?.[source]?.rating ?? 0) - (a.ratings?.[source]?.rating ?? 0));
},
bestOfGenre(items, { genre, minRating = 7.5, minVotes = 300, source = 'simkl' } = {}) {
return items.filter(i => i.genres?.includes(genre) &&
(i.ratings?.[source]?.rating ?? 0) >= minRating &&
(i.ratings?.[source]?.votes ?? 0) >= minVotes)
.sort((a, b) => (b.ratings?.[source]?.rating ?? 0) - (a.ratings?.[source]?.rating ?? 0));
},
// Filters (all use { ...opts } so they're JSON-describable)
byGenre (items, { genre } = {}) { return items.filter(i => i.genres?.includes(genre)); },
byCountry (items, { country } = {}) { const c = String(country || '').toLowerCase(); return items.filter(i => i.country === c); },
byNetwork (items, { network } = {}) { return items.filter(i => i.network === network); },
byStatus (items, { status } = {}) { return items.filter(i => i.status === status); },
byAnimeType(items, { animeType } = {}) { return items.filter(i => i.anime_type === animeType); },
withTrailer(items) { return items.filter(i => i.trailer); },
ongoing(items) { return items.filter(i => i.status === 'ongoing'); },
recentlyEnded(items) { return items.filter(i => i.status === 'ended'); },
// `status === 'premiere'` flags items whose first episode JUST aired (typically within ~30 days).
// It is NOT an "upcoming premieres" filter — those items have status 'ongoing' or no status set.
justPremiered(items) { return items.filter(i => i.status === 'premiere'); },
premiering(items) { return items.filter(i => i.status === 'premiere'); }, // alias of justPremiered (kept for back-compat)
miniSeries(items, { maxEpisodes = 6 } = {}) { return items.filter(i => (i.total_episodes ?? Infinity) <= maxEpisodes); },
justReleased(items, { withinDays = 30 } = {}) {
const cutoff = Date.now() - withinDays * 86_400_000;
return items.filter(i => i.release_date && +new Date(i.release_date) >= cutoff);
},
inTheatersNow(items, { withinDays = 30 } = {}) { // movies only
const cutoff = Date.now() - withinDays * 86_400_000;
return items.filter(i => i.theater && +new Date(i.theater) >= cutoff);
},
recentlyOnDvd(items, { withinDays = 30 } = {}) { // movies only
const cutoff = Date.now() - withinDays * 86_400_000;
return items.filter(i => i.dvd_date && +new Date(i.dvd_date) >= cutoff);
},
// Era filters — release_date in trending feeds is "MM/DD/YYYY"; the regex grabs the 4-digit year either way.
byYear (items, { year } = {}) { return items.filter(i => +String(i.release_date||'').match(/\d{4}/)?.[0] === year); },
byDecade (items, { decade } = {}) { return items.filter(i => { const y = +String(i.release_date||'').match(/\d{4}/)?.[0]; return y >= decade && y < decade + 10; }); },
currentYear(items) { const y = new Date().getFullYear(); return items.filter(i => +String(i.release_date||'').match(/\d{4}/)?.[0] === y); },
lastYear (items) { const y = new Date().getFullYear()-1; return items.filter(i => +String(i.release_date||'').match(/\d{4}/)?.[0] === y); },
bestOfDecade(items, { decade, source = 'simkl', minVotes = 100 } = {}) { // decade + rating sort, mirrors bestOfNetwork/bestOfGenre
return items.filter(i => { const y = +String(i.release_date||'').match(/\d{4}/)?.[0];
return y >= decade && y < decade + 10 && (i.ratings?.[source]?.votes ?? 0) >= minVotes; })
.sort((a, b) => (b.ratings?.[source]?.rating ?? 0) - (a.ratings?.[source]?.rating ?? 0));
},
quickWatches(items, { maxMinutes = 90 } = {}) { // movies only
const m = s => { if (!s) return Infinity;
const x = s.match(/^(?:(\d+)h)?\s*(?:(\d+)m)?$/);
return x ? ((parseInt(x[1]||'0')*60) + parseInt(x[2]||'0')) || Infinity : Infinity; };
return items.filter(i => m(i.runtime) <= maxMinutes);
},
};
/* === Sample usage (put this in a SEPARATE file — app.js / main.mjs / etc.) ===
import data from './simkl-trending-urls.json' with { type: 'json' };
import { SimklTrendingClient } from './simkl-trending-client.js';
const client = new SimklTrendingClient({ clientId: 'YOUR_CLIENT_ID', appName: 'my-app', appVersion: '1.0' }, data);
client.subscribeCatalogs(rows => {
for (const row of rows) console.log(`${row.title}: ${row.items.length} items`);
});
await client.start();
// process.on('SIGINT', () => { client.stop(); process.exit(0); });
=== Background scheduling (server-side runtimes) ===
For long-running servers, the in-process loop is enough. For serverless / cron-triggered
runtimes (Cloudflare Workers, AWS Lambda, GitHub Actions), wire `await client.tick()` to
your scheduler instead of calling `start()`. For browser usage, see the Browser tab above. */
```
```python Python theme={"theme":{"light":"github-light","dark":"vesper"}}
import email.utils, json, os, random, threading, time, urllib.parse
import requests
# === Load the URL catalog ===
# 1. Save the "Copy-paste JSON" block above as `simkl-trending-urls.json` in your project.
# 2. Load it into SIMKL_TRENDING_URLS:
# with open('simkl-trending-urls.json') as f: SIMKL_TRENDING_URLS = json.load(f)
# Or for a packaged module:
# import importlib.resources, json
# SIMKL_TRENDING_URLS = json.loads(importlib.resources.files('your_pkg').joinpath('simkl-trending-urls.json').read_text())
# You can also pass it explicitly: SimklTrendingClient(urls=urls_dict)
# SimklTrendingClient raises SimklTrendingError if it can't find the catalog in either place.
# === SIMKL_TRENDING_DEFAULTS — tune to match your app ===
SIMKL_TRENDING_DEFAULTS = {
# What to load
'supported_types': ['movies', 'tv', 'anime', 'dvd'], # movies / tv / anime / dvd — drop ones your app doesn't surface
'timeframes': ['today'], # today / week / month (ignored for dvd)
'size': 100, # 100 or 500 — use 500 if you render derived catalogs
# HTTP — replace with YOUR values
'client_id': 'YOUR_CLIENT_ID',
'app_name': 'my-app-name',
'app_version': '1.0',
'user_agent': 'my-app-name/1.0',
'request_timeout_s': 8,
'min_interval_s': 0.25, # process-wide floor between consecutive requests
'fetch_retry': {
'max_attempts': 3,
'base_backoff_s': 1,
'max_backoff_s': 16,
'retry_statuses': [408, 425, 429, 500, 502, 503, 504], # never retry other 4xx — they're permanent
},
# Refresh
'regen_retry_s': [300, 900, 1800], # 5 / 15 / 30 min after a 304 (server not regenerated yet)
'jitter_percent': 0.05, # ±5% so all clients don't hit the CDN at exactly :00
# Persistent cache — set persist_path to None to disable. For server-side runtimes,
# swap _persist / _restore_persisted for a Redis / KV adapter.
'persist_path': './.simkl-trending-cache.json',
'persist_max_age_s': 7 * 86400, # discard cache older than 7d
# Catalogs — the rows on your home screen. Each becomes a {'id', 'title', 'items'} dict
# returned by client.catalogs() / pushed to client.subscribe_catalogs(cb). If None,
# one row per supported_type with the JSON's pre-composed title.
# { 'id', 'source', 'timeframe'?, 'title'?, 'transform'? }
'catalogs': None,
}
class SimklTrendingError(Exception):
def __init__(self, message, status=None, retry_after_s=None, permanent=False):
super().__init__(message)
self.status = status
self.retry_after_s = retry_after_s
self.permanent = bool(permanent)
class SimklTrendingClient:
def __init__(self, config=None, data=None):
data = data or globals().get('SIMKL_TRENDING_URLS')
if not data:
raise SimklTrendingError('Pass the URL catalog (SIMKL_TRENDING_URLS) as the second arg or define it globally.')
self.cfg = {**SIMKL_TRENDING_DEFAULTS, **(config or {})}
self.urls = data.get('urls', data) # unwrap {urls: {...}, catalogs: [...]} shape
self.default_catalogs = data.get('catalogs') # pre-defined rows from JSON
self._cache = {}
self._inflight = {}
self._subs = {}
self._lock = threading.RLock()
self._stopped = threading.Event()
self._threads = []
self._last_request_at = 0.0
# === Public API ===
def start(self):
self._restore_persisted()
with self._lock:
for t in list(self._subs):
self._notify(t) # paint immediately from restored cache
for entry in self._plan():
th = threading.Thread(target=self._loop, args=(entry,), daemon=True)
th.start()
self._threads.append(th)
def stop(self):
self._stopped.set()
# Wipe in-memory + persistent cache and notify subscribers (so UI repaints empty).
# Call client.tick() after this to immediately re-fetch.
def clear_cache(self):
with self._lock:
self._cache.clear()
subs_snapshot = {t: list(cbs) for t, cbs in self._subs.items()}
if self.cfg.get('persist_path'):
try: os.remove(self.cfg['persist_path'])
except OSError: pass # no file — fine
for t, cbs in subs_snapshot.items():
for cb in cbs: cb([])
def get(self, type_):
with self._lock:
for v in self._cache.values():
if type_ in v['items_by_type']:
return v['items_by_type'][type_]
return []
def subscribe(self, type_, cb):
with self._lock:
self._subs.setdefault(type_, []).append(cb)
items = self.get(type_)
if items:
cb(items) # immediate paint from cache
def unsub():
with self._lock:
if cb in self._subs.get(type_, []): self._subs[type_].remove(cb)
return unsub
# Resolve all configured catalogs to [{'id','title','items'}, ...] — TV-home-screen rows.
def catalogs(self):
return [self._build_catalog(c) for c in self._catalog_configs()]
# Subscribe to ALL catalog rows at once. cb([{...}, ...]) fires on every refresh.
def subscribe_catalogs(self, cb):
sources = {c['source'] for c in self._catalog_configs()}
unsubs = []
def fire(_): cb(self.catalogs())
for s in sources:
unsubs.append(self.subscribe(s, fire))
if not any(self.get(s) for s in sources):
cb(self.catalogs()) # empty cache — fire once for skeleton paint
def unsub():
for u in unsubs: u()
return unsub
# Call from a background scheduler (cron, Celery beat, APScheduler) for a single pass.
def tick(self):
self._restore_persisted()
for entry in self._plan():
with self._lock: cached = self._cache.get(entry['url'])
r = self._fetch(entry, cached and cached['last_modified'])
if not r['not_modified']: self._store_and_notify(entry, r)
self._persist()
# === Internals ===
def _required_query(self):
c = self.cfg
return (f"client_id={c['client_id']}"
f"&app-name={urllib.parse.quote(c['app_name'])}"
f"&app-version={urllib.parse.quote(c['app_version'])}")
def _plan(self):
c = self.cfg
if c['size'] not in (100, 500): raise SimklTrendingError('size must be 100 or 500')
key = f"top_{c['size']}"
tvma = [t for t in c['supported_types'] if t in ('movies', 'tv', 'anime')]
use_combined = len(tvma) >= 2 # 1 round-trip < 2-3 per timeframe
plan = []
if tvma:
for tf in c['timeframes']:
f = (self.urls['combined'][tf][key] if use_combined
else self.urls[tvma[0]][tf][key])
plan.append({**f, 'types': tvma})
if 'dvd' in c['supported_types']:
plan.append({**self.urls['dvd'][key], 'types': ['dvd']})
return plan
# Process-wide rate-limit floor (so N entries refreshing at once don't fire N simultaneous requests).
def _throttle(self):
with self._lock:
gap = time.monotonic() - self._last_request_at
if gap < self.cfg['min_interval_s']:
if self._stopped.wait(self.cfg['min_interval_s'] - gap): return False
self._last_request_at = time.monotonic()
return True
@staticmethod
def _parse_retry_after(value):
if not value: return None
try: return max(0.0, float(value))
except (TypeError, ValueError): pass
try:
dt = email.utils.parsedate_to_datetime(value)
return max(0.0, dt.timestamp() - time.time())
except (TypeError, ValueError): return None
# _fetch — dedup + retry + timeout + If-Modified-Since + Retry-After. Normalizes list vs dict into items_by_type.
def _fetch(self, entry, last_modified):
sep = '&' if '?' in entry['url'] else '?'
url = entry['url'] + sep + self._required_query()
with self._lock:
inflight = self._inflight.get(url)
if inflight:
inflight['done'].wait()
if inflight.get('error'): raise inflight['error']
return inflight['result']
done = threading.Event()
slot = {'done': done, 'result': None, 'error': None}
with self._lock: self._inflight[url] = slot
try:
headers = {'User-Agent': self.cfg['user_agent'], 'Accept': 'application/json'}
if last_modified: headers['If-Modified-Since'] = last_modified
retry_cfg = self.cfg['fetch_retry']
last_err = None
for attempt in range(retry_cfg['max_attempts']):
self._throttle()
try:
res = requests.get(url, headers=headers, timeout=self.cfg['request_timeout_s'])
if res.status_code == 304:
slot['result'] = {'items_by_type': None, 'last_modified': last_modified, 'not_modified': True}
return slot['result']
if not res.ok:
permanent = res.status_code not in retry_cfg['retry_statuses']
retry_after_s = self._parse_retry_after(res.headers.get('Retry-After'))
raise SimklTrendingError(f"{res.status_code} {res.reason} — {entry['title']}",
status=res.status_code, retry_after_s=retry_after_s, permanent=permanent)
raw = res.json()
items_by_type = ({entry['types'][0]: raw} if isinstance(raw, list) # per-type / dvd → list
else {t: raw.get(t, []) for t in entry['types']}) # combined → dict, filter to wanted types
slot['result'] = {'items_by_type': items_by_type,
'last_modified': res.headers.get('Last-Modified'),
'not_modified': False}
return slot['result']
except SimklTrendingError as e:
if e.permanent: slot['error'] = e; raise # don't waste backoff on permanent errors
last_err = e
honored = e.retry_after_s if e.retry_after_s is not None else None
backoff = honored if honored is not None else min(retry_cfg['base_backoff_s'] * (2 ** attempt), retry_cfg['max_backoff_s'])
backoff += backoff * (random.random() * 0.2 - 0.1) # ±10% jitter
if self._stopped.wait(max(0, backoff)): raise RuntimeError('stopped')
except Exception as e:
if self._stopped.is_set(): raise
last_err = e
backoff = min(retry_cfg['base_backoff_s'] * (2 ** attempt), retry_cfg['max_backoff_s'])
if self._stopped.wait(backoff): raise RuntimeError('stopped')
slot['error'] = last_err
raise last_err
finally:
done.set()
with self._lock: self._inflight.pop(url, None)
def _loop(self, entry):
with self._lock: c = self._cache.get(entry['url'])
last_modified = c['last_modified'] if c else None
retry = 0
while not self._stopped.is_set():
try:
r = self._fetch(entry, last_modified)
except SimklTrendingError as e:
if self._stopped.is_set(): return
if e.permanent: # 404 / 401 / etc — halt this entry's loop
print(f"{entry['title']}: {e} (permanent, halting loop)")
return
print(f"{entry['title']}: {e}")
if self._stopped.wait(30): return
continue
except Exception as e:
if self._stopped.is_set(): return
print(f"{entry['title']}: {e}")
if self._stopped.wait(30): return
continue
if r['not_modified']:
wait = self.cfg['regen_retry_s'][min(retry, len(self.cfg['regen_retry_s']) - 1)]
retry += 1
else:
self._store_and_notify(entry, r)
last_modified = r['last_modified']
retry = 0
self._persist()
j = self.cfg['jitter_percent']
wait = round(entry['refresh_seconds'] * (1 + (random.random() - 0.5) * 2 * j))
if self._stopped.wait(wait): return
def _store_and_notify(self, entry, r):
with self._lock:
self._cache[entry['url']] = {**r, 'types': entry['types'], 'fetched_at': time.time()}
for t in entry['types']: self._notify(t)
def _notify(self, type_):
with self._lock:
cbs = list(self._subs.get(type_, []))
items = self.get(type_)
for cb in cbs: cb(items)
# === Catalog helpers ===
def _catalog_configs(self):
if self.cfg.get('catalogs'): return self.cfg['catalogs'] # explicit override wins
if self.default_catalogs: return self.default_catalogs # JSON's catalogs (recommended path)
tf0 = (self.cfg.get('timeframes') or ['today'])[0] # last resort: auto-build
out = []
for t in self.cfg['supported_types']:
if t == 'dvd': out.append({'id': 'dvd', 'source': 'dvd'})
else: out.append({'id': f'{t}_{tf0}', 'source': t, 'timeframe': tf0})
return out
def _file_title_for(self, source, timeframe=None):
size_key = f"top_{self.cfg['size']}"
if source == 'dvd':
file = (self.urls.get('dvd') or {}).get(size_key)
else:
file = ((self.urls.get(source) or {}).get(timeframe) or {}).get(size_key)
return (file or {}).get('title', f'Trending {source} on Simkl')
def _build_catalog(self, c):
if not c.get('id') or not c.get('source'):
raise SimklTrendingError("catalogs[] entries need 'id' and 'source'")
items = self.get(c['source'])
transform = c.get('transform')
out = transform(items) if callable(transform) else items
title = c.get('title') or self._file_title_for(c['source'], c.get('timeframe') or (self.cfg.get('timeframes') or ['today'])[0])
return {'id': c['id'], 'title': title, 'items': out}
# Persistence — swap these two methods for a Redis / KV adapter in server-side runtimes.
def _persist(self):
if not self.cfg['persist_path']: return
try:
with self._lock: dump = json.dumps(self._cache)
tmp = self.cfg['persist_path'] + '.tmp'
with open(tmp, 'w') as f: f.write(dump)
os.replace(tmp, self.cfg['persist_path'])
except Exception: pass # best-effort
def _restore_persisted(self):
if not self.cfg['persist_path'] or not os.path.exists(self.cfg['persist_path']): return
try:
with open(self.cfg['persist_path']) as f: raw = json.load(f)
cutoff = time.time() - self.cfg['persist_max_age_s']
with self._lock:
for url, val in raw.items():
if val.get('fetched_at', 0) >= cutoff: self._cache[url] = val
except Exception: pass # no cache yet — fine
# === Run ===
client = SimklTrendingClient()
client.subscribe_catalogs(lambda rows: [print(f"{r['title']}: {len(r['items'])} items") for r in rows])
client.start()
# import signal; signal.signal(signal.SIGINT, lambda *_: (client.stop(), exit(0)))
# === Background scheduling (server-side) ===
# For long-running processes, the in-process threads are enough. For cron-style runtimes
# (Celery beat, APScheduler, GitHub Actions), wire `client.tick()` to your scheduler instead of start().
```
```swift Swift (iOS) theme={"theme":{"light":"github-light","dark":"vesper"}}
import Foundation
// === Load the URL catalog ===
// 1. Save the "Copy-paste JSON" block above as `simkl-trending-urls.json` in your app bundle (e.g.
// drag it into Xcode and tick "Copy items if needed" + your app target's "Copy Bundle Resources").
// 2. Load it into a global `SIMKL_TRENDING_URLS: [String: Any]` (and pass it to the constructor, or
// let the actor pick it up from the global on first use):
// let url = Bundle.main.url(forResource: "simkl-trending-urls", withExtension: "json")!
// let data = try Data(contentsOf: url)
// let SIMKL_TRENDING_URLS = try JSONSerialization.jsonObject(with: data) as! [String: Any]
// SimklTrendingClient(config:, trending: SIMKL_TRENDING_URLS) — fatalErrors if neither global nor arg is set.
// === CONFIG — tune to match your app ===
struct SimklTrendingConfig {
// What to load
var supportedTypes: [String] = ["movies", "tv", "anime", "dvd"] // movies / tv / anime / dvd — drop ones your app doesn't surface
var timeframes: [String] = ["today"] // today / week / month (ignored for dvd)
var size: Int = 100 // 100 or 500 — use 500 if you render derived catalogs (see "Build your own catalogs" below)
// HTTP — replace with YOUR values
var clientId: String = "YOUR_CLIENT_ID"
var appName: String = "my-app-name"
var appVersion: String = "1.0"
var userAgent: String = "my-app-name/1.0"
var requestTimeout: TimeInterval = 8
var minIntervalMs: Int = 250 // process-wide floor between consecutive requests
var fetchMaxAttempts: Int = 3
var fetchBaseBackoff: TimeInterval = 1
var fetchMaxBackoff: TimeInterval = 16
var retryStatuses: Set = [408, 425, 429, 500, 502, 503, 504] // never retry other 4xx — they're permanent
// Refresh
var regenRetrySeconds: [Int] = [300, 900, 1800] // 5 / 15 / 30 min after a 304
var jitterPercent: Double = 0.05 // ±5%
// Persistent cache — set persistFile to nil to disable
var persistFile: URL? = FileManager.default
.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("simkl-trending-cache.json")
var persistMaxAgeSeconds: TimeInterval = 7 * 86_400
// Catalogs — the rows on your home screen. Nil = auto-build one per supportedType.
var catalogs: [SimklTrendingCatalogConfig]? = nil
}
struct SimklTrendingCatalogConfig {
let id: String // unique row key
let source: String // 'movies' | 'tv' | 'anime' | 'dvd'
var timeframe: String? = nil // defaults to cfg.timeframes[0]
var title: String? = nil // defaults to JSON's pre-composed title
var transform: (([[String: Any]]) -> [[String: Any]])? = nil
}
struct SimklTrendingCatalog {
let id: String
let title: String
let items: [[String: Any]]
}
struct SimklTrendingError: Error {
let message: String
let status: Int?
let retryAfter: TimeInterval?
let permanent: Bool
var localizedDescription: String { message }
}
struct SimklTrendingEntry: Codable {
let title: String, url: String
let refreshSeconds: Int
let types: [String]
}
actor SimklTrendingClient {
private var cfg: SimklTrendingConfig
private let trending: [String: Any]
private var cache: [String: _SimklCached] = [:]
private var inflight: [String: Task] = [:]
private var subs: [String: [UUID: ([[String: Any]]) -> Void]] = [:]
private var loops: [Task] = []
private var stopped: Bool = false
private var lastRequestAt: Date? = nil // for minIntervalMs floor
struct _SimklCached { var itemsByType: [String: [[String: Any]]]; var lastModified: String?; var types: [String]; var fetchedAt: Date }
struct SimklTrendingFetchResult { let itemsByType: [String: [[String: Any]]]?; let lastModified: String?; let notModified: Bool }
private var defaultCatalogs: [[String: Any]]? = nil
init(config: SimklTrendingConfig = SimklTrendingConfig(), trending: [String: Any]? = nil) {
self.cfg = config
let data = trending ?? (SIMKL_TRENDING_URLS as [String: Any])
// Accept both shapes: { urls: {...}, catalogs: [...] } from JSON OR the legacy unwrapped { combined, movies, ... }.
self.trending = (data["urls"] != nil) ? data : ["urls": data]
self.defaultCatalogs = data["catalogs"] as? [[String: Any]]
}
// === Public API ===
func start() async {
restorePersisted()
for t in subs.keys { notify(type: t) } // paint immediately from restored cache
for entry in plan() {
let task = Task { await loop(entry) }
loops.append(task)
}
}
func stop() {
stopped = true
for task in loops { task.cancel() }
for (_, t) in inflight { t.cancel() }
loops.removeAll(); inflight.removeAll()
}
// Wipe in-memory + persistent cache and notify subscribers (so UI repaints empty).
// Call client.tick() after this to immediately re-fetch.
func clearCache() {
cache.removeAll()
if let url = cfg.persistFile { try? FileManager.default.removeItem(at: url) }
for (t, _) in subs { notify(type: t) }
}
func get(_ type: String) -> [[String: Any]] {
for v in cache.values { if let arr = v.itemsByType[type] { return arr } }
return []
}
func subscribe(_ type: String, _ cb: @escaping ([[String: Any]]) -> Void) -> UUID {
let id = UUID()
subs[type, default: [:]][id] = cb
let items = get(type); if !items.isEmpty { cb(items) } // immediate paint from cache
return id
}
func unsubscribe(_ type: String, _ id: UUID) { subs[type]?.removeValue(forKey: id) }
// === Catalogs — rows on the home screen ===
func catalogs() -> [SimklTrendingCatalog] {
return catalogConfigs().map { buildCatalog($0) }
}
func subscribeCatalogs(_ cb: @escaping ([SimklTrendingCatalog]) -> Void) -> [UUID] {
let sources = Set(catalogConfigs().map { $0.source })
var ids: [UUID] = []
for source in sources {
ids.append(subscribe(source) { _ in cb(self.catalogs()) })
}
if sources.allSatisfy({ get($0).isEmpty }) { cb(catalogs()) } // empty cache — skeleton paint
return ids
}
private func catalogConfigs() -> [SimklTrendingCatalogConfig] {
if let cs = cfg.catalogs { return cs } // explicit override wins
if let raw = defaultCatalogs { // JSON's catalogs[] (recommended)
return raw.compactMap { d in
guard let id = d["id"] as? String, let src = d["source"] as? String else { return nil }
return SimklTrendingCatalogConfig(id: id, source: src,
timeframe: d["timeframe"] as? String, title: d["title"] as? String, transform: nil)
}
}
let tf0 = cfg.timeframes.first ?? "today" // last resort: auto-build
return cfg.supportedTypes.map { t in
t == "dvd"
? SimklTrendingCatalogConfig(id: "dvd", source: "dvd")
: SimklTrendingCatalogConfig(id: "\(t)_\(tf0)", source: t, timeframe: tf0)
}
}
private func fileTitleFor(_ source: String, _ timeframe: String?) -> String {
let sizeKey = "top_\(cfg.size)"
let urls = trending["urls"] as? [String: Any] ?? [:]
let file: [String: Any]?
if source == "dvd" {
file = (urls["dvd"] as? [String: Any])?[sizeKey] as? [String: Any]
} else {
let bucket = (urls[source] as? [String: Any])?[timeframe ?? "today"] as? [String: Any]
file = bucket?[sizeKey] as? [String: Any]
}
return (file?["title"] as? String) ?? "Trending \(source) on Simkl"
}
private func buildCatalog(_ c: SimklTrendingCatalogConfig) -> SimklTrendingCatalog {
let items = get(c.source)
let out = c.transform?(items) ?? items
let title = c.title ?? fileTitleFor(c.source, c.timeframe ?? cfg.timeframes.first)
return SimklTrendingCatalog(id: c.id, title: title, items: out)
}
// Call from BGTaskScheduler for a single revalidate-and-persist pass.
func tick() async {
restorePersisted()
for entry in plan() {
let lm = cache[entry.url]?.lastModified
if let r = try? await fetchDedupd(entry, lastModified: lm), !r.notModified, r.itemsByType != nil {
storeAndNotify(entry: entry, result: r)
}
}
persist()
}
// === Internals ===
private func requiredQuery() -> String {
let an = cfg.appName.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? cfg.appName
let av = cfg.appVersion.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? cfg.appVersion
return "client_id=\(cfg.clientId)&app-name=\(an)&app-version=\(av)"
}
private func plan() -> [SimklTrendingEntry] {
precondition(cfg.size == 100 || cfg.size == 500, "size must be 100 or 500")
let key = "top_\(cfg.size)"
let urls = trending["urls"] as! [String: Any]
let tvma = cfg.supportedTypes.filter { ["movies","tv","anime"].contains($0) }
let useCombined = tvma.count >= 2 // 1 round-trip < 2-3 per timeframe
func make(_ raw: Any, types: [String]) -> SimklTrendingEntry {
let d = raw as! [String: Any]
return SimklTrendingEntry(title: d["title"] as! String, url: d["url"] as! String,
refreshSeconds: d["refresh_seconds"] as! Int, types: types)
}
var out: [SimklTrendingEntry] = []
if !tvma.isEmpty {
for tf in cfg.timeframes {
let bucket = (useCombined ? urls["combined"] : urls[tvma[0]]) as! [String: Any]
out.append(make((bucket[tf] as! [String: Any])[key]!, types: tvma))
}
}
if cfg.supportedTypes.contains("dvd") {
out.append(make((urls["dvd"] as! [String: Any])[key]!, types: ["dvd"]))
}
return out
}
// In-flight dedup: actor-serialized lookup of inflight tasks.
private func fetchDedupd(_ entry: SimklTrendingEntry, lastModified: String?) async throws -> SimklTrendingFetchResult {
let sep = entry.url.contains("?") ? "&" : "?"
let url = entry.url + sep + requiredQuery()
if let t = inflight[url] { return try await t.value }
let task = Task { try await self.runFetch(url: url, entry: entry, lastModified: lastModified) }
inflight[url] = task
defer { inflight[url] = nil }
return try await task.value
}
// Process-wide rate-limit floor (so N entries refreshing at once don't fire N simultaneous requests).
private func throttle() async {
if let last = lastRequestAt {
let gapMs = Date().timeIntervalSince(last) * 1_000
let floor = Double(cfg.minIntervalMs)
if gapMs < floor { try? await Task.sleep(nanoseconds: UInt64((floor - gapMs) * 1_000_000)) }
}
lastRequestAt = Date()
}
private static func parseRetryAfter(_ value: String?) -> TimeInterval? {
guard let v = value else { return nil }
if let s = TimeInterval(v) { return max(0, s) }
let f = DateFormatter(); f.dateFormat = "EEE, dd MMM yyyy HH:mm:ss zzz"; f.locale = Locale(identifier: "en_US_POSIX")
if let d = f.date(from: v) { return max(0, d.timeIntervalSinceNow) }
return nil
}
// runFetch — throttle + retry with exponential backoff + Retry-After + permanent-error classify. Normalizes both response shapes.
private func runFetch(url: String, entry: SimklTrendingEntry, lastModified: String?) async throws -> SimklTrendingFetchResult {
var lastErr: Error?
for attempt in 0..= cutoff,
let ibt = v["itemsByType"] as? [String: [[String: Any]]],
let types = v["types"] as? [String] else { continue }
cache[k] = _SimklCached(itemsByType: ibt, lastModified: v["lastModified"] as? String,
types: types, fetchedAt: Date(timeIntervalSince1970: fa))
}
}
}
// === Run ===
Task {
let client = SimklTrendingClient()
_ = await client.subscribeCatalogs { rows in
for row in rows { print("\(row.title): \(row.items.count) items") }
}
await client.start()
}
// === Background scheduling (iOS) ===
// Register a BGAppRefreshTask in your AppDelegate. In the handler call `await client.tick()`
// to perform a single revalidate-and-persist pass — Task.sleep timers do not fire while the
// app is suspended, so BGTaskScheduler is what keeps the cache warm in the background.
```
```kotlin Kotlin (Android) theme={"theme":{"light":"github-light","dark":"vesper"}}
import kotlinx.coroutines.*
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import okhttp3.OkHttpClient
import okhttp3.Request
import org.json.JSONArray
import org.json.JSONObject
import java.io.File
import java.net.URLEncoder
import java.util.concurrent.TimeUnit
import kotlin.random.Random
import kotlin.math.pow
// === Load the URL catalog ===
// 1. Save the "Copy-paste JSON" block above as `simkl-trending-urls.json` in `app/src/main/assets/`.
// 2. Load it into a global `SIMKL_TRENDING_URLS: JSONObject` once at app startup
// (e.g. in your Application.onCreate or DI module):
// val SIMKL_TRENDING_URLS = JSONObject(
// context.assets.open("simkl-trending-urls.json").bufferedReader().use { it.readText() }
// )
// You can also skip the global and pass it directly: `SimklTrendingClient(cfg, trending = urls)`.
// The constructor throws if neither the global nor the `trending` arg is provided.
// === CONFIG — tune to match your app ===
data class SimklTrendingConfig(
// What to load
val supportedTypes: List = listOf("movies", "tv", "anime", "dvd"), // movies / tv / anime / dvd — drop ones your app doesn't surface
val timeframes: List = listOf("today"), // today / week / month (ignored for dvd)
val size: Int = 100, // 100 or 500 — use 500 if you render derived catalogs (see "Build your own catalogs" below)
// HTTP — replace with YOUR values
val clientId: String = "YOUR_CLIENT_ID",
val appName: String = "my-app-name",
val appVersion: String = "1.0",
val userAgent: String = "my-app-name/1.0",
val requestTimeoutMs: Long = 8_000,
val minIntervalMs: Long = 250, // process-wide floor between consecutive requests
val fetchMaxAttempts: Int = 3,
val fetchBaseBackoffMs: Long = 1_000,
val fetchMaxBackoffMs: Long = 16_000,
val retryStatuses: Set = setOf(408, 425, 429, 500, 502, 503, 504), // never retry other 4xx — they're permanent
// Refresh
val regenRetrySeconds: List = listOf(300, 900, 1_800), // 5 / 15 / 30 min after a 304
val jitterPercent: Double = 0.05, // ±5%
// Persistent cache — set persistFile to null to disable. In Android pass context.cacheDir.
val persistFile: File? = null, // e.g. File(context.cacheDir, "simkl-trending-cache.json")
val persistMaxAgeMs: Long = 7L * 86_400 * 1_000,
// Catalogs — the rows on your home screen. Null = auto-build one per supportedType.
val catalogs: List? = null,
)
data class SimklTrendingCatalogConfig(
val id: String,
val source: String, // "movies" | "tv" | "anime" | "dvd"
val timeframe: String? = null, // defaults to cfg.timeframes[0]
val title: String? = null, // defaults to JSON's pre-composed title
val transform: ((List) -> List)? = null,
)
data class SimklTrendingCatalog(val id: String, val title: String, val items: List)
data class SimklTrendingEntry(val title: String, val url: String, val refreshSeconds: Int, val types: List)
class SimklTrendingError(message: String, val status: Int? = null, val retryAfterMs: Long? = null, val permanent: Boolean = false) : RuntimeException(message)
class SimklTrendingClient(
private val cfg: SimklTrendingConfig = SimklTrendingConfig(),
data: JSONObject = SIMKL_TRENDING_URLS,
) {
// Accept both JSON shapes: { urls: {...}, catalogs: [...] } OR the legacy unwrapped { combined, movies, ... }.
private val trending: JSONObject = if (data.has("urls")) data else JSONObject().put("urls", data)
private val defaultCatalogs: JSONArray? = data.optJSONArray("catalogs")
private data class _SimklCached(val itemsByType: Map, val lastModified: String?, val types: List, val fetchedAt: Long)
data class SimklTrendingFetchResult(val itemsByType: Map?, val lastModified: String?, val notModified: Boolean)
private val http = OkHttpClient.Builder()
.callTimeout(cfg.requestTimeoutMs, TimeUnit.MILLISECONDS).build()
private val cache = mutableMapOf()
private val inflight = mutableMapOf>()
private val subs = mutableMapOf) -> Unit>>()
private var subIdSeq = 0
private val mu = Mutex()
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private var lastRequestAt = 0L // for minIntervalMs floor
// === Public API ===
suspend fun start() {
restorePersisted()
mu.withLock { subs.keys.toList() }.forEach { notify(it) } // paint immediately from restored cache
plan().forEach { entry -> scope.launch { loop(entry) } }
}
fun stop() { scope.cancel() }
/** Wipe in-memory + persistent cache and notify subscribers (UI repaints empty). Call client.tick() after to re-fetch. */
suspend fun clearCache() {
val snapshot = mu.withLock {
cache.clear()
subs.mapValues { it.value.values.toList() }
}
cfg.persistFile?.takeIf { it.exists() }?.delete()
snapshot.forEach { (_, cbs) -> cbs.forEach { it(emptyList()) } }
}
suspend fun get(type: String): List = mu.withLock {
for (v in cache.values) v.itemsByType[type]?.let { return@withLock listOf(it) }
emptyList()
}
suspend fun subscribe(type: String, cb: (List) -> Unit): Int {
val id = mu.withLock {
val i = ++subIdSeq
subs.getOrPut(type) { mutableMapOf() }[i] = cb
i
}
val items = get(type); if (items.isNotEmpty()) cb(items) // immediate paint from cache
return id
}
suspend fun unsubscribe(type: String, id: Int) { mu.withLock { subs[type]?.remove(id) } }
// === Catalogs — rows on the home screen ===
suspend fun catalogs(): List = catalogConfigs().map { buildCatalog(it) }
suspend fun subscribeCatalogs(cb: (List) -> Unit): List {
val sources = catalogConfigs().map { it.source }.toSet()
val ids = mutableListOf()
for (s in sources) {
ids.add(subscribe(s) { _ -> scope.launch { cb(catalogs()) } })
}
if (sources.all { get(it).isEmpty() }) cb(catalogs()) // empty cache — skeleton paint
return ids
}
private fun catalogConfigs(): List {
cfg.catalogs?.let { return it } // explicit override wins
defaultCatalogs?.let { arr -> // JSON's catalogs[]
return (0 until arr.length()).mapNotNull { i ->
val o = arr.optJSONObject(i) ?: return@mapNotNull null
val id = o.optString("id"); val src = o.optString("source")
if (id.isEmpty() || src.isEmpty()) null
else SimklTrendingCatalogConfig(id = id, source = src,
timeframe = if (o.isNull("timeframe")) null else o.optString("timeframe"),
title = if (o.isNull("title")) null else o.optString("title"),
transform = null)
}
}
val tf0 = cfg.timeframes.firstOrNull() ?: "today" // last resort: auto-build
return cfg.supportedTypes.map { t ->
if (t == "dvd") SimklTrendingCatalogConfig(id = "dvd", source = "dvd")
else SimklTrendingCatalogConfig(id = "${t}_${tf0}", source = t, timeframe = tf0)
}
}
private fun fileTitleFor(source: String, timeframe: String?): String {
val sizeKey = "top_${cfg.size}"
val urls = trending.optJSONObject("urls") ?: return "Trending $source on Simkl"
val file = if (source == "dvd") urls.optJSONObject("dvd")?.optJSONObject(sizeKey)
else urls.optJSONObject(source)?.optJSONObject(timeframe ?: "today")?.optJSONObject(sizeKey)
return file?.optString("title") ?: "Trending $source on Simkl"
}
private suspend fun buildCatalog(c: SimklTrendingCatalogConfig): SimklTrendingCatalog {
val items = get(c.source)
val out = c.transform?.invoke(items) ?: items
val title = c.title ?: fileTitleFor(c.source, c.timeframe ?: cfg.timeframes.firstOrNull())
return SimklTrendingCatalog(c.id, title, out)
}
// Call from a WorkManager PeriodicWorker for a single revalidate-and-persist pass.
suspend fun tick() {
restorePersisted()
plan().forEach { entry ->
val lm = mu.withLock { cache[entry.url]?.lastModified }
runCatching { fetchDedupd(entry, lm) }.getOrNull()?.let { r ->
if (!r.notModified && r.itemsByType != null) storeAndNotify(entry, r)
}
}
persist()
}
// === Internals ===
private fun requiredQuery(): String =
"client_id=${cfg.clientId}" +
"&app-name=${URLEncoder.encode(cfg.appName, "UTF-8")}" +
"&app-version=${URLEncoder.encode(cfg.appVersion, "UTF-8")}"
private fun plan(): List {
require(cfg.size == 100 || cfg.size == 500) { "size must be 100 or 500" }
val key = "top_${cfg.size}"
val urls = trending.getJSONObject("urls")
val tvma = cfg.supportedTypes.filter { it in listOf("movies", "tv", "anime") }
val useCombined = tvma.size >= 2 // 1 round-trip < 2-3 per timeframe
fun make(o: JSONObject, types: List) =
SimklTrendingEntry(o.getString("title"), o.getString("url"), o.getInt("refresh_seconds"), types)
val out = mutableListOf()
if (tvma.isNotEmpty()) cfg.timeframes.forEach { tf ->
val bucket = if (useCombined) urls.getJSONObject("combined") else urls.getJSONObject(tvma[0])
out.add(make(bucket.getJSONObject(tf).getJSONObject(key), tvma))
}
if ("dvd" in cfg.supportedTypes)
out.add(make(urls.getJSONObject("dvd").getJSONObject(key), listOf("dvd")))
return out
}
// In-flight dedup: only one outstanding network call per URL.
private suspend fun fetchDedupd(entry: SimklTrendingEntry, lastModified: String?): SimklTrendingFetchResult {
val sep = if (entry.url.contains('?')) "&" else "?"
val url = entry.url + sep + requiredQuery()
val existing = mu.withLock { inflight[url] }
if (existing != null) return existing.await()
val task = scope.async { runFetch(url, entry, lastModified) }
mu.withLock { inflight[url] = task }
try { return task.await() } finally { mu.withLock { inflight.remove(url) } }
}
// Process-wide rate-limit floor (so N entries refreshing at once don't fire N simultaneous requests).
private suspend fun throttle() {
val gap = System.currentTimeMillis() - lastRequestAt
if (gap < cfg.minIntervalMs) delay(cfg.minIntervalMs - gap)
lastRequestAt = System.currentTimeMillis()
}
private fun parseRetryAfter(value: String?): Long? {
if (value == null) return null
value.toLongOrNull()?.let { return maxOf(0L, it * 1000L) }
return runCatching {
val fmt = java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME
val dt = java.time.ZonedDateTime.parse(value, fmt)
maxOf(0L, dt.toInstant().toEpochMilli() - System.currentTimeMillis())
}.getOrNull()
}
// runFetch — throttle + retry with exponential backoff + Retry-After + permanent-error classify. Normalizes both response shapes.
private suspend fun runFetch(url: String, entry: SimklTrendingEntry, lastModified: String?): SimklTrendingFetchResult {
var lastErr: Exception? = null
for (attempt in 0 until cfg.fetchMaxAttempts) {
throttle()
try {
val req = Request.Builder().url(url)
.header("User-Agent", cfg.userAgent)
.header("Accept", "application/json")
.apply { lastModified?.let { header("If-Modified-Since", it) } }.build()
http.newCall(req).execute().use { res ->
if (res.code == 304) return SimklTrendingFetchResult(null, lastModified, notModified = true)
if (!res.isSuccessful) {
val retryAfter = parseRetryAfter(res.header("Retry-After"))
val permanent = res.code !in cfg.retryStatuses
throw SimklTrendingError("${res.code} ${res.message} — ${entry.title}",
status = res.code, retryAfterMs = retryAfter, permanent = permanent)
}
val body = res.body!!.string().trim()
val ibt = mutableMapOf()
if (body.startsWith("[")) ibt[entry.types[0]] = JSONArray(body) // per-type / dvd → array
else { // combined → object, filter to wanted types
val obj = JSONObject(body)
entry.types.forEach { ibt[it] = obj.optJSONArray(it) ?: JSONArray() }
}
return SimklTrendingFetchResult(ibt, res.header("Last-Modified"), notModified = false)
}
} catch (e: CancellationException) { throw e }
catch (e: SimklTrendingError) {
if (e.permanent) throw e // don't waste backoff on permanent errors
lastErr = e
val honored = e.retryAfterMs
val backoff = honored ?: (cfg.fetchBaseBackoffMs * 2.0.pow(attempt.toDouble())).toLong().coerceAtMost(cfg.fetchMaxBackoffMs)
val jitter = (backoff * (Random.nextDouble() * 0.2 - 0.1)).toLong() // ±10% jitter
delay(maxOf(0L, backoff + jitter))
}
catch (e: Exception) {
lastErr = e
val backoff = (cfg.fetchBaseBackoffMs * 2.0.pow(attempt.toDouble())).toLong().coerceAtMost(cfg.fetchMaxBackoffMs)
delay(backoff)
}
}
throw lastErr ?: SimklTrendingError("fetch failed")
}
private suspend fun loop(entry: SimklTrendingEntry) {
var lastModified = mu.withLock { cache[entry.url]?.lastModified }
var retry = 0
while (scope.isActive) {
val r = try { fetchDedupd(entry, lastModified) }
catch (e: CancellationException) { return }
catch (e: SimklTrendingError) {
if (e.permanent) { println("${entry.title}: ${e.message} (permanent, halting loop)"); return }
delay(30_000L); continue
}
catch (e: Exception) { delay(30_000L); continue }
val wait = if (r.notModified) {
cfg.regenRetrySeconds[minOf(retry, cfg.regenRetrySeconds.lastIndex)].also { retry++ }
} else {
storeAndNotify(entry, r)
lastModified = r.lastModified; retry = 0
persist()
(entry.refreshSeconds * (1.0 + (Random.nextDouble() - 0.5) * 2 * cfg.jitterPercent)).toInt()
}
delay(wait * 1_000L)
}
}
private suspend fun storeAndNotify(entry: SimklTrendingEntry, r: SimklTrendingFetchResult) {
mu.withLock {
cache[entry.url] = _SimklCached(r.itemsByType ?: emptyMap(), r.lastModified, entry.types, System.currentTimeMillis())
}
entry.types.forEach { notify(it) }
}
private suspend fun notify(type: String) {
val items = get(type)
val cbs = mu.withLock { subs[type]?.values?.toList().orEmpty() }
cbs.forEach { it(items) }
}
private suspend fun persist() {
val f = cfg.persistFile ?: return
val dump = JSONObject()
mu.withLock {
for ((url, v) in cache) {
val o = JSONObject()
val ibt = JSONObject(); v.itemsByType.forEach { (k, arr) -> ibt.put(k, arr) }
o.put("itemsByType", ibt)
o.put("lastModified", v.lastModified ?: JSONObject.NULL)
o.put("types", JSONArray(v.types)); o.put("fetchedAt", v.fetchedAt)
dump.put(url, o)
}
}
runCatching {
val tmp = File(f.parentFile, f.name + ".tmp")
tmp.writeText(dump.toString()); tmp.renameTo(f)
}
}
private suspend fun restorePersisted() {
val f = cfg.persistFile ?: return
if (!f.exists()) return
runCatching {
val raw = JSONObject(f.readText())
val cutoff = System.currentTimeMillis() - cfg.persistMaxAgeMs
mu.withLock {
raw.keys().forEach { url ->
val o = raw.getJSONObject(url)
val fa = o.optLong("fetchedAt", 0L); if (fa < cutoff) return@forEach
val ibtO = o.getJSONObject("itemsByType")
val ibt = mutableMapOf()
ibtO.keys().forEach { t -> ibt[t] = ibtO.getJSONArray(t) }
val types = (0 until o.getJSONArray("types").length()).map { o.getJSONArray("types").getString(it) }
cache[url] = _SimklCached(ibt, if (o.isNull("lastModified")) null else o.getString("lastModified"), types, fa)
}
}
}
}
}
// === Run ===
runBlocking {
val client = SimklTrendingClient()
client.subscribeCatalogs { rows ->
for (row in rows) println("${row.title}: ${row.items.size} items")
}
client.start()
}
// === Background scheduling (Android) ===
// In-process coroutines stop when the app process is killed. Schedule a
// WorkManager.PeriodicWorkRequest whose Worker calls `client.tick()` to keep the cache warm
// in the background. Use cfg.persistFile = File(context.cacheDir, "simkl-trending-cache.json")
// so the next foreground launch can paint the UI from disk instantly.
```
```dart Dart (Flutter) theme={"theme":{"light":"github-light","dark":"vesper"}}
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:math';
import 'package:http/http.dart' as http;
// import 'package:path_provider/path_provider.dart'; // for persistFile in Flutter — see Run block
// === Load the URL catalog ===
// 1. Save the "Copy-paste JSON" block above as `assets/simkl-trending-urls.json`, then declare
// it under `flutter: assets:` in pubspec.yaml.
// 2. Load it into a global `SIMKL_TRENDING_URLS` once at app startup (after WidgetsFlutterBinding.ensureInitialized):
// final SIMKL_TRENDING_URLS = jsonDecode(
// await rootBundle.loadString('assets/simkl-trending-urls.json')
// ) as Map;
// For pure-Dart (server / CLI / tests), use:
// final SIMKL_TRENDING_URLS = jsonDecode(await File('simkl-trending-urls.json').readAsString()) as Map;
// You can also skip the global and pass it explicitly: `SimklTrendingClient(trending: urlsMap)`.
// === CONFIG — tune to match your app ===
class SimklTrendingConfig {
// What to load
final List supportedTypes; // movies / tv / anime / dvd
final List timeframes; // today / week / month (ignored for dvd)
final int size; // 100 or 500 — use 500 if you render derived catalogs (see "Build your own catalogs" below)
// HTTP — replace with YOUR values
final String clientId, appName, appVersion, userAgent;
final Duration requestTimeout;
final Duration minInterval; // process-wide floor between consecutive requests
final int fetchMaxAttempts;
final Duration fetchBaseBackoff, fetchMaxBackoff;
final Set retryStatuses; // never retry other 4xx — they're permanent
// Refresh
final List regenRetrySeconds; // 5 / 15 / 30 min after a 304
final double jitterPercent; // ±5%
// Persistent cache — set persistFile to null to disable.
final File? persistFile;
final Duration persistMaxAge;
// Catalogs — the rows on your home screen. Null = auto-build one per supportedType.
final List? catalogs;
const SimklTrendingConfig({
this.supportedTypes = const ['movies', 'tv', 'anime', 'dvd'],
this.timeframes = const ['today'],
this.size = 100,
this.clientId = 'YOUR_CLIENT_ID',
this.appName = 'my-app-name',
this.appVersion = '1.0',
this.userAgent = 'my-app-name/1.0',
this.requestTimeout = const Duration(seconds: 8),
this.minInterval = const Duration(milliseconds: 250),
this.fetchMaxAttempts = 3,
this.fetchBaseBackoff = const Duration(seconds: 1),
this.fetchMaxBackoff = const Duration(seconds: 16),
this.retryStatuses = const {408, 425, 429, 500, 502, 503, 504},
this.regenRetrySeconds = const [300, 900, 1800],
this.jitterPercent = 0.05,
this.persistFile,
this.persistMaxAge = const Duration(days: 7),
this.catalogs,
});
}
class SimklTrendingCatalogConfig {
final String id;
final String source; // 'movies' | 'tv' | 'anime' | 'dvd'
final String? timeframe; // defaults to cfg.timeframes.first
final String? title; // defaults to JSON's pre-composed title
final List Function(List)? transform;
SimklTrendingCatalogConfig({required this.id, required this.source, this.timeframe, this.title, this.transform});
}
class SimklTrendingCatalog {
final String id, title;
final List items;
SimklTrendingCatalog(this.id, this.title, this.items);
}
class SimklTrendingError implements Exception {
final String message;
final int? status;
final Duration? retryAfter;
final bool permanent;
SimklTrendingError(this.message, {this.status, this.retryAfter, this.permanent = false});
@override String toString() => 'SimklTrendingError($status): $message';
}
class SimklTrendingEntry {
final String title, url;
final int refreshSeconds;
final List types;
SimklTrendingEntry(this.title, this.url, this.refreshSeconds, this.types);
}
class _SimklCached {
Map> itemsByType; String? lastModified; List types; int fetchedAt;
_SimklCached(this.itemsByType, this.lastModified, this.types, this.fetchedAt);
}
class SimklTrendingFetchResult {
final Map>? itemsByType;
final String? lastModified;
final bool notModified;
SimklTrendingFetchResult({this.itemsByType, this.lastModified, this.notModified = false});
}
class SimklTrendingClient {
final SimklTrendingConfig cfg;
final Map trending;
final _cache = {};
final _inflight = >{};
final _subs = )>>{};
final _client = http.Client();
int _subIdSeq = 0;
bool _stopped = false;
int _lastRequestAt = 0; // for minInterval floor
final _loops = >[];
final List? defaultCatalogs;
// Accept both JSON shapes: { urls: {...}, catalogs: [...] } OR the legacy unwrapped { combined, movies, ... }.
factory SimklTrendingClient({SimklTrendingConfig? config, Map? trending}) {
final raw = trending ?? (SIMKL_TRENDING_URLS as Map);
final urlsMap = raw['urls'] != null ? raw : { 'urls': raw };
return SimklTrendingClient._internal(config ?? const SimklTrendingConfig(),
urlsMap as Map,
raw['catalogs'] as List?);
}
SimklTrendingClient._internal(this.cfg, this.trending, this.defaultCatalogs);
// === Public API ===
Future start() async {
await _restorePersisted();
for (final t in _subs.keys.toList()) _notify(t); // paint immediately from restored cache
for (final entry in _plan()) _loops.add(_loop(entry));
}
void stop() { _stopped = true; _client.close(); }
/// Wipe in-memory + persistent cache and notify subscribers (UI repaints empty). Call client.tick() after to re-fetch.
Future clearCache() async {
_cache.clear();
final f = cfg.persistFile;
if (f != null && await f.exists()) { try { await f.delete(); } catch (_) {/* ignore */} }
for (final t in _subs.keys.toList()) _notify(t);
}
List get(String type) {
for (final v in _cache.values) if (v.itemsByType.containsKey(type)) return v.itemsByType[type]!;
return const [];
}
int subscribe(String type, void Function(List) cb) {
final id = ++_subIdSeq;
(_subs[type] ??= {})[id] = cb;
final items = get(type); if (items.isNotEmpty) cb(items); // immediate paint from cache
return id;
}
void unsubscribe(String type, int id) { _subs[type]?.remove(id); }
// === Catalogs — rows on the home screen ===
List catalogs() => _catalogConfigs().map(_buildCatalog).toList();
List subscribeCatalogs(void Function(List) cb) {
final sources = _catalogConfigs().map((c) => c.source).toSet();
final ids = [];
for (final s in sources) {
ids.add(subscribe(s, (_) => cb(catalogs())));
}
if (sources.every((s) => get(s).isEmpty)) cb(catalogs()); // empty cache — skeleton paint
return ids;
}
List _catalogConfigs() {
if (cfg.catalogs != null) return cfg.catalogs!; // explicit override wins
if (defaultCatalogs != null) { // JSON's catalogs[]
return defaultCatalogs!.whereType