diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 91f3e0c..26df06b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,11 +23,12 @@ env: PAGES_PROJECT: flow-ailist WRANGLER_VERSION: 4.156.0 DATA_BRANCH: data + CACHE_BRANCH: resolve-cache jobs: publish: runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 30 steps: - name: Check out main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -49,17 +50,41 @@ jobs: git worktree add --orphan -b "$DATA_BRANCH" site fi + - name: Check out the lookup cache branch into cache/ + run: | + if git ls-remote --exit-code --heads origin "$CACHE_BRANCH" > /dev/null; then + git fetch --no-tags --depth=1 origin "$CACHE_BRANCH" + git worktree add -B "$CACHE_BRANCH" cache FETCH_HEAD + else + echo "No $CACHE_BRANCH branch yet: starting with an empty cache." + git worktree add --orphan -b "$CACHE_BRANCH" cache + fi + - name: Build, sign and self-check env: AILIST_SIGNING_KEY: ${{ secrets.AILIST_SIGNING_KEY }} GITHUB_TOKEN: ${{ github.token }} FORCE_FLAG: ${{ inputs.force && '--force' || '' }} run: | - node src/publish.js --out site \ + node src/publish.js --out site --cache cache \ --summary "$RUNNER_TEMP/summary.json" \ --report "$RUNNER_TEMP/guard-report.md" \ $FORCE_FLAG + - name: Commit and push the lookup cache + if: always() + run: | + cd cache + git add -A + if git diff --cached --quiet; then + echo "Cache unchanged: nothing to commit." + exit 0 + fi + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit --quiet -m "cache: id lookups $(date -u +%F)" + git push origin "HEAD:refs/heads/$CACHE_BRANCH" + - name: Open or update the publish-blocked issue if: failure() env: diff --git a/.gitignore b/.gitignore index 8f72d3b..11a3239 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ node_modules/ *.pem !keys/*.pub.pem .wrangler/ +/cache/ diff --git a/README.md b/README.md index 0c964bd..ed840c7 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,10 @@ canonical JSON is needed. Decoded payload: ``` - `handles` are normalized (see below), stored without the `@`, deduplicated, sorted. -- `ids` are channel IDs matching `^UC[A-Za-z0-9_-]{22}$`, deduplicated, sorted. They are empty - until channel IDs are resolved; clients must already check them. +- `ids` are channel IDs matching `^UC[A-Za-z0-9_-]{22}$`, deduplicated, sorted. They hold the + IDs the source lists directly plus the channel ID of every listed handle the pipeline has + resolved (see [Channel IDs](#channel-ids)). A handle and its ID are both published, in the + same list. - An entry in `block` never also appears in `warn`. ### delta/<version>.json @@ -192,7 +194,7 @@ or a failed verification means: keep the current list, do nothing else. ### Matching - A channel is listed when its normalized handle is in `handles` or its channel ID is in `ids`. - Check both from day one; `ids` is empty for now. + Check both: many screens (YouTube Music in particular) show a channel ID but no handle. - `block` and `warn` are separate sets with separate user switches. - An item with neither a handle nor a listed ID is not filtered. - Matching is exact string equality against a hash set. Never use prefix or substring matching: @@ -245,19 +247,43 @@ shared fixture in all three codebases. 1. Checks out `main` and the `data` branch (the published tree) into `site/`. 2. Pins AiSList to its current commit and downloads both list files from that commit. -3. Parses, normalizes and deduplicates; removes `overrides/allow.txt` entries from both lists; - removes anything in `block` from `warn`. +3. Parses, normalizes and deduplicates; looks up channel IDs for listed handles within the daily + budget (see [Channel IDs](#channel-ids)); removes `overrides/allow.txt` entries from both + lists; removes anything in `block` from `warn`. 4. Runs the guards from `sources/aislist.json`: minimum block size, reject ratio, `Total Channels` header mismatch, and (against the previous snapshot) maximum shrink and growth. 5. If the lists changed, writes a new snapshot and delta and prunes old ones. 6. Signs the manifest, regenerates `index.json`, copies `static/`, and self-checks the whole tree. -7. Commits `site/` to `data` when it changed, deploys it to Cloudflare Pages, and smoke-tests the - live site (signature, hashes, headers, and a real 404 for a missing path). +7. Commits the lookup cache to `resolve-cache` (even when a guard stopped the run), commits + `site/` to `data` when it changed, deploys it to Cloudflare Pages, and smoke-tests the live + site (signature, hashes, headers, and a real 404 for a missing path). **When a guard aborts**, the run fails, the live list stays at the last good version, and an issue labelled `publish-blocked` is opened (or commented on) with the numbers. After a human look, re-run `publish.yml` with `force: true`, which skips only the shrink and growth guards. +### Channel IDs + +A source that has a `resolve` block in its config gets the channel ID of each listed handle +from YouTube's public `navigation/resolve_url` endpoint. Lookups are cached in +`cache/.tsv` on the `resolve-cache` branch (`handle`, `channel ID or -`, `last checked`), +so each handle costs one request and then a recheck: + +- Each run looks up at most `daily_budget` handles: handles new since the last version first, + then handles never looked up, then cached entries due a recheck, oldest first. +- A found ID is rechecked after `recheck_found_days`, so a handle that moved to another channel + is corrected; a handle with no channel is retried after `recheck_missing_days`. +- Lookups stop at `max_minutes`, or at once on a 429, and the run publishes what it has. + A failed lookup is not cached and is retried next run. +- The cache keeps only handles still listed. When a guard stops the run, it is saved unpruned, + so a bad source never costs the lookups already made. +- An allow-listed handle is never looked up, and its ID is never published. +- The growth guard leaves out looked-up IDs that are new since the last version, since the + budget already bounds them; growth from the source itself is still checked. + +Losing the cache branch drops the resolved IDs on the next run, which the shrink guard stops. +Restore the branch rather than forcing the run. + **False positives** go in `overrides/allow.txt`, one `@handle` or channel ID per line, through a pull request. They are removed from both lists on the next run. @@ -267,12 +293,13 @@ Requires Node 22 or newer. There are no dependencies to install. ```sh node --test # unit and end-to-end tests, no network -node src/publish.js --dry-run --out ./out # real AiSList source, throwaway signing key +node src/publish.js --dry-run --out ./out --resolve-budget 0 # real source, throwaway key, no ID lookups node src/publish.js --help # all options ``` A dry run downloads the real source, builds and self-checks the tree in `./out`, and touches -nothing remote. Run it again on the same folder to see the unchanged or delta path. +nothing remote. Without `--resolve-budget 0` it also looks up channel IDs against YouTube and +writes the cache to `./cache`. Run it again on the same folder to see the unchanged or delta path. | Path | Responsibility | | --- | --- | @@ -281,6 +308,8 @@ nothing remote. Run it again on the same folder to see the unchanged or delta pa | `src/normalize.js` | The handle rules above | | `src/guards.js` | Size and change-rate checks | | `src/build.js` | Lists, snapshot, delta, pruning, manifest payload, index | +| `src/resolve-cache.js` | Lookup cache format, lookup planning and merging | +| `src/resolve.js` | Handle to channel ID lookups against YouTube, with a budget and a stop on 429 | | `src/sign.js` | Ed25519 envelope sign and verify | | `src/verify-tree.js` | Self-check of a built tree, also used for the live smoke test | | `src/publish.js` | Entry point; runs the steps in order and owns all file access | diff --git a/sources/aislist.json b/sources/aislist.json index e2a3634..ee92888 100644 --- a/sources/aislist.json +++ b/sources/aislist.json @@ -17,5 +17,11 @@ "max_growth_ratio": 0.25, "max_reject_ratio": 0.01, "max_header_mismatch_ratio": 0.01 + }, + "resolve": { + "daily_budget": 5000, + "recheck_found_days": 90, + "recheck_missing_days": 30, + "max_minutes": 12 } } diff --git a/src/constants.js b/src/constants.js index 6728dd7..2330e42 100644 --- a/src/constants.js +++ b/src/constants.js @@ -38,6 +38,7 @@ export const DEFAULT_PATHS = Object.freeze({ static: 'static', keys: 'keys', out: 'site', + cache: 'cache', }); /** Published tree layout. Tree paths are relative to the site root. */ diff --git a/src/guards.js b/src/guards.js index 5ce4879..a42e936 100644 --- a/src/guards.js +++ b/src/guards.js @@ -48,9 +48,11 @@ const percent = (ratio) => `${(ratio * 100).toFixed(2)}%`; * * blockEntries: block size (handles + ids) after overrides. * previousBlockEntries: block size of the previous snapshot, or null on the first run. + * lookupGrowth: ids from lookups that the previous version did not have. They are bounded + * by the lookup budget, not by the source, so the growth guard leaves them out. * force: skip only the shrink and growth guards. */ -export function checkGuards({ limits, stats, blockEntries, previousBlockEntries, force = false }) { +export function checkGuards({ limits, stats, blockEntries, previousBlockEntries, lookupGrowth = 0, force = false }) { validateLimits(limits); const failures = []; const skipped = []; @@ -80,9 +82,10 @@ export function checkGuards({ limits, stats, blockEntries, previousBlockEntries, if (shrink > previousBlockEntries * limits.max_shrink_ratio) { fail('shrink', `block shrank from ${previousBlockEntries} to ${blockEntries} (-${percent(shrink / previousBlockEntries)}), more than ${percent(limits.max_shrink_ratio)}`); } - const growth = blockEntries - previousBlockEntries; + const growth = blockEntries - previousBlockEntries - lookupGrowth; if (growth > previousBlockEntries * limits.max_growth_ratio) { - fail('growth', `block grew from ${previousBlockEntries} to ${blockEntries} (+${percent(growth / previousBlockEntries)}), more than ${percent(limits.max_growth_ratio)}`); + const fromLookups = lookupGrowth > 0 ? `, ${lookupGrowth} of them new lookups` : ''; + fail('growth', `block grew from ${previousBlockEntries} to ${blockEntries}${fromLookups} (+${percent(growth / previousBlockEntries)} from the source), more than ${percent(limits.max_growth_ratio)}`); } } diff --git a/src/publish.js b/src/publish.js index bd4840e..54b3616 100644 --- a/src/publish.js +++ b/src/publish.js @@ -37,6 +37,8 @@ import { assembleLists, planRelease, makeIndex, encodeJson, snapshotLists, kindS import { signEnvelope, readEnvelopeUnverified, generateThrowawayKey } from './sign.js'; import { verifyTree, checkLiveHeaders } from './verify-tree.js'; import { downloadSource, createHttpReader } from './fetch.js'; +import { parseCache, serializeCache, pruneCache, planLookups, mergeResults, cachedIds, isoDay } from './resolve-cache.js'; +import { resolveHandle, runLookups } from './resolve.js'; const SIGNING_KEY_ENV = 'AILIST_SIGNING_KEY'; const GITHUB_TOKEN_ENV = 'GITHUB_TOKEN'; @@ -44,6 +46,7 @@ const SMOKE_ATTEMPTS = 5; const SMOKE_DELAY_MS = 30_000; const PLACEHOLDER_COMMIT = '0'.repeat(40); const REQUIRED_CONFIG_FIELDS = ['id', 'name', 'description', 'homepage', 'repo', 'branch', 'files', 'format', 'license', 'guards']; +const RESOLVE_LIMITS = ['daily_budget', 'recheck_found_days', 'recheck_missing_days', 'max_minutes']; const CLI_OPTIONS = { 'dry-run': { type: 'boolean', default: false }, @@ -53,6 +56,9 @@ const CLI_OPTIONS = { allow: { type: 'string', default: DEFAULT_PATHS.allow }, static: { type: 'string', default: DEFAULT_PATHS.static }, keys: { type: 'string', default: DEFAULT_PATHS.keys }, + cache: { type: 'string', default: DEFAULT_PATHS.cache }, + 'resolve-budget': { type: 'string' }, + 'resolve-fixture': { type: 'string' }, 'source-dir': { type: 'string' }, 'source-commit': { type: 'string', default: PLACEHOLDER_COMMIT }, summary: { type: 'string' }, @@ -73,6 +79,9 @@ Options: --allow overrides allow list (default: ${DEFAULT_PATHS.allow}) --static static files copied into the tree (default: ${DEFAULT_PATHS.static}) --keys public keys, named .pub.pem (default: ${DEFAULT_PATHS.keys}) + --cache id lookup cache, one .tsv per list (default: ${DEFAULT_PATHS.cache}) + --resolve-budget most id lookups per list this run (default: the config's daily_budget; 0 = none) + --resolve-fixture answer id lookups from a JSON file of { key: id or null } instead of YouTube --source-dir read source files from / instead of downloading --source-commit commit recorded for --source-dir (default: ${PLACEHOLDER_COMMIT}) --summary write a JSON run summary, including the data commit message @@ -131,11 +140,18 @@ export async function loadConfigs(dir) { } getParser(config.format); validateLimits(config.guards); + if (config.resolve !== undefined) validateResolve(config.resolve, where); configs.push(config); } return configs; } +function validateResolve(resolve, where) { + for (const key of RESOLVE_LIMITS) { + if (!Number.isInteger(resolve?.[key]) || resolve[key] < 0) throw new Error(`${where}: resolve.${key} must be a non-negative integer`); + } +} + /** Allow-list entries, normalized. An unreadable line is an error, never silently skipped. */ export async function loadAllow(file) { const text = (await readOptional(file))?.toString('utf8') ?? ''; @@ -193,6 +209,82 @@ async function acquireSource(config, options) { return { commit: options['source-commit'], texts }; } +// ---------------------------------------------------------------- id resolution + +const cachePath = (options, listId) => path.join(options.cache, `${listId}.tsv`); + +/** The lookup function for this run: a fixture file in tests, YouTube otherwise. */ +async function createLookup(options) { + if (!options['resolve-fixture']) return (handle) => resolveHandle(handle); + const answers = JSON.parse(await readFile(options['resolve-fixture'], 'utf8')); + return async (handle) => { + if (!Object.hasOwn(answers, handle)) throw new Error(`no fixture answer for ${handle}`); + return { id: answers[handle] }; + }; +} + +/** + * Look up channel ids for the listed handles within the run's budget, and add + * every id the cache knows to the same list kind as its handle. Allow-listed + * handles are never looked up and never contribute an id. + * + * Returns { cache, prunedCache, lookupGrowth }. The caller saves the pruned + * cache once the guards pass and the full one when they do not, so a bad source + * never costs the lookups already made. lookupGrowth counts block ids that come + * from lookups and were not in the previous version, which the growth guard + * leaves out: the lookup budget bounds them, and counting them would keep + * blocking every run after one aborted with lookups cached. + */ +async function resolveListIds(config, { normalized, allow, previous, options }) { + const allowed = new Set(allow.handles); + const handlesByKind = Object.fromEntries(KINDS.map((kind) => [kind, [...normalized[kind].handles].filter((h) => !allowed.has(h))])); + const listed = [...new Set(KINDS.flatMap((kind) => handlesByKind[kind]))]; + const previousHandles = new Set(previous ? KINDS.flatMap((kind) => previous.lists[kind].handles) : []); + const today = isoDay(new Date()); + + let cache = parseCache((await readOptional(cachePath(options, config.id)))?.toString('utf8')); + const budget = options['resolve-budget'] === undefined ? config.resolve.daily_budget : Number(options['resolve-budget']); + if (!Number.isInteger(budget) || budget < 0) throw new Error('--resolve-budget must be a non-negative integer'); + const plan = planLookups({ + keys: listed, + priority: previous ? listed.filter((h) => !previousHandles.has(h)) : [], + cache, + today, + budget, + recheckFoundDays: config.resolve.recheck_found_days, + recheckMissingDays: config.resolve.recheck_missing_days, + }); + if (plan.length > 0) { + const run = await runLookups(plan, await createLookup(options), { + deadline: Date.now() + config.resolve.max_minutes * 60_000, + spacingMs: options['resolve-fixture'] ? 0 : undefined, + }); + cache = mergeResults(cache, run.results, today); + const found = [...run.results.values()].filter((r) => r.id).length; + log( + `${config.id}: looked up ${run.results.size} of ${plan.length} planned handles: ${found} found, ` + + `${run.results.size - found} missing, ${run.failed} failed` + + (run.rateLimited ? ', stopped on a rate limit' : '') + + (run.timedOut ? ', stopped at the time limit' : ''), + ); + } + + const sourceBlockIds = new Set(normalized.block.ids); + for (const kind of KINDS) { + for (const id of cachedIds(cache, handlesByKind[kind])) normalized[kind].ids.add(id); + } + const previousBlockIds = new Set(previous?.lists.block.ids ?? []); + const lookupGrowth = [...normalized.block.ids].filter((id) => !sourceBlockIds.has(id) && !previousBlockIds.has(id)).length; + const resolved = listed.filter((h) => cache.get(h)?.id).length; + const unchecked = listed.filter((h) => !cache.has(h)).length; + log(`${config.id}: ${resolved} of ${listed.length} handles have a channel id, ${unchecked} not looked up yet`); + return { cache, prunedCache: pruneCache(cache, listed), lookupGrowth }; +} + +async function saveCache(options, listId, cache) { + await writeFileDeep(cachePath(options, listId), serializeCache(cache, `${listId}: normalized handle, channel id or -, last checked (UTC)`)); +} + // ---------------------------------------------------------------- reporting function describeVersion(listId, version, changes, commit) { @@ -245,14 +337,16 @@ async function prepareList(config, { options, allow, out }) { ); } - const lists = assembleLists(normalized, allow); const previous = await readPrevious(out, config.id); + const resolution = config.resolve ? await resolveListIds(config, { normalized, allow, previous, options }) : null; + const lists = assembleLists(normalized, allow); const stats = measureSource(parsed, normalized); const guard = checkGuards({ limits: config.guards, stats, blockEntries: kindSize(lists, 'block'), previousBlockEntries: previous ? kindSize(previous.lists, 'block') : null, + lookupGrowth: resolution?.lookupGrowth ?? 0, force: options.force, }); log(`${config.id}: after overrides and overlap: block ${kindSize(lists, 'block')}, warn ${kindSize(lists, 'warn')}`); @@ -271,7 +365,7 @@ async function prepareList(config, { options, allow, out }) { generatedAt: new Date(), }); } - return { config, commit, lists, previous, stats, guard, plan }; + return { config, commit, lists, previous, stats, guard, plan, resolution }; } /** @@ -299,7 +393,11 @@ async function publish(options) { const prepared = []; for (const config of configs) prepared.push(await prepareList(config, { options, allow, out })); - if (prepared.some((p) => p.guard.failures.length > 0)) { + const blocked = prepared.some((p) => p.guard.failures.length > 0); + for (const { config, resolution } of prepared) { + if (resolution) await saveCache(options, config.id, blocked ? resolution.cache : resolution.prunedCache); + } + if (blocked) { if (options.report) await writeFileDeep(options.report, guardReport(prepared, options.force)); throw new GuardAbort('A guard aborted the run. Nothing was written.'); } diff --git a/src/resolve-cache.js b/src/resolve-cache.js new file mode 100644 index 0000000..c89c030 --- /dev/null +++ b/src/resolve-cache.js @@ -0,0 +1,106 @@ +// The lookup cache: which key (a normalized handle, or an upstream artist id) +// resolved to which YouTube id, and when it was last checked. Pure functions: +// text in, data out. The pipeline keeps the file on its own branch so each key +// costs one request, then a recheck every few weeks. +// +// File format, one entry per line, sorted by key: +// \t\t +// `-` records a lookup that found nothing. Lines starting with `#` are comments. + +import { CHANNEL_ID_PATTERN } from './constants.js'; + +const MISSING = '-'; +const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/; +const DAY_MS = 86_400_000; + +/** `YYYY-MM-DD` in UTC. */ +export const isoDay = (date) => date.toISOString().slice(0, 10); + +const daysBetween = (from, to) => Math.floor((Date.parse(to) - Date.parse(from)) / DAY_MS); + +/** Parse cache text into a Map of key to { id: string | null, checked: 'YYYY-MM-DD' }. Bad lines throw. */ +export function parseCache(text) { + const cache = new Map(); + for (const [index, line] of (text ?? '').split('\n').entries()) { + if (line === '' || line.startsWith('#')) continue; + const fields = line.split('\t'); + const [key, id, checked] = fields; + const valid = + fields.length === 3 && + key !== '' && + (id === MISSING || CHANNEL_ID_PATTERN.test(id)) && + DATE_PATTERN.test(checked); + if (!valid) throw new Error(`cache line ${index + 1} is malformed: ${JSON.stringify(line)}`); + cache.set(key, { id: id === MISSING ? null : id, checked }); + } + return cache; +} + +/** Cache text with a header comment, entries sorted by key. */ +export function serializeCache(cache, header) { + const keys = [...cache.keys()].sort(); + const lines = keys.map((key) => { + const { id, checked } = cache.get(key); + return `${key}\t${id ?? MISSING}\t${checked}`; + }); + return `${header ? `# ${header}\n` : ''}${lines.join('\n')}${lines.length > 0 ? '\n' : ''}`; +} + +/** A copy of the cache holding only the keys still listed, so it never grows past the lists. */ +export function pruneCache(cache, keys) { + const keep = new Set(keys); + return new Map([...cache].filter(([key]) => keep.has(key))); +} + +/** + * Choose which keys to look up this run, at most `budget` of them, in this order: + * 1. keys in `priority` (new on the source since the last version) not yet cached; + * 2. every other key not yet cached, sorted; + * 3. cached keys due a recheck, oldest check first. + * A found id is rechecked after `recheckFoundDays`, so a handle that moved to + * another channel is corrected; a miss is retried after `recheckMissingDays`. + */ +export function planLookups({ keys, priority = [], cache, today, budget, recheckFoundDays, recheckMissingDays }) { + if (budget <= 0) return []; + const listed = new Set(keys); + const plan = []; + const planned = new Set(); + const take = (key) => { + if (plan.length >= budget || planned.has(key)) return; + planned.add(key); + plan.push(key); + }; + + for (const key of priority) if (listed.has(key) && !cache.has(key)) take(key); + for (const key of [...listed].sort()) if (!cache.has(key)) take(key); + + const due = [...listed] + .filter((key) => cache.has(key)) + .map((key) => ({ key, entry: cache.get(key) })) + .filter(({ entry }) => daysBetween(entry.checked, today) >= (entry.id ? recheckFoundDays : recheckMissingDays)) + .sort((a, b) => a.entry.checked.localeCompare(b.entry.checked) || a.key.localeCompare(b.key)); + for (const { key } of due) take(key); + + return plan; +} + +/** + * Merge lookup results into a copy of the cache. `results` maps key to + * { id: string | null }; keys whose lookup failed are simply absent and keep + * their old entry. + */ +export function mergeResults(cache, results, today) { + const merged = new Map(cache); + for (const [key, { id }] of results) merged.set(key, { id, checked: today }); + return merged; +} + +/** The ids the cache knows for `keys`, skipping misses and unknown keys. */ +export function cachedIds(cache, keys) { + const ids = new Set(); + for (const key of keys) { + const id = cache.get(key)?.id; + if (id) ids.add(id); + } + return ids; +} diff --git a/src/resolve.js b/src/resolve.js new file mode 100644 index 0000000..4356d13 --- /dev/null +++ b/src/resolve.js @@ -0,0 +1,79 @@ +// Network edge for id resolution: turns a normalized handle into its YouTube +// channel id through YouTube's public `navigation/resolve_url` endpoint, and +// runs a batch of lookups with bounded concurrency, a deadline, and a full stop +// on the first rate-limit answer. Uses the built-in fetch only. + +import { CHANNEL_ID_PATTERN } from './constants.js'; + +const RESOLVE_URL = 'https://www.youtube.com/youtubei/v1/navigation/resolve_url?prettyPrint=false'; +const WEB_CLIENT = Object.freeze({ clientName: 'WEB', clientVersion: '2.20251113.00.00', hl: 'en', gl: 'US' }); +const REQUEST_TIMEOUT_MS = 20_000; + +/** Thrown when YouTube answers 429: the run stops looking up and keeps what it has. */ +export class RateLimited extends Error {} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** POST a JSON body and return { status, json }. */ +async function postJson(fetchImpl, url, body, headers = {}) { + const response = await fetchImpl(url, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (response.status === 429) throw new RateLimited(`${url} answered 429`); + const json = response.ok ? await response.json() : null; + return { status: response.status, json }; +} + +/** + * Resolve one normalized handle. Returns { id } where id is a channel id, or + * null when YouTube knows no channel by that handle (404 or no channel in the + * answer). Throws on anything else, so a transient failure is retried next run + * instead of being cached as a miss. + */ +export async function resolveHandle(handle, { fetchImpl = fetch } = {}) { + const url = `https://www.youtube.com/@${encodeURIComponent(handle)}`; + const { status, json } = await postJson(fetchImpl, RESOLVE_URL, { context: { client: WEB_CLIENT }, url }); + if (status === 404) return { id: null }; + if (status !== 200) throw new Error(`resolve_url for @${handle} answered ${status}`); + const browseId = json?.endpoint?.browseEndpoint?.browseId; + return { id: CHANNEL_ID_PATTERN.test(browseId ?? '') ? browseId : null }; +} + +/** + * Look up `keys` with `lookup(key)`, at most `concurrency` at a time, each + * worker pausing `spacingMs` between requests. Stops starting new lookups at + * `deadline` (epoch ms) or after the first RateLimited. + * + * Returns { results: Map, failed, rateLimited, timedOut }. + */ +export async function runLookups(keys, lookup, { concurrency = 4, spacingMs = 150, deadline = Infinity, now = Date.now } = {}) { + const results = new Map(); + let failed = 0; + let rateLimited = false; + let timedOut = false; + let next = 0; + + async function worker() { + while (next < keys.length && !rateLimited) { + if (now() >= deadline) { + timedOut = true; + return; + } + const key = keys[next]; + next += 1; + try { + results.set(key, await lookup(key)); + } catch (error) { + if (error instanceof RateLimited) rateLimited = true; + else failed += 1; + } + if (spacingMs > 0) await sleep(spacingMs); + } + } + + await Promise.all(Array.from({ length: Math.max(1, concurrency) }, worker)); + return { results, failed, rateLimited, timedOut }; +} diff --git a/static/index.html b/static/index.html index 99ea528..fddb44b 100644 --- a/static/index.html +++ b/static/index.html @@ -215,7 +215,7 @@

Snapshot

  • handles Normalized (see Match a channel), without @, deduplicated and sorted.
  • -
  • ids Channel IDs matching ^UC[A-Za-z0-9_-]{22}$. Empty for now; check them anyway.
  • +
  • ids Channel IDs matching ^UC[A-Za-z0-9_-]{22}$. They include the resolved ID of each listed handle, so check both.
  • An entry in block never also appears in warn.
@@ -373,8 +373,8 @@

Data and licence

const m = decodePayload(envelope.payload); const updated = new Date(m.generated_at); $('s-version').textContent = `v${m.version}`; - $('s-block').textContent = number.format(m.counts.block_handles + m.counts.block_ids); - $('s-warn').textContent = number.format(m.counts.warn_handles + m.counts.warn_ids); + $('s-block').textContent = number.format(m.counts.block_handles); + $('s-warn').textContent = number.format(m.counts.warn_handles); $('s-updated').textContent = relative(updated); $('s-updated').title = updated.toUTCString(); const commit = document.createElement('a'); diff --git a/test/e2e.test.js b/test/e2e.test.js index dc2add7..b139734 100644 --- a/test/e2e.test.js +++ b/test/e2e.test.js @@ -161,3 +161,51 @@ test('a guard abort exits non-zero, writes a report and leaves the tree unchange const forced = await publishFixture(work, 'v4-shrunk', ['--force']); assert.equal(forced.lists[0].version, 2); }); + +test('channel ids fill in over runs within the lookup budget, and the cache survives a guard abort', async (t) => { + const work = await mkdtemp(path.join(tmpdir(), 'ailist-e2e-')); + t.after(() => rm(work, { recursive: true, force: true })); + const out = path.join(work, 'out'); + const cacheDir = path.join(work, 'cache'); + const publish = (version, extra = []) => + publishFixture(work, version, [ + '--sources', path.join(FIXTURES, 'sources-resolve'), + '--cache', cacheDir, + '--resolve-fixture', path.join(FIXTURES, 'resolve-answers.json'), + ...extra, + ]); + const cacheLines = async () => + (await readFile(path.join(cacheDir, `${LIST}.tsv`), 'utf8')).split('\n').filter((l) => l && !l.startsWith('#')); + + // Budget 5 per run: the first run resolves the first five handles in sorted order. + await publish('v1'); + const v1 = await readSnapshotLists(out, 1); + assert.equal((await cacheLines()).length, 5); + assert.ok(v1.block.ids.includes('UCabcdefghijklmnopqrstuv'), 'source ids are kept'); + assert.ok(v1.block.ids.length > 1, 'resolved ids are published'); + + // v2 adds handles: new ones are looked up before the backlog. + const s2 = await publish('v2'); + assert.equal(s2.lists[0].version, 2); + const v2 = await readSnapshotLists(out, 2); + for (const id of ['UCnewoneAAAAAAAAAAAAAAAA', 'UCnewtwoAAAAAAAAAAAAAAAA']) assert.ok(v2.block.ids.includes(id), `${id} resolved first`); + + // Repeated runs drain the backlog; the allow-listed handle is never looked up or published. + for (let i = 0; i < 4; i += 1) await publish('v2'); + const lines = await cacheLines(); + assert.ok(!lines.some((l) => l.startsWith('allowedfalsepositive\t'))); + assert.ok(lines.some((l) => l.startsWith('소소한작업실-z8d\t-\t')), 'a miss is cached as -'); + const manifest = JSON.parse(await readFile(path.join(out, manifestPath(LIST)), 'utf8')); + const counts = JSON.parse(Buffer.from(manifest.payload, 'base64').toString('utf8')).counts; + assert.equal(counts.block_ids, 14, '13 resolved plus the source id'); + assert.equal(counts.warn_ids, 2); + const latest = await readSnapshotLists(out, JSON.parse(Buffer.from(manifest.payload, 'base64').toString('utf8')).version); + assert.ok(latest.block.ids.includes('UCwarntwoAAAAAAAAAAAAAAA'), 'warn-two moved to block in v2, and its id with it'); + assert.ok(!latest.warn.ids.includes('UCwarntwoAAAAAAAAAAAAAAA'), 'block wins over warn for ids too'); + assert.ok(!latest.block.ids.includes('UCallowedfalseAAAAAAAAAA')); + + // A guard abort keeps every cached lookup, even for handles the bad source dropped. + const before = await cacheLines(); + await assert.rejects(publish('v4-shrunk')); + assert.deepEqual(await cacheLines(), before); +}); diff --git a/test/fixtures/resolve-answers.json b/test/fixtures/resolve-answers.json new file mode 100644 index 0000000..bd37e37 --- /dev/null +++ b/test/fixtures/resolve-answers.json @@ -0,0 +1,22 @@ +{ + "chickofhonor-l": "UCchickofhonorAAAAAAAAAA", + "dpcartoonài-v6k": "UCdpcartoonAAAAAAAAAAAAA", + "hfy_en_español": "UChfyAAAAAAAAAAAAAAAAAAA", + "소소한작업실-z8d": null, + "eltíofrontal": "UCeltiofrontalAAAAAAAAAA", + "músicabíblicainstrumental": "UCmusicabiblicaAAAAAAAAA", + "aiomoあいおも": "UCaiomoAAAAAAAAAAAAAAAAA", + "drew": "UCdrewAAAAAAAAAAAAAAAAAA", + "somechannel": "UCsomechannelAAAAAAAAAAA", + "another.one_2": "UCanotheroneAAAAAAAAAAAA", + "dup-case": "UCdupcaseAAAAAAAAAAAAAAA", + "mover": "UCmoverAAAAAAAAAAAAAAAAA", + "maybechannel": "UCmaybechannelAAAAAAAAAA", + "warn-two": "UCwarntwoAAAAAAAAAAAAAAA", + "warn-three": "UCwarnthreeAAAAAAAAAAAAA", + "warn-four": "UCwarnfourAAAAAAAAAAAAAA", + "newone": "UCnewoneAAAAAAAAAAAAAAAA", + "newtwo": "UCnewtwoAAAAAAAAAAAAAAAA", + "third-wave": "UCthirdwaveAAAAAAAAAAAAA", + "allowedfalsepositive": "UCallowedfalseAAAAAAAAAA" +} diff --git a/test/fixtures/sources-resolve/aislist.json b/test/fixtures/sources-resolve/aislist.json new file mode 100644 index 0000000..9e4219d --- /dev/null +++ b/test/fixtures/sources-resolve/aislist.json @@ -0,0 +1,27 @@ +{ + "id": "aislist", + "name": "AiSList", + "description": "Community list of YouTube channels that mainly publish AI-generated content", + "homepage": "https://aisloplist.com", + "repo": "Override92/AiSList", + "branch": "main", + "files": { + "block": "AiSList/aislist_blocklist.txt", + "warn": "AiSList/aislist_warnlist.txt" + }, + "format": "aislist-txt", + "license": "CC BY-NC 4.0", + "guards": { + "min_block_entries": 5, + "max_shrink_ratio": 0.2, + "max_growth_ratio": 0.5, + "max_reject_ratio": 0.05, + "max_header_mismatch_ratio": 0.05 + }, + "resolve": { + "daily_budget": 5, + "recheck_found_days": 90, + "recheck_missing_days": 30, + "max_minutes": 5 + } +} diff --git a/test/guards.test.js b/test/guards.test.js index 29277e6..3923e59 100644 --- a/test/guards.test.js +++ b/test/guards.test.js @@ -109,3 +109,13 @@ test('missing thresholds are a config error', () => { assert.throws(() => validateLimits({ ...LIMITS, max_growth_ratio: undefined }), /guards\.max_growth_ratio/); assert.throws(() => validateLimits(undefined), /guards\.min_block_entries/); }); + +test('ids found by this run\'s lookups do not count as growth, source growth still does', () => { + const limits = { min_block_entries: 1, max_shrink_ratio: 0.1, max_growth_ratio: 0.25, max_reject_ratio: 0.5, max_header_mismatch_ratio: 0.5 }; + const stats = { entryLines: 10, rejected: 0, headers: [] }; + const base = { limits, stats, previousBlockEntries: 100 }; + assert.deepEqual(checkGuards({ ...base, blockEntries: 300, lookupGrowth: 190 }).failures, []); + const failed = checkGuards({ ...base, blockEntries: 300, lookupGrowth: 50 }).failures; + assert.equal(failed.length, 1); + assert.match(failed[0].message, /50 of them new lookups/); +}); diff --git a/test/resolve.test.js b/test/resolve.test.js new file mode 100644 index 0000000..4d40785 --- /dev/null +++ b/test/resolve.test.js @@ -0,0 +1,97 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseCache, serializeCache, pruneCache, planLookups, mergeResults, cachedIds } from '../src/resolve-cache.js'; +import { resolveHandle, runLookups, RateLimited } from '../src/resolve.js'; + +const ID_A = 'UCaaaaaaaaaaaaaaaaaaaaaa'; +const ID_B = 'UCbbbbbbbbbbbbbbbbbbbbbb'; +const RECHECK = { recheckFoundDays: 90, recheckMissingDays: 30 }; + +test('cache text round-trips, sorted, with misses as -', () => { + const cache = new Map([ + ['zeta', { id: null, checked: '2026-11-01' }], + ['alpha', { id: ID_A, checked: '2026-11-02' }], + ]); + const text = serializeCache(cache, 'header'); + assert.equal(text, `# header\nalpha\t${ID_A}\t2026-11-02\nzeta\t-\t2026-11-01\n`); + assert.deepEqual(parseCache(text), new Map([...cache].sort())); + assert.deepEqual(parseCache(undefined), new Map()); +}); + +test('a malformed cache line throws instead of being skipped', () => { + assert.throws(() => parseCache(`alpha\tUCshort\t2026-11-01\n`), /line 1/); + assert.throws(() => parseCache(`alpha\t${ID_A}\n`), /line 1/); + assert.throws(() => parseCache(`alpha\t${ID_A}\t11/01/2026\n`), /line 1/); +}); + +test('lookups take new keys first, then the backlog, then rechecks, within the budget', () => { + const cache = new Map([ + ['found-old', { id: ID_A, checked: '2026-07-01' }], + ['found-new', { id: ID_B, checked: '2026-11-01' }], + ['miss-old', { id: null, checked: '2026-10-01' }], + ['miss-new', { id: null, checked: '2026-11-10' }], + ]); + const keys = ['backlog-b', 'backlog-a', 'fresh', 'found-old', 'found-new', 'miss-old', 'miss-new']; + const base = { keys, priority: ['fresh', 'unlisted'], cache, today: '2026-11-15', ...RECHECK }; + assert.deepEqual(planLookups({ ...base, budget: 10 }), ['fresh', 'backlog-a', 'backlog-b', 'found-old', 'miss-old']); + assert.deepEqual(planLookups({ ...base, budget: 2 }), ['fresh', 'backlog-a']); + assert.deepEqual(planLookups({ ...base, budget: 0 }), []); +}); + +test('merge keeps failed keys, prune drops unlisted ones, cachedIds skips misses', () => { + const cache = new Map([ + ['kept', { id: ID_A, checked: '2026-10-01' }], + ['gone', { id: ID_B, checked: '2026-10-01' }], + ]); + const merged = mergeResults(cache, new Map([['new', { id: null }], ['kept', { id: ID_B }]]), '2026-11-15'); + assert.deepEqual(merged.get('kept'), { id: ID_B, checked: '2026-11-15' }); + assert.deepEqual(merged.get('new'), { id: null, checked: '2026-11-15' }); + assert.equal(cache.get('kept').id, ID_A, 'the input cache is not changed'); + const pruned = pruneCache(merged, ['kept', 'new']); + assert.deepEqual([...pruned.keys()].sort(), ['kept', 'new']); + assert.deepEqual(cachedIds(pruned, ['kept', 'new', 'unknown']), new Set([ID_B])); +}); + +const jsonResponse = (status, body) => ({ status, ok: status >= 200 && status < 300, json: async () => body }); + +test('resolveHandle reads the browse id, treats 404 as a miss, and throws on other errors', async () => { + let sent; + const found = async (url, init) => { + sent = JSON.parse(init.body); + return jsonResponse(200, { endpoint: { browseEndpoint: { browseId: ID_A } } }); + }; + assert.deepEqual(await resolveHandle('músicabíblica', { fetchImpl: found }), { id: ID_A }); + assert.equal(sent.url, 'https://www.youtube.com/@m%C3%BAsicab%C3%ADblica'); + assert.deepEqual(await resolveHandle('gone', { fetchImpl: async () => jsonResponse(404, null) }), { id: null }); + const notAChannel = async () => jsonResponse(200, { endpoint: { browseEndpoint: { browseId: 'FEwhat_to_watch' } } }); + assert.deepEqual(await resolveHandle('odd', { fetchImpl: notAChannel }), { id: null }); + await assert.rejects(resolveHandle('x', { fetchImpl: async () => jsonResponse(500, null) }), /answered 500/); + await assert.rejects(resolveHandle('x', { fetchImpl: async () => jsonResponse(429, null) }), RateLimited); +}); + +test('runLookups counts failures and stops starting lookups after a rate limit', async () => { + const seen = []; + const lookup = async (key) => { + seen.push(key); + if (key === 'bad') throw new Error('boom'); + if (key === 'limit') throw new RateLimited('429'); + return { id: null }; + }; + const run = await runLookups(['a', 'bad', 'limit', 'never', 'never2'], lookup, { concurrency: 1, spacingMs: 0 }); + assert.deepEqual(seen, ['a', 'bad', 'limit']); + assert.deepEqual([...run.results.keys()], ['a']); + assert.equal(run.failed, 1); + assert.equal(run.rateLimited, true); +}); + +test('runLookups stops at the deadline', async () => { + let clock = 0; + const run = await runLookups(['a', 'b', 'c'], async () => ({ id: null }), { + concurrency: 1, + spacingMs: 0, + deadline: 2, + now: () => clock++, + }); + assert.equal(run.results.size, 2); + assert.equal(run.timedOut, true); +});