From e01b05ac20097723dcc8604c8e482d124414cd95 Mon Sep 17 00:00:00 2001 From: HugoXK Date: Tue, 17 Nov 2026 06:42:33 +0000 Subject: [PATCH 1/2] feat(retention): read the prune windows from config and preview them (#281) Every age window `pruneOldData` enforced was a literal inside one function, so no deployment could read, change or preview one without a code change and a release. The windows now live in `scripts/lib/retention.mjs` with today's number as the effective default; `loadConfig` builds `config.retention.windows` from `RETENTION_*_DAYS`, where unset means the shipped default rather than "keep forever"; `pruneOldData` takes the windows and keeps every predicate. The new read-only `scripts/retention.mjs --dry-run --days N` counts what a window would delete through the same predicate, and `/admin` gains a read-only Data retention card with the effective window, a `default` mark and the current count. Sessions (expiry-based) and API tokens (no age window) are named instead of guessed. Docs: docs/architecture.md, docs/deploy.md, docs/security.md, .env.example. Written-by: Fledge Platform Dev-3 --- .env.example | 20 +++ changelog.d/281.added.md | 1 + docs/architecture.md | 19 +++ docs/deploy.md | 42 ++++++ docs/security.md | 15 +++ scripts/lib/retention.d.mts | 57 ++++++++ scripts/lib/retention.mjs | 195 +++++++++++++++++++++++++++ scripts/retention.mjs | 83 ++++++++++++ src/app.ts | 10 ++ src/config.ts | 24 ++++ src/db.ts | 46 ++++--- src/jobs/service.ts | 2 +- src/views/pages.ts | 43 ++++++ test/retention.test.ts | 253 ++++++++++++++++++++++++++++++++++++ test/sectionNav.test.ts | 2 +- 15 files changed, 792 insertions(+), 20 deletions(-) create mode 100644 changelog.d/281.added.md create mode 100644 scripts/lib/retention.d.mts create mode 100644 scripts/lib/retention.mjs create mode 100644 scripts/retention.mjs create mode 100644 test/retention.test.ts diff --git a/.env.example b/.env.example index 2252cdda..e1b92169 100644 --- a/.env.example +++ b/.env.example @@ -111,6 +111,26 @@ ARTIFACT_RETENTION_KEEP=3 ARTIFACT_RETENTION_MAX_AGE_DAYS=14 ARTIFACT_RETENTION_ROOT= +# ── Data retention (#281) ──────────────────────────────────────────────── +# Age windows for the scheduler's prune pass, in days. Unset takes the built-in +# value shown after the name, and that is what the service deletes by today: an +# empty line never means "keep forever". A value that is not an integer or is +# below 1 falls back to the default rather than becoming 0 ("delete everything"). +# Preview a change first: node scripts/retention.mjs --dry-run --days N +# RETENTION_OAUTH_STATES_DAYS=7 +# RETENTION_APPROVAL_TOKENS_DAYS=30 +# RETENTION_NOTIFICATIONS_READ_DAYS=30 +# RETENTION_NOTIFICATIONS_UNREAD_DAYS=90 +# RETENTION_EVENTS_DAYS=90 +# RETENTION_JOB_EVENTS_DAYS=365 +# RETENTION_WEBHOOK_DELIVERIES_DAYS=30 +# RETENTION_OUTBOX_SENT_DAYS=30 +# RETENTION_OUTBOX_DEAD_DAYS=90 +# RETENTION_OUTBOX_UNKNOWN_DAYS=365 +# RETENTION_JOBS_DAYS=365 +# RETENTION_AUDIT_LOG_DAYS=365 +# RETENTION_QUOTA_LEDGER_DAYS=730 + # ── Storage ────────────────────────────────────────────────────────────── DATA_DIR=./data # Optional; defaults to DATA_DIR/backups. diff --git a/changelog.d/281.added.md b/changelog.d/281.added.md new file mode 100644 index 00000000..6448f828 --- /dev/null +++ b/changelog.d/281.added.md @@ -0,0 +1 @@ +Operators can set each data-retention window with a `RETENTION_*_DAYS` environment variable and preview a change before making it: the new read-only `node scripts/retention.mjs --dry-run --days N` prints, per table, how many rows that window would delete without deleting anything, and the `/admin` page gains a **Data retention** card with the effective window per class and its current count. diff --git a/docs/architecture.md b/docs/architecture.md index 5462b3a5..05e617a9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2201,6 +2201,25 @@ all, so the console can report it only as staleness plus a mismatch; the drill prints its verdict and records nothing, so the card says "restore drill: none recorded" rather than a green it cannot prove. +### Data retention windows (#281) + +`Store.pruneOldData` is the one age-based delete pass, run from the scheduler +tick (at most hourly) and otherwise unchanged. Every window now comes from +`config.retention.windows`, built in `loadConfig` from `scripts/lib/retention.mjs` +— one table that also feeds each predicate and the preview, so the rows a preview +counts are exactly the rows a deploy deletes. Unset keeps the literal this +service has always used (OAuth states 7 days; approval tokens, read +notifications, webhook deliveries and sent outbox 30; unread notifications and +events 90; dead/blocked outbox 90; job events, finished jobs, audit log and +unknown outbox 365; quota ledger 730), and `configured` is false for a fallback +so the page can mark it `default`. Unset never means "keep forever": that would +silently stop deleting rows the service deletes today. `sessions` are removed by +`expires_at` and `api_tokens` have no age window; both are reported as such +rather than given a number. `Store.retentionCounts` counts at the configured +windows for the `/admin` card, which is read-only and has no window control; the +operator tool `scripts/retention.mjs --dry-run --days N` counts any window on a +host copy. + ### Operational alerts A failed job also feeds the ops signal: three consecutive failures in one diff --git a/docs/deploy.md b/docs/deploy.md index a29a4433..63dea082 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -1442,6 +1442,48 @@ Run it from a timer only after a dry run has shown the candidates: 10 4 * * * cd /opt/fledge && node scripts/artifact-retention.mjs --root /var/lib/fledge/flybot-users --apply ``` +### Data retention windows + +`pruneOldData` runs in the scheduler tick (at most hourly) and deletes rows older +than a per-class window. Each window is an environment variable; unset takes the +built-in value, which is the number the service already enforced: + +| Variable | Class | Default | +|---|---|---| +| `RETENTION_OAUTH_STATES_DAYS` | `oauth_states` | 7 | +| `RETENTION_APPROVAL_TOKENS_DAYS` | `approval_tokens` (expired or used) | 30 | +| `RETENTION_NOTIFICATIONS_READ_DAYS` | read `notifications` | 30 | +| `RETENTION_NOTIFICATIONS_UNREAD_DAYS` | unread `notifications` | 90 | +| `RETENTION_EVENTS_DAYS` | `events` | 90 | +| `RETENTION_JOB_EVENTS_DAYS` | `job_events` | 365 | +| `RETENTION_WEBHOOK_DELIVERIES_DAYS` | `webhook_deliveries` | 30 | +| `RETENTION_OUTBOX_SENT_DAYS` | sent `notification_outbox` | 30 | +| `RETENTION_OUTBOX_DEAD_DAYS` | dead/blocked `notification_outbox` | 90 | +| `RETENTION_OUTBOX_UNKNOWN_DAYS` | unknown `notification_outbox` | 365 | +| `RETENTION_JOBS_DAYS` | finished `jobs` | 365 | +| `RETENTION_AUDIT_LOG_DAYS` | `audit_log` | 365 | +| `RETENTION_QUOTA_LEDGER_DAYS` | `quota_ledger` | 730 | + +An unset window is the default, never "keep forever", and a value that is not an +integer or is below 1 falls back to the default rather than being read as zero +(zero here would mean "delete every row"). `sessions` are removed by +`expires_at` and `api_tokens` have no age window; neither is configurable here. + +Preview any change before making it. The tool opens the database read-only, holds +no write lock, prints one `count=` line per class and two states instead of a +number: + +```sh +sudo -u fledge DB_PATH=/var/lib/fledge/fledge.db node scripts/retention.mjs --dry-run --days 30 +``` + +`--days` is required; it never guesses a window. `--dry-run` is accepted for +symmetry with `npm run artifacts:prune`, and the tool never deletes rows with or +without it. The output is counts only — never a row body, login, email or token +value — so the preview can be pasted into a log. The same counts for the +configured windows appear on `/admin`'s **Data retention** card, with an unset +window marked `default`. + ## 10. Go-live checklist - [ ] HTTPS `BASE_URL`; OAuth callback matches exactly. diff --git a/docs/security.md b/docs/security.md index 11b043e2..8dbbd6e9 100644 --- a/docs/security.md +++ b/docs/security.md @@ -619,6 +619,21 @@ whose duplicate risk a person still has to judge; a human resend is audited as deletes the GitHub installation row with the account; it does not call GitHub to uninstall the App installation. +- **Row retention windows (#281).** The scheduler's age pass deletes + `oauth_states`, `approval_tokens`, read and unread `notifications`, `events`, + `job_events`, `webhook_deliveries`, the three `notification_outbox` states, + finished `jobs`, `audit_log` and `quota_ledger` by a per-class window; the + numbers are configuration (`RETENTION_*_DAYS`) with the shipped literal as the + effective default. An unset window is never "keep forever": that would stop a + deletion the service performs today. A value that is not an integer or is + below 1 falls back to the default rather than becoming zero, because zero here + would mean "delete every row". `sessions` age out by `expires_at` and + `api_tokens` are revoked or removed with their account; the preview names both + states instead of printing a count. Before a window is shortened, + `node scripts/retention.mjs --dry-run --days N` counts what it would delete — + read-only, printing counts and never a row body, login, email or token value. + The `/admin` card shows the same counts for the configured windows. + ### Platform emergency stop - `POST /admin/platform/stop` and `POST /admin/platform/resume` are the diff --git a/scripts/lib/retention.d.mts b/scripts/lib/retention.d.mts new file mode 100644 index 00000000..31ed4fb0 --- /dev/null +++ b/scripts/lib/retention.d.mts @@ -0,0 +1,57 @@ +export type RetentionId = + | 'oauthStates' + | 'approvalTokens' + | 'notificationsRead' + | 'notificationsUnread' + | 'events' + | 'jobEvents' + | 'webhookDeliveries' + | 'outboxSent' + | 'outboxDead' + | 'outboxUnknown' + | 'jobs' + | 'auditLog' + | 'quotaLedger' + +export type RetentionWindows = Record + +export interface RetentionAt { + now: string + cutoff: string +} + +export interface RetentionWhere { + sql: string + params: string[] +} + +export interface RetentionClass { + id: RetentionId + key: string + table: string + env: string + defaultDays: number + label: readonly [string, string] + where: (at: RetentionAt) => RetentionWhere +} + +export interface RetentionIdleTable { + id: 'sessions' | 'api_tokens' + state: 'expiry-based' | 'no-age-window' + label: readonly [string, string] +} + +export interface RetentionCount { + id: RetentionId + key: string + days: number + count: number +} + +export const RETENTION_CLASSES: readonly RetentionClass[] +export const RETENTION_IDLE_TABLES: readonly RetentionIdleTable[] +export function retentionDefaultWindows(): RetentionWindows +export function retentionWindow(raw: string | undefined | null, fallback: number): { days: number; configured: boolean } +export function retentionDeleteSql(cls: RetentionClass, at: RetentionAt): { sql: string; params: string[] } +export function retentionCountSql(cls: RetentionClass, at: RetentionAt): { sql: string; params: string[] } +export function retentionCountsAt(count: (sql: string, params: string[]) => number, days: number, now?: number): RetentionCount[] diff --git a/scripts/lib/retention.mjs b/scripts/lib/retention.mjs new file mode 100644 index 00000000..4da5717f --- /dev/null +++ b/scripts/lib/retention.mjs @@ -0,0 +1,195 @@ +// #281: the age windows `pruneOldData` enforces, in one table. +// +// The store, the read-only dry-run tool and the admin page all read this file. +// A window that can delete rows and a window that a preview reports must be the +// same number and the same predicate, or the preview is a guess; keeping them in +// one place is what makes `scripts/retention.mjs --days N` honest about what a +// deploy will do. +// +// `defaultDays` is the literal this service shipped with, and it is also the +// effective value when the env var is unset: "unset" must never mean "keep +// forever", because that would silently stop deleting rows that are deleted +// today. + +/** + * One retention class. `where` returns the SQL after `WHERE` plus its params; + * it receives the run instant and the cutoff so a predicate that compares + * against "now" (an expired approval token) cannot accidentally compare against + * the cutoff instead. + */ +export const RETENTION_CLASSES = [ + { + id: 'oauthStates', + key: 'oauth_states', + table: 'oauth_states', + env: 'RETENTION_OAUTH_STATES_DAYS', + defaultDays: 7, + label: ['OAuth 状态', 'OAuth states'], + where: ({ cutoff }) => ({ sql: 'created_at < ?', params: [cutoff] }), + }, + { + id: 'approvalTokens', + key: 'approval_tokens', + table: 'approval_tokens', + env: 'RETENTION_APPROVAL_TOKENS_DAYS', + defaultDays: 30, + label: ['审批令牌', 'Approval tokens'], + where: ({ now, cutoff }) => ({ sql: '(expires_at < ? OR used_at IS NOT NULL) AND created_at < ?', params: [now, cutoff] }), + }, + { + id: 'notificationsRead', + key: 'notifications_read', + table: 'notifications', + env: 'RETENTION_NOTIFICATIONS_READ_DAYS', + defaultDays: 30, + label: ['已读通知', 'Read notifications'], + where: ({ cutoff }) => ({ sql: 'read_at IS NOT NULL AND created_at < ?', params: [cutoff] }), + }, + { + id: 'notificationsUnread', + key: 'notifications_unread', + table: 'notifications', + env: 'RETENTION_NOTIFICATIONS_UNREAD_DAYS', + defaultDays: 90, + label: ['未读通知', 'Unread notifications'], + where: ({ cutoff }) => ({ sql: 'read_at IS NULL AND created_at < ?', params: [cutoff] }), + }, + { + id: 'events', + key: 'events', + table: 'events', + env: 'RETENTION_EVENTS_DAYS', + defaultDays: 90, + label: ['事件', 'Events'], + where: ({ cutoff }) => ({ sql: 'at < ?', params: [cutoff] }), + }, + { + id: 'jobEvents', + key: 'job_events', + table: 'job_events', + env: 'RETENTION_JOB_EVENTS_DAYS', + defaultDays: 365, + label: ['任务事件', 'Job events'], + where: ({ cutoff }) => ({ sql: 'at < ?', params: [cutoff] }), + }, + { + id: 'webhookDeliveries', + key: 'webhook_deliveries', + table: 'webhook_deliveries', + env: 'RETENTION_WEBHOOK_DELIVERIES_DAYS', + defaultDays: 30, + label: ['入站 Webhook 投递', 'Inbound webhook deliveries'], + where: ({ cutoff }) => ({ sql: 'received_at < ?', params: [cutoff] }), + }, + // #544 slice 7: a delivered notification is a receipt, so it goes with the + // other 30-day receipts. The unresolved states are kept longer because they + // are what a human still has to look at; `unknown` keeps its full window so + // the duplicate risk stays visible instead of ageing out with the sent rows. + { + id: 'outboxSent', + key: 'outbox_sent', + table: 'notification_outbox', + env: 'RETENTION_OUTBOX_SENT_DAYS', + defaultDays: 30, + label: ['发件箱 · 已发送', 'Outbox · sent'], + where: ({ cutoff }) => ({ sql: "state = 'sent' AND created_at < ?", params: [cutoff] }), + }, + { + id: 'outboxDead', + key: 'outbox_dead', + table: 'notification_outbox', + env: 'RETENTION_OUTBOX_DEAD_DAYS', + defaultDays: 90, + label: ['发件箱 · 受阻', 'Outbox · dead or blocked'], + where: ({ cutoff }) => ({ sql: "state IN ('dead', 'blocked') AND created_at < ?", params: [cutoff] }), + }, + { + id: 'outboxUnknown', + key: 'outbox_unknown', + table: 'notification_outbox', + env: 'RETENTION_OUTBOX_UNKNOWN_DAYS', + defaultDays: 365, + label: ['发件箱 · 结果未知', 'Outbox · unknown'], + where: ({ cutoff }) => ({ sql: "state = 'unknown' AND created_at < ?", params: [cutoff] }), + }, + { + id: 'jobs', + key: 'jobs', + table: 'jobs', + env: 'RETENTION_JOBS_DAYS', + defaultDays: 365, + label: ['任务', 'Jobs'], + where: ({ cutoff }) => ({ sql: "status IN ('succeeded', 'failed', 'quota_exhausted') AND finished_at IS NOT NULL AND finished_at < ?", params: [cutoff] }), + }, + { + id: 'auditLog', + key: 'audit_log', + table: 'audit_log', + env: 'RETENTION_AUDIT_LOG_DAYS', + defaultDays: 365, + label: ['审计日志', 'Audit log'], + where: ({ cutoff }) => ({ sql: 'at < ?', params: [cutoff] }), + }, + { + id: 'quotaLedger', + key: 'quota_ledger', + table: 'quota_ledger', + env: 'RETENTION_QUOTA_LEDGER_DAYS', + defaultDays: 730, + label: ['额度流水', 'Quota ledger'], + where: ({ cutoff }) => ({ sql: 'at < ?', params: [cutoff] }), + }, +] + +// These two have no age window at all, and reporting a fabricated number for +// them would be worse than saying so: sessions are reclaimed by `expires_at` +// and `api_tokens` are revoked, or removed by the account cascade. Adding a +// delete path for either is a behaviour change, not a config value. +export const RETENTION_IDLE_TABLES = [ + { id: 'sessions', state: 'expiry-based', label: ['会话', 'Sessions'] }, + { id: 'api_tokens', state: 'no-age-window', label: ['API 令牌', 'API tokens'] }, +] + +/** Every window at its built-in value. The defaults must equal today's literals. */ +export function retentionDefaultWindows() { + return Object.fromEntries(RETENTION_CLASSES.map((cls) => [cls.id, cls.defaultDays])) +} + +/** + * Parse one configured window. Empty, non-numeric or below one day means the + * built-in value, and `configured` false is what the operator surface marks as + * `default`. A window of zero is refused rather than read as "delete + * everything": that is a data-loss instruction, not a retention window. + */ +export function retentionWindow(raw, fallback) { + const text = raw === undefined || raw === null ? '' : String(raw).trim() + if (text === '') return { days: fallback, configured: false } + const n = Number(text) + if (!Number.isInteger(n) || n < 1) return { days: fallback, configured: false } + return { days: n, configured: true } +} + +/** The DELETE the scheduler runs for one class. */ +export function retentionDeleteSql(cls, at) { + const { sql, params } = cls.where(at) + return { sql: `DELETE FROM ${cls.table} WHERE ${sql}`, params } +} + +/** The matching COUNT, so a preview cannot select different rows than the delete. */ +export function retentionCountSql(cls, at) { + const { sql, params } = cls.where(at) + return { sql: `SELECT COUNT(*) AS n FROM ${cls.table} WHERE ${sql}`, params } +} + +/** + * Count what each window would delete at `days`, through the same predicates. + * `count(sql, params)` returns the scalar, so the store and the plain-node tool + * share this loop instead of each keeping its own. + */ +export function retentionCountsAt(count, days, now = Date.now()) { + const at = { now: new Date(now).toISOString(), cutoff: new Date(now - days * 86_400_000).toISOString() } + return RETENTION_CLASSES.map((cls) => { + const { sql, params } = retentionCountSql(cls, at) + return { id: cls.id, key: cls.key, days, count: Number(count(sql, params)) } + }) +} diff --git a/scripts/retention.mjs b/scripts/retention.mjs new file mode 100644 index 00000000..78276341 --- /dev/null +++ b/scripts/retention.mjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node +// #281: report what the retention windows would delete — and nothing else. +// +// The windows live in `scripts/lib/retention.mjs`, the same table `Store.pruneOldData` +// runs, so this tool cannot count different rows than a deploy deletes. It opens +// the database read-only, holds no write lock, prints one count per class and +// never a row body, login, email or token value: a preview that leaks the data +// it is about to delete is worse than no preview. +// +// `--days` is required on purpose. A guessed default in a tool whose output is +// "how many rows will disappear" is how a window gets shortened by accident, so +// the operator states the number they are asking about. +// +// node scripts/retention.mjs --dry-run --days 30 +// DB_PATH=/data/fledge.db node scripts/retention.mjs --dry-run --days 90 +import { DatabaseSync } from 'node:sqlite' +import { join } from 'node:path' +import { RETENTION_IDLE_TABLES, retentionCountsAt } from './lib/retention.mjs' + +const USAGE = 'usage: node scripts/retention.mjs --dry-run --days [--db ]' + +function usageError(message) { + console.error(`${USAGE}\n${message}`) + process.exit(2) +} + +let days +let dryRun = false +let dbPath = process.env.DB_PATH ?? join(process.env.DATA_DIR ?? './data', 'fledge.db') + +for (let index = 2; index < process.argv.length; index += 1) { + const arg = process.argv[index] + if (arg === '--dry-run') { + dryRun = true + } else if (arg === '--days') { + const raw = process.argv[index + 1] + if (raw === undefined) usageError('--days needs a value: --days ') + const n = Number(raw) + if (!Number.isInteger(n) || n < 1) usageError(`--days must be an integer >= 1, got "${raw}"`) + days = n + index += 1 + } else if (arg === '--db') { + const raw = process.argv[index + 1] + if (raw === undefined || raw === '') usageError('--db needs a value: --db ') + dbPath = raw + index += 1 + } else if (arg === '--help' || arg === '-h') { + console.log(USAGE) + process.exit(0) + } else { + usageError(`unknown argument: ${arg}`) + } +} + +if (days === undefined) usageError('--days is required: the window this preview counts at') + +if (!dryRun) { + // The tool is read-only either way; `--dry-run` is the explicit spelling the + // runbook asks for. Saying so on stderr keeps stdout a plain count report. + console.error('note: this tool never deletes rows; --dry-run is accepted for symmetry with other operator scripts.') +} + +let db +try { + db = new DatabaseSync(dbPath, { readOnly: true }) +} catch (error) { + console.error(`cannot open ${dbPath} read-only: ${error instanceof Error ? error.message : String(error)}`) + process.exit(1) +} + +try { + const count = (sql, params) => Number(db.prepare(sql).get(...params)?.n ?? 0) + for (const row of retentionCountsAt(count, days)) { + console.log(`${row.key} count=${row.count} window=${days}d`) + } + // Two tables have no age window. Naming the reason is better than printing a + // fabricated zero that reads as "nothing to delete". + for (const table of RETENTION_IDLE_TABLES) { + console.log(`${table.id} state=${table.state}`) + } +} finally { + db.close() +} diff --git a/src/app.ts b/src/app.ts index 16806c89..3c839531 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,6 +8,7 @@ import { Readable } from 'node:stream' import type { Config } from './config.js' import { assertProductionConfig } from './config.js' import { openStore, type GateSessionRow, type JobRow, type ProjectMemoryVersionRow, type RepoRow, type Store, type UserRow, type WorkflowFailureStatus } from './db.js' +import { RETENTION_IDLE_TABLES } from '../scripts/lib/retention.mjs' import { createAnnotatedTag, parseVersion, releasePreflight, releaseRefusals } from './releases/release.js' import { reconcileRelease, trackingUpdate } from './releases/tracking.js' import { collectReleaseNotes, releaseNotesSourceJson } from './releases/notes.js' @@ -5539,6 +5540,15 @@ export async function buildApp(options: BuildOptions): Promise { auditPage, auditTotal, perPage, + // #281: the effective window per class and what it would delete now, plus + // the two tables that have no age window so the page never implies one. + retention: [ + ...store.retentionCounts(config.retention.windows).map((row) => ({ + ...row, + configured: config.retention.configured[row.id], + })), + ...RETENTION_IDLE_TABLES.map((table) => ({ id: table.id, days: null, configured: false, count: null, state: table.state })), + ], platformStop: jobs.platformStop(), superadmin, preview: false, diff --git a/src/config.ts b/src/config.ts index 1fe12e42..27cf31d0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,6 +8,7 @@ import { DEFAULT_PAT_EXPIRY_DAYS } from './lib/tokens.js' import { registerRedactionSecrets } from './lib/secrets.js' import { DEFAULT_RATE_LIMIT_BUDGETS, type RateLimitBudgets } from './lib/rateLimit.js' import { parseManagementService, type ManagementServiceMode } from './management/mode.js' +import { RETENTION_CLASSES, retentionDefaultWindows, retentionWindow, type RetentionId, type RetentionWindows } from '../scripts/lib/retention.mjs' export type RunnerMode = 'mock' | 'flybot' export type MailTransport = 'console' | 'smtp' @@ -41,6 +42,15 @@ export interface Config { // #708: the per-class windows mutating routes take. A route names its class; // the numbers live here so a deployment tightens a limit without a rebuild. rateLimit: RateLimitBudgets + // #281: the age windows `pruneOldData` enforces, read from config so a deploy + // can change one without a code change. `configured` is false when the env var + // is unset, so the operator surface can mark the built-in value as the default. + // Unset means today's window, never "keep forever": the rows are deleted at + // that age today, and stopping that quietly would be a privacy regression. + retention: { + windows: RetentionWindows + configured: Record + } // #637: the comment names that count as a trigger. `@` in an issue or // pull request comment starts a review; a deployment renames or adds names // here rather than in code, and an empty list disables the trigger. @@ -416,6 +426,16 @@ export function loadConfig(overrides: Partial = {}): Config { const keyB64 = text('TOKEN_ENCRYPTION_KEY') const encryptionKey = keyB64 === '' ? randomBytes(32).toString('base64') : keyB64 const dataDir = resolve(text('DATA_DIR', './data')) + // #281: one window per retention class, with the shipped literal as the + // effective default. `retentionWindow` refuses 0 and non-integers rather than + // reading them as "delete everything", and marks a fallback `configured:false`. + const retentionWindows = retentionDefaultWindows() + const retentionConfigured = {} as Record + for (const cls of RETENTION_CLASSES) { + const parsed = retentionWindow(process.env[cls.env], cls.defaultDays) + retentionWindows[cls.id] = parsed.days + retentionConfigured[cls.id] = parsed.configured + } const config: Config = { env, port: number('PORT', 8080), @@ -446,6 +466,10 @@ export function loadConfig(overrides: Partial = {}): Config { windowMs: Math.max(1_000, number('RATE_LIMIT_EXPENSIVE_WINDOW_MS', DEFAULT_RATE_LIMIT_BUDGETS.expensive.windowMs)), }, }, + retention: { + windows: retentionWindows, + configured: retentionConfigured, + }, triggers: { mentions: text('FLEDGE_MENTION_NAMES', 'fledge') .split(',') diff --git a/src/db.ts b/src/db.ts index c644616d..9ebc1c20 100644 --- a/src/db.ts +++ b/src/db.ts @@ -28,6 +28,7 @@ export interface UsageDetail { import { artifactKey, candidateRefusal, type CandidateRefusal } from './lib/evolution/artifacts.js' import { BUDGET_SCOPE_ACTIONS, BUDGET_SCOPE_KINDS, isBudgetScopeAction, isBudgetScopeKind } from './lib/budgetScope.js' import { capabilitiesForRole, deriveRepoRole, identityPrincipalId, legacyTenantId } from './identity/authorization.js' +import { RETENTION_CLASSES, retentionCountSql, retentionDefaultWindows, retentionDeleteSql, type RetentionAt, type RetentionId, type RetentionWindows } from '../scripts/lib/retention.mjs' // #355: one settings row marks the PAT expiry backfill as done, so a later // operator token (explicitly minted without an expiry) is never given one by a @@ -6980,27 +6981,36 @@ export class Store { return { retries, followups, jobEvents, instances } } - pruneOldData(): void { + /** + * #281: how many rows each configured window would delete right now, through + * the same predicate `pruneOldData` runs. The admin page renders these; the + * dry-run tool prints them from the same table. Read-only by construction. + */ + retentionCounts(windows: RetentionWindows = retentionDefaultWindows()): Array<{ id: RetentionId; days: number; count: number }> { + const now = Date.now() + return RETENTION_CLASSES.map((cls) => { + const days = windows[cls.id] ?? cls.defaultDays + const at: RetentionAt = { now: nowIso(), cutoff: new Date(now - days * 86_400_000).toISOString() } + const { sql, params } = retentionCountSql(cls, at) + return { id: cls.id, days, count: Number(this.get<{ n: number }>(sql, ...params)?.n ?? 0) } + }) + } + + // #281: every window comes from config, with the shipped literal as the + // default, and the predicates live beside the dry-run tool in + // `scripts/lib/retention.mjs` so a preview cannot count different rows than + // the delete removes. `deleteExpiredSessions` stays: sessions age out by + // `expires_at`, not by a day count. No new delete path and no default changed. + pruneOldData(windows: RetentionWindows = retentionDefaultWindows()): void { const now = Date.now() const cutoff = (days: number): string => new Date(now - days * 86_400_000).toISOString() + const at = (days: number): RetentionAt => ({ now: nowIso(), cutoff: cutoff(days) }) this.deleteExpiredSessions() - this.run('DELETE FROM oauth_states WHERE created_at < ?', cutoff(7)) - this.run('DELETE FROM approval_tokens WHERE (expires_at < ? OR used_at IS NOT NULL) AND created_at < ?', nowIso(), cutoff(30)) - this.run('DELETE FROM notifications WHERE read_at IS NOT NULL AND created_at < ?', cutoff(30)) - this.run('DELETE FROM notifications WHERE read_at IS NULL AND created_at < ?', cutoff(90)) - this.run('DELETE FROM events WHERE at < ?', cutoff(90)) - this.run('DELETE FROM job_events WHERE at < ?', cutoff(365)) - this.run('DELETE FROM webhook_deliveries WHERE received_at < ?', cutoff(30)) - // #544 slice 7: a delivered notification is a receipt, so it goes with the - // other 30-day receipts. The unresolved states are kept longer because they - // are what a human still has to look at; `unknown` keeps its full window so - // the duplicate risk stays visible instead of ageing out with the sent rows. - this.run("DELETE FROM notification_outbox WHERE state = 'sent' AND created_at < ?", cutoff(30)) - this.run("DELETE FROM notification_outbox WHERE state IN ('dead', 'blocked') AND created_at < ?", cutoff(90)) - this.run("DELETE FROM notification_outbox WHERE state = 'unknown' AND created_at < ?", cutoff(365)) - this.run("DELETE FROM jobs WHERE status IN ('succeeded', 'failed', 'quota_exhausted') AND finished_at IS NOT NULL AND finished_at < ?", cutoff(365)) - this.run('DELETE FROM audit_log WHERE at < ?', cutoff(365)) - this.run('DELETE FROM quota_ledger WHERE at < ?', cutoff(730)) + for (const cls of RETENTION_CLASSES) { + const days = windows[cls.id] ?? cls.defaultDays + const { sql, params } = retentionDeleteSql(cls, at(days)) + this.run(sql, ...params) + } // #419: the prefix list and the cutoff table were two hand-kept copies of // one mapping, so a key could be written without either half ever selecting // it. `plan_expired:` and `plan_reminded:` are written per unapproved plan diff --git a/src/jobs/service.ts b/src/jobs/service.ts index 9e587573..0493fcda 100644 --- a/src/jobs/service.ts +++ b/src/jobs/service.ts @@ -4358,7 +4358,7 @@ export class Jobs { this.store.deleteExpiredJobTokens() if (Date.now() - this.lastPruneAt > 60 * 60 * 1_000) { this.lastPruneAt = Date.now() - this.store.pruneOldData() + this.store.pruneOldData(this.config.retention.windows) } if (this.config.env !== 'test' && this.instances !== undefined) { await this.instances.healthSweep().catch((error) => log.warn('instance health sweep failed', { error: String(error) })) diff --git a/src/views/pages.ts b/src/views/pages.ts index f61540a2..53659166 100644 --- a/src/views/pages.ts +++ b/src/views/pages.ts @@ -43,6 +43,7 @@ import { readReleaseNotesSources } from '../releases/notes.js' import { readReleaseAnalysis } from '../releases/analysis.js' import { compareRunning, type ReleaseStatusSnapshot } from '../lib/releaseStatus.js' import { readBackupStatus, readRestoreDrill, SHA256_MAX_BYTES, type BackupCopyStatus, type BackupStatus, type RestoreDrillStatus } from '../lib/backupStatus.js' +import { RETENTION_CLASSES, RETENTION_IDLE_TABLES, type RetentionId } from '../../scripts/lib/retention.mjs' import { classifyFailure, failureLabel, failureSentence, type FailureAction, type FailureKind } from '../lib/failure.js' import { filterTrials, trialCounts, trialStateOf, TRIAL_DAYS, type TrialCandidateSource, type TrialFilter } from '../lib/trials.js' import { STAGE_MODES } from '../lib/stages.js' @@ -2594,6 +2595,16 @@ function pager(path: string, pageParam: string, page: number, total: number, per return `
${prev}${page} / ${pages} · ${total}${next}
` } +/** One row of the admin retention card: an age window and what it would delete now. */ +export interface RetentionRow { + id: RetentionId | 'sessions' | 'api_tokens' + /** Null for a table with no age window; `state` says why. */ + days: number | null + configured: boolean + count: number | null + state?: string +} + export function adminPage(input: { config: Config locale: Locale @@ -2606,6 +2617,8 @@ export function adminPage(input: { auditPage: number auditTotal: number perPage: number + /** #281: the effective window per retention class and its current count. */ + retention?: RetentionRow[] activity?: Map mailFailedUsers?: Set superadmin?: boolean @@ -2846,6 +2859,34 @@ export function adminPage(input: {

${escapeHtml(t('只读:本卡片列出备份目录并核对清单,不会打开正在备份的数据库,也没有备份、恢复或删除按钮。', 'Read-only: this card lists the backup directory and checks the manifests. It never opens the database being backed up, and carries no backup, restore or delete control.'))} ${escapeHtml(t('只列出真正不符或无法读取的副本;旧副本只按大小核对,sha256 只核对最新副本。', 'Only copies that differ or cannot be read are listed; older copies are compared by size, and sha256 only on the newest.'))}${newest === null ? ` ${escapeHtml(t('在主机上运行', 'Run'))} npm run db:backup ${escapeHtml(t('后,下一份副本会出现在这里。', 'on the host; the next copy appears here.'))}` : ''}

` + // #281: the operator surface for the windows `pruneOldData` enforces. The + // count is what the configured window would delete right now, read through the + // same predicate the scheduler runs; the card is read-only and names a + // fallback window as `default` so a configured number cannot be mistaken for + // the built-in one. + const retentionRows = input.retention ?? [] + const retentionLabel = (id: string): string => { + const timed = RETENTION_CLASSES.find((entry) => entry.id === id) + if (timed !== undefined) return locale === 'zh' ? timed.label[0] : timed.label[1] + const idle = RETENTION_IDLE_TABLES.find((entry) => entry.id === id) + return idle === undefined ? id : locale === 'zh' ? idle.label[0] : idle.label[1] + } + const retentionCard = `
+
+

${escapeHtml(t('数据保留', 'Data retention'))}

+ ${escapeHtml(t('只读预览', 'read-only preview'))} +
+
+ ${retentionRows.map((row) => { + const window = row.days === null + ? escapeHtml(row.state ?? '—') + : `${escapeHtml(t(`${row.days} 天`, `${row.days} days`))}${row.configured ? '' : ` ${escapeHtml(t('默认', 'default'))}`}` + const count = row.count === null ? `${escapeHtml(t('不适用', 'not applicable'))}` : `${escapeHtml(String(row.count))}` + return `
${escapeHtml(retentionLabel(row.id))}${window}
${count}
` + }).join('')} +
+

${escapeHtml(t('只读:窗口来自配置,未配置时是内置默认值(标为 default)。在主机的数据库副本上运行 node scripts/retention.mjs --dry-run --days N,可以在改动窗口前预览任意 N 天会删除多少行。', 'Read-only: the windows come from config, and an unset one is the built-in default (marked default). Run node scripts/retention.mjs --dry-run --days N against a host database to preview any N before changing a window.'))}

+
` // #317: the stop route reports its result once in a redirect notice, and a // reload drops that line. // The record is read fresh per request, so the banner states who stopped the @@ -2898,6 +2939,7 @@ export function adminPage(input: { ${showAudit ? `${escapeHtml(t('审计', 'Audit'))}` : ''} ${escapeHtml(t('发布', 'Release'))} ${escapeHtml(t('备份', 'Backups'))} + ${escapeHtml(t('数据保留', 'Retention'))} ${escapeHtml(t('运维', 'Operations'))}
@@ -2920,6 +2962,7 @@ export function adminPage(input: {
` : ''} ${releaseCard} ${backupCard} + ${retentionCard}

${escapeHtml(t('运维工具', 'Operations'))}

${escapeHtml(t('独立运维页面', 'separate operator pages'))}
diff --git a/test/retention.test.ts b/test/retention.test.ts new file mode 100644 index 00000000..ded12d68 --- /dev/null +++ b/test/retention.test.ts @@ -0,0 +1,253 @@ +import assert from 'node:assert/strict' +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { fileURLToPath } from 'node:url' +import { loadConfig } from '../src/config.js' +import { Store } from '../src/db.js' +import { adminPage } from '../src/views/pages.js' +import { makeUser, testConfig } from './helpers.js' + +// #281: `pruneOldData` enforced ten age windows as literals inside one function, +// so nothing outside the code could read or preview one. This suite pins the +// first slice: the windows come from config with today's literal as the default, +// and `scripts/retention.mjs` reports — read-only — how many rows each window +// would delete at `--days N`. +const TOOL = fileURLToPath(new URL('../scripts/retention.mjs', import.meta.url)) + +// Spelled out rather than imported from the module under test: a regression in +// the id list or a default must show up here as a failure, not follow along. +const DEFAULT_WINDOWS = { + oauthStates: 7, + approvalTokens: 30, + notificationsRead: 30, + notificationsUnread: 90, + events: 90, + jobEvents: 365, + webhookDeliveries: 30, + outboxSent: 30, + outboxDead: 90, + outboxUnknown: 365, + jobs: 365, + auditLog: 365, + quotaLedger: 730, +} + +const DAY = 86_400_000 + +function runTool(dbPath: string, args: string[]): { status: number | null; stdout: string; stderr: string } { + const result = spawnSync(process.execPath, [TOOL, ...args], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, DB_PATH: dbPath }, + }) + return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' } +} + +function count(store: Store, table: string): number { + return store.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ${table}`)?.n ?? 0 +} + +/** One old row and one recent row per class, so a count that includes the recent row is visible. */ +function seed(dbPath: string): void { + const store = new Store(dbPath) + try { + const user = makeUser(store, { login: 'retention-user', status: 'approved' }) + const repo = store.createRepo({ id: 'repo_retention', userId: user.id, owner: 'o', name: 'retention', defaultBranch: 'main', tokenEnc: 'x', permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440 }) + const old = new Date(Date.now() - 40 * DAY).toISOString() + const recent = new Date(Date.now() - 1 * DAY).toISOString() + for (const [id, finishedAt] of [['job_old', old], ['job_new', recent]] as const) { + store.createJob({ id, repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'g', callbackHash: `cb_${id}`, platformTokenHash: '' }) + store.updateJob(id, { status: 'succeeded', finished_at: finishedAt }) + } + const uid = user.id + const sql = ` + INSERT INTO job_events (at, job_id, kind) VALUES ('${old}', 'job_old', 'note'), ('${recent}', 'job_new', 'note'); + INSERT INTO oauth_states (state_hash, created_at, expires_at) VALUES ('h_old', '${old}', '${old}'), ('h_new', '${recent}', '${recent}'); + INSERT INTO approval_tokens (token_hash, user_id, action, created_at, expires_at, used_at) VALUES ('t_old', '${uid}', 'approve', '${old}', '${old}', NULL), ('t_new', '${uid}', 'approve', '${recent}', '${recent}', NULL); + INSERT INTO notifications (id, user_id, kind, message, created_at, read_at) VALUES + ('n_old_read', '${uid}', 'test', 'old read', '${old}', '${old}'), + ('n_new_read', '${uid}', 'test', 'new read', '${recent}', '${recent}'), + ('n_old_unread', '${uid}', 'test', 'old unread', '${old}', NULL), + ('n_new_unread', '${uid}', 'test', 'new unread', '${recent}', NULL); + INSERT INTO events (at, name) VALUES ('${old}', 'test.old'), ('${recent}', 'test.new'); + INSERT INTO webhook_deliveries (delivery_id, event, received_at) VALUES ('d_old', 'push', '${old}'), ('d_new', 'push', '${recent}'); + INSERT INTO quota_ledger (at, user_id, cost_usd) VALUES ('${old}', '${uid}', 0.1), ('${recent}', '${uid}', 0.1); + INSERT INTO audit_log (at, action, detail) VALUES + ('${old}', 'test.old', 'login=leaky-user email=leaky@example.com token=leaky-token-value body=leaky-row-body'), + ('${recent}', 'test.new', 'nothing to see'); + INSERT INTO notification_outbox (id, transport, destination, payload_json, idempotency_key, state, created_at, updated_at) VALUES + ('o_sent_old', 'mail', 'a@example.com', '{}', 'k_sent_old', 'sent', '${old}', '${old}'), + ('o_sent_new', 'mail', 'a@example.com', '{}', 'k_sent_new', 'sent', '${recent}', '${recent}'), + ('o_dead_old', 'mail', 'a@example.com', '{}', 'k_dead_old', 'dead', '${old}', '${old}'), + ('o_unknown_old', 'mail', 'a@example.com', '{}', 'k_unknown_old', 'unknown', '${old}', '${old}'), + ('o_unknown_new', 'mail', 'a@example.com', '{}', 'k_unknown_new', 'unknown', '${recent}', '${recent}'); + ` + store.db.exec(sql) + } finally { + store.close() + } +} + +test('the dry run reports the count each window would delete and changes nothing', () => { + const dir = mkdtempSync(join(tmpdir(), 'fledge-retention-')) + const dbPath = join(dir, 'fledge.db') + try { + seed(dbPath) + const before = (() => { + const store = new Store(dbPath) + try { + return ['jobs', 'job_events', 'audit_log', 'webhook_deliveries', 'quota_ledger', 'oauth_states', 'approval_tokens', 'notifications', 'events', 'notification_outbox'].map((table) => count(store, table)) + } finally { + store.close() + } + })() + + const first = runTool(dbPath, ['--dry-run', '--days', '30']) + assert.equal(first.status, 0, first.stderr) + const expected: Record = { + oauth_states: 1, + approval_tokens: 1, + notifications_read: 1, + notifications_unread: 1, + events: 1, + job_events: 1, + webhook_deliveries: 1, + outbox_sent: 1, + outbox_dead: 1, + outbox_unknown: 1, + jobs: 1, + audit_log: 1, + quota_ledger: 1, + } + for (const [id, n] of Object.entries(expected)) { + const match = new RegExp(`^${id} count=(\\d+) window=30d$`, 'm').exec(first.stdout) + assert.ok(match, `no line for ${id} in:\n${first.stdout}`) + assert.equal(Number(match[1]), n, `${id} counted the recent row as well or missed the old one`) + } + + const second = runTool(dbPath, ['--dry-run', '--days', '30']) + assert.equal(second.status, 0, second.stderr) + assert.equal(second.stdout, first.stdout, 'two consecutive dry runs differ') + + const after = (() => { + const store = new Store(dbPath) + try { + return ['jobs', 'job_events', 'audit_log', 'webhook_deliveries', 'quota_ledger', 'oauth_states', 'approval_tokens', 'notifications', 'events', 'notification_outbox'].map((table) => count(store, table)) + } finally { + store.close() + } + })() + assert.deepEqual(after, before, 'the dry run deleted rows') + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('--days is required rather than defaulted', () => { + const dir = mkdtempSync(join(tmpdir(), 'fledge-retention-')) + const dbPath = join(dir, 'fledge.db') + try { + seed(dbPath) + const result = runTool(dbPath, ['--dry-run']) + assert.notEqual(result.status, 0, 'a dry run without --days exited 0') + assert.match(`${result.stdout}${result.stderr}`, /--days/, 'the usage error does not name --days') + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('an unset window reports today\'s literal as the effective default', () => { + const config = loadConfig({}) + assert.deepEqual(config.retention.windows, DEFAULT_WINDOWS) + for (const [id, configured] of Object.entries(config.retention.configured)) { + assert.equal(configured, false, `${id} is marked configured with no env var set`) + } +}) + +test('pruneOldData deletes by the configured window, not the built-in one', () => { + const store = new Store(':memory:') + try { + const user = makeUser(store, { login: 'retention-window-user', status: 'approved' }) + const repo = store.createRepo({ id: 'repo_retention_window', userId: user.id, owner: 'o', name: 'w', defaultBranch: 'main', tokenEnc: 'x', permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440 }) + for (const [id, ageDays] of [['job_configured_old', 40], ['job_configured_new', 20]] as const) { + store.createJob({ id, repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'g', callbackHash: `cb_${id}`, platformTokenHash: '' }) + store.updateJob(id, { status: 'succeeded', finished_at: new Date(Date.now() - ageDays * DAY).toISOString() }) + } + store.pruneOldData({ ...DEFAULT_WINDOWS, jobs: 30 }) + assert.equal(store.getJob('job_configured_old'), undefined, 'a 40-day job survived a configured 30-day window') + assert.notEqual(store.getJob('job_configured_new'), undefined, 'a 20-day job was deleted by a configured 30-day window') + } finally { + store.close() + } +}) + +test('the tool names the two tables it has no age window for', () => { + const dir = mkdtempSync(join(tmpdir(), 'fledge-retention-')) + const dbPath = join(dir, 'fledge.db') + try { + seed(dbPath) + const result = runTool(dbPath, ['--days', '30']) + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^sessions state=expiry-based$/m, 'sessions is not reported as expiry-based') + assert.match(result.stdout, /^api_tokens state=no-age-window$/m, 'api_tokens is not reported as having no age window') + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('the dry run prints counts only, never a row body, login, email or token', () => { + const dir = mkdtempSync(join(tmpdir(), 'fledge-retention-')) + const dbPath = join(dir, 'fledge.db') + try { + seed(dbPath) + const result = runTool(dbPath, ['--dry-run', '--days', '30']) + assert.equal(result.status, 0, result.stderr) + for (const secret of ['leaky-user', 'leaky@example.com', 'leaky-token-value', 'leaky-row-body']) { + assert.equal(result.stdout.includes(secret), false, `the dry run printed ${secret}`) + } + assert.equal(/^\S+ count=\d+ window=30d$/m.test(result.stdout), true, 'the per-class lines are not counts') + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('the admin page shows the effective window, marks a default and names the count', () => { + const store = new Store(':memory:') + try { + const user = makeUser(store, { login: 'retention-admin', status: 'approved', superadmin: true }) + const html = adminPage({ + config: testConfig(), + locale: 'en', + viewer: { user: store.userById(user.id)!, csrfToken: 'csrf' }, + users: [], + jobs: [], + jobsPage: 1, + jobsTotal: 0, + audit: [], + auditPage: 1, + auditTotal: 0, + perPage: 10, + retention: [ + { id: 'jobs', days: 365, configured: false, count: 2 }, + { id: 'quotaLedger', days: 60, configured: true, count: 4 }, + ], + }) + const card = /]*id="retention"[\s\S]*?<\/section>/.exec(html) + assert.ok(card !== null, 'the admin page has no retention card') + assert.match(card[0], /365/) + assert.match(card[0], /default/, 'the built-in window is not marked default') + assert.match(card[0], /60/, 'the configured window is not shown') + assert.match(card[0], />24 { const pages = [ { picker: 'settings' as const, ids: ['account', 'model-providers', 'notifications', 'tokens', 'sessions', 'feedback'] }, - { picker: 'admin' as const, ids: ['pending', 'users', 'jobs', 'audit', 'release', 'backups', 'operations'] }, + { picker: 'admin' as const, ids: ['pending', 'users', 'jobs', 'audit', 'release', 'backups', 'retention', 'operations'] }, ] for (const page of pages) { const html = render(page.picker) From f267615e8192a0a450a2c3bcc665bbea60653394 Mon Sep 17 00:00:00 2001 From: HugoXK Date: Wed, 18 Nov 2026 03:33:18 +0000 Subject: [PATCH 2/2] test(retention): prove the env var reaches the scheduler and refuse a zero window (#281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two mutations survived the suite: replacing `retentionWindow(process.env[cls.env], cls.defaultDays)` with `retentionWindow(undefined, …)` and replacing `pruneOldData(this.config.retention.windows)` with `pruneOldData()` both left every case green, so the config→scheduler wiring had no test and the PR's headline behaviour was unproven. One case now sets `RETENTION_JOBS_DAYS=30` and asserts the window and `configured.jobs`; one spies on the store and asserts the tick hands the configured windows to the prune call. `Store.pruneOldData` also accepted any `RetentionWindows`, so `{ jobs: 0 }` — a value `loadConfig` refuses — would have deleted every finished job. The store now refuses a window that is not a whole number of days of at least 1, before the first delete, so a call that cannot be honoured removes nothing. Docs name both, and the changelog fragment carries the refusal. Written-by: Fledge Platform Dev-3 --- changelog.d/281.added.md | 2 +- docs/architecture.md | 5 ++- docs/deploy.md | 6 ++++ src/db.ts | 10 ++++++ test/retention.test.ts | 72 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/changelog.d/281.added.md b/changelog.d/281.added.md index 6448f828..bdf6cbd1 100644 --- a/changelog.d/281.added.md +++ b/changelog.d/281.added.md @@ -1 +1 @@ -Operators can set each data-retention window with a `RETENTION_*_DAYS` environment variable and preview a change before making it: the new read-only `node scripts/retention.mjs --dry-run --days N` prints, per table, how many rows that window would delete without deleting anything, and the `/admin` page gains a **Data retention** card with the effective window per class and its current count. +Operators can set each data-retention window with a `RETENTION_*_DAYS` environment variable and preview a change before making it: the new read-only `node scripts/retention.mjs --dry-run --days N` prints, per table, how many rows that window would delete without deleting anything, and the `/admin` page gains a **Data retention** card with the effective window per class and its current count. A window below one day is refused by the delete path as well as by the config, so a direct `pruneOldData({ jobs: 0 })` throws instead of deleting every finished job. diff --git a/docs/architecture.md b/docs/architecture.md index 05e617a9..e27cfa90 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2218,7 +2218,10 @@ silently stop deleting rows the service deletes today. `sessions` are removed by rather than given a number. `Store.retentionCounts` counts at the configured windows for the `/admin` card, which is read-only and has no window control; the operator tool `scripts/retention.mjs --dry-run --days N` counts any window on a -host copy. +host copy. The delete path refuses a window that is not a whole number of days of +at least 1, before the first delete: `loadConfig` already refuses such an env +value, and `pruneOldData({ jobs: 0 })` is a direct caller the config cannot see, +so the store checks the same invariant rather than trusting its input. ### Operational alerts diff --git a/docs/deploy.md b/docs/deploy.md index 63dea082..827ac1a8 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -1469,6 +1469,12 @@ integer or is below 1 falls back to the default rather than being read as zero (zero here would mean "delete every row"). `sessions` are removed by `expires_at` and `api_tokens` have no age window; neither is configurable here. +The store refuses as well as the config. `Store.pruneOldData` throws on a window +that is not a whole number of days of at least 1 — including a caller that passes +`{ jobs: 0 }` directly, which without the guard would delete every finished job — +and it refuses before the first delete, so a call that cannot be honoured removes +nothing. + Preview any change before making it. The tool opens the database read-only, holds no write lock, prints one `count=` line per class and two states instead of a number: diff --git a/src/db.ts b/src/db.ts index 9ebc1c20..5999a24a 100644 --- a/src/db.ts +++ b/src/db.ts @@ -7002,6 +7002,16 @@ export class Store { // the delete removes. `deleteExpiredSessions` stays: sessions age out by // `expires_at`, not by a day count. No new delete path and no default changed. pruneOldData(windows: RetentionWindows = retentionDefaultWindows()): void { + // #281: `loadConfig` refuses a zero-day env window, but the store is a + // public surface too, and `pruneOldData({ jobs: 0 })` would delete every + // finished job. The refusal is here, before the first delete, so a call + // that cannot be honoured removes nothing rather than half-pruning. + for (const cls of RETENTION_CLASSES) { + const configured = windows[cls.id] ?? cls.defaultDays + if (!Number.isInteger(configured) || configured < 1) { + throw new Error(`retention window ${cls.id} must be a whole number of days >= 1, got ${String(configured)}`) + } + } const now = Date.now() const cutoff = (days: number): string => new Date(now - days * 86_400_000).toISOString() const at = (days: number): RetentionAt => ({ now: nowIso(), cutoff: cutoff(days) }) diff --git a/test/retention.test.ts b/test/retention.test.ts index ded12d68..80fad050 100644 --- a/test/retention.test.ts +++ b/test/retention.test.ts @@ -7,8 +7,11 @@ import test from 'node:test' import { fileURLToPath } from 'node:url' import { loadConfig } from '../src/config.js' import { Store } from '../src/db.js' +import { Jobs } from '../src/jobs/service.js' +import type { Runner } from '../src/jobs/runner.js' import { adminPage } from '../src/views/pages.js' -import { makeUser, testConfig } from './helpers.js' +import type { RetentionWindows } from '../scripts/lib/retention.mjs' +import { MemoryMailer, makeUser, testConfig } from './helpers.js' // #281: `pruneOldData` enforced ten age windows as literals inside one function, // so nothing outside the code could read or preview one. This suite pins the @@ -247,6 +250,73 @@ test('the admin page shows the effective window, marks a default and names the c } }) +// #281 rework: two mutations survived this suite, so nothing proved the env var +// reached the scheduler. `retentionWindow(process.env[cls.env], …)` replaced by +// `retentionWindow(undefined, …)` and `pruneOldData(this.config.retention.windows)` +// replaced by `pruneOldData()` both left every case green. These two pin the +// wiring itself: one reads a real env var through `loadConfig`, the other +// watches what the tick hands the store. +test('a RETENTION_*_DAYS env var becomes the window and is marked configured', () => { + const previous = process.env.RETENTION_JOBS_DAYS + process.env.RETENTION_JOBS_DAYS = '30' + try { + const config = loadConfig({}) + assert.equal(config.retention.windows.jobs, 30, 'the env var did not reach the jobs window') + assert.equal(config.retention.configured.jobs, true, 'a window read from the env var is not marked configured') + // The other classes stay on their literal: one env var cannot move another. + assert.equal(config.retention.windows.events, DEFAULT_WINDOWS.events) + assert.equal(config.retention.configured.events, false) + } finally { + if (previous === undefined) delete process.env.RETENTION_JOBS_DAYS + else process.env.RETENTION_JOBS_DAYS = previous + } +}) + +test('the scheduler tick prunes by config.retention.windows, not by the built-in literals', async () => { + const windows: RetentionWindows = { ...DEFAULT_WINDOWS, jobs: 30, events: 45 } + const config = testConfig() + config.retention = { windows, configured: config.retention.configured } + const store = new Store(':memory:') + const calls: RetentionWindows[] = [] + const original = store.pruneOldData.bind(store) + // The tick must hand over the configured windows. A call with no argument + // would take the store's built-in defaults, so that is a failure here rather + // than a call to record. + store.pruneOldData = (passed?: RetentionWindows) => { + if (passed === undefined) throw new Error('the tick pruned without passing the configured windows') + calls.push(passed) + original(passed) + } + const silentRunner: Runner = { name: 'silent', async run() { return { status: 'running' } } } + const jobs = new Jobs(store, config, silentRunner, new MemoryMailer()) + try { + await jobs.tick() + assert.equal(calls.length, 1, 'the first tick did not prune') + const passed = calls[0] + assert.ok(passed !== undefined, 'the tick recorded no prune call') + assert.equal(passed.jobs, 30, 'the tick pruned jobs with a window config did not carry') + assert.equal(passed.events, 45, 'the tick pruned events with a window config did not carry') + assert.deepEqual(passed, config.retention.windows) + } finally { + store.close() + } +}) + +test('pruneOldData refuses a zero-day window instead of deleting every row in it', () => { + const store = new Store(':memory:') + try { + const user = makeUser(store, { login: 'retention-zero-user', status: 'approved' }) + const repo = store.createRepo({ id: 'repo_retention_zero', userId: user.id, owner: 'o', name: 'zero', defaultBranch: 'main', tokenEnc: 'x', permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440 }) + store.createJob({ id: 'job_zero_old', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'g', callbackHash: 'cb_zero', platformTokenHash: '' }) + store.updateJob('job_zero_old', { status: 'succeeded', finished_at: new Date(Date.now() - 400 * DAY).toISOString() }) + assert.throws(() => store.pruneOldData({ ...DEFAULT_WINDOWS, jobs: 0 }), /retention window jobs must be a whole number of days >= 1, got 0/) + assert.notEqual(store.getJob('job_zero_old'), undefined, 'a refused window still deleted the job') + assert.throws(() => store.pruneOldData({ ...DEFAULT_WINDOWS, jobs: 1.5 }), /retention window jobs/) + } finally { + store.close() + } +}) + // Keep the compiler honest about the helper this file relies on; execFileSync is // imported for parity with the other tool tests and used here so lint does not // read it as dead weight.