From 0ab00ff39bc2e923af468d8d9bb95dfd7f59bc46 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Tue, 22 Sep 2026 14:32:57 +0100 Subject: [PATCH] fix(npmjs): new versions silently missed due to non-monotonic changes feed The npm replica's `_changes` feed occasionally makes change entries visible behind positions the follower has already read past, and the registry occasionally serves packuments older than the revision the feed announced. Both paths silently dropped package versions until the affected package published again. The follower no longer tracks a single high-water mark. It keeps a receipt for every change entry it has received, and every run re-scans a trailing window of the feed (newest slice first), processing any entry it has no receipt for. A periodic deep scan covers the full state retention so insertion delays beyond the regular window are still observed. When the registry serves a packument behind the announced revision (a laggy packument), the served versions are processed immediately and the announced revision is re-checked every run until it appears or the expectation ages out. State (receipts, checkpoints, laggy packuments) lives in a single scannable, newline-delimited S3 object. Existing deployments are migrated by seeding the first checkpoint from the legacy transaction marker. New LateChangeCount/LateChangeLag and LaggyPackuments metrics feed a Feed Reliability dashboard widget and two low-severity alarms. Internal reference: V2376798198 --- API.md | 85 ++ docs/application-overview.md | 26 +- docs/npm-sync-protocol.md | 171 ++++ docs/operator-runbook.md | 76 ++ .../__snapshots__/construct-hub.test.ts.snap | 750 +++++++++++++--- .../__snapshots__/snapshot.test.ts.snap | 158 +++- .../follower-state.lambda-shared.test.ts | 227 +++++ .../npmjs/npm-js-follower.lambda.test.ts | 537 ++++++++++++ src/package-sources/npmjs.ts | 176 +++- .../npmjs/constants.lambda-shared.ts | 75 ++ .../npmjs/couch-changes.lambda-shared.ts | 235 +++-- .../npmjs/follower-state.lambda-shared.ts | 355 ++++++++ .../npmjs/npm-js-follower.lambda.ts | 809 +++++++++++++----- 13 files changed, 3200 insertions(+), 480 deletions(-) create mode 100644 docs/npm-sync-protocol.md create mode 100644 src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts create mode 100644 src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts create mode 100644 src/package-sources/npmjs/follower-state.lambda-shared.ts diff --git a/API.md b/API.md index 9c61d1136..c15fb177e 100644 --- a/API.md +++ b/API.md @@ -2880,7 +2880,12 @@ new sources.NpmJs(props?: NpmJsProps) | bind | Binds the package source to a scope and target queue. | | metricBatchProcessingTime | The average time it took to process a changes batch. | | metricChangeCount | The total count of changes that were processed. | +| metricLaggyPackumentGiveUps | The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. | +| metricLaggyPackuments | The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. | +| metricLaggyPackumentsRecovered | The number of laggy packuments for which the registry caught up with the revision announced by the `_changes` feed. | | metricLastSeq | The last sequence number that was processed. | +| metricLateChangeCount | The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. | +| metricLateChangeLag | For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. | | metricNpmJsChangeAge | *No description.* | | metricPackageVersionAge | The age of the oldest package version that was processed. | | metricPackageVersionCount | The total count of package versions that were inspected. | @@ -2938,6 +2943,55 @@ The total count of changes that were processed. --- +##### `metricLaggyPackumentGiveUps` + +```typescript +public metricLaggyPackumentGiveUps(opts?: MetricOptions): Metric +``` + +The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. + +A version announced by the feed may be missing until +the affected package publishes again. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricLaggyPackuments` + +```typescript +public metricLaggyPackuments(opts?: MetricOptions): Metric +``` + +The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. + +The versions served so far +have been processed; the follower keeps re-checking for the announced +revision. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricLaggyPackumentsRecovered` + +```typescript +public metricLaggyPackumentsRecovered(opts?: MetricOptions): Metric +``` + +The number of laggy packuments for which the registry caught up with the revision announced by the `_changes` feed. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricLastSeq` ```typescript @@ -2955,6 +3009,37 @@ discover when a sequence reset has happened in the CouchDB instance. --- +##### `metricLateChangeCount` + +```typescript +public metricLateChangeCount(opts?: MetricOptions): Metric +``` + +The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricLateChangeLag` + +```typescript +public metricLateChangeLag(opts?: MetricOptions): Metric +``` + +For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. + +This is a lower bound on the feed's insertion lag, +and can be used to tune the scan window. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricNpmJsChangeAge` ```typescript diff --git a/docs/application-overview.md b/docs/application-overview.md index b01a8d366..12521402d 100644 --- a/docs/application-overview.md +++ b/docs/application-overview.md @@ -37,18 +37,30 @@ ConstructHub provides two package source implementations: `NpmJs` and `CodeArtifact`. * The `NpmJs` source interfaces with the `npmjs.com` CouchDB replica (which is - at `replicate.npmjs.com/registry`) by following it's `_changes` stream in + at `replicate.npmjs.com/registry`) by scanning it's `_changes` stream in search of relevant packages. When such a package is identified, a stager function is invoked, which stages the package tarball into an S3 bucket then notifies the ConstructHub ingestion SQS queue. The CouchDB follower is - scheduled to run every `5 minutes`, and stores the current CouchDB sequence ID - in a specific object in the S3 bucket used for staging package tarballs. + scheduled to run every `5 minutes`, and stores its state (receipts for the + change entries received, time/sequence checkpoints, and laggy packument + expectations) in a specific object in the S3 bucket used for staging package + tarballs. See [npm-sync-protocol.md](./npm-sync-protocol.md) for the + guarantees the feed does and does not give, and how the follower compensates. > Back-filling is automatic for the `NpmJs` source. Upon initial deployment, - > it will start scanning the CouchDB `_changes` stream. Should there be a need - > to re-run a backfill of this source, the transaction marker object in S3 can - > be deleted to roll back to that initial transaction. The marker object is - > linked from the backend dashboard. + > it will start scanning the CouchDB `_changes` stream from the beginning, + > progressing incrementally across runs. Should there be a need to re-process + > a range of the stream, replace the follower state object in S3 with one + > that seeds a checkpoint at the desired starting sequence number: + > + > ```text + > #chfollower/1 + > w - + > c + > ``` + > + > The next scan then covers everything from that position to the feed head. + > The state object is linked from the backend dashboard. - A **high-severity** alarm triggers if the NpmJs Follower is not running at the scheduled cadence, or if it encounters failures for more than diff --git a/docs/npm-sync-protocol.md b/docs/npm-sync-protocol.md new file mode 100644 index 000000000..36fc5cc0c --- /dev/null +++ b/docs/npm-sync-protocol.md @@ -0,0 +1,171 @@ +# The npm sync protocol: what it does and does not guarantee + +This document describes the protocol ConstructHub's `NpmJs` package source uses to discover new package versions, +the guarantees npm gives (and explicitly does not give) for that protocol, +and how ConstructHub compensates for each gap. +Claims cite public npm statements where they exist; +anything marked *(our observation)* we established ourselves through incident investigation. + +## The protocol + +Syncing from npm uses two endpoints: + +1. **The changes feed** at `https://replicate.npmjs.com/registry/_changes`. + A paginated stream of change entries, each `{ seq, id, changes: [{ rev }] }`, read with `?since=&limit=`. + A consumer pages forward by passing the previous response's `last_seq` as the next `since`. +2. **The registry** at `https://registry.npmjs.org/`. + Change entries carry no package content, + so a consumer fetches the full metadata document (the "packument") from the registry, + and derives new versions from its `time` map. + +Since May 2025 the feed is no longer a real CouchDB. +npm replaced it with a purpose-built service (the root endpoint reports `engine: "npm-replicate"`) +that keeps the CouchDB API shape but supports only a subset of it +([migration guide][152515], [deprecation notice][changelog]). +Several of the non-guarantees below date from this migration. + +## What npm guarantees + +- **Pagination by sequence number.** + `_changes?since=&limit=` returns change entries with `seq > since` in ascending order, + plus a `last_seq` to continue from. + `limit` defaults to 1,000 and caps at 10,000 ([migration guide][152515]). +- **A head-of-feed pointer.** + The root endpoint (`/`) reports the current `update_seq`; + npm recommends it as the starting point for a new consumer ([npm engineering response][152515-updateseq]). +- **One entry per package.** + A package has at most one entry in the feed: + a new change replaces the package's previous entry, at a new sequence number. + npm confirmed this is intended ([npm: "That is indeed normal feed behavior"][152515-late]). +- **Metadata lives in the registry, not the feed.** + `include_docs` is not supported; + npm's documented guidance is to fetch packuments from `registry.npmjs.org` separately ([migration guide][152515]). + +That is the entire public contract. +Notably absent: any statement about *when* a change becomes visible in the feed, +or that it becomes visible at all. + +## What npm does not guarantee + +Each heading below states a property a consumer might reasonably expect, but that npm does not guarantee. +For ease of use, each gap also gets a name; +the code and the operator runbook reference the same names. + +### Changes become visible in order + +Change entries can appear in the feed *behind* a previously returned `last_seq`; +we name this gap *late insertion*. +A consumer that reads strictly forward will never see such an entry. +Community measurement found ~4 late-inserted entries per 1,024-entry window over two hours; +npm's response was that entries appearing behind `last_seq` is expected +([report][152515-late-report], [npm confirmation][152515-late]). + +*(our observation)* The delay can be far larger than the minutes suggested in that thread. +On 2026-09-21 we read past a probe package's sequence range within ~3 minutes of its entry's sequence number +being assigned, and the entry was not served; +a manual re-read of the same range two hours later returned it. +Five consecutive versions of the package were missed this way over ~15 hours. +See the [appendix](#appendix-a-late-insertion-reconstructed) for a reconstructed timeline. + +### The registry is up-to-date with the changes feed + +The registry can serve a packument revision *older* than the revision the feed announced for the same package, +sometimes for extended periods; +we name this gap *laggy packuments*. +The feed and the registry are separate systems and are not consistent with each other +([community description][152515-stale], [metadata out of sync][128412]). + +A consumer can detect this: +the packument carries no sequence number, +but the feed entry announces the revision(s) it covers and the packument carries its current revision (`_rev`). +Comparing the numeric prefix of the two tells a consumer whether the registry has caught up +with the change it is processing. + +*(our observation)* This is not rare. +E.g. on 2026-09-21 the follower logged ~66,000 revision-mismatch retries across unrelated packages in a single day. + +### Every change becomes visible + +Occasionally a change never becomes visible in the feed at any sequence number; +we name this gap *lost events*. +Because a package has at most one entry, there is nothing to find by re-reading the feed: +the package's only entry still describes its previous publish. +The change only surfaces when the package publishes again and a fresh entry replaces the old one. +This was reported for `patch-package@7.0.0` (missing for a month, recovered only by the next publish) +and for `zod-validation-error@1.5.0`; +deletions have also been observed not to propagate, in the thousands +([lost events][128412], [version skipped a revision][128412-lost], [missing deletions][128412-deletions]). + +### Sequence numbers are dense + +Sequence numbers are opaque cursors, not counters. +The sequence space has gaps, +and values have jumped by tens of millions within weeks during the 2025 migration ([seq jumps][152515-jumps]). +A consumer must not derive meaning from the distance between two sequence numbers. + +### The feed head only moves forward + +Change entries can disappear from the end of the feed, +in effect moving the head (`update_seq`) *backwards*; +we name this gap *head regression*. +During the 2025 migration the head regressed by ~20 million, +and previously returned entries above the new head no longer existed ([seq regression][152515-regression]). +The pre-migration feed also once froze its `update_seq` entirely for hours ([stuck update_seq][128412-stuck]). + +## How ConstructHub compensates + +The *NpmJs Follower* runs on a regular schedule and fetches packuments from the registry to identify new versions. +It does not track a single high-water mark: +it keeps a receipt for every change entry it has received, +and every run re-reads a trailing window of the feed, +processing any entry it has no receipt for. +Rolling checkpoints map times to feed positions, and derive the window's starting position. +On top of that baseline, each gap has a dedicated mitigation: + +| Gap | Mitigation | Detection | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | +| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm | +| Laggy packuments | Process every packument the registry serves immediately; record the announced revision and re-check it every run until it appears or ages out | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` metrics; `LaggyPackumentGiveUps` alarm | +| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package | +| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric | + +**The design idea behind all of this is that the feed data are just hints without real guarantees.** +The follower treats every change entry as "this package probably changed around here", +and keeps enough state (receipts, checkpoints, known versions) to re-derive correctness from re-reads. + +## Appendix: a late insertion, reconstructed + +The timeline below reconstructs the 2026-09-21 incident from log evidence and manual feed reads +*(our observation)*. +Times are UTC; "seq X" is the sequence number assigned to the probe package's change entry. + +```text +time consumer feed +───── ─────────────────────────────────── ────────────────────────────────────── +15:09 package version is published entry assigned seq X, NOT yet visible +15:12 reads past seq X, entry not served entry still not visible +15:?? entry becomes visible at seq X, which + is behind the consumer's position +17:10 manually re-reads the range at seq X entry is served ("late insertion") +``` + +A consumer that never re-reads previously covered ranges has no way to notice that anything happened. + +## References + +- Operator runbook sections for the alarms named above: [operator-runbook.md](./operator-runbook.md) +- Follower implementation: `src/package-sources/npmjs/npm-js-follower.lambda.ts` +- Incident that motivated the windowed scan: internal reference V2376798198 *(our observation)* + +[152515]: https://github.com/orgs/community/discussions/152515 +[changelog]: https://github.blog/changelog/2025-02-27-changes-and-deprecation-notice-for-npm-replication-apis +[152515-updateseq]: https://github.com/orgs/community/discussions/152515#discussioncomment-12713712 +[152515-late-report]: https://github.com/orgs/community/discussions/152515#discussioncomment-13561875 +[152515-late]: https://github.com/orgs/community/discussions/152515#discussioncomment-13563633 +[152515-stale]: https://github.com/orgs/community/discussions/152515#discussioncomment-15298851 +[128412-lost]: https://github.com/orgs/community/discussions/128412#discussioncomment-7445623 +[152515-jumps]: https://github.com/orgs/community/discussions/152515#discussioncomment-12731335 +[152515-regression]: https://github.com/orgs/community/discussions/152515#discussioncomment-12754432 +[128412]: https://github.com/orgs/community/discussions/128412 +[128412-deletions]: https://github.com/orgs/community/discussions/128412#discussioncomment-6755605 +[128412-stuck]: https://github.com/orgs/community/discussions/128412#discussioncomment-7653871 diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 794fb44f1..9773cc1d0 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -582,6 +582,82 @@ but is disabled by default. Once all messages have cleared from the dead-letter queue, do not forget to disable the SQS Trigger again. +### `ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh` + +#### Description + +This alarm is only provisioned when the `NpmJs` package source is configured. It +triggers when the *NpmJs Follower* discovers changes that were inserted into the +CouchDB `_changes` feed with a delay approaching the scan window. Should the +delay exceed the window, the regular scan will miss those changes, and only the +daily deep scan would still discover them. + +#### Investigation + +The `_changes` feed occasionally makes change entries visible behind positions +the follower has already read past (npm considers this normal feed behavior). +The follower compensates by re-reading a trailing window of the feed on every +run, and the `LateChangeLag` metric records how far behind each late entry was +discovered. See [npm-sync-protocol.md](./npm-sync-protocol.md) for the details +of the feed's behavior. + +Review the *Feed Reliability* widget on the backend dashboard. A short burst of +high-delay insertions usually indicates turbulence in the npmjs.com replication +pipelines, and resolves on its own. Look at `npmjs.com` status updates and +announcements. + +#### Resolution + +The alarm will automatically go back to green once the insertion delays +subside. If elevated delays persist over multiple days, the scan window should +be increased: adjust `SCAN_WINDOW_MS` in +`src/package-sources/npmjs/constants.lambda-shared.ts` (keeping it below +`STATE_RETENTION_MS`) and deploy the updated construct. + +### `ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps` + +#### Description + +This alarm is only provisioned when the `NpmJs` package source is configured. It +triggers when the npm registry never served the packument revision announced by +the CouchDB `_changes` feed for one or more packages, even after the follower +re-checked for a prolonged period. + +All package versions the registry did serve have been processed. However, a +version announced by the changes feed may be missing from ConstructHub until +the affected package publishes its next version. + +#### Investigation + +The *NpmJs Follower* fetches package metadata from `registry.npmjs.org` for +each change announced by the CouchDB `_changes` feed. The registry occasionally +lags behind the feed. When it does, the follower processes the versions that +were served, records the announced revision as a "laggy packument", and +re-checks it on every run until the revision appears or the expectation ages +out. This alarm fires for the latter. + +Review the logs of the *NpmJs Follower* for `Giving up on laggy packument` +entries; they identify the affected package and the revision that never +appeared. Verify what the registry currently serves for the package: + +```console +$ curl https://registry.npmjs.org/ | jq ._rev +``` + +If the served revision is still older than the expected one, the problem is +with upstream npmjs.com. Look at `npmjs.com` status updates and announcements. + +If you suspect the problem is with upstream npmjs.com, you can cut them a ticket. Log +in to , visit the [Support Page](https://www.npmjs.com/support) and +select *There is a problem with the npm registry*. + +#### Resolution + +The missed package versions will automatically be discovered once the affected +packages publish their next version. To ingest a missed version immediately, +invoke the *ReStagePackageVersion* Lambda function with the package name and +version from the follower logs. + -------------------------------------------------------------------------------- ## :repeat: Bulk Re-processing diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index e27f18236..3d8828f39 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -632,7 +632,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -703,7 +707,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -728,11 +736,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -740,7 +748,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -763,7 +771,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -775,7 +783,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -800,11 +808,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -829,7 +837,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -860,7 +868,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -876,7 +884,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -896,7 +904,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -921,7 +929,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -952,7 +960,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -963,7 +971,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -975,7 +983,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -995,11 +1003,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -1007,7 +1015,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -1019,11 +1027,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -9875,6 +9883,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -9958,7 +9974,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -10412,6 +10428,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -10654,6 +10711,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -10765,7 +10859,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -11021,6 +11115,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -14609,7 +14711,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -14680,7 +14786,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -14705,11 +14815,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -14717,7 +14827,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -14740,7 +14850,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -14752,7 +14862,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -14777,11 +14887,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -14806,7 +14916,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -14837,7 +14947,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -14853,7 +14963,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -14873,7 +14983,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -14898,7 +15008,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -14929,7 +15039,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -14940,7 +15050,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -14952,7 +15062,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -14972,11 +15082,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -14984,7 +15094,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -14996,11 +15106,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -24226,6 +24336,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -24309,7 +24427,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -24763,6 +24881,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -25005,6 +25164,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -25116,7 +25312,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -25372,6 +25568,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -28898,7 +29102,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -28969,7 +29177,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -28994,11 +29206,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -29006,7 +29218,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -29029,7 +29241,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -29041,7 +29253,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -29066,11 +29278,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -29095,7 +29307,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -29126,7 +29338,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -29142,7 +29354,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -29162,7 +29374,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -29187,7 +29399,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -29218,7 +29430,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -29229,7 +29441,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -29241,7 +29453,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -29261,11 +29473,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -29273,7 +29485,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -29285,11 +29497,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -38141,6 +38353,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -38224,7 +38444,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -38678,6 +38898,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -38920,6 +39181,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -39031,7 +39329,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -39287,6 +39585,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -42961,7 +43267,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -43032,7 +43342,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -43057,11 +43371,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -43069,7 +43383,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -43092,7 +43406,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -43104,7 +43418,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -43129,11 +43443,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -43158,7 +43472,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -43189,7 +43503,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -43205,7 +43519,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -43225,7 +43539,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -43250,7 +43564,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -43281,7 +43595,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -43292,7 +43606,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -43304,7 +43618,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -43324,11 +43638,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -43336,7 +43650,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -43348,11 +43662,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -52213,6 +52527,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -52296,7 +52618,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -52737,6 +53059,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -52979,6 +53342,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -53090,7 +53490,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -53346,6 +53746,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -57209,7 +57617,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -57280,7 +57692,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -57305,11 +57721,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -57317,7 +57733,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -57340,7 +57756,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -57352,7 +57768,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -57377,11 +57793,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -57406,7 +57822,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -57437,7 +57853,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -57453,7 +57869,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -57473,7 +57889,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -57498,7 +57914,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -57529,7 +57945,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -57540,7 +57956,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -57552,7 +57968,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -57572,11 +57988,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -57584,7 +58000,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -57596,11 +58012,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -66268,6 +66684,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -66351,7 +66775,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -66805,6 +67229,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -67047,6 +67512,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -67158,7 +67660,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -67414,6 +67916,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 6fd98f419..9e2159b43 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -694,7 +694,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -765,7 +769,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -790,11 +798,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -802,7 +810,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -825,7 +833,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -837,7 +845,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -862,11 +870,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -891,7 +899,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -922,7 +930,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -938,7 +946,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -958,7 +966,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -983,7 +991,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -1014,7 +1022,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -1025,7 +1033,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -1037,7 +1045,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -1057,11 +1065,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -1069,7 +1077,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -1081,11 +1089,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -1093,7 +1101,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"1 minutes (Lambda timeout)","value":60,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":135,"properties":{"markdown":"# Release Notes\\n\\n[button:primary:StateMachine](/states/home#/statemachines/view/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"1 minutes (Lambda timeout)","value":60,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":141,"properties":{"markdown":"# Release Notes\\n\\n[button:primary:StateMachine](/states/home#/statemachines/view/", { "Ref": "ConstructHubReleaseNotesStateMachine8C711CC7", }, @@ -1154,15 +1162,15 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":137,"properties":{"view":"timeSeries","title":"Number of release Notes","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":143,"properties":{"view":"timeSeries","title":"Number of release Notes","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with release notes","expression":"FILL(m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d, REPEAT)"}],["ConstructHub/ReleaseNotes","PackageWithChangeLog",{"label":"Packages with release notes","stat":"Sum","visible":false,"id":"m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":137,"properties":{"view":"timeSeries","title":"Release notes generation Errors","region":"", + "","metrics":[[{"label":"Packages with release notes","expression":"FILL(m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d, REPEAT)"}],["ConstructHub/ReleaseNotes","PackageWithChangeLog",{"label":"Packages with release notes","stat":"Sum","visible":false,"id":"m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":143,"properties":{"view":"timeSeries","title":"Release notes generation Errors","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/ReleaseNotes","AllErrors",{"label":"All Errors","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnknownError",{"label":"UnknownError","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidCredentials",{"label":"InvalidCredentials","stat":"Maximum"}],["ConstructHub/ReleaseNotes","RequestQuotaExhausted",{"label":"RequestQuotaExhausted","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnSupportedRepo",{"label":"UnSupportedRepo","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidPackageJson",{"label":"InvalidPackageJson","stat":"Maximum"}],["ConstructHub/ReleaseNotes","ChangelogFetchError",{"label":"ChangeLogFetchError","stat":"Maximum"}]],"yAxis":{}}},{"type":"metric","width":12,"height":6,"x":0,"y":143,"properties":{"view":"timeSeries","title":"GitHub API Rate Limits","region":"", + "","metrics":[["ConstructHub/ReleaseNotes","AllErrors",{"label":"All Errors","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnknownError",{"label":"UnknownError","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidCredentials",{"label":"InvalidCredentials","stat":"Maximum"}],["ConstructHub/ReleaseNotes","RequestQuotaExhausted",{"label":"RequestQuotaExhausted","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnSupportedRepo",{"label":"UnSupportedRepo","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidPackageJson",{"label":"InvalidPackageJson","stat":"Maximum"}],["ConstructHub/ReleaseNotes","ChangelogFetchError",{"label":"ChangeLogFetchError","stat":"Maximum"}]],"yAxis":{}}},{"type":"metric","width":12,"height":6,"x":0,"y":149,"properties":{"view":"timeSeries","title":"GitHub API Rate Limits","region":"", { "Ref": "AWS::Region", }, @@ -11150,6 +11158,14 @@ Direct link to the function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -11233,7 +11249,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -11684,6 +11700,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -11923,6 +11980,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -12034,7 +12128,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -12290,6 +12384,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { diff --git a/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts new file mode 100644 index 000000000..cd4698f2a --- /dev/null +++ b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts @@ -0,0 +1,227 @@ +import { + DEEP_SCAN_INTERVAL_MS, + SCAN_WINDOW_MS, + STATE_RETENTION_MS, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import { FollowerState } from '../../../package-sources/npmjs/follower-state.lambda-shared'; + +const T0 = 1_700_000_000_000; +const HOUR = 60 * 60 * 1_000; + +describe('text format', () => { + test('produces the exact documented golden output', () => { + const state = new FollowerState(); + state.addCheckpoint(90, T0); + state.addCheckpoint(300, T0 + HOUR); + state.addSeqs([100, 250]); + state.recordLaggyPackument('@scope/laggy-package', 5, 123, T0); + state.completeDeepScan(T0 + HOUR); + + expect(state.toText()).toBe( + [ + '#chfollower/1', + `w ${T0 + HOUR}`, + `c ${T0} 90`, + `c ${T0 + HOUR} 300`, + `l ${T0} 5 123 @scope/laggy-package`, + 's 100', + 's 250', + '', // trailing newline + ].join('\n') + ); + }); + + test('round-trips through the text format', () => { + const state = new FollowerState(); + state.addCheckpoint(90, T0); + state.addSeqs([100, 105, 250]); + state.recordLaggyPackument('laggy-package', 7, undefined, T0); + state.completeDeepScan(T0); + + const restored = FollowerState.fromText(state.toText()); + + for (const seq of [100, 105, 250]) { + expect(restored.has(seq)).toBe(true); + } + expect(restored.has(101)).toBe(false); + expect(restored.receiptCount).toBe(3); + expect(restored.checkpointCount).toBe(1); + expect(restored.lastDeepScanAt).toBe(T0); + expect(restored.laggyPackuments()).toEqual([ + { name: 'laggy-package', expectedRev: 7, seq: undefined, firstSeen: T0 }, + ]); + }); + + test('a fresh state round-trips', () => { + const restored = FollowerState.fromText(new FollowerState().toText()); + expect(restored.receiptCount).toBe(0); + expect(restored.checkpointCount).toBe(0); + expect(restored.laggyPackumentCount).toBe(0); + expect(restored.lastDeepScanAt).toBeUndefined(); + }); + + test('serialization is stable (parse . print is identity)', () => { + const state = new FollowerState(); + state.addCheckpoint(90, T0); + state.addSeqs([100, 250, 251]); + state.recordLaggyPackument('laggy-package', 7, 200, T0); + + const once = state.toText(); + const twice = FollowerState.fromText(once).toText(); + expect(twice).toBe(once); + }); + + test('handles large sequence numbers (beyond 2^31)', () => { + const bigSeq = 131_711_133_000; + const state = new FollowerState(); + state.addCheckpoint(bigSeq - 10, T0); + state.addSeqs([bigSeq]); + const restored = FollowerState.fromText(state.toText()); + expect(restored.has(bigSeq)).toBe(true); + }); + + test('round-trips a large receipt set intact', () => { + const state = new FollowerState(); + state.addCheckpoint(0, T0); + state.addSeqs( + Array.from({ length: 10_000 }, (_, i) => 131_000_000 + i * 3) + ); + const restored = FollowerState.fromText(state.toText()); + expect(restored.receiptCount).toBe(10_000); + expect(restored.has(131_000_000)).toBe(true); + expect(restored.has(131_000_001)).toBe(false); + }); + + test('tolerates blank lines, surrounding whitespace, and CRLF line endings', () => { + const text = [ + '#chfollower/1', + 'w -', + '', + `c ${T0} 90 `, + ' s 100', + 's 250\r', + '', + ].join('\n'); + + const restored = FollowerState.fromText(text.replace(/\n/g, '\r\n')); + expect(restored.checkpointCount).toBe(1); + expect(restored.has(100)).toBe(true); + expect(restored.has(250)).toBe(true); + }); + + test.each([ + ['empty input', ''], + ['missing header', 'w -\ns 100\n'], + ['JSON input', '{"seqs":[100]}'], + ['wrong version', '#chfollower/2\ns 100\n'], + ])('rejects %s', (_name, text) => { + expect(() => FollowerState.fromText(text)).toThrow( + /Unexpected follower state header/ + ); + }); + + test('rejects unknown record types', () => { + expect(() => + FollowerState.fromText('#chfollower/1\nx something\n') + ).toThrow(/Unexpected follower state line: x something/); + }); +}); + +describe('checkpoints and receipts', () => { + test('floorSeq maps the window through checkpoints', () => { + const state = new FollowerState(); + state.addCheckpoint(100, T0 - SCAN_WINDOW_MS - HOUR); + state.addCheckpoint(200, T0 - SCAN_WINDOW_MS); + state.addCheckpoint(900, T0); + expect(state.floorSeq(SCAN_WINDOW_MS, T0)).toBe(200); + }); + + test('floorSeq falls back to the newest checkpoint when history is younger than the window', () => { + const state = new FollowerState(); + state.addCheckpoint(150, T0 - HOUR); + state.addCheckpoint(900, T0 - 30 * 60_000); + // No checkpoint is older than the window: coverage below the newest one + // is guaranteed, so the scan resumes from there. + expect(state.floorSeq(SCAN_WINDOW_MS, T0)).toBe(900); + expect(new FollowerState().floorSeq(SCAN_WINDOW_MS, T0)).toBeUndefined(); + }); + + test('timeCrossed reports when a completed scan first passed a seq', () => { + const state = new FollowerState(); + state.addCheckpoint(100, T0); + state.addCheckpoint(200, T0 + HOUR); + state.addCheckpoint(300, T0 + 2 * HOUR); + + expect(state.timeCrossed(150)).toBe(T0 + HOUR); + expect(state.timeCrossed(100)).toBe(T0); + expect(state.timeCrossed(999)).toBeUndefined(); + }); + + test('prunes checkpoints and receipts outside the retention window', () => { + const state = new FollowerState(); + state.addCheckpoint(50, T0 - STATE_RETENTION_MS - 2 * HOUR); + state.addCheckpoint(450, T0 - STATE_RETENTION_MS - HOUR); + state.addSeqs([460, 500, 600]); + state.addCheckpoint(700, T0); + + // The checkpoint at seq=450 is the last one at/before the cut-off and is + // retained; the older one is dropped, along with receipts below seq=450. + expect(state.checkpointCount).toBe(2); + expect(state.has(460)).toBe(true); + expect(state.has(500)).toBe(true); + }); + + test('resetPosition clears receipts and checkpoints but keeps laggy packuments', () => { + const state = new FollowerState(); + state.addCheckpoint(100, T0); + state.addSeqs([150]); + state.recordLaggyPackument('laggy-package', 5, 120, T0); + + state.resetPosition(); + + expect(state.checkpointCount).toBe(0); + expect(state.receiptCount).toBe(0); + expect(state.newestCheckpointSeq()).toBeUndefined(); + expect(state.laggyPackumentCount).toBe(1); + }); +}); + +describe('deep scans', () => { + test('the first ever scan is deep', () => { + expect(new FollowerState().deepScanDue(T0)).toBe(true); + }); + + test('a deep scan is due again after the interval', () => { + const state = new FollowerState(); + state.completeDeepScan(T0); + expect(state.deepScanDue(T0 + DEEP_SCAN_INTERVAL_MS - 1)).toBe(false); + expect(state.deepScanDue(T0 + DEEP_SCAN_INTERVAL_MS)).toBe(true); + }); +}); + +describe('laggy packuments', () => { + test('keeps the earliest firstSeen and the highest expected revision', () => { + const state = new FollowerState(); + state.recordLaggyPackument('laggy-package', 5, 100, T0); + state.recordLaggyPackument('laggy-package', 7, 200, T0 + HOUR); + state.recordLaggyPackument('laggy-package', 6, 300, T0 + 2 * HOUR); + + expect(state.laggyPackuments()).toEqual([ + { name: 'laggy-package', expectedRev: 7, seq: 100, firstSeen: T0 }, + ]); + }); + + test('lists expectations oldest first and removes them', () => { + const state = new FollowerState(); + state.recordLaggyPackument('second', 2, undefined, T0 + HOUR); + state.recordLaggyPackument('first', 1, undefined, T0); + + expect(state.laggyPackuments().map((l) => l.name)).toEqual([ + 'first', + 'second', + ]); + + state.removeLaggyPackument('first'); + expect(state.laggyPackumentCount).toBe(1); + }); +}); diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts new file mode 100644 index 000000000..7390f042f --- /dev/null +++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts @@ -0,0 +1,537 @@ +import { gunzipSync } from 'zlib'; +import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; +import { + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import type { metricScope, MetricsLogger } from 'aws-embedded-metrics'; +import { Unit } from 'aws-embedded-metrics'; +import type { Context, ScheduledEvent } from 'aws-lambda'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import nock from 'nock'; +import { + ENV_DENY_LIST_BUCKET_NAME, + ENV_DENY_LIST_OBJECT_KEY, +} from '../../../backend/deny-list/constants'; +import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; +import { + FOLLOWER_STATE_FILE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, + MetricName, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import { FollowerState } from '../../../package-sources/npmjs/follower-state.lambda-shared'; +import { handler } from '../../../package-sources/npmjs/npm-js-follower.lambda'; +import { stringToStream } from '../../streams'; + +jest.setTimeout(30_000); + +jest.mock('aws-embedded-metrics'); + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const mockMetricScope = require('aws-embedded-metrics') + .metricScope as jest.MockedFunction; +const mockPutMetric = jest + .fn() + .mockName('MetricsLogger.putMetric') as jest.MockedFunction< + MetricsLogger['putMetric'] +>; +const mockMetrics: MetricsLogger = { + putMetric: mockPutMetric, + setDimensions: (...args: any[]) => expect(args).toEqual([{}]), + setProperty: jest.fn(), +} as any; +mockMetricScope.mockImplementation((cb) => { + const impl = cb(mockMetrics); + return async (...args) => impl(...args); +}); + +const REPLICA = 'https://replicate.npmjs.com'; +const REGISTRY = 'https://registry.npmjs.org'; + +const MOCK_BUCKET = 'mock-staging-bucket'; +const MOCK_STAGING_FUNCTION = 'mock-staging-function'; +const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; +const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; +const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name'; +const MOCK_LICENSE_LIST_OBJECT = 'license-list.json'; + +const HOUR = 60 * 60 * 1_000; + +const mockS3 = mockClient(S3Client); +const mockLambda = mockClient(LambdaClient); + +const context: Context = { + awsRequestId: 'mock-request-id', + logGroupName: 'mock-log-group', + logStreamName: 'mock-log-stream', + getRemainingTimeInMillis: () => 300_000, +} as any; + +const event: ScheduledEvent = {} as any; + +beforeEach(() => { + mockS3.reset(); + mockLambda.reset(); + mockPutMetric.mockClear(); + + process.env.BUCKET_NAME = MOCK_BUCKET; + process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION; + process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; + process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; + process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET; + process.env[LicenseListEnv.OBJECT_KEY] = MOCK_LICENSE_LIST_OBJECT; + + // Empty deny list + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_DENY_LIST_BUCKET, + Key: MOCK_DENY_LIST_OBJECT, + }) + .resolves({ Body: stringToStream(JSON.stringify({})) }); + // License list allowing Apache-2.0 + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_LICENSE_LIST_BUCKET, + Key: MOCK_LICENSE_LIST_OBJECT, + }) + .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) }); + // Empty known versions + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET, + Key: KNOWN_VERSIONS_FILE_NAME, + }) + .resolves({ Body: stringToStream(JSON.stringify({ knownVersions: {} })) }); + // No legacy marker by default + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME }) + .rejects(noSuchKey()); + + mockS3.on(PutObjectCommand).resolves({}); + mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 }); +}); + +afterEach(() => { + delete process.env.BUCKET_NAME; + delete process.env.FUNCTION_NAME; + delete process.env[ENV_DENY_LIST_BUCKET_NAME]; + delete process.env[ENV_DENY_LIST_OBJECT_KEY]; + delete process.env[LicenseListEnv.BUCKET_NAME]; + delete process.env[LicenseListEnv.OBJECT_KEY]; + nock.cleanAll(); +}); + +function noSuchKey() { + return Object.assign(new Error('no such key'), { name: 'NoSuchKey' }); +} + +function givenHead(updateSeq: number) { + nock(REPLICA).get('/').reply(200, { + db_name: 'registry', + engine: 'npm-replicate', + doc_count: 1, + update_seq: updateSeq, + }); +} + +function givenNoStateFile() { + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET, + Key: FOLLOWER_STATE_FILE_NAME, + }) + .rejects(noSuchKey()); +} + +function givenStateFile(state: FollowerState) { + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET, + Key: FOLLOWER_STATE_FILE_NAME, + }) + .resolves({ Body: stringToStream(state.toText()) }); +} + +function givenLegacyMarker(marker: number) { + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME }) + .resolves({ Body: stringToStream(JSON.stringify({ marker })) }); +} + +function savedState(): FollowerState { + const calls = mockS3 + .commandCalls(PutObjectCommand) + .filter((call) => call.args[0].input.Key === FOLLOWER_STATE_FILE_NAME); + expect(calls.length).toBeGreaterThan(0); + const input = calls[calls.length - 1].args[0].input; + let body = input.Body as Buffer; + if (input.ContentEncoding === 'gzip') { + body = gunzipSync(body); + } + return FollowerState.fromText(body.toString('utf-8')); +} + +function packageDoc(name: string, rev: string, ...versions: string[]) { + const time: Record = { + created: '2023-01-01T00:00:00.000Z', + modified: '2023-06-01T00:00:00.000Z', + }; + const versionInfos: Record = {}; + for (const [i, version] of versions.entries()) { + time[version] = `2023-06-01T00:0${i}:00.000Z`; + versionInfos[version] = { + dependencies: { constructs: '^10.0.0' }, + dist: { + shasum: 'mock-shasum', + tarball: `${REGISTRY}/${name}/-/${name}-${version}.tgz`, + }, + jsii: {}, + keywords: ['cdk'], + license: 'Apache-2.0', + name, + version, + }; + } + return { + _id: name, + _rev: rev, + 'dist-tags': { latest: versions[versions.length - 1] }, + name, + time, + versions: versionInfos, + }; +} + +function stagedPackages(): Array<{ name: string; version: string }> { + return mockLambda.commandCalls(InvokeCommand).map((call) => { + const payload = JSON.parse( + Buffer.from(call.args[0].input.Payload! as Uint8Array).toString('utf-8') + ); + return { name: payload.name, version: payload.version }; + }); +} + +test('seeds from the legacy marker and processes fresh changes', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + + // The scan starts at the seeded marker position and finds one new entry. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(200, { + results: [ + { + seq: 1_005, + id: 'fresh-package', + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + }, + ], + last_seq: 1_005, + }); + nock(REGISTRY) + .get('/fresh-package') + .reply(200, packageDoc('fresh-package', '1-aaaaaa', '1.0.0')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'fresh-package', version: '1.0.0' }, + ]); + // Entries above the previously covered position are fresh, not late. + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LATE_CHANGE_COUNT, + 0, + Unit.Count + ); + + const state = savedState(); + expect(state.has(1_005)).toBe(true); + // A completed scan records a checkpoint at the highest position covered. + expect(state.newestCheckpointSeq()).toBe(1_010); + // The first ever scan doubles as a deep scan. + expect(state.lastDeepScanAt).toBeDefined(); +}); + +test('discovers and stages late-inserted changes', async () => { + const now = Date.now(); + givenHead(1_000); + + // A completed scan passed seq 500 seven hours ago and seq 1000 five minutes + // ago. Seq 550 was received; seq 600 was NOT (it appeared in the feed + // late). Only a regular scan is due. + const state = new FollowerState(); + state.addCheckpoint(500, now - 7 * HOUR); + state.addCheckpoint(1_000, now - 5 * 60_000); + state.addSeqs([550]); + state.completeDeepScan(now - 2 * HOUR); + givenStateFile(state); + + // Regular scan: floor is the checkpoint at/bordering (now - window), i.e. + // seq 500. Only the late entry's packument may be fetched. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '500' }) + .reply(200, { + results: [ + { + seq: 550, + id: 'already-seen-package', + changes: [{ rev: '1-cccccc' }], + deleted: false, + }, + { + seq: 600, + id: 'late-package', + changes: [{ rev: '2-dddddd' }], + deleted: false, + }, + ], + last_seq: 600, + }); + nock(REGISTRY) + .get('/late-package') + .reply(200, packageDoc('late-package', '2-dddddd', '1.2.3')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'late-package', version: '1.2.3' }, + ]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LATE_CHANGE_COUNT, + 1, + Unit.Count + ); + // The lag is measured against the checkpoint that first crossed seq 600. + const lagCalls = mockPutMetric.mock.calls.filter( + ([name]) => name === MetricName.LATE_CHANGE_LAG + ); + expect(lagCalls).toHaveLength(1); + expect(lagCalls[0][1]).toBeGreaterThanOrEqual(5 * 60_000); + expect(lagCalls[0][1]).toBeLessThan(HOUR); + + const saved = savedState(); + expect(saved.has(600)).toBe(true); + expect(saved.newestCheckpointSeq()).toBe(1_000); +}); + +test('laggy packument: processes served versions and records the expectation', async () => { + givenHead(2_010); + givenNoStateFile(); + givenLegacyMarker(2_000); + + // The feed announces rev 5, but the registry only serves rev 3. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '2000' }) + .reply(200, { + results: [ + { + seq: 2_005, + id: 'laggy-package', + changes: [{ rev: '5-eeeeee' }], + deleted: false, + }, + ], + last_seq: 2_005, + }); + nock(REGISTRY) + .get('/laggy-package') + .reply(200, packageDoc('laggy-package', '3-ffffff', '1.0.0')); + + await handler(event, context); + + // The versions the registry did serve are processed anyway. + expect(stagedPackages()).toEqual([ + { name: 'laggy-package', version: '1.0.0' }, + ]); + // ... and the expectation of rev 5 is recorded for later re-checks. + const state = savedState(); + expect(state.laggyPackuments()).toEqual([ + expect.objectContaining({ + name: 'laggy-package', + expectedRev: 5, + seq: 2_005, + }), + ]); +}); + +test('laggy packument: recovered once the registry catches up', async () => { + const now = Date.now(); + givenHead(3_000); + + const state = new FollowerState(); + state.addCheckpoint(3_000, now - 5 * 60_000); + state.completeDeepScan(now - HOUR); + state.recordLaggyPackument('recovered-package', 4, 1_500, now - HOUR); + givenStateFile(state); + + // The registry has now caught up (rev 4 >= expected 4), serving a version + // we don't know yet. + nock(REGISTRY) + .get('/recovered-package') + .reply(200, packageDoc('recovered-package', '4-gggggg', '2.0.0')); + // The scan itself is empty. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '3000' }) + .reply(200, { results: [], last_seq: 3_000 }); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'recovered-package', version: '2.0.0' }, + ]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + 1, + Unit.Count + ); + expect(savedState().laggyPackumentCount).toBe(0); +}); + +test('laggy packument: gives up after the maximum age, still processing what is served', async () => { + const now = Date.now(); + givenHead(3_000); + + const state = new FollowerState(); + state.addCheckpoint(3_000, now - 5 * 60_000); + state.completeDeepScan(now - HOUR); + // First seen 25 hours ago: beyond the give-up age. + state.recordLaggyPackument('stuck-package', 9, 1_600, now - 25 * HOUR); + givenStateFile(state); + + // The registry still serves rev 7 < expected 9 - but with a version we + // don't know, which is processed regardless. + nock(REGISTRY) + .get('/stuck-package') + .reply(200, packageDoc('stuck-package', '7-hhhhhh', '3.0.0')); + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '3000' }) + .reply(200, { results: [], last_seq: 3_000 }); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'stuck-package', version: '3.0.0' }, + ]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); + expect(savedState().laggyPackumentCount).toBe(0); +}); + +test('seeds at the beginning of the feed on first deployment (automatic backfill)', async () => { + givenHead(1_010); + givenNoStateFile(); + // No legacy marker either: this is a brand-new deployment. + + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '0' }) + .reply(200, { + results: [ + { + seq: 1_005, + id: 'historic-package', + changes: [{ rev: '1-bbbbbb' }], + deleted: false, + }, + ], + last_seq: 1_005, + }); + nock(REGISTRY) + .get('/historic-package') + .reply(200, packageDoc('historic-package', '1-bbbbbb', '1.0.0')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'historic-package', version: '1.0.0' }, + ]); + const state = savedState(); + expect(state.has(1_005)).toBe(true); + expect(state.newestCheckpointSeq()).toBe(1_010); +}); + +test('records interim checkpoints so an interrupted scan resumes mid-window', async () => { + const now = Date.now(); + givenHead(50_000); + + // A backfill in progress: the last interim checkpoint is at seq 0, and the + // first full page of the window has already been received (receipts exist). + const seqs = Array.from({ length: 10_000 }, (_, i) => (i + 1) * 2); + const state = new FollowerState(); + state.addCheckpoint(0, now - 5 * 60_000); + state.addSeqs(seqs); + givenStateFile(state); + + // The first page is full (10,000 entries, all receipted), so the scan wants + // to continue - but the time budget runs out right after this page. + let remaining = 300_000; + (context as any).getRemainingTimeInMillis = () => remaining; + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '0' }) + .reply(200, () => { + remaining = 30_000; // exhaust the budget after this page + return { + results: seqs.map((seq) => ({ + seq, + id: `package-${seq}`, + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + })), + last_seq: 20_000, + }; + }); + + try { + await handler(event, context); + } finally { + (context as any).getRemainingTimeInMillis = () => 300_000; + } + + // Nothing needed staging (everything was receipted), and no packument was + // fetched (no registry nock exists). The interim checkpoint was recorded, + // so the next run resumes from seq 20000 instead of the floor. + expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand); + const saved = savedState(); + expect(saved.newestCheckpointSeq()).toBe(20_000); + // The interrupted (deep) scan did not complete. + expect(saved.lastDeepScanAt).toBeUndefined(); +}); + +test('resets its position when the feed head regresses below the newest checkpoint', async () => { + const now = Date.now(); + // The feed head (3000) is below our newest checkpoint (5000). + givenHead(3_000); + + const state = new FollowerState(); + state.addCheckpoint(5_000, now - 5 * 60_000); + state.addSeqs([4_500]); + state.completeDeepScan(now - HOUR); + givenStateFile(state); + + // After the reset, the scan starts from the new head. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '3000' }) + .reply(200, { results: [], last_seq: 3_000 }); + + await handler(event, context); + + expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand); + const saved = savedState(); + expect(saved.newestCheckpointSeq()).toBe(3_000); + expect(saved.has(4_500)).toBe(false); +}); diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index f7c3bbd1a..aae04325c 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -10,6 +10,7 @@ import { Metric, MetricOptions, Statistic, + Stats, TreatMissingData, } from 'aws-cdk-lib/aws-cloudwatch'; import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; @@ -30,11 +31,13 @@ import { fillMetric } from '../metric-utils'; import { addAlarm } from '../monitoring'; import { NpmJsPackageCanary } from './npmjs/canary'; import { + FOLLOWER_STATE_FILE_NAME, KNOWN_VERSIONS_FILE_NAME, MARKER_FILE_NAME, METRICS_NAMESPACE, MetricName, S3KeyPrefix, + SCAN_WINDOW_MS, } from './npmjs/constants.lambda-shared'; import { NpmJsFollower } from './npmjs/npm-js-follower'; import { StageAndNotify } from './npmjs/stage-and-notify'; @@ -158,6 +161,12 @@ export class NpmJs implements IPackageSource { noncurrentVersionExpiration: Duration.days(30), expiredObjectDeleteMarker: true, }, + // Permanently delete follower state updates after 1 day (we don't need these) + { + prefix: FOLLOWER_STATE_FILE_NAME, + noncurrentVersionExpiration: Duration.days(1), + expiredObjectDeleteMarker: true, + }, ], }); bucket.grantRead(ingestion); @@ -208,6 +217,7 @@ export class NpmJs implements IPackageSource { bucket.grantReadWrite(follower, MARKER_FILE_NAME); bucket.grantReadWrite(follower, KNOWN_VERSIONS_FILE_NAME); + bucket.grantReadWrite(follower, FOLLOWER_STATE_FILE_NAME); denyList?.grantRead(follower); licenseList.grantRead(follower); stager.grantInvoke(follower); @@ -258,7 +268,14 @@ export class NpmJs implements IPackageSource { url: lambdaFunctionUrl(follower), primary: true, }, - { name: 'Marker Object', url: s3ObjectUrl(bucket, MARKER_FILE_NAME) }, + { + name: 'Follower State', + url: s3ObjectUrl(bucket, FOLLOWER_STATE_FILE_NAME), + }, + { + name: 'Marker Object (legacy)', + url: s3ObjectUrl(bucket, MARKER_FILE_NAME), + }, { name: 'Known Versions', url: s3ObjectUrl(bucket, KNOWN_VERSIONS_FILE_NAME), @@ -349,6 +366,42 @@ export class NpmJs implements IPackageSource { period: Duration.minutes(5), }), ], + [ + new GraphWidget({ + height: 6, + width: 12, + title: 'Feed Reliability', + left: [ + fillMetric( + this.metricLateChangeCount({ label: 'Late Changes' }), + 0 + ), + fillMetric( + this.metricLaggyPackuments({ label: 'Laggy Packuments' }), + 0 + ), + fillMetric( + this.metricLaggyPackumentGiveUps({ + label: 'Laggy Packument Give-Ups', + }), + 0 + ), + ], + leftYAxis: { min: 0 }, + right: [ + this.metricLateChangeLag({ label: 'Late Change Lag (max)' }), + ], + rightAnnotations: [ + { + color: '#ff0000', + label: 'Scan Window', + value: SCAN_WINDOW_MS, + }, + ], + rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false }, + period: Duration.minutes(5), + }), + ], [ new GraphWidget({ height: 6, @@ -433,6 +486,82 @@ export class NpmJs implements IPackageSource { }); } + /** + * The number of change entries discovered by a scan that were inserted into + * the `_changes` feed behind a position the follower had already read past. + */ + public metricLateChangeCount(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.LATE_CHANGE_COUNT, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * For each late change entry, the time elapsed between the moment the + * follower first read past the entry's sequence number and the moment the + * entry was discovered. This is a lower bound on the feed's insertion lag, + * and can be used to tune the scan window. + */ + public metricLateChangeLag(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.MAXIMUM, + ...opts, + metricName: MetricName.LATE_CHANGE_LAG, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of packages for which the registry packument is still behind + * the revision announced by the `_changes` feed. The versions served so far + * have been processed; the follower keeps re-checking for the announced + * revision. + */ + public metricLaggyPackuments(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.MAXIMUM, + ...opts, + metricName: MetricName.LAGGY_PACKUMENTS, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of laggy packuments for which the registry caught up with the + * revision announced by the `_changes` feed. + */ + public metricLaggyPackumentsRecovered(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.LAGGY_PACKUMENTS_RECOVERED, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of laggy packuments the follower gave up on: the registry + * never served the revision announced by the `_changes` feed within the + * maximum retry age. A version announced by the feed may be missing until + * the affected package publishes again. + */ + public metricLaggyPackumentGiveUps(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.LAGGY_PACKUMENT_GIVE_UPS, + namespace: METRICS_NAMESPACE, + }); + } + public metricNpmJsChangeAge(opts?: MetricOptions): Metric { return new Metric({ period: Duration.minutes(1), @@ -609,6 +738,51 @@ export class NpmJs implements IPackageSource { dlqNotEmptyAlarm ); + const lateChangeLagAlarm = this.metricLateChangeLag({ + period: Duration.hours(1), + }).createAlarm(scope, 'NpmJs/Follower/LateChangeLagHigh', { + alarmName: `${scope.node.path}/NpmJs/Follower/LateChangeLagHigh`, + alarmDescription: [ + 'The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed', + 'with a delay approaching the scan window. If the delay exceeds the window, the regular scan', + 'will miss those changes (only the daily deep scan would catch them). Consider increasing', + 'the scan window (SCAN_WINDOW_MS).', + '', + `Runbook: ${RUNBOOK_URL}`, + ].join('\n'), + comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD, + evaluationPeriods: 1, + threshold: 0.8 * SCAN_WINDOW_MS, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + monitoring.addLowSeverityAlarm( + 'NpmJs/Follower Late Change Lag approaching scan window', + lateChangeLagAlarm + ); + + const laggyPackumentGiveUpsAlarm = this.metricLaggyPackumentGiveUps({ + period: Duration.hours(1), + }).createAlarm(scope, 'NpmJs/Follower/LaggyPackumentGiveUps', { + alarmName: `${scope.node.path}/NpmJs/Follower/LaggyPackumentGiveUps`, + alarmDescription: [ + 'The npm registry never served the packument revision announced by the CouchDB changes feed', + 'for one or more packages, even after repeated re-checks. All versions the registry did serve', + 'have been processed, but a version announced by the feed may be missing until the affected', + 'package publishes again. The follower logs identify the affected packages, which can be', + 'manually ingested using the ReStagePackageVersion function.', + '', + `Runbook: ${RUNBOOK_URL}`, + ].join('\n'), + comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + evaluationPeriods: 1, + threshold: 1, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + monitoring.addLowSeverityAlarm( + 'NpmJs/Follower gave up on laggy packuments', + laggyPackumentGiveUpsAlarm + ); + // Finally - the "not running" alarm depends on the schedule (it won't run until the schedule // exists!), and the schedule depends on the failure alarm existing (we don't want it to run // before we can know it is failing). This means the returned `IDependable` effectively ensures diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index 560e23ac0..caf354cd8 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -16,6 +16,44 @@ export const enum MetricName { * Number of package versions that were skipped since they were in the deny list. */ DENY_LISTED_COUNT = 'DenyListedCount', + + /** + * Number of change entries discovered by a scan that were inserted into the + * `_changes` feed behind a position the follower had already read past + * ("late insertion"). + */ + LATE_CHANGE_COUNT = 'LateChangeCount', + + /** + * For each late change entry, the time elapsed between the moment the + * follower first read past the entry's sequence number and the moment the + * entry was discovered. This is a lower bound on the feed's insertion lag, + * and is the quantity the scan window must exceed. + */ + LATE_CHANGE_LAG = 'LateChangeLag', + + /** + * The number of packages for which the registry packument is still behind + * the revision announced by the `_changes` feed ("laggy packuments"). The + * versions served so far have been processed; the follower keeps re-checking + * for the announced revision. + */ + LAGGY_PACKUMENTS = 'LaggyPackuments', + + /** + * Number of laggy packuments for which the registry caught up with the + * revision announced by the `_changes` feed. + */ + LAGGY_PACKUMENTS_RECOVERED = 'LaggyPackumentsRecovered', + + /** + * Number of laggy packuments the follower gave up on: the registry never + * served the revision announced by the `_changes` feed within the maximum + * retry age. Any versions the registry did serve have been processed, but a + * version announced by the feed may be missing until the package publishes + * again. + */ + LAGGY_PACKUMENT_GIVE_UPS = 'LaggyPackumentGiveUps', } export const enum S3KeyPrefix { @@ -39,3 +77,40 @@ export const MARKER_FILE_NAME = 'couchdb-last-transaction-id.2'; * The name of the object that contains the list of known versions. */ export const KNOWN_VERSIONS_FILE_NAME = 'couchdb-known-versions.2'; + +/** + * The name of the object that contains the follower state: receipts for the + * change entries received from the `_changes` feed, rolling time/sequence + * checkpoints, laggy packument expectations, and deep scan bookkeeping. + */ +export const FOLLOWER_STATE_FILE_NAME = 'couchdb-follower-state'; + +/** + * How far behind the head of the `_changes` feed each regular scan re-reads. + * The feed occasionally makes change entries visible *behind* positions that + * were already read (npm calls this "normal feed behavior"), so a follower + * that only ever reads forward silently misses those entries. Every scan + * covers this window and processes any entry it has no receipt for. + */ +export const SCAN_WINDOW_MS = 6 * 60 * 60 * 1_000; // 6 hours + +/** + * How often a "deep" scan runs. The deep scan covers the entire state + * retention window (instead of just `SCAN_WINDOW_MS`), so that insertion lag + * *beyond* the regular scan window is still observed (via the `LateChangeLag` + * metric) and can inform tuning of the window. + */ +export const DEEP_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1_000; // 24 hours + +/** + * How long receipts and checkpoints are retained. This is the upper bound of + * what any scan can cover. + */ +export const STATE_RETENTION_MS = 24 * 60 * 60 * 1_000; // 24 hours + +/** + * How long the follower keeps re-checking a laggy packument (a package whose + * registry packument is behind the revision announced by the `_changes` feed) + * before giving up on the announced revision. + */ +export const LAGGY_PACKUMENT_GIVE_UP_MS = 24 * 60 * 60 * 1_000; // 24 hours diff --git a/src/package-sources/npmjs/couch-changes.lambda-shared.ts b/src/package-sources/npmjs/couch-changes.lambda-shared.ts index da9f36744..dc15e114a 100644 --- a/src/package-sources/npmjs/couch-changes.lambda-shared.ts +++ b/src/package-sources/npmjs/couch-changes.lambda-shared.ts @@ -16,8 +16,6 @@ const DEFAULT_BATCH_SIZE = 100; const MAX_CONNS_PER_HOST = 100; -const MAX_PACKAGE_SERVER_LAG_MS = 30_000; // 30 seconds - /** * A utility class that helps with traversing CouchDB database changes streams * in a promise-based, page-by-page manner. @@ -55,12 +53,15 @@ export class CouchChanges extends EventEmitter { } /** - * Obtains a batch of changes from the database. + * Obtains a batch of change entries from the database, without fetching the + * associated package metadata. Entries are deduplicated against previously + * received ones (via the follower state receipts) before any metadata is + * fetched, which is why this method does not fetch it eagerly. * * @param since the sequence value since when history should be fetched. - * @param batchSize the maximum amount of changes to return in a single page. + * @param batchSize the maximum amount of entries to return in a single page. * - * @returns a page of changes. + * @returns a page of change entries. */ public async changes( since: string | number, @@ -74,95 +75,109 @@ export class CouchChanges extends EventEmitter { const result = (await this.https('get', changesUrl)) as any; - const last_seq = result.last_seq; - const results = await this.fetchAndFilterAllMetadata(result.results); - + const results: DatabaseChange[] = result.results ?? []; return { - last_seq, - actionableResults: results, - totalCount: result.results.length, + last_seq: result.last_seq, + results, + seqs: results + .map((change) => Number(change.seq)) + .filter((seq) => !isNaN(seq)), + totalCount: results.length, }; } /** - * Fetch the metadata associated with a change. The change comes associated with a revision number, - * which can be compared to the revision number of the metadata to determine if the replica is - * lagging behind the changes stream. If so, we retry until the replica is up-to-date or until - * 30 seconds elapsed after which we return the potentially stale metadata. + * Fetches the current metadata document for the provided package name from + * the npm registry. + * + * @returns the metadata document, or `undefined` if the package does not + * exist (HTTP 404). */ - private async fetchAndFilterMetadata(change: DatabaseChange) { + public async getPackageDoc( + packageName: string + ): Promise<{ readonly [key: string]: unknown } | undefined> { + const metadataUrl = new URL(packageName, NPM_REGISTRY_URL); + try { + return await this.https('get', metadataUrl); + } catch (e: any) { + if (e.message?.includes('HTTP 404')) { + return undefined; + } + throw e; + } + } + + /** + * Fetches the registry metadata for each of the provided change entries and + * attaches it to the entry. Entries for deleted or unreachable packages are + * dropped. + * + * The registry is a separate system from the changes feed and can serve a + * packument revision older than the revision the feed announced (a "laggy + * packument"). Such entries are returned with `laggy: true`: the caller + * should process the versions that were served, and separately track the + * expectation that the announced revision will eventually appear. + */ + public async attachAllMetadata( + changes: readonly DatabaseChange[] + ): Promise { + const outcomes = await Promise.all( + changes.map((change) => this.attachMetadata(change)) + ); + return outcomes.filter( + (outcome): outcome is AttachedChange => outcome !== undefined + ); + } + + private async attachMetadata( + change: DatabaseChange + ): Promise { // Filter out deleted packages or null ids if (change.deleted || !change.id) { console.log(`Skipping ${change.id}: deleted or null id`); - return; + return undefined; } - const latestChangesRev = getMaxSequentialRevision(change); - const metadataUrl = new URL(change.id, NPM_REGISTRY_URL); - console.log(`Fetching metadata for ${change.id}: ${metadataUrl}`); - - // Retry configuration - const baseDelay = 1_000; // 1 second - const maxDelay = 8_000; // 8 seconds max - let attempt = 0; - const startTime = Date.now(); - // note: this function should not throw validation errors as // it may cause a poison pill - whereby a single corrupt package will // fail the entire lambda execution and prevent us from ingesting any package. - // instead, log the violation and return undefined. - - while (Date.now() - startTime < MAX_PACKAGE_SERVER_LAG_MS) { - try { - const meta = await this.https('get', metadataUrl); - if (!meta) { - // can happen if a package was removed from npm - console.log(`Skipping ${change.id} because no metadata found`); - return; - } - if (!meta._rev) { - // can happen if a package was removed from npm - console.log(`Skipping ${change.id} because no _rev found in metadata`); - return; - } - const latestReplicaRev = parseSequentialRevision(meta._rev as string); - - change.doc = meta; // add metadata to the change object - - // Happy path: replica is up-to-date - if (latestReplicaRev >= latestChangesRev) { - return change; - } - - // Unhappy path: replica is behind. Calculate delay and retry - const delay = Math.floor(Math.random() * Math.min(baseDelay * Math.pow(2, attempt), maxDelay)); - console.log(`${change.id}: package _rev ${latestReplicaRev} < expected replication rev ${latestChangesRev}, retrying in ${delay} ms`); - await new Promise(resolve => setTimeout(resolve, delay)); - attempt++; - } catch (e: any) { - if (e.message?.includes('HTTP 404')) { - console.log( - `Skipping ${change.id} because of HTTP 404 (Not Found) error` - ); - return; - } - throw e; + // instead, log the violation and return undefined. + const metadataUrl = new URL(change.id, NPM_REGISTRY_URL); + console.log(`Fetching metadata for ${change.id}: ${metadataUrl}`); + let meta; + try { + meta = await this.https('get', metadataUrl); + } catch (e: any) { + if (e.message?.includes('HTTP 404')) { + console.log( + `Skipping ${change.id} because of HTTP 404 (Not Found) error` + ); + return undefined; } + throw e; } - // Timeout reached, proceed with stale data - console.log(`Timeout reached for ${change.id}, replica may be stale`); - return change; - } + if (!meta) { + // can happen if a package was removed from npm + console.log(`Skipping ${change.id} because no metadata found`); + return undefined; + } + if (!meta._rev) { + // can happen if a package was removed from npm + console.log(`Skipping ${change.id} because no _rev found in metadata`); + return undefined; + } - private async fetchAndFilterAllMetadata( - changes: DatabaseChange[] - ): Promise { - return ( - await Promise.all( - changes.map((change) => this.fetchAndFilterMetadata(change)) - ) - ).filter((change): change is DatabaseChange => change !== undefined); + const announcedRev = getMaxSequentialRevision(change); + const servedRev = parseSequentialRevision(meta._rev as string); + change.doc = meta; // add metadata to the change object + + return { + change, + announcedRev, + servedRev, + laggy: servedRev < announcedRev, + }; } /** @@ -226,7 +241,8 @@ export class CouchChanges extends EventEmitter { } console.log( - `Response: ${method.toUpperCase()} ${url} => HTTP ${res.statusCode + `Response: ${method.toUpperCase()} ${url} => HTTP ${ + res.statusCode } (${res.statusMessage})` ); @@ -282,7 +298,7 @@ function readResponseJson( }); } -class RetryableError extends Error { } +class RetryableError extends Error {} function isRetryableError(e: Error): boolean { return e instanceof RetryableError || (e as any).code === 'ECONNRESET'; @@ -292,37 +308,78 @@ async function sleep(ms: number) { return new Promise((ok) => setTimeout(ok, ms)); } -function parseSequentialRevision(rev: string): number { +/** + * Parses the sequential (numeric) prefix of a CouchDB revision string (e.g. + * `42` for `42-0bf6e0fa87ae20bc7245f96216263817`). + */ +export function parseSequentialRevision(rev: string): number { return parseInt(rev.split('-')[0]); } -function getMaxSequentialRevision(change: DatabaseChange): number { - return Math.max(...change.changes - .map(change => parseSequentialRevision(change.rev)) - .filter(num => !isNaN(num)) +/** + * The highest sequential revision announced by a change entry. + */ +export function getMaxSequentialRevision(change: DatabaseChange): number { + return Math.max( + ...change.changes + .map((c) => parseSequentialRevision(c.rev)) + .filter((num) => !isNaN(num)) ); } export interface DatabaseChanges { /** - * The last sequence ID from this change set. This is the value that should be - * passed to the subsequent `.changes` call to fetch the next page. + * The last sequence ID from this change set. For ascending requests, this + * is the value that should be passed to a subsequent `.changes` call to + * fetch the next page. */ readonly last_seq: string | number; /** - * The actionable changes that are part of this batch. - * This has deleted and unreachable packages removed. + * The change entries that are part of this batch (no metadata attached). */ - readonly actionableResults: readonly DatabaseChange[]; + readonly results: readonly DatabaseChange[]; /** - * The total count of changes in this batch. This includes unprocessable changes. + * The sequence numbers of all the change entries received in this batch. + */ + readonly seqs: readonly number[]; + + /** + * The total count of change entries in this batch. For ascending requests, * 0 indicates we are up to date with "now". */ readonly totalCount: number; } +/** + * A change entry with its registry metadata attached. + */ +export interface AttachedChange { + /** + * The change entry, with the packument attached as `doc`. + */ + readonly change: DatabaseChange; + + /** + * The highest sequential revision announced by the changes feed for this + * entry. + */ + readonly announcedRev: number; + + /** + * The sequential revision of the packument the registry actually served. + */ + readonly servedRev: number; + + /** + * Whether the served packument is behind the announced revision (a "laggy + * packument"). The served versions can (and should) still be processed, but + * the announced revision has not been observed yet. + */ + readonly laggy: boolean; +} + export interface DatabaseChange { /** * The set of revisions to the object that were resolved as part of this diff --git a/src/package-sources/npmjs/follower-state.lambda-shared.ts b/src/package-sources/npmjs/follower-state.lambda-shared.ts new file mode 100644 index 000000000..fdf9af06d --- /dev/null +++ b/src/package-sources/npmjs/follower-state.lambda-shared.ts @@ -0,0 +1,355 @@ +import { + DEEP_SCAN_INTERVAL_MS, + STATE_RETENTION_MS, +} from './constants.lambda-shared'; + +/** + * A point-in-time record of a feed position the follower has fully read up + * to: "at time `at`, a completed scan had received everything the feed served + * up to sequence number `seq`". + */ +export interface Checkpoint { + /** + * Epoch milliseconds at which the checkpoint was taken. + */ + readonly at: number; + + /** + * The sequence number the completed scan had reached. + */ + readonly seq: number; +} + +/** + * A package whose registry packument is behind the revision announced by the + * `_changes` feed. The versions the registry did serve have been processed; + * this expectation tracks the revision we are still waiting for. + */ +export interface LaggyPackument { + /** + * The package name. + */ + readonly name: string; + + /** + * The revision (sequential prefix) announced by the changes feed. + */ + readonly expectedRev: number; + + /** + * The sequence number of the change entry that announced the revision, if + * known. + */ + readonly seq?: number; + + /** + * Epoch milliseconds at which the expectation was first recorded. Drives + * the give-up policy. + */ + readonly firstSeen: number; +} + +/** + * The persisted state of the *NpmJs Follower*, in a newline-delimited + * plain-text format so it can be scanned cheaply (e.g. `grep '^s 131711133$'`) + * and parsed without materializing a large JSON document. Each line is a + * record identified by its first token: + * + * ```text + * #chfollower/1 <- header (format magic + version) + * w <- deep scan bookkeeping ("-" when unset) + * c <- checkpoint (repeated, ascending by time) + * l <- laggy packument (repeated; seq is "-" when unknown) + * s <- received sequence number (repeated, ascending) + * ``` + * + * The receipts are an exact set on purpose: any form of range encoding would + * claim coverage of sequence numbers that were never observed, and a late + * inserted change entry at such a position would be skipped - which is the + * exact failure mode the receipts exist to catch. + */ +const STATE_FORMAT_HEADER = '#chfollower/1'; + +/** + * Tracks which `_changes` feed entries (by sequence number) the follower has + * already received, rolling time/seq checkpoints mapping times to feed + * positions, and the set of laggy packuments awaiting a registry catch-up. + * + * The npm `_changes` feed does not guarantee that entries only ever become + * visible above previously returned `last_seq` values. The receipt set + * upgrades the follower's position tracking from "I have processed everything + * up to X" to "here is exactly what I received up to X", enabling cheap + * re-scans of a trailing window that only process entries with no receipt. + */ +export class FollowerState { + public static fromText(text: string): FollowerState { + const state = new FollowerState(); + const lines = text.split('\n'); + if (lines[0]?.trim() !== STATE_FORMAT_HEADER) { + throw new Error( + `Unexpected follower state header: ${JSON.stringify(lines[0])}` + ); + } + for (let i = 1; i < lines.length; i++) { + const line = lines[i].trim(); + if (line === '') { + continue; + } + const parts = line.split(' '); + switch (parts[0]) { + case 's': + state.seqs.add(Number(parts[1])); + break; + case 'c': + state.checkpoints.push({ + at: Number(parts[1]), + seq: Number(parts[2]), + }); + break; + case 'l': + state.laggy.set(parts.slice(4).join(' '), { + name: parts.slice(4).join(' '), + firstSeen: Number(parts[1]), + expectedRev: Number(parts[2]), + seq: parts[3] === '-' ? undefined : Number(parts[3]), + }); + break; + case 'w': + state.lastDeepScanAt = + parts[1] === '-' ? undefined : Number(parts[1]); + break; + default: + throw new Error(`Unexpected follower state line: ${line}`); + } + } + return state; + } + + public lastDeepScanAt?: number; + + private checkpoints: Checkpoint[] = []; + private readonly seqs = new Set(); + private readonly laggy = new Map(); + // The threshold below which receipts have already been pruned; avoids + // re-iterating the (potentially large) receipt set on every checkpoint. + private prunedBelow = 0; + + //#region Receipts + /** + * Whether a receipt exists for the provided sequence number. + */ + public has(seq: number): boolean { + return this.seqs.has(seq); + } + + /** + * Records receipts for the provided sequence numbers. + */ + public addSeqs(seqs: Iterable): void { + for (const seq of seqs) { + this.seqs.add(seq); + } + } + + /** + * The number of sequence number receipts currently retained. + */ + public get receiptCount(): number { + return this.seqs.size; + } + //#endregion + + //#region Checkpoints + /** + * Records a checkpoint at the current time for the provided sequence + * number, then prunes checkpoints and receipts that have aged out of the + * retention window. + */ + public addCheckpoint(seq: number, now: number = Date.now()): void { + this.checkpoints.push({ at: now, seq }); + this.prune(now); + } + + /** + * The number of checkpoints currently retained. + */ + public get checkpointCount(): number { + return this.checkpoints.length; + } + + /** + * The sequence number of the newest checkpoint, i.e. the highest feed + * position a completed scan has read up to. Entries discovered below this + * position were inserted late. Returns `undefined` when no checkpoints + * exist. + */ + public newestCheckpointSeq(): number | undefined { + return this.checkpoints[this.checkpoints.length - 1]?.seq; + } + + /** + * The feed position a scan should re-read from, to cover the provided + * trailing window: the newest checkpoint at least `windowMs` old. If the + * checkpoint history does not reach that far back (a young deployment, or a + * backfill in progress), the newest checkpoint is used: coverage below it + * is guaranteed, and starting any lower would make long scans restart from + * the beginning on every run. Returns `undefined` when no checkpoints exist + * at all. + */ + public floorSeq( + windowMs: number, + now: number = Date.now() + ): number | undefined { + let candidate: Checkpoint | undefined; + for (const checkpoint of this.checkpoints) { + if (checkpoint.at > now - windowMs) { + break; + } + candidate = checkpoint; + } + return (candidate ?? this.checkpoints[this.checkpoints.length - 1])?.seq; + } + + /** + * The (approximate) time at which a completed scan first read past the + * provided sequence number: the timestamp of the earliest checkpoint whose + * sequence number is `>= seq`. Returns `undefined` if no completed scan has + * passed that sequence number yet. + */ + public timeCrossed(seq: number): number | undefined { + for (const checkpoint of this.checkpoints) { + if (checkpoint.seq >= seq) { + return checkpoint.at; + } + } + return undefined; + } + + /** + * Discards all position state (receipts and checkpoints). Used when the + * feed's sequence space is no longer compatible with the recorded state + * (e.g. the feed head regressed below our newest checkpoint). Laggy + * packument expectations are package-keyed and remain valid. + */ + public resetPosition(): void { + this.checkpoints = []; + this.seqs.clear(); + this.prunedBelow = 0; + } + //#endregion + + //#region Deep scans + /** + * Whether a deep scan (covering the full state retention window) is due. + */ + public deepScanDue(now: number = Date.now()): boolean { + return ( + this.lastDeepScanAt == null || + now - this.lastDeepScanAt >= DEEP_SCAN_INTERVAL_MS + ); + } + + /** + * Marks a deep scan as completed at the provided time. + */ + public completeDeepScan(now: number = Date.now()): void { + this.lastDeepScanAt = now; + } + //#endregion + + //#region Laggy packuments + /** + * Records (or extends) the expectation that the registry packument for the + * provided package should reach the provided revision. The earliest + * `firstSeen` is retained (so the give-up policy measures from the first + * occurrence), and the highest announced revision wins. + */ + public recordLaggyPackument( + name: string, + expectedRev: number, + seq: number | undefined, + now: number = Date.now() + ): void { + const existing = this.laggy.get(name); + this.laggy.set(name, { + name, + expectedRev: Math.max(expectedRev, existing?.expectedRev ?? 0), + seq: existing?.seq ?? seq, + firstSeen: existing?.firstSeen ?? now, + }); + } + + /** + * Removes the expectation for the provided package (recovered, gone, or + * given up). + */ + public removeLaggyPackument(name: string): void { + this.laggy.delete(name); + } + + /** + * All laggy packument expectations, oldest first. + */ + public laggyPackuments(): LaggyPackument[] { + return Array.from(this.laggy.values()).sort( + (a, b) => a.firstSeen - b.firstSeen + ); + } + + /** + * The number of laggy packument expectations currently tracked. + */ + public get laggyPackumentCount(): number { + return this.laggy.size; + } + //#endregion + + public toText(): string { + const lines = new Array(); + lines.push(STATE_FORMAT_HEADER); + lines.push(`w ${this.lastDeepScanAt ?? '-'}`); + for (const { at, seq } of this.checkpoints) { + lines.push(`c ${at} ${seq}`); + } + for (const laggy of this.laggyPackuments()) { + lines.push( + `l ${laggy.firstSeen} ${laggy.expectedRev} ${laggy.seq ?? '-'} ${ + laggy.name + }` + ); + } + for (const seq of Array.from(this.seqs).sort((a, b) => a - b)) { + lines.push(`s ${seq}`); + } + lines.push(''); // trailing newline + return lines.join('\n'); + } + + private prune(now: number): void { + const cutOff = now - STATE_RETENTION_MS; + // Always keep at least one checkpoint at or before the cut-off, so that + // `floorSeq` remains well-defined over the whole retention window. + let firstRetained = 0; + for (let i = 0; i < this.checkpoints.length; i++) { + if (this.checkpoints[i].at <= cutOff) { + firstRetained = i; + } else { + break; + } + } + this.checkpoints = this.checkpoints.slice(firstRetained); + + // Receipts below the earliest retained checkpoint's sequence number can + // no longer be reached by any scan, so drop them. + const minSeq = this.checkpoints[0]?.seq ?? 0; + if (minSeq <= this.prunedBelow) { + return; + } + for (const seq of this.seqs) { + if (seq < minSeq) { + this.seqs.delete(seq); + } + } + this.prunedBelow = minSeq; + } +} diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index e021c1d53..69f0f6612 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -24,8 +24,18 @@ import { MARKER_FILE_NAME, METRICS_NAMESPACE, KNOWN_VERSIONS_FILE_NAME, + FOLLOWER_STATE_FILE_NAME, + SCAN_WINDOW_MS, + STATE_RETENTION_MS, + LAGGY_PACKUMENT_GIVE_UP_MS, } from './constants.lambda-shared'; -import { CouchChanges, DatabaseChange } from './couch-changes.lambda-shared'; +import { + CouchChanges, + DatabaseChange, + DatabaseChanges, + parseSequentialRevision, +} from './couch-changes.lambda-shared'; +import { FollowerState } from './follower-state.lambda-shared'; import { PackageVersion } from './stage-and-notify.lambda'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; @@ -33,7 +43,10 @@ import { LAMBDA_CLIENT, S3_CLIENT, } from '../../backend/shared/aws.lambda-shared'; -import { decompressContent } from '../../backend/shared/compress-content.lambda-shared'; +import { + compressContent, + decompressContent, +} from '../../backend/shared/compress-content.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; // eslint-disable-next-line @typescript-eslint/no-require-imports const normalizeNPMMetadata = require('normalize-registry-metadata'); @@ -50,11 +63,37 @@ const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/'; /** * The release date of `aws-cdk@0.8.0`. Anything earlier than this basically is * not a relevant package, as it cannot possibly be a constructs-based package. - * This is used to fast-forward over boring stuff when the sequence number is - * reset. + * This is used to fast-forward over boring stuff when the follower state is + * seeded far in the past. */ const DAWN_OF_CONSTRUCTS = new Date('2018-07-31T13:43:04.615Z'); +/** + * The page size used when scanning the `_changes` feed. Receipts discard + * already-received entries before any packument is fetched, so scans can + * afford the feed's maximum page size (10,000). + */ +const SCAN_BATCH_SIZE = 10_000; + +/** + * A scan batch is only started when at least this much time remains in the + * Lambda execution. An interrupted scan simply resumes on the next run: the + * receipts make re-reads cheap, and no checkpoint is recorded for an + * incomplete scan. + */ +const SCAN_TIME_BUDGET_MS = 60_000; + +/** + * Laggy packuments are only re-checked while at least this much time remains + * in the Lambda execution, so that re-checks cannot starve the scan. + */ +const LAGGY_RETRY_TIME_BUDGET_MS = 240_000; + +/** + * The maximum number of laggy packuments re-checked per run. + */ +const MAX_LAGGY_RETRIES_PER_RUN = 50; + // Configure embedded metrics format Configuration.namespace = METRICS_NAMESPACE; @@ -65,13 +104,30 @@ captureHTTPsGlobal(require('https')); captureHTTPsGlobal(require('http')); /** - * This function triggers on a fixed schedule and reads a stream of changes from npmjs couchdb _changes endpoint. - * Upon invocation the function starts reading from a sequence stored in an s3 object - the `marker`. - * If the marker fails to load (or do not exist), the stream will start from `now` - the latest change. - * For each change: - * - the package version tarball will be copied from the npm registry to a stating bucket. - * - a message will be sent to an sqs queue + * This function triggers on a fixed schedule and scans the npmjs CouchDB + * `_changes` feed for new package versions. + * + * The feed gives no guarantee that change entries become visible in sequence + * order (entries occasionally appear *behind* previously returned positions), + * so the follower does not track a single high-water mark. Instead it keeps a + * receipt for every change entry it has received, and every run re-reads a + * trailing window of the feed, processing any entry it has no receipt for: + * + * 1. Rolling checkpoints map times to feed positions a completed scan has + * fully covered. The scan floor is the checkpointed position from + * `SCAN_WINDOW_MS` ago (a periodic deep scan covers the full retention + * window instead). + * 2. Each scan reads the window ascending from the floor. Receipts make + * re-reads of already-covered ranges nearly free. + * 3. For each new entry, the packument is fetched from the registry and every + * version it contains is processed (deny list, license, and known-versions + * checks apply). If the packument is behind the revision announced by the + * feed (a "laggy packument"), the served versions are still processed, and + * the follower records the expectation and keeps re-checking on subsequent + * runs until the announced revision appears or the expectation ages out. + * * npm registry API docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md + * * @param context a Lambda execution context */ export async function handler(event: ScheduledEvent, context: Context) { @@ -85,154 +141,424 @@ export async function handler(event: ScheduledEvent, context: Context) { const npm = new CouchChanges(NPM_REPLICA_REGISTRY_URL, 'registry/_changes'); - const { - marker: initialMarker, - knownVersions, - didLoadVersionsFromLegacy, - } = await loadFollowerPosition(stagingBucket, npm); + const head = Number((await npm.info()).update_seq); - // If we loaded from the old location, write to the new + const { knownVersions, didLoadVersionsFromLegacy } = await loadKnownVersions( + stagingBucket + ); let writeKnownVersionsFile = didLoadVersionsFromLegacy; - // The last written marker seq id. - let updatedMarker = initialMarker; + const state = await loadFollowerState(stagingBucket, head); - // The slowest batch processing time so far (starts at 60 seconds). This is how much time should - // be left before timeout if a new batch is to be fetched. - let maxBatchProcessingTime = 60_000; - // Whether we should continue reading more items or not... This is set to false when the current - // latest change is reached (i.e: next page of changes is empty). - let shouldContinue = true; + // The feed's sequence space can change (entries can disappear from the end, + // moving the head backwards). Our position state is then meaningless. + const newestCheckpoint = state.newestCheckpointSeq(); + if (newestCheckpoint != null && newestCheckpoint > head) { + console.warn( + `Feed head (${head}) is below our newest checkpoint (${newestCheckpoint}); the sequence space changed. Resetting position state.` + ); + state.resetPosition(); + state.addCheckpoint(head); + } - do { - await metricScope((metrics) => async () => { - console.log('Polling changes from npm replica'); - const changes = await npm.changes(updatedMarker); + // Re-check laggy packuments (packages whose registry packument was behind + // the revision announced by the feed when we first saw them). + const laggyStaged = await retryLaggyPackuments( + context, + npm, + state, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + writeKnownVersionsFile ||= laggyStaged > 0; - // Clear automatically set dimensions - we don't need them (see https://github.com/awslabs/aws-embedded-metrics-node/issues/73) - metrics.setDimensions({}); + // Scan the trailing window of the feed. + const scanStaged = await runScan( + context, + npm, + state, + head, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + writeKnownVersionsFile ||= scanStaged > 0; - // Recording current seq range and updating the `updatedMarker`. - metrics.setProperty('StartSeq', updatedMarker); - updatedMarker = Number(changes.last_seq); - metrics.setProperty('EndSeq', updatedMarker); - - const startTime = Date.now(); - - try { - const batch = changes.actionableResults as readonly Change[]; - - // The most recent "modified" timestamp observed in the batch. - let lastModified: Date | undefined; - // Emit npm.js replication lag - for (const { doc } of batch) { - if (doc?.time?.modified) { - const modified = new Date(doc.time.modified); - metrics.putMetric( - MetricName.NPMJS_CHANGE_AGE, - startTime - modified.getTime(), - Unit.Milliseconds - ); - if (lastModified == null || lastModified < modified) { - lastModified = modified; - } - } - } + await Promise.all([ + saveFollowerState(context, stagingBucket, state), + ...(writeKnownVersionsFile + ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] + : []), + ]); - console.log( - `Received a batch of ${changes.totalCount} element(s), ${batch.length} after filtering` - ); - metrics.putMetric(MetricName.CHANGE_COUNT, batch.length, Unit.Count); + console.log('All done here, we have success!'); - if (lastModified && lastModified < DAWN_OF_CONSTRUCTS) { - console.log( - `Skipping batch as the latest modification is ${lastModified}, which is pre-Constructs` - ); - } else if (changes.totalCount === 0) { - console.log('Received 0 changes, caught up to "now", exiting...'); - shouldContinue = false; - } else { - // Obtain the modified package version from the update event, and filter - // out packages that are not of interest to us (not construct libraries). - const versionInfos = getRelevantVersionInfos( - batch, - metrics, - denyList, - licenseList, - knownVersions - ); - console.log( - `Identified ${versionInfos.length} relevant package version update(s)` - ); - metrics.putMetric( - MetricName.RELEVANT_PACKAGE_VERSIONS, - versionInfos.length, - Unit.Count - ); + return { head }; +} - // If we have versionInfos, we already added them to the knownVersions map and we should save it again - writeKnownVersionsFile ||= versionInfos.length > 0; - - // Process all remaining updates - await Promise.all( - versionInfos.map(async ({ infos, modified, seq }) => { - const invokeArgs: PackageVersion = { - integrity: infos.dist.shasum, - modified: modified.toISOString(), - name: infos.name, - seq: seq?.toString(), - tarballUrl: infos.dist.tarball, - version: infos.version, - }; - // "Fire-and-forget" invocation here. - console.log(`Sending ${invokeArgs.tarballUrl} for staging`); - await LAMBDA_CLIENT.send( - new InvokeCommand({ - FunctionName: stagingFunction, - InvocationType: 'Event', - Payload: Buffer.from(JSON.stringify(invokeArgs)), - }) - ); - // Record that this is now a "known" version (no need to re-discover) - knownVersions.set(`${infos.name}@${infos.version}`, modified); - }) - ); +//#region Scan +/** + * Re-reads the trailing window of the feed and processes every change entry + * for which no receipt exists. A checkpoint is only recorded when the scan + * covered the entire window; an interrupted scan resumes naturally on the + * next run (receipts make the re-read cheap). + * + * @returns the number of package versions sent for staging. + */ +async function runScan( + context: Context, + npm: CouchChanges, + state: FollowerState, + head: number, + stagingFunction: string, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: Map +): Promise { + const now = Date.now(); + const deep = state.deepScanDue(now); + const windowMs = deep ? STATE_RETENTION_MS : SCAN_WINDOW_MS; + const floor = state.floorSeq(windowMs, now); + if (floor == null) { + // Cannot happen after seeding, but guards the type. + console.warn('No checkpoint history; skipping scan'); + return 0; + } + // Entries discovered at or below this position were inserted into the feed + // behind a position we had already read past. + const lateBoundary = state.newestCheckpointSeq() ?? floor; + + console.log( + `Starting ${ + deep ? 'deep ' : '' + }scan of (${floor}, ${head}] (late boundary: ${lateBoundary})` + ); + + let staged = 0; + let completed = false; + let maxSeqSeen = head; + + // Read the window ascending from the floor. Receipts make re-reads of + // already-covered ranges nearly free (no packuments are fetched for them). + let cursor = floor; + while ( + !completed && + context.getRemainingTimeInMillis() > SCAN_TIME_BUDGET_MS + ) { + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + metrics.setProperty('StartSeq', cursor); + const batch = await npm.changes(cursor, { batchSize: SCAN_BATCH_SIZE }); + const next = Number(batch.last_seq); + metrics.setProperty('EndSeq', next); + if (batch.seqs.length > 0) { + maxSeqSeen = Math.max(maxSeqSeen, ...batch.seqs); + } + staged += await processBatch( + batch, + metrics, + context, + npm, + state, + lateBoundary, + head, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + if (batch.totalCount < SCAN_BATCH_SIZE || next <= cursor) { + // A short page means we are caught up with the feed head. + completed = true; + } else { + cursor = next; + // Coverage from the floor up to `next` is contiguous, so this is a + // valid checkpoint. Recording it makes an interrupted scan (most + // importantly: a backfill) resume from here instead of the floor. + if (next > (state.newestCheckpointSeq() ?? 0)) { + state.addCheckpoint(next); } + } + })(); + } + + if (completed) { + // The scan covered the entire window: everything the feed served up to + // `maxSeqSeen` now has a receipt. + state.addCheckpoint(maxSeqSeen); + if (deep) { + state.completeDeepScan(); + console.log(`Completed deep scan at seq ${maxSeqSeen}`); + } + } else { + console.log( + 'Scan ran out of time; it will resume on the next run (no checkpoint recorded)' + ); + } + return staged; +} - // Updating the S3 stored marker with the new seq id and known versions concurrently - await Promise.all([ - // always save the transaction marker - saveLastTransactionMarker(context, stagingBucket, updatedMarker), - // conditionally save known versions - ...(writeKnownVersionsFile - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ]); - } finally { - metrics.putMetric(MetricName.LAST_SEQ, updatedMarker, Unit.None); +/** + * Processes one page of change entries: discards everything we already have a + * receipt for, fetches packuments for the remainder, stages new relevant + * package versions, records laggy packument expectations, and finally records + * receipts for every entry received. + * + * @returns the number of package versions sent for staging. + */ +async function processBatch( + batch: DatabaseChanges, + metrics: MetricsLogger, + context: Context, + npm: CouchChanges, + state: FollowerState, + lateBoundary: number, + head: number, + stagingFunction: string, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: Map +): Promise { + const startTime = Date.now(); + let staged = 0; + try { + const fresh = batch.results.filter( + (change) => change.seq != null && !state.has(Number(change.seq)) + ); + console.log( + `Received ${batch.totalCount} change entr(ies), ${fresh.length} without a receipt` + ); + metrics.putMetric(MetricName.CHANGE_COUNT, fresh.length, Unit.Count); + + // Entries below the late boundary were inserted into the feed behind a + // position a completed scan had already read past. + const late = fresh.filter((change) => Number(change.seq) <= lateBoundary); + metrics.putMetric(MetricName.LATE_CHANGE_COUNT, late.length, Unit.Count); + for (const change of late) { + const crossedAt = state.timeCrossed(Number(change.seq)); + if (crossedAt != null) { metrics.putMetric( - MetricName.BATCH_PROCESSING_TIME, - Date.now() - startTime, + MetricName.LATE_CHANGE_LAG, + Date.now() - crossedAt, Unit.Milliseconds ); + } + console.log( + `[late] Change entry discovered at seq ${change.seq} (behind ${lateBoundary}): ${change.id}` + ); + } + + const attached = await npm.attachAllMetadata(fresh); + + // The most recent "modified" timestamp observed in the batch. + let lastModified: Date | undefined; + // Emit npm.js replication lag + for (const { change } of attached) { + const doc = change.doc as Change['doc'] | undefined; + if (doc?.time?.modified) { + const modified = new Date(doc.time.modified); metrics.putMetric( - MetricName.REMAINING_TIME, - context.getRemainingTimeInMillis(), + MetricName.NPMJS_CHANGE_AGE, + startTime - modified.getTime(), Unit.Milliseconds ); + if (lastModified == null || lastModified < modified) { + lastModified = modified; + } } - })(); - } while ( - shouldContinue && - context.getRemainingTimeInMillis() >= maxBatchProcessingTime - ); + } - console.log('All done here, we have success!'); + if (lastModified && lastModified < DAWN_OF_CONSTRUCTS) { + console.log( + `Skipping batch as the latest modification is ${lastModified}, which is pre-Constructs` + ); + } else { + // Laggy packuments: the registry served an older revision than the feed + // announced. Process the versions we did get, and record the + // expectation so the announced revision is re-checked on later runs. + for (const laggy of attached.filter((a) => a.laggy)) { + console.log( + `${laggy.change.id}: registry packument rev ${laggy.servedRev} is behind announced rev ${laggy.announcedRev}, recording laggy packument` + ); + state.recordLaggyPackument( + laggy.change.id, + laggy.announcedRev, + isNaN(Number(laggy.change.seq)) ? undefined : Number(laggy.change.seq) + ); + } - return { initialMarker, updatedMarker }; + const versionInfos = getRelevantVersionInfos( + attached.map((a) => a.change) as unknown as readonly Change[], + metrics, + denyList, + licenseList, + knownVersions + ); + console.log( + `Identified ${versionInfos.length} relevant package version update(s)` + ); + metrics.putMetric( + MetricName.RELEVANT_PACKAGE_VERSIONS, + versionInfos.length, + Unit.Count + ); + await stageVersions(versionInfos, stagingFunction, knownVersions); + staged = versionInfos.length; + } + + // Record receipts for every entry received (including deleted or + // unreachable packages: they were received and deliberately skipped). + state.addSeqs(batch.seqs); + } finally { + metrics.putMetric(MetricName.LAST_SEQ, head, Unit.None); + metrics.putMetric( + MetricName.BATCH_PROCESSING_TIME, + Date.now() - startTime, + Unit.Milliseconds + ); + metrics.putMetric( + MetricName.REMAINING_TIME, + context.getRemainingTimeInMillis(), + Unit.Milliseconds + ); + } + return staged; } +//#endregion -//#region Last known versions and marker +//#region Laggy packuments +/** + * Re-checks laggy packuments: packages whose registry packument was behind + * the revision announced by the changes feed. Each re-check processes + * whatever the registry serves now (new versions are staged), and the + * expectation is cleared when the announced revision appears - or aged out + * after `LAGGY_PACKUMENT_GIVE_UP_MS`, at which point a version announced by + * the feed may be missing until the package publishes again. + * + * @returns the number of package versions sent for staging. + */ +async function retryLaggyPackuments( + context: Context, + npm: CouchChanges, + state: FollowerState, + stagingFunction: string, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: Map +): Promise { + let staged = 0; + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + let recovered = 0; + let gaveUp = 0; + const now = Date.now(); + for (const laggy of state + .laggyPackuments() + .slice(0, MAX_LAGGY_RETRIES_PER_RUN)) { + if (context.getRemainingTimeInMillis() < LAGGY_RETRY_TIME_BUDGET_MS) { + break; + } + const doc = await npm.getPackageDoc(laggy.name); + if (doc == null || doc._rev == null) { + console.log( + `Laggy packument for ${laggy.name} is no longer available in the registry, dropping the expectation` + ); + state.removeLaggyPackument(laggy.name); + continue; + } + + // Process whatever the registry serves now; knownVersions dedupes. + const change: DatabaseChange = { + changes: [{ rev: doc._rev as string }], + deleted: false, + id: laggy.name, + seq: laggy.seq, + doc, + }; + const versionInfos = getRelevantVersionInfos( + [change] as unknown as readonly Change[], + metrics, + denyList, + licenseList, + knownVersions + ); + await stageVersions(versionInfos, stagingFunction, knownVersions); + staged += versionInfos.length; + + if (parseSequentialRevision(doc._rev as string) >= laggy.expectedRev) { + console.log( + `Laggy packument for ${laggy.name} caught up (rev ${doc._rev} >= ${laggy.expectedRev})` + ); + state.removeLaggyPackument(laggy.name); + recovered += 1; + } else if (now - laggy.firstSeen > LAGGY_PACKUMENT_GIVE_UP_MS) { + console.warn( + `Giving up on laggy packument for ${ + laggy.name + }: the registry never served rev ${ + laggy.expectedRev + } (first seen ${new Date( + laggy.firstSeen + ).toISOString()}). A version announced by the changes feed may be missing until the package publishes again.` + ); + state.removeLaggyPackument(laggy.name); + gaveUp += 1; + } + } + metrics.putMetric( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + recovered, + Unit.Count + ); + metrics.putMetric(MetricName.LAGGY_PACKUMENT_GIVE_UPS, gaveUp, Unit.Count); + metrics.putMetric( + MetricName.LAGGY_PACKUMENTS, + state.laggyPackumentCount, + Unit.Count + ); + })(); + return staged; +} +//#endregion + +/** + * Sends the provided package version updates to the staging function + * ("fire-and-forget"), and records them as known versions. + */ +async function stageVersions( + versionInfos: readonly UpdatedVersion[], + stagingFunction: string, + knownVersions: Map +): Promise { + await Promise.all( + versionInfos.map(async ({ infos, modified, seq }) => { + const invokeArgs: PackageVersion = { + integrity: infos.dist.shasum, + modified: modified.toISOString(), + name: infos.name, + seq: seq?.toString(), + tarballUrl: infos.dist.tarball, + version: infos.version, + }; + // "Fire-and-forget" invocation here. + console.log(`Sending ${invokeArgs.tarballUrl} for staging`); + await LAMBDA_CLIENT.send( + new InvokeCommand({ + FunctionName: stagingFunction, + InvocationType: 'Event', + Payload: Buffer.from(JSON.stringify(invokeArgs)), + }) + ); + // Record that this is now a "known" version (no need to re-discover) + knownVersions.set(`${infos.name}@${infos.version}`, modified); + }) + ); +} + +//#region State and known versions /** * Common function to load data from an S3 file with error handling * @@ -270,113 +596,143 @@ async function loadContentFromS3( } } -async function loadFollowerPosition( +/** + * Loads the follower state from S3. When the state file does not exist (or + * cannot be parsed), a fresh state is seeded: from the legacy transaction + * marker if one exists (so existing deployments continue where the previous + * follower left off), or from the current feed head otherwise. + */ +async function loadFollowerState( stagingBucket: string, - registry: CouchChanges -): Promise<{ - marker: number; - knownVersions: Map; - didLoadVersionsFromLegacy: boolean; -}> { - const [markerFile, knownVersions] = await Promise.all([ - loadLastTransactionMarker(stagingBucket, registry), - loadLastKnownVersions(stagingBucket), - ]); - - if (!knownVersions) { - // For legacy reasons, knownVersions could come from the transaction marker file - // If it does, then we take that as knownVersions but will migrate knownVersions - // to the modern file location - return { - marker: markerFile.marker, - knownVersions: markerFile.knownVersions ?? new Map(), - didLoadVersionsFromLegacy: true, - }; + head: number +): Promise { + const warningMessage = `Follower state object (s3://${stagingBucket}/${FOLLOWER_STATE_FILE_NAME}) does not exist, seeding a fresh state`; + const content = await loadContentFromS3( + stagingBucket, + FOLLOWER_STATE_FILE_NAME, + warningMessage + ); + if (content != null) { + try { + const state = FollowerState.fromText(content); + console.log( + `Loaded follower state: ${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s)` + ); + return state; + } catch (error) { + console.warn(`Could not parse follower state, seeding fresh: ${error}`); + } } - return { - marker: markerFile.marker, - knownVersions, - didLoadVersionsFromLegacy: false, - }; + const state = new FollowerState(); + const seed = await loadLegacyMarker(stagingBucket); + if (seed != null && seed <= head) { + console.log(`Seeding follower state from legacy marker: ${seed}`); + state.addCheckpoint(seed); + } else { + // Brand-new deployment: start at the beginning of the feed, so the + // instance backfills the entire history automatically (interim + // checkpoints make the backfill incremental across runs). + console.log( + 'Seeding follower state at the beginning of the feed (automatic backfill)' + ); + state.addCheckpoint(0); + } + return state; } /** - * Loads the last known versions from S3. - * - * @returns the value of the last known versions. + * Reads the sequence number from the legacy transaction marker file, if + * present. The previous follower implementation stored its position there. */ -async function loadLastKnownVersions( +async function loadLegacyMarker( stagingBucket: string -): Promise | undefined> { - const warningMessage = `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist, starting from scratch`; +): Promise { const content = await loadContentFromS3( stagingBucket, - KNOWN_VERSIONS_FILE_NAME, - warningMessage + MARKER_FILE_NAME, + `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` ); - - // Known Versions does not exist, starting from scratch if (content === null) { return undefined; } - - // Known Versions exists, generating map from the data - const contentsObj: KnownVersionsFileSchema = JSON.parse(content); - - console.log(`Loaded last known versions data`); - - return dateMapFromObj(contentsObj.knownVersions); + try { + const parsed: MarkerFileSchema = JSON.parse(content); + const marker = typeof parsed === 'number' ? parsed : Number(parsed.marker); + return isNaN(marker) ? undefined : marker; + } catch (error) { + console.warn(`Could not parse legacy marker: ${error}`); + return undefined; + } } /** - * Loads the last transaction marker from S3. - * - * @param registry a Nano database corresponding to the Npmjs.com CouchDB instance. - * - * @returns the value of the last transaction marker. + * Persists the follower state to S3 (gzip-compressed when large enough to + * warrant it). */ -async function loadLastTransactionMarker( +async function saveFollowerState( + context: Context, stagingBucket: string, - registry: CouchChanges -): Promise<{ marker: number; knownVersions?: Map }> { - const warningMessage = `Marker object (s3://${stagingBucket}/${MARKER_FILE_NAME}) does not exist, starting from scratch`; + state: FollowerState +) { + const { buffer, contentEncoding } = compressContent( + Buffer.from(state.toText(), 'utf-8') + ); + console.log( + `Updating follower state (${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s))` + ); + await putObject(context, stagingBucket, FOLLOWER_STATE_FILE_NAME, buffer, { + ContentType: 'text/plain', + ContentEncoding: contentEncoding, + }); + console.log('Successfully updated follower state'); +} + +/** + * Loads the known versions from S3. For legacy reasons, the map may also be + * embedded in the transaction marker file; when the dedicated file does not + * exist, that location is used and the map is migrated on the next save. + */ +async function loadKnownVersions(stagingBucket: string): Promise<{ + knownVersions: Map; + didLoadVersionsFromLegacy: boolean; +}> { + const warningMessage = `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist, starting from scratch`; const content = await loadContentFromS3( stagingBucket, - MARKER_FILE_NAME, + KNOWN_VERSIONS_FILE_NAME, warningMessage ); - - // Last transaction marker does not exist - if (content === null) { - return { marker: 0 }; - } - - const parsedFile: MarkerFileSchema = JSON.parse(content); - - let marker: number; - let knownVersions: Map | undefined; - if (typeof parsedFile === 'number') { - marker = parsedFile; - } else { - marker = Number(parsedFile.marker); - knownVersions = parsedFile.knownVersions - ? dateMapFromObj(parsedFile.knownVersions) - : undefined; + if (content != null) { + const contentsObj: KnownVersionsFileSchema = JSON.parse(content); + console.log('Loaded last known versions data'); + return { + knownVersions: dateMapFromObj(contentsObj.knownVersions), + didLoadVersionsFromLegacy: false, + }; } - // amazonq-ignore-next-line - console.log(`Read last transaction marker: ${marker}`); - - const dbUpdateSeq = Number((await registry.info()).update_seq); - if (dbUpdateSeq < marker) { - console.warn( - `Current DB update_seq (${dbUpdateSeq}) is lower than marker (CouchDB instance was likely replaced), resetting to 0!` - ); - return { marker: 0 }; + // Legacy location: embedded in the transaction marker file. + const markerContent = await loadContentFromS3( + stagingBucket, + MARKER_FILE_NAME, + `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` + ); + if (markerContent != null) { + try { + const parsed: MarkerFileSchema = JSON.parse(markerContent); + if (typeof parsed !== 'number' && parsed.knownVersions) { + console.log('Loaded known versions from legacy marker file'); + return { + knownVersions: dateMapFromObj(parsed.knownVersions), + didLoadVersionsFromLegacy: true, + }; + } + } catch (error) { + console.warn(`Could not parse legacy marker: ${error}`); + } } - - return { marker, knownVersions }; + return { knownVersions: new Map(), didLoadVersionsFromLegacy: true }; } /** @@ -405,24 +761,6 @@ async function saveLastKnownVersions( ); console.log('Successfully updated known versions'); } - -/** - * Updates the last transaction marker in S3. - * - * @param marker the last transaction marker value - */ -async function saveLastTransactionMarker( - context: Context, - stagingBucket: string, - marker: number -) { - const contents = JSON.stringify({ marker }); - console.log(`Updating last transaction marker to ${marker}`); - await putObject(context, stagingBucket, MARKER_FILE_NAME, contents, { - ContentType: 'application/json', - }); - console.log('Successfully updated marker'); -} //#endregion //#region Asynchronous Primitives @@ -698,7 +1036,8 @@ interface UpdatedVersion { } /** - * We are accounting for a good bit of legacy data modeling here + * The legacy transaction marker file format, kept around to seed the follower + * state (and known versions) of existing deployments. * * The file can be a just a number, or a combination of a sequence number (which * is potentially encoded as a string) and a set of known versions and the dates