diff --git a/API.md b/API.md index 9c61d1136..c15fb177e 100644 --- a/API.md +++ b/API.md @@ -2880,7 +2880,12 @@ new sources.NpmJs(props?: NpmJsProps) | bind | Binds the package source to a scope and target queue. | | metricBatchProcessingTime | The average time it took to process a changes batch. | | metricChangeCount | The total count of changes that were processed. | +| metricLaggyPackumentGiveUps | The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. | +| metricLaggyPackuments | The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. | +| metricLaggyPackumentsRecovered | The number of laggy packuments for which the registry caught up with the revision announced by the `_changes` feed. | | metricLastSeq | The last sequence number that was processed. | +| metricLateChangeCount | The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. | +| metricLateChangeLag | For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. | | metricNpmJsChangeAge | *No description.* | | metricPackageVersionAge | The age of the oldest package version that was processed. | | metricPackageVersionCount | The total count of package versions that were inspected. | @@ -2938,6 +2943,55 @@ The total count of changes that were processed. --- +##### `metricLaggyPackumentGiveUps` + +```typescript +public metricLaggyPackumentGiveUps(opts?: MetricOptions): Metric +``` + +The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. + +A version announced by the feed may be missing until +the affected package publishes again. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricLaggyPackuments` + +```typescript +public metricLaggyPackuments(opts?: MetricOptions): Metric +``` + +The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. + +The versions served so far +have been processed; the follower keeps re-checking for the announced +revision. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricLaggyPackumentsRecovered` + +```typescript +public metricLaggyPackumentsRecovered(opts?: MetricOptions): Metric +``` + +The number of laggy packuments for which the registry caught up with the revision announced by the `_changes` feed. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricLastSeq` ```typescript @@ -2955,6 +3009,37 @@ discover when a sequence reset has happened in the CouchDB instance. --- +##### `metricLateChangeCount` + +```typescript +public metricLateChangeCount(opts?: MetricOptions): Metric +``` + +The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricLateChangeLag` + +```typescript +public metricLateChangeLag(opts?: MetricOptions): Metric +``` + +For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. + +This is a lower bound on the feed's insertion lag, +and can be used to tune the scan window. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricNpmJsChangeAge` ```typescript diff --git a/docs/application-overview.md b/docs/application-overview.md index b01a8d366..12521402d 100644 --- a/docs/application-overview.md +++ b/docs/application-overview.md @@ -37,18 +37,30 @@ ConstructHub provides two package source implementations: `NpmJs` and `CodeArtifact`. * The `NpmJs` source interfaces with the `npmjs.com` CouchDB replica (which is - at `replicate.npmjs.com/registry`) by following it's `_changes` stream in + at `replicate.npmjs.com/registry`) by scanning it's `_changes` stream in search of relevant packages. When such a package is identified, a stager function is invoked, which stages the package tarball into an S3 bucket then notifies the ConstructHub ingestion SQS queue. The CouchDB follower is - scheduled to run every `5 minutes`, and stores the current CouchDB sequence ID - in a specific object in the S3 bucket used for staging package tarballs. + scheduled to run every `5 minutes`, and stores its state (receipts for the + change entries received, time/sequence checkpoints, and laggy packument + expectations) in a specific object in the S3 bucket used for staging package + tarballs. See [npm-sync-protocol.md](./npm-sync-protocol.md) for the + guarantees the feed does and does not give, and how the follower compensates. > Back-filling is automatic for the `NpmJs` source. Upon initial deployment, - > it will start scanning the CouchDB `_changes` stream. Should there be a need - > to re-run a backfill of this source, the transaction marker object in S3 can - > be deleted to roll back to that initial transaction. The marker object is - > linked from the backend dashboard. + > it will start scanning the CouchDB `_changes` stream from the beginning, + > progressing incrementally across runs. Should there be a need to re-process + > a range of the stream, replace the follower state object in S3 with one + > that seeds a checkpoint at the desired starting sequence number: + > + > ```text + > #chfollower/1 + > w - + > c + > ``` + > + > The next scan then covers everything from that position to the feed head. + > The state object is linked from the backend dashboard. - A **high-severity** alarm triggers if the NpmJs Follower is not running at the scheduled cadence, or if it encounters failures for more than diff --git a/docs/npm-sync-protocol.md b/docs/npm-sync-protocol.md new file mode 100644 index 000000000..36fc5cc0c --- /dev/null +++ b/docs/npm-sync-protocol.md @@ -0,0 +1,171 @@ +# The npm sync protocol: what it does and does not guarantee + +This document describes the protocol ConstructHub's `NpmJs` package source uses to discover new package versions, +the guarantees npm gives (and explicitly does not give) for that protocol, +and how ConstructHub compensates for each gap. +Claims cite public npm statements where they exist; +anything marked *(our observation)* we established ourselves through incident investigation. + +## The protocol + +Syncing from npm uses two endpoints: + +1. **The changes feed** at `https://replicate.npmjs.com/registry/_changes`. + A paginated stream of change entries, each `{ seq, id, changes: [{ rev }] }`, read with `?since=&limit=`. + A consumer pages forward by passing the previous response's `last_seq` as the next `since`. +2. **The registry** at `https://registry.npmjs.org/`. + Change entries carry no package content, + so a consumer fetches the full metadata document (the "packument") from the registry, + and derives new versions from its `time` map. + +Since May 2025 the feed is no longer a real CouchDB. +npm replaced it with a purpose-built service (the root endpoint reports `engine: "npm-replicate"`) +that keeps the CouchDB API shape but supports only a subset of it +([migration guide][152515], [deprecation notice][changelog]). +Several of the non-guarantees below date from this migration. + +## What npm guarantees + +- **Pagination by sequence number.** + `_changes?since=&limit=` returns change entries with `seq > since` in ascending order, + plus a `last_seq` to continue from. + `limit` defaults to 1,000 and caps at 10,000 ([migration guide][152515]). +- **A head-of-feed pointer.** + The root endpoint (`/`) reports the current `update_seq`; + npm recommends it as the starting point for a new consumer ([npm engineering response][152515-updateseq]). +- **One entry per package.** + A package has at most one entry in the feed: + a new change replaces the package's previous entry, at a new sequence number. + npm confirmed this is intended ([npm: "That is indeed normal feed behavior"][152515-late]). +- **Metadata lives in the registry, not the feed.** + `include_docs` is not supported; + npm's documented guidance is to fetch packuments from `registry.npmjs.org` separately ([migration guide][152515]). + +That is the entire public contract. +Notably absent: any statement about *when* a change becomes visible in the feed, +or that it becomes visible at all. + +## What npm does not guarantee + +Each heading below states a property a consumer might reasonably expect, but that npm does not guarantee. +For ease of use, each gap also gets a name; +the code and the operator runbook reference the same names. + +### Changes become visible in order + +Change entries can appear in the feed *behind* a previously returned `last_seq`; +we name this gap *late insertion*. +A consumer that reads strictly forward will never see such an entry. +Community measurement found ~4 late-inserted entries per 1,024-entry window over two hours; +npm's response was that entries appearing behind `last_seq` is expected +([report][152515-late-report], [npm confirmation][152515-late]). + +*(our observation)* The delay can be far larger than the minutes suggested in that thread. +On 2026-09-21 we read past a probe package's sequence range within ~3 minutes of its entry's sequence number +being assigned, and the entry was not served; +a manual re-read of the same range two hours later returned it. +Five consecutive versions of the package were missed this way over ~15 hours. +See the [appendix](#appendix-a-late-insertion-reconstructed) for a reconstructed timeline. + +### The registry is up-to-date with the changes feed + +The registry can serve a packument revision *older* than the revision the feed announced for the same package, +sometimes for extended periods; +we name this gap *laggy packuments*. +The feed and the registry are separate systems and are not consistent with each other +([community description][152515-stale], [metadata out of sync][128412]). + +A consumer can detect this: +the packument carries no sequence number, +but the feed entry announces the revision(s) it covers and the packument carries its current revision (`_rev`). +Comparing the numeric prefix of the two tells a consumer whether the registry has caught up +with the change it is processing. + +*(our observation)* This is not rare. +E.g. on 2026-09-21 the follower logged ~66,000 revision-mismatch retries across unrelated packages in a single day. + +### Every change becomes visible + +Occasionally a change never becomes visible in the feed at any sequence number; +we name this gap *lost events*. +Because a package has at most one entry, there is nothing to find by re-reading the feed: +the package's only entry still describes its previous publish. +The change only surfaces when the package publishes again and a fresh entry replaces the old one. +This was reported for `patch-package@7.0.0` (missing for a month, recovered only by the next publish) +and for `zod-validation-error@1.5.0`; +deletions have also been observed not to propagate, in the thousands +([lost events][128412], [version skipped a revision][128412-lost], [missing deletions][128412-deletions]). + +### Sequence numbers are dense + +Sequence numbers are opaque cursors, not counters. +The sequence space has gaps, +and values have jumped by tens of millions within weeks during the 2025 migration ([seq jumps][152515-jumps]). +A consumer must not derive meaning from the distance between two sequence numbers. + +### The feed head only moves forward + +Change entries can disappear from the end of the feed, +in effect moving the head (`update_seq`) *backwards*; +we name this gap *head regression*. +During the 2025 migration the head regressed by ~20 million, +and previously returned entries above the new head no longer existed ([seq regression][152515-regression]). +The pre-migration feed also once froze its `update_seq` entirely for hours ([stuck update_seq][128412-stuck]). + +## How ConstructHub compensates + +The *NpmJs Follower* runs on a regular schedule and fetches packuments from the registry to identify new versions. +It does not track a single high-water mark: +it keeps a receipt for every change entry it has received, +and every run re-reads a trailing window of the feed, +processing any entry it has no receipt for. +Rolling checkpoints map times to feed positions, and derive the window's starting position. +On top of that baseline, each gap has a dedicated mitigation: + +| Gap | Mitigation | Detection | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | +| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm | +| Laggy packuments | Process every packument the registry serves immediately; record the announced revision and re-check it every run until it appears or ages out | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` metrics; `LaggyPackumentGiveUps` alarm | +| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package | +| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric | + +**The design idea behind all of this is that the feed data are just hints without real guarantees.** +The follower treats every change entry as "this package probably changed around here", +and keeps enough state (receipts, checkpoints, known versions) to re-derive correctness from re-reads. + +## Appendix: a late insertion, reconstructed + +The timeline below reconstructs the 2026-09-21 incident from log evidence and manual feed reads +*(our observation)*. +Times are UTC; "seq X" is the sequence number assigned to the probe package's change entry. + +```text +time consumer feed +───── ─────────────────────────────────── ────────────────────────────────────── +15:09 package version is published entry assigned seq X, NOT yet visible +15:12 reads past seq X, entry not served entry still not visible +15:?? entry becomes visible at seq X, which + is behind the consumer's position +17:10 manually re-reads the range at seq X entry is served ("late insertion") +``` + +A consumer that never re-reads previously covered ranges has no way to notice that anything happened. + +## References + +- Operator runbook sections for the alarms named above: [operator-runbook.md](./operator-runbook.md) +- Follower implementation: `src/package-sources/npmjs/npm-js-follower.lambda.ts` +- Incident that motivated the windowed scan: internal reference V2376798198 *(our observation)* + +[152515]: https://github.com/orgs/community/discussions/152515 +[changelog]: https://github.blog/changelog/2025-02-27-changes-and-deprecation-notice-for-npm-replication-apis +[152515-updateseq]: https://github.com/orgs/community/discussions/152515#discussioncomment-12713712 +[152515-late-report]: https://github.com/orgs/community/discussions/152515#discussioncomment-13561875 +[152515-late]: https://github.com/orgs/community/discussions/152515#discussioncomment-13563633 +[152515-stale]: https://github.com/orgs/community/discussions/152515#discussioncomment-15298851 +[128412-lost]: https://github.com/orgs/community/discussions/128412#discussioncomment-7445623 +[152515-jumps]: https://github.com/orgs/community/discussions/152515#discussioncomment-12731335 +[152515-regression]: https://github.com/orgs/community/discussions/152515#discussioncomment-12754432 +[128412]: https://github.com/orgs/community/discussions/128412 +[128412-deletions]: https://github.com/orgs/community/discussions/128412#discussioncomment-6755605 +[128412-stuck]: https://github.com/orgs/community/discussions/128412#discussioncomment-7653871 diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 794fb44f1..9773cc1d0 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -582,6 +582,82 @@ but is disabled by default. Once all messages have cleared from the dead-letter queue, do not forget to disable the SQS Trigger again. +### `ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh` + +#### Description + +This alarm is only provisioned when the `NpmJs` package source is configured. It +triggers when the *NpmJs Follower* discovers changes that were inserted into the +CouchDB `_changes` feed with a delay approaching the scan window. Should the +delay exceed the window, the regular scan will miss those changes, and only the +daily deep scan would still discover them. + +#### Investigation + +The `_changes` feed occasionally makes change entries visible behind positions +the follower has already read past (npm considers this normal feed behavior). +The follower compensates by re-reading a trailing window of the feed on every +run, and the `LateChangeLag` metric records how far behind each late entry was +discovered. See [npm-sync-protocol.md](./npm-sync-protocol.md) for the details +of the feed's behavior. + +Review the *Feed Reliability* widget on the backend dashboard. A short burst of +high-delay insertions usually indicates turbulence in the npmjs.com replication +pipelines, and resolves on its own. Look at `npmjs.com` status updates and +announcements. + +#### Resolution + +The alarm will automatically go back to green once the insertion delays +subside. If elevated delays persist over multiple days, the scan window should +be increased: adjust `SCAN_WINDOW_MS` in +`src/package-sources/npmjs/constants.lambda-shared.ts` (keeping it below +`STATE_RETENTION_MS`) and deploy the updated construct. + +### `ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps` + +#### Description + +This alarm is only provisioned when the `NpmJs` package source is configured. It +triggers when the npm registry never served the packument revision announced by +the CouchDB `_changes` feed for one or more packages, even after the follower +re-checked for a prolonged period. + +All package versions the registry did serve have been processed. However, a +version announced by the changes feed may be missing from ConstructHub until +the affected package publishes its next version. + +#### Investigation + +The *NpmJs Follower* fetches package metadata from `registry.npmjs.org` for +each change announced by the CouchDB `_changes` feed. The registry occasionally +lags behind the feed. When it does, the follower processes the versions that +were served, records the announced revision as a "laggy packument", and +re-checks it on every run until the revision appears or the expectation ages +out. This alarm fires for the latter. + +Review the logs of the *NpmJs Follower* for `Giving up on laggy packument` +entries; they identify the affected package and the revision that never +appeared. Verify what the registry currently serves for the package: + +```console +$ curl https://registry.npmjs.org/ | jq ._rev +``` + +If the served revision is still older than the expected one, the problem is +with upstream npmjs.com. Look at `npmjs.com` status updates and announcements. + +If you suspect the problem is with upstream npmjs.com, you can cut them a ticket. Log +in to , visit the [Support Page](https://www.npmjs.com/support) and +select *There is a problem with the npm registry*. + +#### Resolution + +The missed package versions will automatically be discovered once the affected +packages publish their next version. To ingest a missed version immediately, +invoke the *ReStagePackageVersion* Lambda function with the package name and +version from the follower logs. + -------------------------------------------------------------------------------- ## :repeat: Bulk Re-processing diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index e27f18236..3d8828f39 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -632,7 +632,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -703,7 +707,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -728,11 +736,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -740,7 +748,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -763,7 +771,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -775,7 +783,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -800,11 +808,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -829,7 +837,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -860,7 +868,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -876,7 +884,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -896,7 +904,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -921,7 +929,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -952,7 +960,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -963,7 +971,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -975,7 +983,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -995,11 +1003,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -1007,7 +1015,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -1019,11 +1027,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -9875,6 +9883,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -9958,7 +9974,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -10412,6 +10428,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -10654,6 +10711,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -10765,7 +10859,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -11021,6 +11115,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -14609,7 +14711,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -14680,7 +14786,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -14705,11 +14815,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -14717,7 +14827,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -14740,7 +14850,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -14752,7 +14862,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -14777,11 +14887,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -14806,7 +14916,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -14837,7 +14947,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -14853,7 +14963,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -14873,7 +14983,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -14898,7 +15008,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -14929,7 +15039,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -14940,7 +15050,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -14952,7 +15062,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -14972,11 +15082,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -14984,7 +15094,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -14996,11 +15106,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -24226,6 +24336,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -24309,7 +24427,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -24763,6 +24881,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -25005,6 +25164,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -25116,7 +25312,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -25372,6 +25568,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -28898,7 +29102,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -28969,7 +29177,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -28994,11 +29206,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -29006,7 +29218,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -29029,7 +29241,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -29041,7 +29253,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -29066,11 +29278,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -29095,7 +29307,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -29126,7 +29338,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -29142,7 +29354,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -29162,7 +29374,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -29187,7 +29399,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -29218,7 +29430,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -29229,7 +29441,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -29241,7 +29453,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -29261,11 +29473,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -29273,7 +29485,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -29285,11 +29497,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -38141,6 +38353,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -38224,7 +38444,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -38678,6 +38898,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -38920,6 +39181,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -39031,7 +39329,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -39287,6 +39585,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -42961,7 +43267,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -43032,7 +43342,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -43057,11 +43371,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -43069,7 +43383,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -43092,7 +43406,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -43104,7 +43418,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -43129,11 +43443,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -43158,7 +43472,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -43189,7 +43503,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -43205,7 +43519,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -43225,7 +43539,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -43250,7 +43564,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -43281,7 +43595,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -43292,7 +43606,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -43304,7 +43618,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -43324,11 +43638,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -43336,7 +43650,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -43348,11 +43662,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -52213,6 +52527,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -52296,7 +52618,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -52737,6 +53059,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -52979,6 +53342,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -53090,7 +53490,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -53346,6 +53746,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -57209,7 +57617,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -57280,7 +57692,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -57305,11 +57721,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -57317,7 +57733,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -57340,7 +57756,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -57352,7 +57768,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -57377,11 +57793,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -57406,7 +57822,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -57437,7 +57853,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -57453,7 +57869,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -57473,7 +57889,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -57498,7 +57914,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -57529,7 +57945,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -57540,7 +57956,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -57552,7 +57968,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -57572,11 +57988,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -57584,7 +58000,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -57596,11 +58012,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -66268,6 +66684,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -66351,7 +66775,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -66805,6 +67229,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -67047,6 +67512,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -67158,7 +67660,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -67414,6 +67916,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 6fd98f419..9e2159b43 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -694,7 +694,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJs15A77D2D", }, - ")\\n[button:Marker Object](/s3/object/", + ")\\n[button:Follower State](/s3/object/", + { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "?prefix=couchdb-follower-state)\\n[button:Marker Object (legacy)](/s3/object/", { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, @@ -765,7 +769,11 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -790,11 +798,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"", + "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", + "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"", { "Ref": "AWS::Region", }, @@ -802,7 +810,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsCanary5558FC45", }, - "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", + "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/", { "Ref": "ConstructHubIngestion407909CE", }, @@ -825,7 +833,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"", { "Ref": "AWS::Region", }, @@ -837,7 +845,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubIngestion407909CE", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"", { "Ref": "AWS::Region", }, @@ -862,11 +870,11 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"", + "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"", { "Ref": "AWS::Region", }, - "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", + "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"", { "Ref": "AWS::Region", }, @@ -891,7 +899,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/", { "Ref": "ConstructHubOrchestration39161A46", }, @@ -922,7 +930,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"", { "Ref": "AWS::Region", }, @@ -938,7 +946,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", + "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"", { "Ref": "AWS::Region", }, @@ -958,7 +966,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubOrchestration39161A46", }, - "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", + "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -983,7 +991,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", + "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/", { "Ref": "ConstructHubDenyListBucket1B3C2C2E", }, @@ -1014,7 +1022,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"", { "Ref": "AWS::Region", }, @@ -1025,7 +1033,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", + "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"", { "Ref": "AWS::Region", }, @@ -1037,7 +1045,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubDenyListPrunePruneHandler30B33551", }, - "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", + "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -1057,11 +1065,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsAggregator3345DB4E", }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -1069,7 +1077,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubStatsStateMachine1A1DEC97", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/", { "Ref": "ConstructHubPackageDataDC5EF35E", }, @@ -1081,11 +1089,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", + "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", + "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"", { "Ref": "AWS::Region", }, @@ -1093,7 +1101,7 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubVersionTrackerD5E8AEAE", }, - "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"1 minutes (Lambda timeout)","value":60,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":135,"properties":{"markdown":"# Release Notes\\n\\n[button:primary:StateMachine](/states/home#/statemachines/view/", + "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"1 minutes (Lambda timeout)","value":60,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":141,"properties":{"markdown":"# Release Notes\\n\\n[button:primary:StateMachine](/states/home#/statemachines/view/", { "Ref": "ConstructHubReleaseNotesStateMachine8C711CC7", }, @@ -1154,15 +1162,15 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":137,"properties":{"view":"timeSeries","title":"Number of release Notes","region":"", + ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":143,"properties":{"view":"timeSeries","title":"Number of release Notes","region":"", { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Packages with release notes","expression":"FILL(m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d, REPEAT)"}],["ConstructHub/ReleaseNotes","PackageWithChangeLog",{"label":"Packages with release notes","stat":"Sum","visible":false,"id":"m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":137,"properties":{"view":"timeSeries","title":"Release notes generation Errors","region":"", + "","metrics":[[{"label":"Packages with release notes","expression":"FILL(m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d, REPEAT)"}],["ConstructHub/ReleaseNotes","PackageWithChangeLog",{"label":"Packages with release notes","stat":"Sum","visible":false,"id":"m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":143,"properties":{"view":"timeSeries","title":"Release notes generation Errors","region":"", { "Ref": "AWS::Region", }, - "","metrics":[["ConstructHub/ReleaseNotes","AllErrors",{"label":"All Errors","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnknownError",{"label":"UnknownError","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidCredentials",{"label":"InvalidCredentials","stat":"Maximum"}],["ConstructHub/ReleaseNotes","RequestQuotaExhausted",{"label":"RequestQuotaExhausted","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnSupportedRepo",{"label":"UnSupportedRepo","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidPackageJson",{"label":"InvalidPackageJson","stat":"Maximum"}],["ConstructHub/ReleaseNotes","ChangelogFetchError",{"label":"ChangeLogFetchError","stat":"Maximum"}]],"yAxis":{}}},{"type":"metric","width":12,"height":6,"x":0,"y":143,"properties":{"view":"timeSeries","title":"GitHub API Rate Limits","region":"", + "","metrics":[["ConstructHub/ReleaseNotes","AllErrors",{"label":"All Errors","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnknownError",{"label":"UnknownError","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidCredentials",{"label":"InvalidCredentials","stat":"Maximum"}],["ConstructHub/ReleaseNotes","RequestQuotaExhausted",{"label":"RequestQuotaExhausted","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnSupportedRepo",{"label":"UnSupportedRepo","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidPackageJson",{"label":"InvalidPackageJson","stat":"Maximum"}],["ConstructHub/ReleaseNotes","ChangelogFetchError",{"label":"ChangeLogFetchError","stat":"Maximum"}]],"yAxis":{}}},{"type":"metric","width":12,"height":6,"x":0,"y":149,"properties":{"view":"timeSeries","title":"GitHub API Rate Limits","region":"", { "Ref": "AWS::Region", }, @@ -11150,6 +11158,14 @@ Direct link to the function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { @@ -11233,7 +11249,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip", + "S3Key": "e147a0a80ebd61b2ff2e67e9ec2c9f6bb1b05f71c8f3c7c4083c2d17816c33e2.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -11684,6 +11700,47 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { + "Properties": { + "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed +for one or more packages, even after repeated re-checks. All versions the registry did serve +have been processed, but a version announced by the feed may be missing until the affected +package publishes again. The follower logs identify the affected packages, which can be +manually ingested using the ReStagePackageVersion function. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "EvaluationPeriods": 1, + "MetricName": "LaggyPackumentGiveUps", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { + "Properties": { + "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed +with a delay approaching the scan window. If the delay exceeds the window, the regular scan +will miss those changes (only the daily deep scan would catch them). Consider increasing +the scan window (SCAN_WINDOW_MS). + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "LateChangeLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 3600, + "Statistic": "Maximum", + "Threshold": 17280000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -11923,6 +11980,43 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-follower-state", + ], + ], + }, + ], + }, { "Action": [ "s3:GetObject*", @@ -12034,7 +12128,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip", + "S3Key": "9949a6fc6d8cf1e0891cc64e355de47a97027d870030596dc77d3f6915e603f5.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -12290,6 +12384,14 @@ Direct link to Lambda function: /lambda/home#/functions/", "Prefix": "couchdb-known-versions.2", "Status": "Enabled", }, + { + "ExpiredObjectDeleteMarker": true, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + }, + "Prefix": "couchdb-follower-state", + "Status": "Enabled", + }, ], }, "PublicAccessBlockConfiguration": { diff --git a/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts new file mode 100644 index 000000000..cd4698f2a --- /dev/null +++ b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts @@ -0,0 +1,227 @@ +import { + DEEP_SCAN_INTERVAL_MS, + SCAN_WINDOW_MS, + STATE_RETENTION_MS, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import { FollowerState } from '../../../package-sources/npmjs/follower-state.lambda-shared'; + +const T0 = 1_700_000_000_000; +const HOUR = 60 * 60 * 1_000; + +describe('text format', () => { + test('produces the exact documented golden output', () => { + const state = new FollowerState(); + state.addCheckpoint(90, T0); + state.addCheckpoint(300, T0 + HOUR); + state.addSeqs([100, 250]); + state.recordLaggyPackument('@scope/laggy-package', 5, 123, T0); + state.completeDeepScan(T0 + HOUR); + + expect(state.toText()).toBe( + [ + '#chfollower/1', + `w ${T0 + HOUR}`, + `c ${T0} 90`, + `c ${T0 + HOUR} 300`, + `l ${T0} 5 123 @scope/laggy-package`, + 's 100', + 's 250', + '', // trailing newline + ].join('\n') + ); + }); + + test('round-trips through the text format', () => { + const state = new FollowerState(); + state.addCheckpoint(90, T0); + state.addSeqs([100, 105, 250]); + state.recordLaggyPackument('laggy-package', 7, undefined, T0); + state.completeDeepScan(T0); + + const restored = FollowerState.fromText(state.toText()); + + for (const seq of [100, 105, 250]) { + expect(restored.has(seq)).toBe(true); + } + expect(restored.has(101)).toBe(false); + expect(restored.receiptCount).toBe(3); + expect(restored.checkpointCount).toBe(1); + expect(restored.lastDeepScanAt).toBe(T0); + expect(restored.laggyPackuments()).toEqual([ + { name: 'laggy-package', expectedRev: 7, seq: undefined, firstSeen: T0 }, + ]); + }); + + test('a fresh state round-trips', () => { + const restored = FollowerState.fromText(new FollowerState().toText()); + expect(restored.receiptCount).toBe(0); + expect(restored.checkpointCount).toBe(0); + expect(restored.laggyPackumentCount).toBe(0); + expect(restored.lastDeepScanAt).toBeUndefined(); + }); + + test('serialization is stable (parse . print is identity)', () => { + const state = new FollowerState(); + state.addCheckpoint(90, T0); + state.addSeqs([100, 250, 251]); + state.recordLaggyPackument('laggy-package', 7, 200, T0); + + const once = state.toText(); + const twice = FollowerState.fromText(once).toText(); + expect(twice).toBe(once); + }); + + test('handles large sequence numbers (beyond 2^31)', () => { + const bigSeq = 131_711_133_000; + const state = new FollowerState(); + state.addCheckpoint(bigSeq - 10, T0); + state.addSeqs([bigSeq]); + const restored = FollowerState.fromText(state.toText()); + expect(restored.has(bigSeq)).toBe(true); + }); + + test('round-trips a large receipt set intact', () => { + const state = new FollowerState(); + state.addCheckpoint(0, T0); + state.addSeqs( + Array.from({ length: 10_000 }, (_, i) => 131_000_000 + i * 3) + ); + const restored = FollowerState.fromText(state.toText()); + expect(restored.receiptCount).toBe(10_000); + expect(restored.has(131_000_000)).toBe(true); + expect(restored.has(131_000_001)).toBe(false); + }); + + test('tolerates blank lines, surrounding whitespace, and CRLF line endings', () => { + const text = [ + '#chfollower/1', + 'w -', + '', + `c ${T0} 90 `, + ' s 100', + 's 250\r', + '', + ].join('\n'); + + const restored = FollowerState.fromText(text.replace(/\n/g, '\r\n')); + expect(restored.checkpointCount).toBe(1); + expect(restored.has(100)).toBe(true); + expect(restored.has(250)).toBe(true); + }); + + test.each([ + ['empty input', ''], + ['missing header', 'w -\ns 100\n'], + ['JSON input', '{"seqs":[100]}'], + ['wrong version', '#chfollower/2\ns 100\n'], + ])('rejects %s', (_name, text) => { + expect(() => FollowerState.fromText(text)).toThrow( + /Unexpected follower state header/ + ); + }); + + test('rejects unknown record types', () => { + expect(() => + FollowerState.fromText('#chfollower/1\nx something\n') + ).toThrow(/Unexpected follower state line: x something/); + }); +}); + +describe('checkpoints and receipts', () => { + test('floorSeq maps the window through checkpoints', () => { + const state = new FollowerState(); + state.addCheckpoint(100, T0 - SCAN_WINDOW_MS - HOUR); + state.addCheckpoint(200, T0 - SCAN_WINDOW_MS); + state.addCheckpoint(900, T0); + expect(state.floorSeq(SCAN_WINDOW_MS, T0)).toBe(200); + }); + + test('floorSeq falls back to the newest checkpoint when history is younger than the window', () => { + const state = new FollowerState(); + state.addCheckpoint(150, T0 - HOUR); + state.addCheckpoint(900, T0 - 30 * 60_000); + // No checkpoint is older than the window: coverage below the newest one + // is guaranteed, so the scan resumes from there. + expect(state.floorSeq(SCAN_WINDOW_MS, T0)).toBe(900); + expect(new FollowerState().floorSeq(SCAN_WINDOW_MS, T0)).toBeUndefined(); + }); + + test('timeCrossed reports when a completed scan first passed a seq', () => { + const state = new FollowerState(); + state.addCheckpoint(100, T0); + state.addCheckpoint(200, T0 + HOUR); + state.addCheckpoint(300, T0 + 2 * HOUR); + + expect(state.timeCrossed(150)).toBe(T0 + HOUR); + expect(state.timeCrossed(100)).toBe(T0); + expect(state.timeCrossed(999)).toBeUndefined(); + }); + + test('prunes checkpoints and receipts outside the retention window', () => { + const state = new FollowerState(); + state.addCheckpoint(50, T0 - STATE_RETENTION_MS - 2 * HOUR); + state.addCheckpoint(450, T0 - STATE_RETENTION_MS - HOUR); + state.addSeqs([460, 500, 600]); + state.addCheckpoint(700, T0); + + // The checkpoint at seq=450 is the last one at/before the cut-off and is + // retained; the older one is dropped, along with receipts below seq=450. + expect(state.checkpointCount).toBe(2); + expect(state.has(460)).toBe(true); + expect(state.has(500)).toBe(true); + }); + + test('resetPosition clears receipts and checkpoints but keeps laggy packuments', () => { + const state = new FollowerState(); + state.addCheckpoint(100, T0); + state.addSeqs([150]); + state.recordLaggyPackument('laggy-package', 5, 120, T0); + + state.resetPosition(); + + expect(state.checkpointCount).toBe(0); + expect(state.receiptCount).toBe(0); + expect(state.newestCheckpointSeq()).toBeUndefined(); + expect(state.laggyPackumentCount).toBe(1); + }); +}); + +describe('deep scans', () => { + test('the first ever scan is deep', () => { + expect(new FollowerState().deepScanDue(T0)).toBe(true); + }); + + test('a deep scan is due again after the interval', () => { + const state = new FollowerState(); + state.completeDeepScan(T0); + expect(state.deepScanDue(T0 + DEEP_SCAN_INTERVAL_MS - 1)).toBe(false); + expect(state.deepScanDue(T0 + DEEP_SCAN_INTERVAL_MS)).toBe(true); + }); +}); + +describe('laggy packuments', () => { + test('keeps the earliest firstSeen and the highest expected revision', () => { + const state = new FollowerState(); + state.recordLaggyPackument('laggy-package', 5, 100, T0); + state.recordLaggyPackument('laggy-package', 7, 200, T0 + HOUR); + state.recordLaggyPackument('laggy-package', 6, 300, T0 + 2 * HOUR); + + expect(state.laggyPackuments()).toEqual([ + { name: 'laggy-package', expectedRev: 7, seq: 100, firstSeen: T0 }, + ]); + }); + + test('lists expectations oldest first and removes them', () => { + const state = new FollowerState(); + state.recordLaggyPackument('second', 2, undefined, T0 + HOUR); + state.recordLaggyPackument('first', 1, undefined, T0); + + expect(state.laggyPackuments().map((l) => l.name)).toEqual([ + 'first', + 'second', + ]); + + state.removeLaggyPackument('first'); + expect(state.laggyPackumentCount).toBe(1); + }); +}); diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts new file mode 100644 index 000000000..7390f042f --- /dev/null +++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts @@ -0,0 +1,537 @@ +import { gunzipSync } from 'zlib'; +import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; +import { + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import type { metricScope, MetricsLogger } from 'aws-embedded-metrics'; +import { Unit } from 'aws-embedded-metrics'; +import type { Context, ScheduledEvent } from 'aws-lambda'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import nock from 'nock'; +import { + ENV_DENY_LIST_BUCKET_NAME, + ENV_DENY_LIST_OBJECT_KEY, +} from '../../../backend/deny-list/constants'; +import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; +import { + FOLLOWER_STATE_FILE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, + MetricName, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import { FollowerState } from '../../../package-sources/npmjs/follower-state.lambda-shared'; +import { handler } from '../../../package-sources/npmjs/npm-js-follower.lambda'; +import { stringToStream } from '../../streams'; + +jest.setTimeout(30_000); + +jest.mock('aws-embedded-metrics'); + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const mockMetricScope = require('aws-embedded-metrics') + .metricScope as jest.MockedFunction; +const mockPutMetric = jest + .fn() + .mockName('MetricsLogger.putMetric') as jest.MockedFunction< + MetricsLogger['putMetric'] +>; +const mockMetrics: MetricsLogger = { + putMetric: mockPutMetric, + setDimensions: (...args: any[]) => expect(args).toEqual([{}]), + setProperty: jest.fn(), +} as any; +mockMetricScope.mockImplementation((cb) => { + const impl = cb(mockMetrics); + return async (...args) => impl(...args); +}); + +const REPLICA = 'https://replicate.npmjs.com'; +const REGISTRY = 'https://registry.npmjs.org'; + +const MOCK_BUCKET = 'mock-staging-bucket'; +const MOCK_STAGING_FUNCTION = 'mock-staging-function'; +const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; +const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; +const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name'; +const MOCK_LICENSE_LIST_OBJECT = 'license-list.json'; + +const HOUR = 60 * 60 * 1_000; + +const mockS3 = mockClient(S3Client); +const mockLambda = mockClient(LambdaClient); + +const context: Context = { + awsRequestId: 'mock-request-id', + logGroupName: 'mock-log-group', + logStreamName: 'mock-log-stream', + getRemainingTimeInMillis: () => 300_000, +} as any; + +const event: ScheduledEvent = {} as any; + +beforeEach(() => { + mockS3.reset(); + mockLambda.reset(); + mockPutMetric.mockClear(); + + process.env.BUCKET_NAME = MOCK_BUCKET; + process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION; + process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; + process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; + process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET; + process.env[LicenseListEnv.OBJECT_KEY] = MOCK_LICENSE_LIST_OBJECT; + + // Empty deny list + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_DENY_LIST_BUCKET, + Key: MOCK_DENY_LIST_OBJECT, + }) + .resolves({ Body: stringToStream(JSON.stringify({})) }); + // License list allowing Apache-2.0 + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_LICENSE_LIST_BUCKET, + Key: MOCK_LICENSE_LIST_OBJECT, + }) + .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) }); + // Empty known versions + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET, + Key: KNOWN_VERSIONS_FILE_NAME, + }) + .resolves({ Body: stringToStream(JSON.stringify({ knownVersions: {} })) }); + // No legacy marker by default + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME }) + .rejects(noSuchKey()); + + mockS3.on(PutObjectCommand).resolves({}); + mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 }); +}); + +afterEach(() => { + delete process.env.BUCKET_NAME; + delete process.env.FUNCTION_NAME; + delete process.env[ENV_DENY_LIST_BUCKET_NAME]; + delete process.env[ENV_DENY_LIST_OBJECT_KEY]; + delete process.env[LicenseListEnv.BUCKET_NAME]; + delete process.env[LicenseListEnv.OBJECT_KEY]; + nock.cleanAll(); +}); + +function noSuchKey() { + return Object.assign(new Error('no such key'), { name: 'NoSuchKey' }); +} + +function givenHead(updateSeq: number) { + nock(REPLICA).get('/').reply(200, { + db_name: 'registry', + engine: 'npm-replicate', + doc_count: 1, + update_seq: updateSeq, + }); +} + +function givenNoStateFile() { + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET, + Key: FOLLOWER_STATE_FILE_NAME, + }) + .rejects(noSuchKey()); +} + +function givenStateFile(state: FollowerState) { + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_BUCKET, + Key: FOLLOWER_STATE_FILE_NAME, + }) + .resolves({ Body: stringToStream(state.toText()) }); +} + +function givenLegacyMarker(marker: number) { + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME }) + .resolves({ Body: stringToStream(JSON.stringify({ marker })) }); +} + +function savedState(): FollowerState { + const calls = mockS3 + .commandCalls(PutObjectCommand) + .filter((call) => call.args[0].input.Key === FOLLOWER_STATE_FILE_NAME); + expect(calls.length).toBeGreaterThan(0); + const input = calls[calls.length - 1].args[0].input; + let body = input.Body as Buffer; + if (input.ContentEncoding === 'gzip') { + body = gunzipSync(body); + } + return FollowerState.fromText(body.toString('utf-8')); +} + +function packageDoc(name: string, rev: string, ...versions: string[]) { + const time: Record = { + created: '2023-01-01T00:00:00.000Z', + modified: '2023-06-01T00:00:00.000Z', + }; + const versionInfos: Record = {}; + for (const [i, version] of versions.entries()) { + time[version] = `2023-06-01T00:0${i}:00.000Z`; + versionInfos[version] = { + dependencies: { constructs: '^10.0.0' }, + dist: { + shasum: 'mock-shasum', + tarball: `${REGISTRY}/${name}/-/${name}-${version}.tgz`, + }, + jsii: {}, + keywords: ['cdk'], + license: 'Apache-2.0', + name, + version, + }; + } + return { + _id: name, + _rev: rev, + 'dist-tags': { latest: versions[versions.length - 1] }, + name, + time, + versions: versionInfos, + }; +} + +function stagedPackages(): Array<{ name: string; version: string }> { + return mockLambda.commandCalls(InvokeCommand).map((call) => { + const payload = JSON.parse( + Buffer.from(call.args[0].input.Payload! as Uint8Array).toString('utf-8') + ); + return { name: payload.name, version: payload.version }; + }); +} + +test('seeds from the legacy marker and processes fresh changes', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + + // The scan starts at the seeded marker position and finds one new entry. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(200, { + results: [ + { + seq: 1_005, + id: 'fresh-package', + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + }, + ], + last_seq: 1_005, + }); + nock(REGISTRY) + .get('/fresh-package') + .reply(200, packageDoc('fresh-package', '1-aaaaaa', '1.0.0')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'fresh-package', version: '1.0.0' }, + ]); + // Entries above the previously covered position are fresh, not late. + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LATE_CHANGE_COUNT, + 0, + Unit.Count + ); + + const state = savedState(); + expect(state.has(1_005)).toBe(true); + // A completed scan records a checkpoint at the highest position covered. + expect(state.newestCheckpointSeq()).toBe(1_010); + // The first ever scan doubles as a deep scan. + expect(state.lastDeepScanAt).toBeDefined(); +}); + +test('discovers and stages late-inserted changes', async () => { + const now = Date.now(); + givenHead(1_000); + + // A completed scan passed seq 500 seven hours ago and seq 1000 five minutes + // ago. Seq 550 was received; seq 600 was NOT (it appeared in the feed + // late). Only a regular scan is due. + const state = new FollowerState(); + state.addCheckpoint(500, now - 7 * HOUR); + state.addCheckpoint(1_000, now - 5 * 60_000); + state.addSeqs([550]); + state.completeDeepScan(now - 2 * HOUR); + givenStateFile(state); + + // Regular scan: floor is the checkpoint at/bordering (now - window), i.e. + // seq 500. Only the late entry's packument may be fetched. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '500' }) + .reply(200, { + results: [ + { + seq: 550, + id: 'already-seen-package', + changes: [{ rev: '1-cccccc' }], + deleted: false, + }, + { + seq: 600, + id: 'late-package', + changes: [{ rev: '2-dddddd' }], + deleted: false, + }, + ], + last_seq: 600, + }); + nock(REGISTRY) + .get('/late-package') + .reply(200, packageDoc('late-package', '2-dddddd', '1.2.3')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'late-package', version: '1.2.3' }, + ]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LATE_CHANGE_COUNT, + 1, + Unit.Count + ); + // The lag is measured against the checkpoint that first crossed seq 600. + const lagCalls = mockPutMetric.mock.calls.filter( + ([name]) => name === MetricName.LATE_CHANGE_LAG + ); + expect(lagCalls).toHaveLength(1); + expect(lagCalls[0][1]).toBeGreaterThanOrEqual(5 * 60_000); + expect(lagCalls[0][1]).toBeLessThan(HOUR); + + const saved = savedState(); + expect(saved.has(600)).toBe(true); + expect(saved.newestCheckpointSeq()).toBe(1_000); +}); + +test('laggy packument: processes served versions and records the expectation', async () => { + givenHead(2_010); + givenNoStateFile(); + givenLegacyMarker(2_000); + + // The feed announces rev 5, but the registry only serves rev 3. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '2000' }) + .reply(200, { + results: [ + { + seq: 2_005, + id: 'laggy-package', + changes: [{ rev: '5-eeeeee' }], + deleted: false, + }, + ], + last_seq: 2_005, + }); + nock(REGISTRY) + .get('/laggy-package') + .reply(200, packageDoc('laggy-package', '3-ffffff', '1.0.0')); + + await handler(event, context); + + // The versions the registry did serve are processed anyway. + expect(stagedPackages()).toEqual([ + { name: 'laggy-package', version: '1.0.0' }, + ]); + // ... and the expectation of rev 5 is recorded for later re-checks. + const state = savedState(); + expect(state.laggyPackuments()).toEqual([ + expect.objectContaining({ + name: 'laggy-package', + expectedRev: 5, + seq: 2_005, + }), + ]); +}); + +test('laggy packument: recovered once the registry catches up', async () => { + const now = Date.now(); + givenHead(3_000); + + const state = new FollowerState(); + state.addCheckpoint(3_000, now - 5 * 60_000); + state.completeDeepScan(now - HOUR); + state.recordLaggyPackument('recovered-package', 4, 1_500, now - HOUR); + givenStateFile(state); + + // The registry has now caught up (rev 4 >= expected 4), serving a version + // we don't know yet. + nock(REGISTRY) + .get('/recovered-package') + .reply(200, packageDoc('recovered-package', '4-gggggg', '2.0.0')); + // The scan itself is empty. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '3000' }) + .reply(200, { results: [], last_seq: 3_000 }); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'recovered-package', version: '2.0.0' }, + ]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + 1, + Unit.Count + ); + expect(savedState().laggyPackumentCount).toBe(0); +}); + +test('laggy packument: gives up after the maximum age, still processing what is served', async () => { + const now = Date.now(); + givenHead(3_000); + + const state = new FollowerState(); + state.addCheckpoint(3_000, now - 5 * 60_000); + state.completeDeepScan(now - HOUR); + // First seen 25 hours ago: beyond the give-up age. + state.recordLaggyPackument('stuck-package', 9, 1_600, now - 25 * HOUR); + givenStateFile(state); + + // The registry still serves rev 7 < expected 9 - but with a version we + // don't know, which is processed regardless. + nock(REGISTRY) + .get('/stuck-package') + .reply(200, packageDoc('stuck-package', '7-hhhhhh', '3.0.0')); + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '3000' }) + .reply(200, { results: [], last_seq: 3_000 }); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'stuck-package', version: '3.0.0' }, + ]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); + expect(savedState().laggyPackumentCount).toBe(0); +}); + +test('seeds at the beginning of the feed on first deployment (automatic backfill)', async () => { + givenHead(1_010); + givenNoStateFile(); + // No legacy marker either: this is a brand-new deployment. + + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '0' }) + .reply(200, { + results: [ + { + seq: 1_005, + id: 'historic-package', + changes: [{ rev: '1-bbbbbb' }], + deleted: false, + }, + ], + last_seq: 1_005, + }); + nock(REGISTRY) + .get('/historic-package') + .reply(200, packageDoc('historic-package', '1-bbbbbb', '1.0.0')); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'historic-package', version: '1.0.0' }, + ]); + const state = savedState(); + expect(state.has(1_005)).toBe(true); + expect(state.newestCheckpointSeq()).toBe(1_010); +}); + +test('records interim checkpoints so an interrupted scan resumes mid-window', async () => { + const now = Date.now(); + givenHead(50_000); + + // A backfill in progress: the last interim checkpoint is at seq 0, and the + // first full page of the window has already been received (receipts exist). + const seqs = Array.from({ length: 10_000 }, (_, i) => (i + 1) * 2); + const state = new FollowerState(); + state.addCheckpoint(0, now - 5 * 60_000); + state.addSeqs(seqs); + givenStateFile(state); + + // The first page is full (10,000 entries, all receipted), so the scan wants + // to continue - but the time budget runs out right after this page. + let remaining = 300_000; + (context as any).getRemainingTimeInMillis = () => remaining; + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '0' }) + .reply(200, () => { + remaining = 30_000; // exhaust the budget after this page + return { + results: seqs.map((seq) => ({ + seq, + id: `package-${seq}`, + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + })), + last_seq: 20_000, + }; + }); + + try { + await handler(event, context); + } finally { + (context as any).getRemainingTimeInMillis = () => 300_000; + } + + // Nothing needed staging (everything was receipted), and no packument was + // fetched (no registry nock exists). The interim checkpoint was recorded, + // so the next run resumes from seq 20000 instead of the floor. + expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand); + const saved = savedState(); + expect(saved.newestCheckpointSeq()).toBe(20_000); + // The interrupted (deep) scan did not complete. + expect(saved.lastDeepScanAt).toBeUndefined(); +}); + +test('resets its position when the feed head regresses below the newest checkpoint', async () => { + const now = Date.now(); + // The feed head (3000) is below our newest checkpoint (5000). + givenHead(3_000); + + const state = new FollowerState(); + state.addCheckpoint(5_000, now - 5 * 60_000); + state.addSeqs([4_500]); + state.completeDeepScan(now - HOUR); + givenStateFile(state); + + // After the reset, the scan starts from the new head. + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '3000' }) + .reply(200, { results: [], last_seq: 3_000 }); + + await handler(event, context); + + expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand); + const saved = savedState(); + expect(saved.newestCheckpointSeq()).toBe(3_000); + expect(saved.has(4_500)).toBe(false); +}); diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index f7c3bbd1a..aae04325c 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -10,6 +10,7 @@ import { Metric, MetricOptions, Statistic, + Stats, TreatMissingData, } from 'aws-cdk-lib/aws-cloudwatch'; import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; @@ -30,11 +31,13 @@ import { fillMetric } from '../metric-utils'; import { addAlarm } from '../monitoring'; import { NpmJsPackageCanary } from './npmjs/canary'; import { + FOLLOWER_STATE_FILE_NAME, KNOWN_VERSIONS_FILE_NAME, MARKER_FILE_NAME, METRICS_NAMESPACE, MetricName, S3KeyPrefix, + SCAN_WINDOW_MS, } from './npmjs/constants.lambda-shared'; import { NpmJsFollower } from './npmjs/npm-js-follower'; import { StageAndNotify } from './npmjs/stage-and-notify'; @@ -158,6 +161,12 @@ export class NpmJs implements IPackageSource { noncurrentVersionExpiration: Duration.days(30), expiredObjectDeleteMarker: true, }, + // Permanently delete follower state updates after 1 day (we don't need these) + { + prefix: FOLLOWER_STATE_FILE_NAME, + noncurrentVersionExpiration: Duration.days(1), + expiredObjectDeleteMarker: true, + }, ], }); bucket.grantRead(ingestion); @@ -208,6 +217,7 @@ export class NpmJs implements IPackageSource { bucket.grantReadWrite(follower, MARKER_FILE_NAME); bucket.grantReadWrite(follower, KNOWN_VERSIONS_FILE_NAME); + bucket.grantReadWrite(follower, FOLLOWER_STATE_FILE_NAME); denyList?.grantRead(follower); licenseList.grantRead(follower); stager.grantInvoke(follower); @@ -258,7 +268,14 @@ export class NpmJs implements IPackageSource { url: lambdaFunctionUrl(follower), primary: true, }, - { name: 'Marker Object', url: s3ObjectUrl(bucket, MARKER_FILE_NAME) }, + { + name: 'Follower State', + url: s3ObjectUrl(bucket, FOLLOWER_STATE_FILE_NAME), + }, + { + name: 'Marker Object (legacy)', + url: s3ObjectUrl(bucket, MARKER_FILE_NAME), + }, { name: 'Known Versions', url: s3ObjectUrl(bucket, KNOWN_VERSIONS_FILE_NAME), @@ -349,6 +366,42 @@ export class NpmJs implements IPackageSource { period: Duration.minutes(5), }), ], + [ + new GraphWidget({ + height: 6, + width: 12, + title: 'Feed Reliability', + left: [ + fillMetric( + this.metricLateChangeCount({ label: 'Late Changes' }), + 0 + ), + fillMetric( + this.metricLaggyPackuments({ label: 'Laggy Packuments' }), + 0 + ), + fillMetric( + this.metricLaggyPackumentGiveUps({ + label: 'Laggy Packument Give-Ups', + }), + 0 + ), + ], + leftYAxis: { min: 0 }, + right: [ + this.metricLateChangeLag({ label: 'Late Change Lag (max)' }), + ], + rightAnnotations: [ + { + color: '#ff0000', + label: 'Scan Window', + value: SCAN_WINDOW_MS, + }, + ], + rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false }, + period: Duration.minutes(5), + }), + ], [ new GraphWidget({ height: 6, @@ -433,6 +486,82 @@ export class NpmJs implements IPackageSource { }); } + /** + * The number of change entries discovered by a scan that were inserted into + * the `_changes` feed behind a position the follower had already read past. + */ + public metricLateChangeCount(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.LATE_CHANGE_COUNT, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * For each late change entry, the time elapsed between the moment the + * follower first read past the entry's sequence number and the moment the + * entry was discovered. This is a lower bound on the feed's insertion lag, + * and can be used to tune the scan window. + */ + public metricLateChangeLag(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.MAXIMUM, + ...opts, + metricName: MetricName.LATE_CHANGE_LAG, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of packages for which the registry packument is still behind + * the revision announced by the `_changes` feed. The versions served so far + * have been processed; the follower keeps re-checking for the announced + * revision. + */ + public metricLaggyPackuments(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.MAXIMUM, + ...opts, + metricName: MetricName.LAGGY_PACKUMENTS, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of laggy packuments for which the registry caught up with the + * revision announced by the `_changes` feed. + */ + public metricLaggyPackumentsRecovered(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.LAGGY_PACKUMENTS_RECOVERED, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of laggy packuments the follower gave up on: the registry + * never served the revision announced by the `_changes` feed within the + * maximum retry age. A version announced by the feed may be missing until + * the affected package publishes again. + */ + public metricLaggyPackumentGiveUps(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.LAGGY_PACKUMENT_GIVE_UPS, + namespace: METRICS_NAMESPACE, + }); + } + public metricNpmJsChangeAge(opts?: MetricOptions): Metric { return new Metric({ period: Duration.minutes(1), @@ -609,6 +738,51 @@ export class NpmJs implements IPackageSource { dlqNotEmptyAlarm ); + const lateChangeLagAlarm = this.metricLateChangeLag({ + period: Duration.hours(1), + }).createAlarm(scope, 'NpmJs/Follower/LateChangeLagHigh', { + alarmName: `${scope.node.path}/NpmJs/Follower/LateChangeLagHigh`, + alarmDescription: [ + 'The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed', + 'with a delay approaching the scan window. If the delay exceeds the window, the regular scan', + 'will miss those changes (only the daily deep scan would catch them). Consider increasing', + 'the scan window (SCAN_WINDOW_MS).', + '', + `Runbook: ${RUNBOOK_URL}`, + ].join('\n'), + comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD, + evaluationPeriods: 1, + threshold: 0.8 * SCAN_WINDOW_MS, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + monitoring.addLowSeverityAlarm( + 'NpmJs/Follower Late Change Lag approaching scan window', + lateChangeLagAlarm + ); + + const laggyPackumentGiveUpsAlarm = this.metricLaggyPackumentGiveUps({ + period: Duration.hours(1), + }).createAlarm(scope, 'NpmJs/Follower/LaggyPackumentGiveUps', { + alarmName: `${scope.node.path}/NpmJs/Follower/LaggyPackumentGiveUps`, + alarmDescription: [ + 'The npm registry never served the packument revision announced by the CouchDB changes feed', + 'for one or more packages, even after repeated re-checks. All versions the registry did serve', + 'have been processed, but a version announced by the feed may be missing until the affected', + 'package publishes again. The follower logs identify the affected packages, which can be', + 'manually ingested using the ReStagePackageVersion function.', + '', + `Runbook: ${RUNBOOK_URL}`, + ].join('\n'), + comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + evaluationPeriods: 1, + threshold: 1, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + monitoring.addLowSeverityAlarm( + 'NpmJs/Follower gave up on laggy packuments', + laggyPackumentGiveUpsAlarm + ); + // Finally - the "not running" alarm depends on the schedule (it won't run until the schedule // exists!), and the schedule depends on the failure alarm existing (we don't want it to run // before we can know it is failing). This means the returned `IDependable` effectively ensures diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index 560e23ac0..caf354cd8 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -16,6 +16,44 @@ export const enum MetricName { * Number of package versions that were skipped since they were in the deny list. */ DENY_LISTED_COUNT = 'DenyListedCount', + + /** + * Number of change entries discovered by a scan that were inserted into the + * `_changes` feed behind a position the follower had already read past + * ("late insertion"). + */ + LATE_CHANGE_COUNT = 'LateChangeCount', + + /** + * For each late change entry, the time elapsed between the moment the + * follower first read past the entry's sequence number and the moment the + * entry was discovered. This is a lower bound on the feed's insertion lag, + * and is the quantity the scan window must exceed. + */ + LATE_CHANGE_LAG = 'LateChangeLag', + + /** + * The number of packages for which the registry packument is still behind + * the revision announced by the `_changes` feed ("laggy packuments"). The + * versions served so far have been processed; the follower keeps re-checking + * for the announced revision. + */ + LAGGY_PACKUMENTS = 'LaggyPackuments', + + /** + * Number of laggy packuments for which the registry caught up with the + * revision announced by the `_changes` feed. + */ + LAGGY_PACKUMENTS_RECOVERED = 'LaggyPackumentsRecovered', + + /** + * Number of laggy packuments the follower gave up on: the registry never + * served the revision announced by the `_changes` feed within the maximum + * retry age. Any versions the registry did serve have been processed, but a + * version announced by the feed may be missing until the package publishes + * again. + */ + LAGGY_PACKUMENT_GIVE_UPS = 'LaggyPackumentGiveUps', } export const enum S3KeyPrefix { @@ -39,3 +77,40 @@ export const MARKER_FILE_NAME = 'couchdb-last-transaction-id.2'; * The name of the object that contains the list of known versions. */ export const KNOWN_VERSIONS_FILE_NAME = 'couchdb-known-versions.2'; + +/** + * The name of the object that contains the follower state: receipts for the + * change entries received from the `_changes` feed, rolling time/sequence + * checkpoints, laggy packument expectations, and deep scan bookkeeping. + */ +export const FOLLOWER_STATE_FILE_NAME = 'couchdb-follower-state'; + +/** + * How far behind the head of the `_changes` feed each regular scan re-reads. + * The feed occasionally makes change entries visible *behind* positions that + * were already read (npm calls this "normal feed behavior"), so a follower + * that only ever reads forward silently misses those entries. Every scan + * covers this window and processes any entry it has no receipt for. + */ +export const SCAN_WINDOW_MS = 6 * 60 * 60 * 1_000; // 6 hours + +/** + * How often a "deep" scan runs. The deep scan covers the entire state + * retention window (instead of just `SCAN_WINDOW_MS`), so that insertion lag + * *beyond* the regular scan window is still observed (via the `LateChangeLag` + * metric) and can inform tuning of the window. + */ +export const DEEP_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1_000; // 24 hours + +/** + * How long receipts and checkpoints are retained. This is the upper bound of + * what any scan can cover. + */ +export const STATE_RETENTION_MS = 24 * 60 * 60 * 1_000; // 24 hours + +/** + * How long the follower keeps re-checking a laggy packument (a package whose + * registry packument is behind the revision announced by the `_changes` feed) + * before giving up on the announced revision. + */ +export const LAGGY_PACKUMENT_GIVE_UP_MS = 24 * 60 * 60 * 1_000; // 24 hours diff --git a/src/package-sources/npmjs/couch-changes.lambda-shared.ts b/src/package-sources/npmjs/couch-changes.lambda-shared.ts index da9f36744..dc15e114a 100644 --- a/src/package-sources/npmjs/couch-changes.lambda-shared.ts +++ b/src/package-sources/npmjs/couch-changes.lambda-shared.ts @@ -16,8 +16,6 @@ const DEFAULT_BATCH_SIZE = 100; const MAX_CONNS_PER_HOST = 100; -const MAX_PACKAGE_SERVER_LAG_MS = 30_000; // 30 seconds - /** * A utility class that helps with traversing CouchDB database changes streams * in a promise-based, page-by-page manner. @@ -55,12 +53,15 @@ export class CouchChanges extends EventEmitter { } /** - * Obtains a batch of changes from the database. + * Obtains a batch of change entries from the database, without fetching the + * associated package metadata. Entries are deduplicated against previously + * received ones (via the follower state receipts) before any metadata is + * fetched, which is why this method does not fetch it eagerly. * * @param since the sequence value since when history should be fetched. - * @param batchSize the maximum amount of changes to return in a single page. + * @param batchSize the maximum amount of entries to return in a single page. * - * @returns a page of changes. + * @returns a page of change entries. */ public async changes( since: string | number, @@ -74,95 +75,109 @@ export class CouchChanges extends EventEmitter { const result = (await this.https('get', changesUrl)) as any; - const last_seq = result.last_seq; - const results = await this.fetchAndFilterAllMetadata(result.results); - + const results: DatabaseChange[] = result.results ?? []; return { - last_seq, - actionableResults: results, - totalCount: result.results.length, + last_seq: result.last_seq, + results, + seqs: results + .map((change) => Number(change.seq)) + .filter((seq) => !isNaN(seq)), + totalCount: results.length, }; } /** - * Fetch the metadata associated with a change. The change comes associated with a revision number, - * which can be compared to the revision number of the metadata to determine if the replica is - * lagging behind the changes stream. If so, we retry until the replica is up-to-date or until - * 30 seconds elapsed after which we return the potentially stale metadata. + * Fetches the current metadata document for the provided package name from + * the npm registry. + * + * @returns the metadata document, or `undefined` if the package does not + * exist (HTTP 404). */ - private async fetchAndFilterMetadata(change: DatabaseChange) { + public async getPackageDoc( + packageName: string + ): Promise<{ readonly [key: string]: unknown } | undefined> { + const metadataUrl = new URL(packageName, NPM_REGISTRY_URL); + try { + return await this.https('get', metadataUrl); + } catch (e: any) { + if (e.message?.includes('HTTP 404')) { + return undefined; + } + throw e; + } + } + + /** + * Fetches the registry metadata for each of the provided change entries and + * attaches it to the entry. Entries for deleted or unreachable packages are + * dropped. + * + * The registry is a separate system from the changes feed and can serve a + * packument revision older than the revision the feed announced (a "laggy + * packument"). Such entries are returned with `laggy: true`: the caller + * should process the versions that were served, and separately track the + * expectation that the announced revision will eventually appear. + */ + public async attachAllMetadata( + changes: readonly DatabaseChange[] + ): Promise { + const outcomes = await Promise.all( + changes.map((change) => this.attachMetadata(change)) + ); + return outcomes.filter( + (outcome): outcome is AttachedChange => outcome !== undefined + ); + } + + private async attachMetadata( + change: DatabaseChange + ): Promise { // Filter out deleted packages or null ids if (change.deleted || !change.id) { console.log(`Skipping ${change.id}: deleted or null id`); - return; + return undefined; } - const latestChangesRev = getMaxSequentialRevision(change); - const metadataUrl = new URL(change.id, NPM_REGISTRY_URL); - console.log(`Fetching metadata for ${change.id}: ${metadataUrl}`); - - // Retry configuration - const baseDelay = 1_000; // 1 second - const maxDelay = 8_000; // 8 seconds max - let attempt = 0; - const startTime = Date.now(); - // note: this function should not throw validation errors as // it may cause a poison pill - whereby a single corrupt package will // fail the entire lambda execution and prevent us from ingesting any package. - // instead, log the violation and return undefined. - - while (Date.now() - startTime < MAX_PACKAGE_SERVER_LAG_MS) { - try { - const meta = await this.https('get', metadataUrl); - if (!meta) { - // can happen if a package was removed from npm - console.log(`Skipping ${change.id} because no metadata found`); - return; - } - if (!meta._rev) { - // can happen if a package was removed from npm - console.log(`Skipping ${change.id} because no _rev found in metadata`); - return; - } - const latestReplicaRev = parseSequentialRevision(meta._rev as string); - - change.doc = meta; // add metadata to the change object - - // Happy path: replica is up-to-date - if (latestReplicaRev >= latestChangesRev) { - return change; - } - - // Unhappy path: replica is behind. Calculate delay and retry - const delay = Math.floor(Math.random() * Math.min(baseDelay * Math.pow(2, attempt), maxDelay)); - console.log(`${change.id}: package _rev ${latestReplicaRev} < expected replication rev ${latestChangesRev}, retrying in ${delay} ms`); - await new Promise(resolve => setTimeout(resolve, delay)); - attempt++; - } catch (e: any) { - if (e.message?.includes('HTTP 404')) { - console.log( - `Skipping ${change.id} because of HTTP 404 (Not Found) error` - ); - return; - } - throw e; + // instead, log the violation and return undefined. + const metadataUrl = new URL(change.id, NPM_REGISTRY_URL); + console.log(`Fetching metadata for ${change.id}: ${metadataUrl}`); + let meta; + try { + meta = await this.https('get', metadataUrl); + } catch (e: any) { + if (e.message?.includes('HTTP 404')) { + console.log( + `Skipping ${change.id} because of HTTP 404 (Not Found) error` + ); + return undefined; } + throw e; } - // Timeout reached, proceed with stale data - console.log(`Timeout reached for ${change.id}, replica may be stale`); - return change; - } + if (!meta) { + // can happen if a package was removed from npm + console.log(`Skipping ${change.id} because no metadata found`); + return undefined; + } + if (!meta._rev) { + // can happen if a package was removed from npm + console.log(`Skipping ${change.id} because no _rev found in metadata`); + return undefined; + } - private async fetchAndFilterAllMetadata( - changes: DatabaseChange[] - ): Promise { - return ( - await Promise.all( - changes.map((change) => this.fetchAndFilterMetadata(change)) - ) - ).filter((change): change is DatabaseChange => change !== undefined); + const announcedRev = getMaxSequentialRevision(change); + const servedRev = parseSequentialRevision(meta._rev as string); + change.doc = meta; // add metadata to the change object + + return { + change, + announcedRev, + servedRev, + laggy: servedRev < announcedRev, + }; } /** @@ -226,7 +241,8 @@ export class CouchChanges extends EventEmitter { } console.log( - `Response: ${method.toUpperCase()} ${url} => HTTP ${res.statusCode + `Response: ${method.toUpperCase()} ${url} => HTTP ${ + res.statusCode } (${res.statusMessage})` ); @@ -282,7 +298,7 @@ function readResponseJson( }); } -class RetryableError extends Error { } +class RetryableError extends Error {} function isRetryableError(e: Error): boolean { return e instanceof RetryableError || (e as any).code === 'ECONNRESET'; @@ -292,37 +308,78 @@ async function sleep(ms: number) { return new Promise((ok) => setTimeout(ok, ms)); } -function parseSequentialRevision(rev: string): number { +/** + * Parses the sequential (numeric) prefix of a CouchDB revision string (e.g. + * `42` for `42-0bf6e0fa87ae20bc7245f96216263817`). + */ +export function parseSequentialRevision(rev: string): number { return parseInt(rev.split('-')[0]); } -function getMaxSequentialRevision(change: DatabaseChange): number { - return Math.max(...change.changes - .map(change => parseSequentialRevision(change.rev)) - .filter(num => !isNaN(num)) +/** + * The highest sequential revision announced by a change entry. + */ +export function getMaxSequentialRevision(change: DatabaseChange): number { + return Math.max( + ...change.changes + .map((c) => parseSequentialRevision(c.rev)) + .filter((num) => !isNaN(num)) ); } export interface DatabaseChanges { /** - * The last sequence ID from this change set. This is the value that should be - * passed to the subsequent `.changes` call to fetch the next page. + * The last sequence ID from this change set. For ascending requests, this + * is the value that should be passed to a subsequent `.changes` call to + * fetch the next page. */ readonly last_seq: string | number; /** - * The actionable changes that are part of this batch. - * This has deleted and unreachable packages removed. + * The change entries that are part of this batch (no metadata attached). */ - readonly actionableResults: readonly DatabaseChange[]; + readonly results: readonly DatabaseChange[]; /** - * The total count of changes in this batch. This includes unprocessable changes. + * The sequence numbers of all the change entries received in this batch. + */ + readonly seqs: readonly number[]; + + /** + * The total count of change entries in this batch. For ascending requests, * 0 indicates we are up to date with "now". */ readonly totalCount: number; } +/** + * A change entry with its registry metadata attached. + */ +export interface AttachedChange { + /** + * The change entry, with the packument attached as `doc`. + */ + readonly change: DatabaseChange; + + /** + * The highest sequential revision announced by the changes feed for this + * entry. + */ + readonly announcedRev: number; + + /** + * The sequential revision of the packument the registry actually served. + */ + readonly servedRev: number; + + /** + * Whether the served packument is behind the announced revision (a "laggy + * packument"). The served versions can (and should) still be processed, but + * the announced revision has not been observed yet. + */ + readonly laggy: boolean; +} + export interface DatabaseChange { /** * The set of revisions to the object that were resolved as part of this diff --git a/src/package-sources/npmjs/follower-state.lambda-shared.ts b/src/package-sources/npmjs/follower-state.lambda-shared.ts new file mode 100644 index 000000000..fdf9af06d --- /dev/null +++ b/src/package-sources/npmjs/follower-state.lambda-shared.ts @@ -0,0 +1,355 @@ +import { + DEEP_SCAN_INTERVAL_MS, + STATE_RETENTION_MS, +} from './constants.lambda-shared'; + +/** + * A point-in-time record of a feed position the follower has fully read up + * to: "at time `at`, a completed scan had received everything the feed served + * up to sequence number `seq`". + */ +export interface Checkpoint { + /** + * Epoch milliseconds at which the checkpoint was taken. + */ + readonly at: number; + + /** + * The sequence number the completed scan had reached. + */ + readonly seq: number; +} + +/** + * A package whose registry packument is behind the revision announced by the + * `_changes` feed. The versions the registry did serve have been processed; + * this expectation tracks the revision we are still waiting for. + */ +export interface LaggyPackument { + /** + * The package name. + */ + readonly name: string; + + /** + * The revision (sequential prefix) announced by the changes feed. + */ + readonly expectedRev: number; + + /** + * The sequence number of the change entry that announced the revision, if + * known. + */ + readonly seq?: number; + + /** + * Epoch milliseconds at which the expectation was first recorded. Drives + * the give-up policy. + */ + readonly firstSeen: number; +} + +/** + * The persisted state of the *NpmJs Follower*, in a newline-delimited + * plain-text format so it can be scanned cheaply (e.g. `grep '^s 131711133$'`) + * and parsed without materializing a large JSON document. Each line is a + * record identified by its first token: + * + * ```text + * #chfollower/1 <- header (format magic + version) + * w <- deep scan bookkeeping ("-" when unset) + * c <- checkpoint (repeated, ascending by time) + * l <- laggy packument (repeated; seq is "-" when unknown) + * s <- received sequence number (repeated, ascending) + * ``` + * + * The receipts are an exact set on purpose: any form of range encoding would + * claim coverage of sequence numbers that were never observed, and a late + * inserted change entry at such a position would be skipped - which is the + * exact failure mode the receipts exist to catch. + */ +const STATE_FORMAT_HEADER = '#chfollower/1'; + +/** + * Tracks which `_changes` feed entries (by sequence number) the follower has + * already received, rolling time/seq checkpoints mapping times to feed + * positions, and the set of laggy packuments awaiting a registry catch-up. + * + * The npm `_changes` feed does not guarantee that entries only ever become + * visible above previously returned `last_seq` values. The receipt set + * upgrades the follower's position tracking from "I have processed everything + * up to X" to "here is exactly what I received up to X", enabling cheap + * re-scans of a trailing window that only process entries with no receipt. + */ +export class FollowerState { + public static fromText(text: string): FollowerState { + const state = new FollowerState(); + const lines = text.split('\n'); + if (lines[0]?.trim() !== STATE_FORMAT_HEADER) { + throw new Error( + `Unexpected follower state header: ${JSON.stringify(lines[0])}` + ); + } + for (let i = 1; i < lines.length; i++) { + const line = lines[i].trim(); + if (line === '') { + continue; + } + const parts = line.split(' '); + switch (parts[0]) { + case 's': + state.seqs.add(Number(parts[1])); + break; + case 'c': + state.checkpoints.push({ + at: Number(parts[1]), + seq: Number(parts[2]), + }); + break; + case 'l': + state.laggy.set(parts.slice(4).join(' '), { + name: parts.slice(4).join(' '), + firstSeen: Number(parts[1]), + expectedRev: Number(parts[2]), + seq: parts[3] === '-' ? undefined : Number(parts[3]), + }); + break; + case 'w': + state.lastDeepScanAt = + parts[1] === '-' ? undefined : Number(parts[1]); + break; + default: + throw new Error(`Unexpected follower state line: ${line}`); + } + } + return state; + } + + public lastDeepScanAt?: number; + + private checkpoints: Checkpoint[] = []; + private readonly seqs = new Set(); + private readonly laggy = new Map(); + // The threshold below which receipts have already been pruned; avoids + // re-iterating the (potentially large) receipt set on every checkpoint. + private prunedBelow = 0; + + //#region Receipts + /** + * Whether a receipt exists for the provided sequence number. + */ + public has(seq: number): boolean { + return this.seqs.has(seq); + } + + /** + * Records receipts for the provided sequence numbers. + */ + public addSeqs(seqs: Iterable): void { + for (const seq of seqs) { + this.seqs.add(seq); + } + } + + /** + * The number of sequence number receipts currently retained. + */ + public get receiptCount(): number { + return this.seqs.size; + } + //#endregion + + //#region Checkpoints + /** + * Records a checkpoint at the current time for the provided sequence + * number, then prunes checkpoints and receipts that have aged out of the + * retention window. + */ + public addCheckpoint(seq: number, now: number = Date.now()): void { + this.checkpoints.push({ at: now, seq }); + this.prune(now); + } + + /** + * The number of checkpoints currently retained. + */ + public get checkpointCount(): number { + return this.checkpoints.length; + } + + /** + * The sequence number of the newest checkpoint, i.e. the highest feed + * position a completed scan has read up to. Entries discovered below this + * position were inserted late. Returns `undefined` when no checkpoints + * exist. + */ + public newestCheckpointSeq(): number | undefined { + return this.checkpoints[this.checkpoints.length - 1]?.seq; + } + + /** + * The feed position a scan should re-read from, to cover the provided + * trailing window: the newest checkpoint at least `windowMs` old. If the + * checkpoint history does not reach that far back (a young deployment, or a + * backfill in progress), the newest checkpoint is used: coverage below it + * is guaranteed, and starting any lower would make long scans restart from + * the beginning on every run. Returns `undefined` when no checkpoints exist + * at all. + */ + public floorSeq( + windowMs: number, + now: number = Date.now() + ): number | undefined { + let candidate: Checkpoint | undefined; + for (const checkpoint of this.checkpoints) { + if (checkpoint.at > now - windowMs) { + break; + } + candidate = checkpoint; + } + return (candidate ?? this.checkpoints[this.checkpoints.length - 1])?.seq; + } + + /** + * The (approximate) time at which a completed scan first read past the + * provided sequence number: the timestamp of the earliest checkpoint whose + * sequence number is `>= seq`. Returns `undefined` if no completed scan has + * passed that sequence number yet. + */ + public timeCrossed(seq: number): number | undefined { + for (const checkpoint of this.checkpoints) { + if (checkpoint.seq >= seq) { + return checkpoint.at; + } + } + return undefined; + } + + /** + * Discards all position state (receipts and checkpoints). Used when the + * feed's sequence space is no longer compatible with the recorded state + * (e.g. the feed head regressed below our newest checkpoint). Laggy + * packument expectations are package-keyed and remain valid. + */ + public resetPosition(): void { + this.checkpoints = []; + this.seqs.clear(); + this.prunedBelow = 0; + } + //#endregion + + //#region Deep scans + /** + * Whether a deep scan (covering the full state retention window) is due. + */ + public deepScanDue(now: number = Date.now()): boolean { + return ( + this.lastDeepScanAt == null || + now - this.lastDeepScanAt >= DEEP_SCAN_INTERVAL_MS + ); + } + + /** + * Marks a deep scan as completed at the provided time. + */ + public completeDeepScan(now: number = Date.now()): void { + this.lastDeepScanAt = now; + } + //#endregion + + //#region Laggy packuments + /** + * Records (or extends) the expectation that the registry packument for the + * provided package should reach the provided revision. The earliest + * `firstSeen` is retained (so the give-up policy measures from the first + * occurrence), and the highest announced revision wins. + */ + public recordLaggyPackument( + name: string, + expectedRev: number, + seq: number | undefined, + now: number = Date.now() + ): void { + const existing = this.laggy.get(name); + this.laggy.set(name, { + name, + expectedRev: Math.max(expectedRev, existing?.expectedRev ?? 0), + seq: existing?.seq ?? seq, + firstSeen: existing?.firstSeen ?? now, + }); + } + + /** + * Removes the expectation for the provided package (recovered, gone, or + * given up). + */ + public removeLaggyPackument(name: string): void { + this.laggy.delete(name); + } + + /** + * All laggy packument expectations, oldest first. + */ + public laggyPackuments(): LaggyPackument[] { + return Array.from(this.laggy.values()).sort( + (a, b) => a.firstSeen - b.firstSeen + ); + } + + /** + * The number of laggy packument expectations currently tracked. + */ + public get laggyPackumentCount(): number { + return this.laggy.size; + } + //#endregion + + public toText(): string { + const lines = new Array(); + lines.push(STATE_FORMAT_HEADER); + lines.push(`w ${this.lastDeepScanAt ?? '-'}`); + for (const { at, seq } of this.checkpoints) { + lines.push(`c ${at} ${seq}`); + } + for (const laggy of this.laggyPackuments()) { + lines.push( + `l ${laggy.firstSeen} ${laggy.expectedRev} ${laggy.seq ?? '-'} ${ + laggy.name + }` + ); + } + for (const seq of Array.from(this.seqs).sort((a, b) => a - b)) { + lines.push(`s ${seq}`); + } + lines.push(''); // trailing newline + return lines.join('\n'); + } + + private prune(now: number): void { + const cutOff = now - STATE_RETENTION_MS; + // Always keep at least one checkpoint at or before the cut-off, so that + // `floorSeq` remains well-defined over the whole retention window. + let firstRetained = 0; + for (let i = 0; i < this.checkpoints.length; i++) { + if (this.checkpoints[i].at <= cutOff) { + firstRetained = i; + } else { + break; + } + } + this.checkpoints = this.checkpoints.slice(firstRetained); + + // Receipts below the earliest retained checkpoint's sequence number can + // no longer be reached by any scan, so drop them. + const minSeq = this.checkpoints[0]?.seq ?? 0; + if (minSeq <= this.prunedBelow) { + return; + } + for (const seq of this.seqs) { + if (seq < minSeq) { + this.seqs.delete(seq); + } + } + this.prunedBelow = minSeq; + } +} diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index e021c1d53..69f0f6612 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -24,8 +24,18 @@ import { MARKER_FILE_NAME, METRICS_NAMESPACE, KNOWN_VERSIONS_FILE_NAME, + FOLLOWER_STATE_FILE_NAME, + SCAN_WINDOW_MS, + STATE_RETENTION_MS, + LAGGY_PACKUMENT_GIVE_UP_MS, } from './constants.lambda-shared'; -import { CouchChanges, DatabaseChange } from './couch-changes.lambda-shared'; +import { + CouchChanges, + DatabaseChange, + DatabaseChanges, + parseSequentialRevision, +} from './couch-changes.lambda-shared'; +import { FollowerState } from './follower-state.lambda-shared'; import { PackageVersion } from './stage-and-notify.lambda'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; @@ -33,7 +43,10 @@ import { LAMBDA_CLIENT, S3_CLIENT, } from '../../backend/shared/aws.lambda-shared'; -import { decompressContent } from '../../backend/shared/compress-content.lambda-shared'; +import { + compressContent, + decompressContent, +} from '../../backend/shared/compress-content.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; // eslint-disable-next-line @typescript-eslint/no-require-imports const normalizeNPMMetadata = require('normalize-registry-metadata'); @@ -50,11 +63,37 @@ const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/'; /** * The release date of `aws-cdk@0.8.0`. Anything earlier than this basically is * not a relevant package, as it cannot possibly be a constructs-based package. - * This is used to fast-forward over boring stuff when the sequence number is - * reset. + * This is used to fast-forward over boring stuff when the follower state is + * seeded far in the past. */ const DAWN_OF_CONSTRUCTS = new Date('2018-07-31T13:43:04.615Z'); +/** + * The page size used when scanning the `_changes` feed. Receipts discard + * already-received entries before any packument is fetched, so scans can + * afford the feed's maximum page size (10,000). + */ +const SCAN_BATCH_SIZE = 10_000; + +/** + * A scan batch is only started when at least this much time remains in the + * Lambda execution. An interrupted scan simply resumes on the next run: the + * receipts make re-reads cheap, and no checkpoint is recorded for an + * incomplete scan. + */ +const SCAN_TIME_BUDGET_MS = 60_000; + +/** + * Laggy packuments are only re-checked while at least this much time remains + * in the Lambda execution, so that re-checks cannot starve the scan. + */ +const LAGGY_RETRY_TIME_BUDGET_MS = 240_000; + +/** + * The maximum number of laggy packuments re-checked per run. + */ +const MAX_LAGGY_RETRIES_PER_RUN = 50; + // Configure embedded metrics format Configuration.namespace = METRICS_NAMESPACE; @@ -65,13 +104,30 @@ captureHTTPsGlobal(require('https')); captureHTTPsGlobal(require('http')); /** - * This function triggers on a fixed schedule and reads a stream of changes from npmjs couchdb _changes endpoint. - * Upon invocation the function starts reading from a sequence stored in an s3 object - the `marker`. - * If the marker fails to load (or do not exist), the stream will start from `now` - the latest change. - * For each change: - * - the package version tarball will be copied from the npm registry to a stating bucket. - * - a message will be sent to an sqs queue + * This function triggers on a fixed schedule and scans the npmjs CouchDB + * `_changes` feed for new package versions. + * + * The feed gives no guarantee that change entries become visible in sequence + * order (entries occasionally appear *behind* previously returned positions), + * so the follower does not track a single high-water mark. Instead it keeps a + * receipt for every change entry it has received, and every run re-reads a + * trailing window of the feed, processing any entry it has no receipt for: + * + * 1. Rolling checkpoints map times to feed positions a completed scan has + * fully covered. The scan floor is the checkpointed position from + * `SCAN_WINDOW_MS` ago (a periodic deep scan covers the full retention + * window instead). + * 2. Each scan reads the window ascending from the floor. Receipts make + * re-reads of already-covered ranges nearly free. + * 3. For each new entry, the packument is fetched from the registry and every + * version it contains is processed (deny list, license, and known-versions + * checks apply). If the packument is behind the revision announced by the + * feed (a "laggy packument"), the served versions are still processed, and + * the follower records the expectation and keeps re-checking on subsequent + * runs until the announced revision appears or the expectation ages out. + * * npm registry API docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md + * * @param context a Lambda execution context */ export async function handler(event: ScheduledEvent, context: Context) { @@ -85,154 +141,424 @@ export async function handler(event: ScheduledEvent, context: Context) { const npm = new CouchChanges(NPM_REPLICA_REGISTRY_URL, 'registry/_changes'); - const { - marker: initialMarker, - knownVersions, - didLoadVersionsFromLegacy, - } = await loadFollowerPosition(stagingBucket, npm); + const head = Number((await npm.info()).update_seq); - // If we loaded from the old location, write to the new + const { knownVersions, didLoadVersionsFromLegacy } = await loadKnownVersions( + stagingBucket + ); let writeKnownVersionsFile = didLoadVersionsFromLegacy; - // The last written marker seq id. - let updatedMarker = initialMarker; + const state = await loadFollowerState(stagingBucket, head); - // The slowest batch processing time so far (starts at 60 seconds). This is how much time should - // be left before timeout if a new batch is to be fetched. - let maxBatchProcessingTime = 60_000; - // Whether we should continue reading more items or not... This is set to false when the current - // latest change is reached (i.e: next page of changes is empty). - let shouldContinue = true; + // The feed's sequence space can change (entries can disappear from the end, + // moving the head backwards). Our position state is then meaningless. + const newestCheckpoint = state.newestCheckpointSeq(); + if (newestCheckpoint != null && newestCheckpoint > head) { + console.warn( + `Feed head (${head}) is below our newest checkpoint (${newestCheckpoint}); the sequence space changed. Resetting position state.` + ); + state.resetPosition(); + state.addCheckpoint(head); + } - do { - await metricScope((metrics) => async () => { - console.log('Polling changes from npm replica'); - const changes = await npm.changes(updatedMarker); + // Re-check laggy packuments (packages whose registry packument was behind + // the revision announced by the feed when we first saw them). + const laggyStaged = await retryLaggyPackuments( + context, + npm, + state, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + writeKnownVersionsFile ||= laggyStaged > 0; - // Clear automatically set dimensions - we don't need them (see https://github.com/awslabs/aws-embedded-metrics-node/issues/73) - metrics.setDimensions({}); + // Scan the trailing window of the feed. + const scanStaged = await runScan( + context, + npm, + state, + head, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + writeKnownVersionsFile ||= scanStaged > 0; - // Recording current seq range and updating the `updatedMarker`. - metrics.setProperty('StartSeq', updatedMarker); - updatedMarker = Number(changes.last_seq); - metrics.setProperty('EndSeq', updatedMarker); - - const startTime = Date.now(); - - try { - const batch = changes.actionableResults as readonly Change[]; - - // The most recent "modified" timestamp observed in the batch. - let lastModified: Date | undefined; - // Emit npm.js replication lag - for (const { doc } of batch) { - if (doc?.time?.modified) { - const modified = new Date(doc.time.modified); - metrics.putMetric( - MetricName.NPMJS_CHANGE_AGE, - startTime - modified.getTime(), - Unit.Milliseconds - ); - if (lastModified == null || lastModified < modified) { - lastModified = modified; - } - } - } + await Promise.all([ + saveFollowerState(context, stagingBucket, state), + ...(writeKnownVersionsFile + ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] + : []), + ]); - console.log( - `Received a batch of ${changes.totalCount} element(s), ${batch.length} after filtering` - ); - metrics.putMetric(MetricName.CHANGE_COUNT, batch.length, Unit.Count); + console.log('All done here, we have success!'); - if (lastModified && lastModified < DAWN_OF_CONSTRUCTS) { - console.log( - `Skipping batch as the latest modification is ${lastModified}, which is pre-Constructs` - ); - } else if (changes.totalCount === 0) { - console.log('Received 0 changes, caught up to "now", exiting...'); - shouldContinue = false; - } else { - // Obtain the modified package version from the update event, and filter - // out packages that are not of interest to us (not construct libraries). - const versionInfos = getRelevantVersionInfos( - batch, - metrics, - denyList, - licenseList, - knownVersions - ); - console.log( - `Identified ${versionInfos.length} relevant package version update(s)` - ); - metrics.putMetric( - MetricName.RELEVANT_PACKAGE_VERSIONS, - versionInfos.length, - Unit.Count - ); + return { head }; +} - // If we have versionInfos, we already added them to the knownVersions map and we should save it again - writeKnownVersionsFile ||= versionInfos.length > 0; - - // Process all remaining updates - await Promise.all( - versionInfos.map(async ({ infos, modified, seq }) => { - const invokeArgs: PackageVersion = { - integrity: infos.dist.shasum, - modified: modified.toISOString(), - name: infos.name, - seq: seq?.toString(), - tarballUrl: infos.dist.tarball, - version: infos.version, - }; - // "Fire-and-forget" invocation here. - console.log(`Sending ${invokeArgs.tarballUrl} for staging`); - await LAMBDA_CLIENT.send( - new InvokeCommand({ - FunctionName: stagingFunction, - InvocationType: 'Event', - Payload: Buffer.from(JSON.stringify(invokeArgs)), - }) - ); - // Record that this is now a "known" version (no need to re-discover) - knownVersions.set(`${infos.name}@${infos.version}`, modified); - }) - ); +//#region Scan +/** + * Re-reads the trailing window of the feed and processes every change entry + * for which no receipt exists. A checkpoint is only recorded when the scan + * covered the entire window; an interrupted scan resumes naturally on the + * next run (receipts make the re-read cheap). + * + * @returns the number of package versions sent for staging. + */ +async function runScan( + context: Context, + npm: CouchChanges, + state: FollowerState, + head: number, + stagingFunction: string, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: Map +): Promise { + const now = Date.now(); + const deep = state.deepScanDue(now); + const windowMs = deep ? STATE_RETENTION_MS : SCAN_WINDOW_MS; + const floor = state.floorSeq(windowMs, now); + if (floor == null) { + // Cannot happen after seeding, but guards the type. + console.warn('No checkpoint history; skipping scan'); + return 0; + } + // Entries discovered at or below this position were inserted into the feed + // behind a position we had already read past. + const lateBoundary = state.newestCheckpointSeq() ?? floor; + + console.log( + `Starting ${ + deep ? 'deep ' : '' + }scan of (${floor}, ${head}] (late boundary: ${lateBoundary})` + ); + + let staged = 0; + let completed = false; + let maxSeqSeen = head; + + // Read the window ascending from the floor. Receipts make re-reads of + // already-covered ranges nearly free (no packuments are fetched for them). + let cursor = floor; + while ( + !completed && + context.getRemainingTimeInMillis() > SCAN_TIME_BUDGET_MS + ) { + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + metrics.setProperty('StartSeq', cursor); + const batch = await npm.changes(cursor, { batchSize: SCAN_BATCH_SIZE }); + const next = Number(batch.last_seq); + metrics.setProperty('EndSeq', next); + if (batch.seqs.length > 0) { + maxSeqSeen = Math.max(maxSeqSeen, ...batch.seqs); + } + staged += await processBatch( + batch, + metrics, + context, + npm, + state, + lateBoundary, + head, + stagingFunction, + denyList, + licenseList, + knownVersions + ); + if (batch.totalCount < SCAN_BATCH_SIZE || next <= cursor) { + // A short page means we are caught up with the feed head. + completed = true; + } else { + cursor = next; + // Coverage from the floor up to `next` is contiguous, so this is a + // valid checkpoint. Recording it makes an interrupted scan (most + // importantly: a backfill) resume from here instead of the floor. + if (next > (state.newestCheckpointSeq() ?? 0)) { + state.addCheckpoint(next); } + } + })(); + } + + if (completed) { + // The scan covered the entire window: everything the feed served up to + // `maxSeqSeen` now has a receipt. + state.addCheckpoint(maxSeqSeen); + if (deep) { + state.completeDeepScan(); + console.log(`Completed deep scan at seq ${maxSeqSeen}`); + } + } else { + console.log( + 'Scan ran out of time; it will resume on the next run (no checkpoint recorded)' + ); + } + return staged; +} - // Updating the S3 stored marker with the new seq id and known versions concurrently - await Promise.all([ - // always save the transaction marker - saveLastTransactionMarker(context, stagingBucket, updatedMarker), - // conditionally save known versions - ...(writeKnownVersionsFile - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ]); - } finally { - metrics.putMetric(MetricName.LAST_SEQ, updatedMarker, Unit.None); +/** + * Processes one page of change entries: discards everything we already have a + * receipt for, fetches packuments for the remainder, stages new relevant + * package versions, records laggy packument expectations, and finally records + * receipts for every entry received. + * + * @returns the number of package versions sent for staging. + */ +async function processBatch( + batch: DatabaseChanges, + metrics: MetricsLogger, + context: Context, + npm: CouchChanges, + state: FollowerState, + lateBoundary: number, + head: number, + stagingFunction: string, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: Map +): Promise { + const startTime = Date.now(); + let staged = 0; + try { + const fresh = batch.results.filter( + (change) => change.seq != null && !state.has(Number(change.seq)) + ); + console.log( + `Received ${batch.totalCount} change entr(ies), ${fresh.length} without a receipt` + ); + metrics.putMetric(MetricName.CHANGE_COUNT, fresh.length, Unit.Count); + + // Entries below the late boundary were inserted into the feed behind a + // position a completed scan had already read past. + const late = fresh.filter((change) => Number(change.seq) <= lateBoundary); + metrics.putMetric(MetricName.LATE_CHANGE_COUNT, late.length, Unit.Count); + for (const change of late) { + const crossedAt = state.timeCrossed(Number(change.seq)); + if (crossedAt != null) { metrics.putMetric( - MetricName.BATCH_PROCESSING_TIME, - Date.now() - startTime, + MetricName.LATE_CHANGE_LAG, + Date.now() - crossedAt, Unit.Milliseconds ); + } + console.log( + `[late] Change entry discovered at seq ${change.seq} (behind ${lateBoundary}): ${change.id}` + ); + } + + const attached = await npm.attachAllMetadata(fresh); + + // The most recent "modified" timestamp observed in the batch. + let lastModified: Date | undefined; + // Emit npm.js replication lag + for (const { change } of attached) { + const doc = change.doc as Change['doc'] | undefined; + if (doc?.time?.modified) { + const modified = new Date(doc.time.modified); metrics.putMetric( - MetricName.REMAINING_TIME, - context.getRemainingTimeInMillis(), + MetricName.NPMJS_CHANGE_AGE, + startTime - modified.getTime(), Unit.Milliseconds ); + if (lastModified == null || lastModified < modified) { + lastModified = modified; + } } - })(); - } while ( - shouldContinue && - context.getRemainingTimeInMillis() >= maxBatchProcessingTime - ); + } - console.log('All done here, we have success!'); + if (lastModified && lastModified < DAWN_OF_CONSTRUCTS) { + console.log( + `Skipping batch as the latest modification is ${lastModified}, which is pre-Constructs` + ); + } else { + // Laggy packuments: the registry served an older revision than the feed + // announced. Process the versions we did get, and record the + // expectation so the announced revision is re-checked on later runs. + for (const laggy of attached.filter((a) => a.laggy)) { + console.log( + `${laggy.change.id}: registry packument rev ${laggy.servedRev} is behind announced rev ${laggy.announcedRev}, recording laggy packument` + ); + state.recordLaggyPackument( + laggy.change.id, + laggy.announcedRev, + isNaN(Number(laggy.change.seq)) ? undefined : Number(laggy.change.seq) + ); + } - return { initialMarker, updatedMarker }; + const versionInfos = getRelevantVersionInfos( + attached.map((a) => a.change) as unknown as readonly Change[], + metrics, + denyList, + licenseList, + knownVersions + ); + console.log( + `Identified ${versionInfos.length} relevant package version update(s)` + ); + metrics.putMetric( + MetricName.RELEVANT_PACKAGE_VERSIONS, + versionInfos.length, + Unit.Count + ); + await stageVersions(versionInfos, stagingFunction, knownVersions); + staged = versionInfos.length; + } + + // Record receipts for every entry received (including deleted or + // unreachable packages: they were received and deliberately skipped). + state.addSeqs(batch.seqs); + } finally { + metrics.putMetric(MetricName.LAST_SEQ, head, Unit.None); + metrics.putMetric( + MetricName.BATCH_PROCESSING_TIME, + Date.now() - startTime, + Unit.Milliseconds + ); + metrics.putMetric( + MetricName.REMAINING_TIME, + context.getRemainingTimeInMillis(), + Unit.Milliseconds + ); + } + return staged; } +//#endregion -//#region Last known versions and marker +//#region Laggy packuments +/** + * Re-checks laggy packuments: packages whose registry packument was behind + * the revision announced by the changes feed. Each re-check processes + * whatever the registry serves now (new versions are staged), and the + * expectation is cleared when the announced revision appears - or aged out + * after `LAGGY_PACKUMENT_GIVE_UP_MS`, at which point a version announced by + * the feed may be missing until the package publishes again. + * + * @returns the number of package versions sent for staging. + */ +async function retryLaggyPackuments( + context: Context, + npm: CouchChanges, + state: FollowerState, + stagingFunction: string, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: Map +): Promise { + let staged = 0; + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + let recovered = 0; + let gaveUp = 0; + const now = Date.now(); + for (const laggy of state + .laggyPackuments() + .slice(0, MAX_LAGGY_RETRIES_PER_RUN)) { + if (context.getRemainingTimeInMillis() < LAGGY_RETRY_TIME_BUDGET_MS) { + break; + } + const doc = await npm.getPackageDoc(laggy.name); + if (doc == null || doc._rev == null) { + console.log( + `Laggy packument for ${laggy.name} is no longer available in the registry, dropping the expectation` + ); + state.removeLaggyPackument(laggy.name); + continue; + } + + // Process whatever the registry serves now; knownVersions dedupes. + const change: DatabaseChange = { + changes: [{ rev: doc._rev as string }], + deleted: false, + id: laggy.name, + seq: laggy.seq, + doc, + }; + const versionInfos = getRelevantVersionInfos( + [change] as unknown as readonly Change[], + metrics, + denyList, + licenseList, + knownVersions + ); + await stageVersions(versionInfos, stagingFunction, knownVersions); + staged += versionInfos.length; + + if (parseSequentialRevision(doc._rev as string) >= laggy.expectedRev) { + console.log( + `Laggy packument for ${laggy.name} caught up (rev ${doc._rev} >= ${laggy.expectedRev})` + ); + state.removeLaggyPackument(laggy.name); + recovered += 1; + } else if (now - laggy.firstSeen > LAGGY_PACKUMENT_GIVE_UP_MS) { + console.warn( + `Giving up on laggy packument for ${ + laggy.name + }: the registry never served rev ${ + laggy.expectedRev + } (first seen ${new Date( + laggy.firstSeen + ).toISOString()}). A version announced by the changes feed may be missing until the package publishes again.` + ); + state.removeLaggyPackument(laggy.name); + gaveUp += 1; + } + } + metrics.putMetric( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + recovered, + Unit.Count + ); + metrics.putMetric(MetricName.LAGGY_PACKUMENT_GIVE_UPS, gaveUp, Unit.Count); + metrics.putMetric( + MetricName.LAGGY_PACKUMENTS, + state.laggyPackumentCount, + Unit.Count + ); + })(); + return staged; +} +//#endregion + +/** + * Sends the provided package version updates to the staging function + * ("fire-and-forget"), and records them as known versions. + */ +async function stageVersions( + versionInfos: readonly UpdatedVersion[], + stagingFunction: string, + knownVersions: Map +): Promise { + await Promise.all( + versionInfos.map(async ({ infos, modified, seq }) => { + const invokeArgs: PackageVersion = { + integrity: infos.dist.shasum, + modified: modified.toISOString(), + name: infos.name, + seq: seq?.toString(), + tarballUrl: infos.dist.tarball, + version: infos.version, + }; + // "Fire-and-forget" invocation here. + console.log(`Sending ${invokeArgs.tarballUrl} for staging`); + await LAMBDA_CLIENT.send( + new InvokeCommand({ + FunctionName: stagingFunction, + InvocationType: 'Event', + Payload: Buffer.from(JSON.stringify(invokeArgs)), + }) + ); + // Record that this is now a "known" version (no need to re-discover) + knownVersions.set(`${infos.name}@${infos.version}`, modified); + }) + ); +} + +//#region State and known versions /** * Common function to load data from an S3 file with error handling * @@ -270,113 +596,143 @@ async function loadContentFromS3( } } -async function loadFollowerPosition( +/** + * Loads the follower state from S3. When the state file does not exist (or + * cannot be parsed), a fresh state is seeded: from the legacy transaction + * marker if one exists (so existing deployments continue where the previous + * follower left off), or from the current feed head otherwise. + */ +async function loadFollowerState( stagingBucket: string, - registry: CouchChanges -): Promise<{ - marker: number; - knownVersions: Map; - didLoadVersionsFromLegacy: boolean; -}> { - const [markerFile, knownVersions] = await Promise.all([ - loadLastTransactionMarker(stagingBucket, registry), - loadLastKnownVersions(stagingBucket), - ]); - - if (!knownVersions) { - // For legacy reasons, knownVersions could come from the transaction marker file - // If it does, then we take that as knownVersions but will migrate knownVersions - // to the modern file location - return { - marker: markerFile.marker, - knownVersions: markerFile.knownVersions ?? new Map(), - didLoadVersionsFromLegacy: true, - }; + head: number +): Promise { + const warningMessage = `Follower state object (s3://${stagingBucket}/${FOLLOWER_STATE_FILE_NAME}) does not exist, seeding a fresh state`; + const content = await loadContentFromS3( + stagingBucket, + FOLLOWER_STATE_FILE_NAME, + warningMessage + ); + if (content != null) { + try { + const state = FollowerState.fromText(content); + console.log( + `Loaded follower state: ${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s)` + ); + return state; + } catch (error) { + console.warn(`Could not parse follower state, seeding fresh: ${error}`); + } } - return { - marker: markerFile.marker, - knownVersions, - didLoadVersionsFromLegacy: false, - }; + const state = new FollowerState(); + const seed = await loadLegacyMarker(stagingBucket); + if (seed != null && seed <= head) { + console.log(`Seeding follower state from legacy marker: ${seed}`); + state.addCheckpoint(seed); + } else { + // Brand-new deployment: start at the beginning of the feed, so the + // instance backfills the entire history automatically (interim + // checkpoints make the backfill incremental across runs). + console.log( + 'Seeding follower state at the beginning of the feed (automatic backfill)' + ); + state.addCheckpoint(0); + } + return state; } /** - * Loads the last known versions from S3. - * - * @returns the value of the last known versions. + * Reads the sequence number from the legacy transaction marker file, if + * present. The previous follower implementation stored its position there. */ -async function loadLastKnownVersions( +async function loadLegacyMarker( stagingBucket: string -): Promise | undefined> { - const warningMessage = `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist, starting from scratch`; +): Promise { const content = await loadContentFromS3( stagingBucket, - KNOWN_VERSIONS_FILE_NAME, - warningMessage + MARKER_FILE_NAME, + `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` ); - - // Known Versions does not exist, starting from scratch if (content === null) { return undefined; } - - // Known Versions exists, generating map from the data - const contentsObj: KnownVersionsFileSchema = JSON.parse(content); - - console.log(`Loaded last known versions data`); - - return dateMapFromObj(contentsObj.knownVersions); + try { + const parsed: MarkerFileSchema = JSON.parse(content); + const marker = typeof parsed === 'number' ? parsed : Number(parsed.marker); + return isNaN(marker) ? undefined : marker; + } catch (error) { + console.warn(`Could not parse legacy marker: ${error}`); + return undefined; + } } /** - * Loads the last transaction marker from S3. - * - * @param registry a Nano database corresponding to the Npmjs.com CouchDB instance. - * - * @returns the value of the last transaction marker. + * Persists the follower state to S3 (gzip-compressed when large enough to + * warrant it). */ -async function loadLastTransactionMarker( +async function saveFollowerState( + context: Context, stagingBucket: string, - registry: CouchChanges -): Promise<{ marker: number; knownVersions?: Map }> { - const warningMessage = `Marker object (s3://${stagingBucket}/${MARKER_FILE_NAME}) does not exist, starting from scratch`; + state: FollowerState +) { + const { buffer, contentEncoding } = compressContent( + Buffer.from(state.toText(), 'utf-8') + ); + console.log( + `Updating follower state (${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s))` + ); + await putObject(context, stagingBucket, FOLLOWER_STATE_FILE_NAME, buffer, { + ContentType: 'text/plain', + ContentEncoding: contentEncoding, + }); + console.log('Successfully updated follower state'); +} + +/** + * Loads the known versions from S3. For legacy reasons, the map may also be + * embedded in the transaction marker file; when the dedicated file does not + * exist, that location is used and the map is migrated on the next save. + */ +async function loadKnownVersions(stagingBucket: string): Promise<{ + knownVersions: Map; + didLoadVersionsFromLegacy: boolean; +}> { + const warningMessage = `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist, starting from scratch`; const content = await loadContentFromS3( stagingBucket, - MARKER_FILE_NAME, + KNOWN_VERSIONS_FILE_NAME, warningMessage ); - - // Last transaction marker does not exist - if (content === null) { - return { marker: 0 }; - } - - const parsedFile: MarkerFileSchema = JSON.parse(content); - - let marker: number; - let knownVersions: Map | undefined; - if (typeof parsedFile === 'number') { - marker = parsedFile; - } else { - marker = Number(parsedFile.marker); - knownVersions = parsedFile.knownVersions - ? dateMapFromObj(parsedFile.knownVersions) - : undefined; + if (content != null) { + const contentsObj: KnownVersionsFileSchema = JSON.parse(content); + console.log('Loaded last known versions data'); + return { + knownVersions: dateMapFromObj(contentsObj.knownVersions), + didLoadVersionsFromLegacy: false, + }; } - // amazonq-ignore-next-line - console.log(`Read last transaction marker: ${marker}`); - - const dbUpdateSeq = Number((await registry.info()).update_seq); - if (dbUpdateSeq < marker) { - console.warn( - `Current DB update_seq (${dbUpdateSeq}) is lower than marker (CouchDB instance was likely replaced), resetting to 0!` - ); - return { marker: 0 }; + // Legacy location: embedded in the transaction marker file. + const markerContent = await loadContentFromS3( + stagingBucket, + MARKER_FILE_NAME, + `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` + ); + if (markerContent != null) { + try { + const parsed: MarkerFileSchema = JSON.parse(markerContent); + if (typeof parsed !== 'number' && parsed.knownVersions) { + console.log('Loaded known versions from legacy marker file'); + return { + knownVersions: dateMapFromObj(parsed.knownVersions), + didLoadVersionsFromLegacy: true, + }; + } + } catch (error) { + console.warn(`Could not parse legacy marker: ${error}`); + } } - - return { marker, knownVersions }; + return { knownVersions: new Map(), didLoadVersionsFromLegacy: true }; } /** @@ -405,24 +761,6 @@ async function saveLastKnownVersions( ); console.log('Successfully updated known versions'); } - -/** - * Updates the last transaction marker in S3. - * - * @param marker the last transaction marker value - */ -async function saveLastTransactionMarker( - context: Context, - stagingBucket: string, - marker: number -) { - const contents = JSON.stringify({ marker }); - console.log(`Updating last transaction marker to ${marker}`); - await putObject(context, stagingBucket, MARKER_FILE_NAME, contents, { - ContentType: 'application/json', - }); - console.log('Successfully updated marker'); -} //#endregion //#region Asynchronous Primitives @@ -698,7 +1036,8 @@ interface UpdatedVersion { } /** - * We are accounting for a good bit of legacy data modeling here + * The legacy transaction marker file format, kept around to seed the follower + * state (and known versions) of existing deployments. * * The file can be a just a number, or a combination of a sequence number (which * is potentially encoded as a string) and a set of known versions and the dates