From f91f97c4062d3cdc19fc893bbec876a2a1f88c28 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Thu, 8 Oct 2026 11:40:56 +0100 Subject: [PATCH 1/3] feat(npmjs): re-check laggy packuments through a queue --- .gitattributes | 1 + .gitignore | 1 + .prettierignore | 1 + .projen/files.json | 1 + .projen/tasks.json | 26 +- API.md | 79 +- docs/application-overview.md | 8 +- docs/npm-sync-protocol.md | 12 +- docs/operator-runbook.md | 118 +- package.json | 2 + .../__snapshots__/construct-hub.test.ts.snap | 2580 +++++++++++++++-- .../__snapshots__/snapshot.test.ts.snap | 508 +++- src/__tests__/package-sources/npmjs.test.ts | 83 + .../follower-state.lambda-shared.test.ts | 55 +- .../npmjs/npm-js-follower.lambda.test.ts | 212 +- .../npmjs/packument-processor.lambda.test.ts | 475 +++ src/package-sources/npmjs.ts | 235 +- .../npmjs/constants.lambda-shared.ts | 74 +- .../npmjs/couch-changes.lambda-shared.ts | 61 +- .../npmjs/follower-state.lambda-shared.ts | 97 +- .../npmjs/npm-js-follower.lambda.ts | 600 +--- .../packument-processing.lambda-shared.ts | 449 +++ .../npmjs/packument-processor.lambda.ts | 255 ++ .../npmjs/packument-processor.ts | 20 + .../npmjs/packument-queue-processor.ts | 100 + .../npmjs/packument-queue.lambda-shared.ts | 138 + .../npmjs/re-stage-package-version.lambda.ts | 2 +- 27 files changed, 5217 insertions(+), 976 deletions(-) create mode 100644 src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts create mode 100644 src/package-sources/npmjs/packument-processing.lambda-shared.ts create mode 100644 src/package-sources/npmjs/packument-processor.lambda.ts create mode 100644 src/package-sources/npmjs/packument-processor.ts create mode 100644 src/package-sources/npmjs/packument-queue-processor.ts create mode 100644 src/package-sources/npmjs/packument-queue.lambda-shared.ts diff --git a/.gitattributes b/.gitattributes index 8def7a98f..90bb428de 100644 --- a/.gitattributes +++ b/.gitattributes @@ -68,6 +68,7 @@ /src/package-sources/npmjs/canary/npmjs-package-canary.ts linguist-generated /src/package-sources/npmjs/known-versions-migration.ts linguist-generated /src/package-sources/npmjs/npm-js-follower.ts linguist-generated +/src/package-sources/npmjs/packument-processor.ts linguist-generated /src/package-sources/npmjs/re-stage-package-version.ts linguist-generated /src/package-sources/npmjs/stage-and-notify.ts linguist-generated /src/spdx-license.ts linguist-generated diff --git a/.gitignore b/.gitignore index ec6ca9dda..0bd84d979 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ lib/__tests__/devapp/cdk.out !/test/tsconfig.json !/src/package-sources/npmjs/stage-and-notify.ts !/src/package-sources/npmjs/re-stage-package-version.ts +!/src/package-sources/npmjs/packument-processor.ts !/src/package-sources/npmjs/npm-js-follower.ts !/src/package-sources/npmjs/known-versions-migration.ts !/src/package-sources/npmjs/canary/npmjs-package-canary.ts diff --git a/.prettierignore b/.prettierignore index c9902bb09..b96b1c5cd 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,6 +1,7 @@ # ~~ Generated by projen. To modify, edit .projenrc.ts and run "yarn projen". src/package-sources/npmjs/stage-and-notify.ts src/package-sources/npmjs/re-stage-package-version.ts +src/package-sources/npmjs/packument-processor.ts src/package-sources/npmjs/npm-js-follower.ts src/package-sources/npmjs/known-versions-migration.ts src/package-sources/npmjs/canary/npmjs-package-canary.ts diff --git a/.projen/files.json b/.projen/files.json index a8dae9bd6..cdb725291 100644 --- a/.projen/files.json +++ b/.projen/files.json @@ -57,6 +57,7 @@ "src/package-sources/npmjs/canary/npmjs-package-canary.ts", "src/package-sources/npmjs/known-versions-migration.ts", "src/package-sources/npmjs/npm-js-follower.ts", + "src/package-sources/npmjs/packument-processor.ts", "src/package-sources/npmjs/re-stage-package-version.ts", "src/package-sources/npmjs/stage-and-notify.ts", "src/spdx-license.ts", diff --git a/.projen/tasks.json b/.projen/tasks.json index b5296b1f6..57a730631 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -79,6 +79,9 @@ { "spawn": "bundle:re-stage-package-version" }, + { + "spawn": "bundle:packument-processor" + }, { "spawn": "bundle:npm-js-follower" }, @@ -389,7 +392,7 @@ "description": "Continuously bundle all AWS Lambda functions", "steps": [ { - "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/known-versions-migration.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json" + "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/packument-processor.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/known-versions-migration.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json" } ] }, @@ -537,6 +540,24 @@ } ] }, + "bundle:packument-processor": { + "name": "bundle:packument-processor", + "description": "Create an AWS Lambda bundle from src/package-sources/npmjs/packument-processor.lambda.ts", + "steps": [ + { + "exec": "esbuild --bundle src/package-sources/npmjs/packument-processor.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/packument-processor.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json" + } + ] + }, + "bundle:packument-processor:watch": { + "name": "bundle:packument-processor:watch", + "description": "Continuously update an AWS Lambda bundle from src/package-sources/npmjs/packument-processor.lambda.ts", + "steps": [ + { + "exec": "esbuild --bundle src/package-sources/npmjs/packument-processor.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/packument-processor.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json --watch" + } + ] + }, "bundle:prune-handler": { "name": "bundle:prune-handler", "description": "Create an AWS Lambda bundle from src/backend/deny-list/prune-handler.lambda.ts", @@ -861,6 +882,9 @@ { "spawn": "bundle:re-stage-package-version" }, + { + "spawn": "bundle:packument-processor" + }, { "spawn": "bundle:npm-js-follower" }, diff --git a/API.md b/API.md index 25f5a18fd..052b32075 100644 --- a/API.md +++ b/API.md @@ -2913,13 +2913,17 @@ new sources.NpmJs(props?: NpmJsProps) | bind | Binds the package source to a scope and target queue. | | metricBatchProcessingTime | The average time it took to process a changes batch. | | metricChangeCount | The total count of changes that were processed. | -| metricLaggyPackumentGiveUps | The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. | -| metricLaggyPackuments | The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. | +| metricLaggyPackumentGiveUps | The number of laggy packuments given up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. | +| metricLaggyPackumentLag | How long laggy packuments took to catch up (or until they were given up on), measured from when the follower first saw them. | +| metricLaggyPackuments | The number of requests in the packument queue, for laggy and missing packuments. | | metricLaggyPackumentsRecovered | The number of laggy packuments for which the registry caught up with the revision announced by the `_changes` feed. | | metricLastSeq | The last sequence number that was processed. | | metricLateChangeCount | The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. | | metricLateChangeLag | For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. | | metricMetadataFetchFailures | The number of change entries whose registry metadata could not be fetched (after transient-error retries). | +| metricMissingPackumentGiveUps | The number of missing packuments given up on: the registry never served a packument for the package within the maximum retry age. | +| metricMissingPackuments | The number of change entries for which the registry had no packument at all (HTTP 404). | +| metricMissingPackumentsRecovered | The number of missing packuments the registry has since served. | | metricNpmJsChangeAge | *No description.* | | metricPackageVersionAge | The age of the oldest package version that was processed. | | metricPackageVersionCount | The total count of package versions that were inspected. | @@ -2983,7 +2987,7 @@ The total count of changes that were processed. public metricLaggyPackumentGiveUps(opts?: MetricOptions): Metric ``` -The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. +The number of laggy packuments given up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. A version announced by the feed may be missing until the affected package publishes again. @@ -2994,17 +2998,30 @@ the affected package publishes again. --- +##### `metricLaggyPackumentLag` + +```typescript +public metricLaggyPackumentLag(opts?: MetricOptions): Metric +``` + +How long laggy packuments took to catch up (or until they were given up on), measured from when the follower first saw them. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricLaggyPackuments` ```typescript public metricLaggyPackuments(opts?: MetricOptions): Metric ``` -The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. +The number of requests in the packument queue, for laggy and missing packuments. -The versions served so far -have been processed; the follower keeps re-checking for the announced -revision. +A package that lagged in several change entries has several +requests. ###### `opts`Optional @@ -3091,6 +3108,54 @@ later scan retries them. --- +##### `metricMissingPackumentGiveUps` + +```typescript +public metricMissingPackumentGiveUps(opts?: MetricOptions): Metric +``` + +The number of missing packuments given up on: the registry never served a packument for the package within the maximum retry age. + +Usually an old +change entry for a package that no longer exists. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricMissingPackuments` + +```typescript +public metricMissingPackuments(opts?: MetricOptions): Metric +``` + +The number of change entries for which the registry had no packument at all (HTTP 404). + +Usually a new package that has not replicated to the +registry yet; the packument processor keeps re-checking for it. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + +##### `metricMissingPackumentsRecovered` + +```typescript +public metricMissingPackumentsRecovered(opts?: MetricOptions): Metric +``` + +The number of missing packuments the registry has since served. + +###### `opts`Optional + +- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions + +--- + ##### `metricNpmJsChangeAge` ```typescript diff --git a/docs/application-overview.md b/docs/application-overview.md index 319a94322..225fd8b40 100644 --- a/docs/application-overview.md +++ b/docs/application-overview.md @@ -42,9 +42,11 @@ ConstructHub provides two package source implementations: `NpmJs` and function is invoked, which stages the package tarball into an S3 bucket then notifies the ConstructHub ingestion SQS queue. The CouchDB follower is scheduled to run every `5 minutes`, and stores its state (receipts for the - change entries received, time/sequence checkpoints, and laggy packument - expectations) in a specific object in the S3 bucket used for staging package - tarballs. + change entries received, and time/sequence checkpoints) in a specific object + in the S3 bucket used for staging package tarballs. + When the registry serves an older revision of a package than the feed + announced, the package is sent to a queue, and a packument processor function + checks it again, with growing waits, for about a day. The package versions it has processed are recorded in a DynamoDB table, so each version is only processed once. Existing deployments kept this list in the S3 bucket; it is copied to the diff --git a/docs/npm-sync-protocol.md b/docs/npm-sync-protocol.md index 36fc5cc0c..923ef5e20 100644 --- a/docs/npm-sync-protocol.md +++ b/docs/npm-sync-protocol.md @@ -122,12 +122,12 @@ processing any entry it has no receipt for. Rolling checkpoints map times to feed positions, and derive the window's starting position. On top of that baseline, each gap has a dedicated mitigation: -| Gap | Mitigation | Detection | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | -| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm | -| Laggy packuments | Process every packument the registry serves immediately; record the announced revision and re-check it every run until it appears or ages out | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` metrics; `LaggyPackumentGiveUps` alarm | -| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package | -| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric | +| Gap | Mitigation | Detection | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm | +| Laggy packuments | Process every packument the registry serves immediately; queue a re-check of the announced revision (also when the registry has no packument yet), retried with back-off for about a day, then moved to a dead-letter queue | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` / `LaggyPackumentLag` and `MissingPackuments` / `MissingPackumentsRecovered` / `MissingPackumentGiveUps` metrics; `LaggyPackumentLagHigh` and `MissingPackumentsHigh` alarms | +| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package | +| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric | **The design idea behind all of this is that the feed data are just hints without real guarantees.** The follower treats every change entry as "this package probably changed around here", diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 9773cc1d0..703c575fd 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -614,49 +614,93 @@ be increased: adjust `SCAN_WINDOW_MS` in `src/package-sources/npmjs/constants.lambda-shared.ts` (keeping it below `STATE_RETENTION_MS`) and deploy the updated construct. -### `ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps` +### `ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh` #### Description This alarm is only provisioned when the `NpmJs` package source is configured. It -triggers when the npm registry never served the packument revision announced by -the CouchDB `_changes` feed for one or more packages, even after the follower -re-checked for a prolonged period. - -All package versions the registry did serve have been processed. However, a -version announced by the changes feed may be missing from ConstructHub until -the affected package publishes its next version. +triggers when the npm registry had no packument (HTTP 404) for an unusually high +number of packages announced by the CouchDB `_changes` feed, within 3 hours. +This usually means the registry has an outage, or is far behind the feed. #### Investigation The *NpmJs Follower* fetches package metadata from `registry.npmjs.org` for -each change announced by the CouchDB `_changes` feed. The registry occasionally -lags behind the feed. When it does, the follower processes the versions that -were served, records the announced revision as a "laggy packument", and -re-checks it on every run until the revision appears or the expectation ages -out. This alarm fires for the latter. +each change announced by the CouchDB `_changes` feed. Some of these always +return 404, usually because a new package has not replicated to the registry +yet (a *missing* packument). The follower sends them to the *Packument Queue*, +where the *Packument Processor* checks them again with growing waits, for about +a day. A normal 3 hour window has a couple of hundred of them. -Review the logs of the *NpmJs Follower* for `Giving up on laggy packument` -entries; they identify the affected package and the revision that never -appeared. Verify what the registry currently serves for the package: +Check the `Missing Packuments` line on the *Feed Reliability* graph of the +backend dashboard, and pick a few affected packages from the logs of the +*NpmJs Follower* (`the registry has no packument for it yet`). Verify whether +the registry serves them now: ```console -$ curl https://registry.npmjs.org/ | jq ._rev +$ curl -s -o /dev/null -w '%{http_code}\n' https://registry.npmjs.org/ ``` -If the served revision is still older than the expected one, the problem is +If most still return 404 while the feed keeps announcing them, the problem is with upstream npmjs.com. Look at `npmjs.com` status updates and announcements. +If you suspect the problem is with upstream npmjs.com, you can cut them a +ticket. Log in to , visit the +[Support Page](https://www.npmjs.com/support) and select +*There is a problem with the npm registry*. -If you suspect the problem is with upstream npmjs.com, you can cut them a ticket. Log -in to , visit the [Support Page](https://www.npmjs.com/support) and -select *There is a problem with the npm registry*. +#### Resolution + +The packument processor processes the packages once the registry serves them. +Packages it gives up on after about a day move to the *Packument DLQ*. Once the +registry is healthy again, redrive the *Packument DLQ* to the *Packument Queue* +(from the SQS console) to check them again. + +-------------------------------------------------------------------------------- + +### `ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh` + +#### Description + +This alarm is only provisioned when the `NpmJs` package source is configured. It +triggers when laggy packuments take unusually long to catch up: in a 3 hour +window, 90% of them took longer than 6 hours. This usually means the registry +has an outage, or is far behind the feed. + +All package versions the registry did serve have been processed. However, +versions announced by the changes feed may be missing from ConstructHub until +the registry catches up. + +#### Investigation + +The registry occasionally serves an older revision of a package than the +changes feed announced (a *laggy* packument). The follower processes the +versions that were served, and sends the package to the *Packument Queue*. The +*Packument Processor* checks it again with growing waits, for about a day, and +reports how long each laggy packument took to catch up. Most catch up within +minutes, and nearly all within 6 hours. + +Check the *Packument Queue* graph of the backend dashboard: how many requests +are waiting, and how many were given up on (*Packument DLQ*). The logs of the +*Packument Processor* show `caught up` and `Giving up on laggy packument` +entries. Verify what the registry serves for a few affected packages: + +```console +$ curl https://registry.npmjs.org/ | jq ._rev +``` + +If the served revision is still older than the expected one, the problem is +with upstream npmjs.com (see the previous alarm for how to reach them). #### Resolution -The missed package versions will automatically be discovered once the affected -packages publish their next version. To ingest a missed version immediately, -invoke the *ReStagePackageVersion* Lambda function with the package name and -version from the follower logs. +The packument processor processes the missed versions once the registry serves +them. Packages it gives up on move to the *Packument DLQ*. Once the registry is +healthy again, redrive the *Packument DLQ* to the *Packument Queue* (from the +SQS console). To ingest a missed version immediately, invoke the +*ReStagePackageVersion* Lambda function with the package name and version. + +The *Packument DLQ* has no alarm of its own. Given-up requests stay in it for +14 days, and it is graphed on the backend dashboard. -------------------------------------------------------------------------------- @@ -811,6 +855,30 @@ When documentation for a package is not available, it could be due to several re -------------------------------------------------------------------------------- +### `ConstructHub/Sources/NpmJs/PackumentProcessor/Failures` + +#### Description + +This alarm is only provisioned when the `NpmJs` package source is configured. It +triggers when the *Packument Processor* function fails or times out. While it +fails, the requests in the *Packument Queue* keep using up their attempts, so +laggy packuments can be given up on (and move to the *Packument DLQ*) earlier +than the usual day. + +#### Investigation + +Review the logs of the *Packument Processor* (linked from the backend +dashboard). Errors while setting up a batch (for example reading the deny list +or license list) fail the whole batch; errors for a single package only fail +that request. Time-outs usually mean the npm registry is slow to respond. + +#### Resolution + +Fix the cause. Requests that were given up on in the meantime can be redriven +from the *Packument DLQ* to the *Packument Queue* (from the SQS console). + +-------------------------------------------------------------------------------- + ### `ConstructHub/Sources/NpmJs/Canary/SLA-Breached` #### Description diff --git a/package.json b/package.json index e3017425b..0026b851c 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,8 @@ "bundle:package-stats-processor:watch": "projen bundle:package-stats-processor:watch", "bundle:package-versions-table-widget-function": "projen bundle:package-versions-table-widget-function", "bundle:package-versions-table-widget-function:watch": "projen bundle:package-versions-table-widget-function:watch", + "bundle:packument-processor": "projen bundle:packument-processor", + "bundle:packument-processor:watch": "projen bundle:packument-processor:watch", "bundle:prune-handler": "projen bundle:prune-handler", "bundle:prune-handler:watch": "projen bundle:prune-handler:watch", "bundle:prune-queue-handler": "projen bundle:prune-queue-handler", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 6851553ee..4d8fcf9e9 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -771,6 +771,40 @@ def sort_filter_rules(json_obj): "QueueName", ], }, + ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + ")\\n[button:Packument Processor](/lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, ")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F", { "Ref": "ConstructHubSourcesNpmJs15A77D2D", @@ -819,7 +853,39 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -9777,6 +9843,20 @@ Request a service quota increase for lambda functions", "QueueName", ], }, + ""},"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "":{"name":"NPM JS Packument DLQ","queueName":"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -10124,7 +10204,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip", + "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -10147,6 +10227,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", @@ -10581,27 +10664,6 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { - "Properties": { - "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed -for one or more packages, even after repeated re-checks. All versions the registry did serve -have been processed, but a version announced by the feed may be missing until the affected -package publishes again. The follower logs identify the affected packages, which can be -manually ingested using the ReStagePackageVersion function. - -Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", - "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "EvaluationPeriods": 1, - "MetricName": "LaggyPackumentGiveUps", - "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", - "Period": 3600, - "Statistic": "Sum", - "Threshold": 1, - "TreatMissingData": "notBreaching", - }, - "Type": "AWS::CloudWatch::Alarm", - }, "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { "Properties": { "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed @@ -10622,6 +10684,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": { + "Properties": { + "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by +the CouchDB changes feed. This usually means the registry has an outage or is far behind +the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "MissingPackuments", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Statistic": "Sum", + "Threshold": 1500, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -10705,7 +10786,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "Event", "ServiceToken": { @@ -10729,7 +10810,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -10743,7 +10824,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -10770,7 +10851,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip", + "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -10938,6 +11019,64 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": { + "Properties": { + "AlarmDescription": { + "Fn::Join": [ + "", + [ + "The packument processor function is failing (errors or time-outs). Requests in the +packument queue use up their attempts while it fails, and may be given up on early. + +Link to the lambda function: /lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + " + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + ], + ], + }, + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "FunctionName", + "Value": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + }, + ], + "EvaluationPeriods": 2, + "MetricName": "Errors", + "Namespace": "AWS/Lambda", + "Period": 900, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": { + "Properties": { + "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions +than the CouchDB changes feed announced, for hours. This usually means the registry has an +outage or is far behind the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "ExtendedStatistic": "p90", + "MetricName": "LaggyPackumentLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Threshold": 21600000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -11209,6 +11348,20 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, ], "Version": "2012-10-17", }, @@ -11234,7 +11387,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip", + "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -11560,10 +11713,21 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, - "ConstructHubSourcesReStagePackageVersion5E1211D1": { + "ConstructHubSourcesPackumentProcessorDLQF401BD77": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 1209600, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesPackumentProcessorFunction0E5F8177": { "DependsOn": [ - "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", - "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", ], "Properties": { "Architectures": [ @@ -11573,31 +11737,52 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip", + "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", }, - "Description": "Manually re-stage a package version", + "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { "Variables": { + "AWS_EMF_ENVIRONMENT": "Local", + "DENY_LIST_BUCKET_NAME": { + "Ref": "ConstructHubDenyListBucket1B3C2C2E", + }, + "DENY_LIST_OBJECT_KEY": "deny-list.json", "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, - "REGISTRY_URL": "https://registry.npmjs.org", + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + "LICENSE_LIST_BUCKET_NAME": { + "Ref": "ConstructHubLicenseListBucket9334047F", + }, + "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", - "MemorySize": 1024, + "MemorySize": 2048, "Role": { "Fn::GetAtt": [ - "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", "Arn", ], }, "Runtime": "nodejs22.x", - "Timeout": 10, + "Timeout": 300, + "TracingConfig": { + "Mode": "Active", + }, }, "Type": "AWS::Lambda::Function", }, - "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": { + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -11628,7 +11813,272 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Role", }, - "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": { + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "Roles": [ + { + "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + "ScalingConfig": { + "MaximumConcurrency": 2, + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "ConstructHubSourcesPackumentProcessorQueue34BDE578": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 345600, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "maxReceiveCount": 9, + }, + "VisibilityTimeout": 1800, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesReStagePackageVersion5E1211D1": { + "DependsOn": [ + "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", + "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip", + }, + "Description": "Manually re-stage a package version", + "Environment": { + "Variables": { + "FUNCTION_NAME": { + "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + }, + "REGISTRY_URL": "https://registry.npmjs.org", + }, + }, + "Handler": "index.handler", + "MemorySize": 1024, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 10, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": { "Properties": { "PolicyDocument": { "Statement": [ @@ -15225,6 +15675,40 @@ def sort_filter_rules(json_obj): "QueueName", ], }, + ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + ")\\n[button:Packument Processor](/lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, ")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F", { "Ref": "ConstructHubSourcesNpmJs15A77D2D", @@ -15273,7 +15757,39 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -24605,6 +25121,20 @@ Request a service quota increase for lambda functions", "QueueName", ], }, + ""},"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "":{"name":"NPM JS Packument DLQ","queueName":"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -24952,7 +25482,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip", + "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -24975,6 +25505,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", @@ -25409,27 +25942,6 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { - "Properties": { - "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed -for one or more packages, even after repeated re-checks. All versions the registry did serve -have been processed, but a version announced by the feed may be missing until the affected -package publishes again. The follower logs identify the affected packages, which can be -manually ingested using the ReStagePackageVersion function. - -Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", - "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "EvaluationPeriods": 1, - "MetricName": "LaggyPackumentGiveUps", - "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", - "Period": 3600, - "Statistic": "Sum", - "Threshold": 1, - "TreatMissingData": "notBreaching", - }, - "Type": "AWS::CloudWatch::Alarm", - }, "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { "Properties": { "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed @@ -25450,6 +25962,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": { + "Properties": { + "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by +the CouchDB changes feed. This usually means the registry has an outage or is far behind +the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "MissingPackuments", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Statistic": "Sum", + "Threshold": 1500, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -25533,7 +26064,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "Event", "ServiceToken": { @@ -25557,7 +26088,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -25571,7 +26102,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -25598,7 +26129,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip", + "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -25766,6 +26297,64 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": { + "Properties": { + "AlarmDescription": { + "Fn::Join": [ + "", + [ + "The packument processor function is failing (errors or time-outs). Requests in the +packument queue use up their attempts while it fails, and may be given up on early. + +Link to the lambda function: /lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + " + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + ], + ], + }, + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "FunctionName", + "Value": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + }, + ], + "EvaluationPeriods": 2, + "MetricName": "Errors", + "Namespace": "AWS/Lambda", + "Period": 900, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": { + "Properties": { + "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions +than the CouchDB changes feed announced, for hours. This usually means the registry has an +outage or is far behind the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "ExtendedStatistic": "p90", + "MetricName": "LaggyPackumentLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Threshold": 21600000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -26037,6 +26626,20 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, ], "Version": "2012-10-17", }, @@ -26062,7 +26665,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip", + "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -26388,6 +26991,303 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesPackumentProcessorDLQF401BD77": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 1209600, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesPackumentProcessorFunction0E5F8177": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + }, + "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", + "Environment": { + "Variables": { + "AWS_EMF_ENVIRONMENT": "Local", + "DENY_LIST_BUCKET_NAME": { + "Ref": "ConstructHubDenyListBucket1B3C2C2E", + }, + "DENY_LIST_OBJECT_KEY": "deny-list.json", + "FUNCTION_NAME": { + "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + "LICENSE_LIST_BUCKET_NAME": { + "Ref": "ConstructHubLicenseListBucket9334047F", + }, + "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 300, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "Roles": [ + { + "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + "ScalingConfig": { + "MaximumConcurrency": 2, + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "ConstructHubSourcesPackumentProcessorQueue34BDE578": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 345600, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "maxReceiveCount": 9, + }, + "VisibilityTimeout": 1800, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, "ConstructHubSourcesReStagePackageVersion5E1211D1": { "DependsOn": [ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", @@ -26401,7 +27301,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip", + "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip", }, "Description": "Manually re-stage a package version", "Environment": { @@ -29991,6 +30891,40 @@ def sort_filter_rules(json_obj): "QueueName", ], }, + ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + ")\\n[button:Packument Processor](/lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, ")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F", { "Ref": "ConstructHubSourcesNpmJs15A77D2D", @@ -30039,7 +30973,39 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -38997,6 +39963,20 @@ Request a service quota increase for lambda functions", "QueueName", ], }, + ""},"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "":{"name":"NPM JS Packument DLQ","queueName":"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -39344,7 +40324,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip", + "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -39367,6 +40347,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", @@ -39801,27 +40784,6 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { - "Properties": { - "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed -for one or more packages, even after repeated re-checks. All versions the registry did serve -have been processed, but a version announced by the feed may be missing until the affected -package publishes again. The follower logs identify the affected packages, which can be -manually ingested using the ReStagePackageVersion function. - -Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", - "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "EvaluationPeriods": 1, - "MetricName": "LaggyPackumentGiveUps", - "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", - "Period": 3600, - "Statistic": "Sum", - "Threshold": 1, - "TreatMissingData": "notBreaching", - }, - "Type": "AWS::CloudWatch::Alarm", - }, "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { "Properties": { "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed @@ -39842,6 +40804,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": { + "Properties": { + "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by +the CouchDB changes feed. This usually means the registry has an outage or is far behind +the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "MissingPackuments", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Statistic": "Sum", + "Threshold": 1500, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -39925,7 +40906,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "Event", "ServiceToken": { @@ -39949,7 +40930,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -39963,7 +40944,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -39990,7 +40971,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip", + "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -40158,6 +41139,64 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": { + "Properties": { + "AlarmDescription": { + "Fn::Join": [ + "", + [ + "The packument processor function is failing (errors or time-outs). Requests in the +packument queue use up their attempts while it fails, and may be given up on early. + +Link to the lambda function: /lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + " + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + ], + ], + }, + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "FunctionName", + "Value": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + }, + ], + "EvaluationPeriods": 2, + "MetricName": "Errors", + "Namespace": "AWS/Lambda", + "Period": 900, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": { + "Properties": { + "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions +than the CouchDB changes feed announced, for hours. This usually means the registry has an +outage or is far behind the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "ExtendedStatistic": "p90", + "MetricName": "LaggyPackumentLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Threshold": 21600000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -40429,6 +41468,20 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, ], "Version": "2012-10-17", }, @@ -40454,7 +41507,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip", + "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -40780,10 +41833,21 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, - "ConstructHubSourcesReStagePackageVersion5E1211D1": { + "ConstructHubSourcesPackumentProcessorDLQF401BD77": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 1209600, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesPackumentProcessorFunction0E5F8177": { "DependsOn": [ - "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", - "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", ], "Properties": { "Architectures": [ @@ -40793,31 +41857,52 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip", + "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", }, - "Description": "Manually re-stage a package version", + "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { "Variables": { + "AWS_EMF_ENVIRONMENT": "Local", + "DENY_LIST_BUCKET_NAME": { + "Ref": "ConstructHubDenyListBucket1B3C2C2E", + }, + "DENY_LIST_OBJECT_KEY": "deny-list.json", "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, - "REGISTRY_URL": "https://registry.npmjs.org", + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + "LICENSE_LIST_BUCKET_NAME": { + "Ref": "ConstructHubLicenseListBucket9334047F", + }, + "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", - "MemorySize": 1024, + "MemorySize": 2048, "Role": { "Fn::GetAtt": [ - "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", "Arn", ], }, "Runtime": "nodejs22.x", - "Timeout": 10, + "Timeout": 300, + "TracingConfig": { + "Mode": "Active", + }, }, "Type": "AWS::Lambda::Function", }, - "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": { + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -40848,7 +41933,272 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Role", }, - "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": { + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "Roles": [ + { + "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + "ScalingConfig": { + "MaximumConcurrency": 2, + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "ConstructHubSourcesPackumentProcessorQueue34BDE578": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 345600, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "maxReceiveCount": 9, + }, + "VisibilityTimeout": 1800, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesReStagePackageVersion5E1211D1": { + "DependsOn": [ + "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", + "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip", + }, + "Description": "Manually re-stage a package version", + "Environment": { + "Variables": { + "FUNCTION_NAME": { + "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + }, + "REGISTRY_URL": "https://registry.npmjs.org", + }, + }, + "Handler": "index.handler", + "MemorySize": 1024, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 10, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": { "Properties": { "PolicyDocument": { "Statement": [ @@ -44531,6 +45881,40 @@ def sort_filter_rules(json_obj): "QueueName", ], }, + ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + ")\\n[button:Packument Processor](/lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, ")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F", { "Ref": "ConstructHubSourcesNpmJs15A77D2D", @@ -44579,7 +45963,39 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -53546,6 +54962,20 @@ Request a service quota increase for lambda functions", "QueueName", ], }, + ""},"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "":{"name":"NPM JS Packument DLQ","queueName":"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -53893,7 +55323,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip", + "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -53916,6 +55346,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", @@ -54337,27 +55770,6 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { - "Properties": { - "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed -for one or more packages, even after repeated re-checks. All versions the registry did serve -have been processed, but a version announced by the feed may be missing until the affected -package publishes again. The follower logs identify the affected packages, which can be -manually ingested using the ReStagePackageVersion function. - -Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", - "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "EvaluationPeriods": 1, - "MetricName": "LaggyPackumentGiveUps", - "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", - "Period": 3600, - "Statistic": "Sum", - "Threshold": 1, - "TreatMissingData": "notBreaching", - }, - "Type": "AWS::CloudWatch::Alarm", - }, "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { "Properties": { "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed @@ -54378,6 +55790,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": { + "Properties": { + "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by +the CouchDB changes feed. This usually means the registry has an outage or is far behind +the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "MissingPackuments", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Statistic": "Sum", + "Threshold": 1500, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -54461,7 +55892,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "Event", "ServiceToken": { @@ -54485,7 +55916,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -54499,7 +55930,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -54526,7 +55957,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip", + "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -54694,6 +56125,64 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": { + "Properties": { + "AlarmDescription": { + "Fn::Join": [ + "", + [ + "The packument processor function is failing (errors or time-outs). Requests in the +packument queue use up their attempts while it fails, and may be given up on early. + +Link to the lambda function: /lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + " + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + ], + ], + }, + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "FunctionName", + "Value": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + }, + ], + "EvaluationPeriods": 2, + "MetricName": "Errors", + "Namespace": "AWS/Lambda", + "Period": 900, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": { + "Properties": { + "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions +than the CouchDB changes feed announced, for hours. This usually means the registry has an +outage or is far behind the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "ExtendedStatistic": "p90", + "MetricName": "LaggyPackumentLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Threshold": 21600000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -54965,6 +56454,20 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, ], "Version": "2012-10-17", }, @@ -54990,7 +56493,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip", + "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -55316,6 +56819,303 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesPackumentProcessorDLQF401BD77": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 1209600, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesPackumentProcessorFunction0E5F8177": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + }, + "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", + "Environment": { + "Variables": { + "AWS_EMF_ENVIRONMENT": "Local", + "DENY_LIST_BUCKET_NAME": { + "Ref": "ConstructHubDenyListBucket1B3C2C2E", + }, + "DENY_LIST_OBJECT_KEY": "deny-list.json", + "FUNCTION_NAME": { + "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + "LICENSE_LIST_BUCKET_NAME": { + "Ref": "ConstructHubLicenseListBucket9334047F", + }, + "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 300, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "Roles": [ + { + "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + "ScalingConfig": { + "MaximumConcurrency": 2, + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "ConstructHubSourcesPackumentProcessorQueue34BDE578": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 345600, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "maxReceiveCount": 9, + }, + "VisibilityTimeout": 1800, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, "ConstructHubSourcesReStagePackageVersion5E1211D1": { "DependsOn": [ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", @@ -55329,7 +57129,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip", + "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip", }, "Description": "Manually re-stage a package version", "Environment": { @@ -59256,6 +61056,40 @@ def sort_filter_rules(json_obj): "QueueName", ], }, + ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + ")\\n[button:Packument Processor](/lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, ")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F", { "Ref": "ConstructHubSourcesNpmJs15A77D2D", @@ -59304,7 +61138,39 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -68016,6 +69882,20 @@ Request a service quota increase for lambda functions", "QueueName", ], }, + ""},"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "":{"name":"NPM JS Packument DLQ","queueName":"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -68425,7 +70305,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip", + "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -68448,6 +70328,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", @@ -68882,27 +70765,6 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { - "Properties": { - "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed -for one or more packages, even after repeated re-checks. All versions the registry did serve -have been processed, but a version announced by the feed may be missing until the affected -package publishes again. The follower logs identify the affected packages, which can be -manually ingested using the ReStagePackageVersion function. - -Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", - "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "EvaluationPeriods": 1, - "MetricName": "LaggyPackumentGiveUps", - "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", - "Period": 3600, - "Statistic": "Sum", - "Threshold": 1, - "TreatMissingData": "notBreaching", - }, - "Type": "AWS::CloudWatch::Alarm", - }, "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { "Properties": { "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed @@ -68923,6 +70785,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": { + "Properties": { + "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by +the CouchDB changes feed. This usually means the registry has an outage or is far behind +the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "MissingPackuments", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Statistic": "Sum", + "Threshold": 1500, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -69006,7 +70887,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "Event", "ServiceToken": { @@ -69030,7 +70911,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -69044,7 +70925,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -69071,7 +70952,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip", + "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -69239,6 +71120,64 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": { + "Properties": { + "AlarmDescription": { + "Fn::Join": [ + "", + [ + "The packument processor function is failing (errors or time-outs). Requests in the +packument queue use up their attempts while it fails, and may be given up on early. + +Link to the lambda function: /lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + " + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + ], + ], + }, + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "FunctionName", + "Value": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + }, + ], + "EvaluationPeriods": 2, + "MetricName": "Errors", + "Namespace": "AWS/Lambda", + "Period": 900, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": { + "Properties": { + "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions +than the CouchDB changes feed announced, for hours. This usually means the registry has an +outage or is far behind the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "ExtendedStatistic": "p90", + "MetricName": "LaggyPackumentLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Threshold": 21600000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -69510,6 +71449,20 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, ], "Version": "2012-10-17", }, @@ -69535,7 +71488,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip", + "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -69861,6 +71814,303 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesPackumentProcessorDLQF401BD77": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 1209600, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesPackumentProcessorFunction0E5F8177": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + }, + "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", + "Environment": { + "Variables": { + "AWS_EMF_ENVIRONMENT": "Local", + "DENY_LIST_BUCKET_NAME": { + "Ref": "ConstructHubDenyListBucket1B3C2C2E", + }, + "DENY_LIST_OBJECT_KEY": "deny-list.json", + "FUNCTION_NAME": { + "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + "LICENSE_LIST_BUCKET_NAME": { + "Ref": "ConstructHubLicenseListBucket9334047F", + }, + "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 300, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "Roles": [ + { + "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + "ScalingConfig": { + "MaximumConcurrency": 2, + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "ConstructHubSourcesPackumentProcessorQueue34BDE578": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 345600, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "maxReceiveCount": 9, + }, + "VisibilityTimeout": 1800, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, "ConstructHubSourcesReStagePackageVersion5E1211D1": { "DependsOn": [ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", @@ -69874,7 +72124,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip", + "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip", }, "Description": "Manually re-stage a package version", "Environment": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 999fe3486..268453366 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -833,6 +833,40 @@ def sort_filter_rules(json_obj): "QueueName", ], }, + ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", + { + "Ref": "AWS::Region", + }, + ".amazonaws.com%2F", + { + "Ref": "AWS::AccountId", + }, + "%2F", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + ")\\n[button:Packument Processor](/lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, ")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F", { "Ref": "ConstructHubSourcesNpmJs15A77D2D", @@ -881,7 +915,39 @@ def sort_filter_rules(json_obj): { "Ref": "AWS::Region", }, - "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"", + { + "Ref": "AWS::Region", + }, + "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "QueueName", + ], + }, + "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -10121,6 +10187,20 @@ Request a service quota increase for lambda functions", "QueueName", ], }, + ""},"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, + "":{"name":"NPM JS Packument DLQ","queueName":"", + { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "QueueName", + ], + }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -11399,7 +11479,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip", + "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -11422,6 +11502,9 @@ Direct link to the function: /lambda/home#/functions/", "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, }, }, "Handler": "index.handler", @@ -11853,27 +11936,6 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": { - "Properties": { - "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed -for one or more packages, even after repeated re-checks. All versions the registry did serve -have been processed, but a version announced by the feed may be missing until the affected -package publishes again. The follower logs identify the affected packages, which can be -manually ingested using the ReStagePackageVersion function. - -Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", - "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps", - "ComparisonOperator": "GreaterThanOrEqualToThreshold", - "EvaluationPeriods": 1, - "MetricName": "LaggyPackumentGiveUps", - "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", - "Period": 3600, - "Statistic": "Sum", - "Threshold": 1, - "TreatMissingData": "notBreaching", - }, - "Type": "AWS::CloudWatch::Alarm", - }, "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": { "Properties": { "AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed @@ -11894,6 +11956,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": { + "Properties": { + "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by +the CouchDB changes feed. This usually means the registry has an outage or is far behind +the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "MetricName": "MissingPackuments", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Statistic": "Sum", + "Threshold": 1500, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": { "Properties": { "AlarmDescription": { @@ -11974,7 +12055,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701fcb8d7a0dc1df7ff16564a6fd439e58c", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701bce79df81d41d50a0ac4ad780930bcd7", }, "InvocationType": "Event", "ServiceToken": { @@ -11998,7 +12079,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701fcb8d7a0dc1df7ff16564a6fd439e58c", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701bce79df81d41d50a0ac4ad780930bcd7", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -12012,7 +12093,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701fcb8d7a0dc1df7ff16564a6fd439e58c": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701bce79df81d41d50a0ac4ad780930bcd7": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -12039,7 +12120,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip", + "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -12207,6 +12288,64 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": { + "Properties": { + "AlarmDescription": { + "Fn::Join": [ + "", + [ + "The packument processor function is failing (errors or time-outs). Requests in the +packument queue use up their attempts while it fails, and may be given up on early. + +Link to the lambda function: /lambda/home#/functions/", + { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + " + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + ], + ], + }, + "AlarmName": "dev/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "FunctionName", + "Value": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + }, + ], + "EvaluationPeriods": 2, + "MetricName": "Errors", + "Namespace": "AWS/Lambda", + "Period": 900, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": { + "Properties": { + "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions +than the CouchDB changes feed announced, for hours. This usually means the registry has an +outage or is far behind the feed. The packument processor keeps re-checking these packages. + +Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md", + "AlarmName": "dev/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh", + "ComparisonOperator": "GreaterThanThreshold", + "EvaluationPeriods": 1, + "ExtendedStatistic": "p90", + "MetricName": "LaggyPackumentLag", + "Namespace": "ConstructHub/PackageSource/NpmJs/Follower", + "Period": 10800, + "Threshold": 21600000, + "TreatMissingData": "notBreaching", + }, + "Type": "AWS::CloudWatch::Alarm", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -12478,6 +12617,20 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, ], "Version": "2012-10-17", }, @@ -12503,7 +12656,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip", + "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -12829,6 +12982,303 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesPackumentProcessorDLQF401BD77": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 1209600, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesPackumentProcessorFunction0E5F8177": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + }, + "Description": "[dev/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", + "Environment": { + "Variables": { + "AWS_EMF_ENVIRONMENT": "Local", + "DENY_LIST_BUCKET_NAME": { + "Ref": "ConstructHubDenyListBucket1B3C2C2E", + }, + "DENY_LIST_OBJECT_KEY": "deny-list.json", + "FUNCTION_NAME": { + "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + "LICENSE_LIST_BUCKET_NAME": { + "Ref": "ConstructHubLicenseListBucket9334047F", + }, + "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_QUEUE_URL": { + "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 300, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubDenyListBucket1B3C2C2E", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubLicenseListBucket9334047F", + "Arn", + ], + }, + "/*", + ], + ], + }, + ], + }, + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4", + "Roles": [ + { + "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourcedevConstructHubSourcesPackumentProcessorQueueB1811D0E3C6BE98A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorQueue34BDE578", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + "ScalingConfig": { + "MaximumConcurrency": 2, + }, + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, + "ConstructHubSourcesPackumentProcessorQueue34BDE578": { + "DeletionPolicy": "Delete", + "Properties": { + "KmsMasterKeyId": "alias/aws/sqs", + "MessageRetentionPeriod": 345600, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "maxReceiveCount": 9, + }, + "VisibilityTimeout": 1800, + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + }, "ConstructHubSourcesReStagePackageVersion5E1211D1": { "DependsOn": [ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40", @@ -12842,7 +13292,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip", + "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip", }, "Description": "Manually re-stage a package version", "Environment": { diff --git a/src/__tests__/package-sources/npmjs.test.ts b/src/__tests__/package-sources/npmjs.test.ts index a5b6b5edd..033d58a93 100644 --- a/src/__tests__/package-sources/npmjs.test.ts +++ b/src/__tests__/package-sources/npmjs.test.ts @@ -110,6 +110,89 @@ test('the known versions migration can be turned off', () => { ).toHaveLength(0); }); +test('retries laggy packuments through a queue that moves give-ups to a dead-letter queue', () => { + const template = synth(new NpmJs()); + + const dlqs = template.findResources('AWS::SQS::Queue', { + Properties: { MessageRetentionPeriod: 14 * 24 * 60 * 60 }, + }); + template.hasResourceProperties('AWS::SQS::Queue', { + MessageRetentionPeriod: 4 * 24 * 60 * 60, + VisibilityTimeout: 30 * 60, + RedrivePolicy: { + deadLetterTargetArn: { + 'Fn::GetAtt': [Match.stringLikeRegexp('PackumentProcessorDLQ'), 'Arn'], + }, + maxReceiveCount: 9, + }, + }); + expect( + Object.keys(dlqs).some((id) => id.includes('PackumentProcessorDLQ')) + ).toBe(true); + + const processors = template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp( + 'Processes packages from the npm registry' + ), + }, + }); + expect(Object.keys(processors)).toHaveLength(1); + template.hasResourceProperties('AWS::Lambda::EventSourceMapping', { + FunctionName: { Ref: Object.keys(processors)[0] }, + BatchSize: 10, + FunctionResponseTypes: ['ReportBatchItemFailures'], + ScalingConfig: { MaximumConcurrency: 2 }, + }); + + // There is no alarm on the dead-letter queue (it is only graphed), and no + // alarm on the give-up metric. + const alarmNames = Object.values( + template.findResources('AWS::CloudWatch::Alarm') + ).map((alarm: any) => String(alarm.Properties.AlarmName)); + expect( + alarmNames.some( + (name) => + name.endsWith('DLQNotEmpty') && name.includes('PackumentProcessor') + ) + ).toBe(false); + expect( + alarmNames.some((name) => name.endsWith('LaggyPackumentGiveUps')) + ).toBe(false); + + // A burst of missing packuments within 3 hours raises an alarm. + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + AlarmName: 'Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh', + MetricName: 'MissingPackuments', + Period: 3 * 60 * 60, + Statistic: 'Sum', + Threshold: 1_500, + }); + // So does a high laggy packument lag. + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + AlarmName: + 'Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh', + MetricName: 'LaggyPackumentLag', + Period: 3 * 60 * 60, + ExtendedStatistic: 'p90', + Threshold: 6 * 60 * 60 * 1_000, + }); + + // Processor failures raise a low severity alarm too. + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + AlarmName: 'Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures', + MetricName: 'Errors', + }); + + // The follower sends laggy packuments to the queue. + template.hasResourceProperties('AWS::Lambda::Function', { + Description: Match.stringLikeRegexp('Periodically query npmjs.com'), + Environment: { + Variables: Match.objectLike({ PACKUMENT_QUEUE_URL: Match.anyValue() }), + }, + }); +}); + test('the billing mode of the known versions table is configurable', () => { const template = synth( new NpmJs({ diff --git a/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts index cd4698f2a..d3d250522 100644 --- a/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts +++ b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts @@ -14,7 +14,6 @@ describe('text format', () => { state.addCheckpoint(90, T0); state.addCheckpoint(300, T0 + HOUR); state.addSeqs([100, 250]); - state.recordLaggyPackument('@scope/laggy-package', 5, 123, T0); state.completeDeepScan(T0 + HOUR); expect(state.toText()).toBe( @@ -23,7 +22,6 @@ describe('text format', () => { `w ${T0 + HOUR}`, `c ${T0} 90`, `c ${T0 + HOUR} 300`, - `l ${T0} 5 123 @scope/laggy-package`, 's 100', 's 250', '', // trailing newline @@ -35,7 +33,6 @@ describe('text format', () => { const state = new FollowerState(); state.addCheckpoint(90, T0); state.addSeqs([100, 105, 250]); - state.recordLaggyPackument('laggy-package', 7, undefined, T0); state.completeDeepScan(T0); const restored = FollowerState.fromText(state.toText()); @@ -47,16 +44,28 @@ describe('text format', () => { expect(restored.receiptCount).toBe(3); expect(restored.checkpointCount).toBe(1); expect(restored.lastDeepScanAt).toBe(T0); - expect(restored.laggyPackuments()).toEqual([ - { name: 'laggy-package', expectedRev: 7, seq: undefined, firstSeen: T0 }, - ]); + }); + + test('ignores the laggy packuments recorded by earlier versions', () => { + const text = [ + '#chfollower/1', + `w ${T0}`, + `c ${T0} 90`, + `l ${T0} 5 123 @scope/laggy-package`, + 's 100', + '', + ].join('\n'); + + const restored = FollowerState.fromText(text); + + expect(restored.has(100)).toBe(true); + expect(restored.toText()).not.toContain('laggy-package'); }); test('a fresh state round-trips', () => { const restored = FollowerState.fromText(new FollowerState().toText()); expect(restored.receiptCount).toBe(0); expect(restored.checkpointCount).toBe(0); - expect(restored.laggyPackumentCount).toBe(0); expect(restored.lastDeepScanAt).toBeUndefined(); }); @@ -64,7 +73,6 @@ describe('text format', () => { const state = new FollowerState(); state.addCheckpoint(90, T0); state.addSeqs([100, 250, 251]); - state.recordLaggyPackument('laggy-package', 7, 200, T0); const once = state.toText(); const twice = FollowerState.fromText(once).toText(); @@ -171,18 +179,16 @@ describe('checkpoints and receipts', () => { expect(state.has(500)).toBe(true); }); - test('resetPosition clears receipts and checkpoints but keeps laggy packuments', () => { + test('resetPosition clears receipts and checkpoints', () => { const state = new FollowerState(); state.addCheckpoint(100, T0); state.addSeqs([150]); - state.recordLaggyPackument('laggy-package', 5, 120, T0); state.resetPosition(); expect(state.checkpointCount).toBe(0); expect(state.receiptCount).toBe(0); expect(state.newestCheckpointSeq()).toBeUndefined(); - expect(state.laggyPackumentCount).toBe(1); }); }); @@ -198,30 +204,3 @@ describe('deep scans', () => { expect(state.deepScanDue(T0 + DEEP_SCAN_INTERVAL_MS)).toBe(true); }); }); - -describe('laggy packuments', () => { - test('keeps the earliest firstSeen and the highest expected revision', () => { - const state = new FollowerState(); - state.recordLaggyPackument('laggy-package', 5, 100, T0); - state.recordLaggyPackument('laggy-package', 7, 200, T0 + HOUR); - state.recordLaggyPackument('laggy-package', 6, 300, T0 + 2 * HOUR); - - expect(state.laggyPackuments()).toEqual([ - { name: 'laggy-package', expectedRev: 7, seq: 100, firstSeen: T0 }, - ]); - }); - - test('lists expectations oldest first and removes them', () => { - const state = new FollowerState(); - state.recordLaggyPackument('second', 2, undefined, T0 + HOUR); - state.recordLaggyPackument('first', 1, undefined, T0); - - expect(state.laggyPackuments().map((l) => l.name)).toEqual([ - 'first', - 'second', - ]); - - state.removeLaggyPackument('first'); - expect(state.laggyPackumentCount).toBe(1); - }); -}); diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts index e93482311..7892b6068 100644 --- a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts +++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts @@ -10,6 +10,11 @@ import { PutObjectCommand, S3Client, } from '@aws-sdk/client-s3'; +import { + GetQueueAttributesCommand, + SendMessageBatchCommand, + SQSClient, +} from '@aws-sdk/client-sqs'; import type { metricScope, MetricsLogger } from 'aws-embedded-metrics'; import { Unit } from 'aws-embedded-metrics'; import type { Context, ScheduledEvent } from 'aws-lambda'; @@ -23,9 +28,11 @@ import { import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; import { ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_QUEUE_URL, FOLLOWER_STATE_FILE_NAME, MARKER_FILE_NAME, MetricName, + PACKUMENT_INITIAL_DELAY_SECONDS, } from '../../../package-sources/npmjs/constants.lambda-shared'; import { FollowerState } from '../../../package-sources/npmjs/follower-state.lambda-shared'; import { handler } from '../../../package-sources/npmjs/npm-js-follower.lambda'; @@ -59,6 +66,8 @@ const REGISTRY = 'https://registry.npmjs.org'; const MOCK_BUCKET = 'mock-staging-bucket'; const MOCK_STAGING_FUNCTION = 'mock-staging-function'; const MOCK_KNOWN_VERSIONS_TABLE = 'mock-known-versions-table'; +const MOCK_PACKUMENT_QUEUE_URL = + 'https://sqs.us-east-1.amazonaws.com/123456789012/mock-packument-queue'; const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name'; @@ -69,6 +78,7 @@ const HOUR = 60 * 60 * 1_000; const mockS3 = mockClient(S3Client); const mockLambda = mockClient(LambdaClient); const mockDynamoDB = mockClient(DynamoDBClient); +const mockSQS = mockClient(SQSClient); const context: Context = { awsRequestId: 'mock-request-id', @@ -83,11 +93,13 @@ beforeEach(() => { mockS3.reset(); mockLambda.reset(); mockDynamoDB.reset(); + mockSQS.reset(); mockPutMetric.mockClear(); process.env.BUCKET_NAME = MOCK_BUCKET; process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION; process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = MOCK_KNOWN_VERSIONS_TABLE; + process.env[ENV_PACKUMENT_QUEUE_URL] = MOCK_PACKUMENT_QUEUE_URL; process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET; @@ -116,6 +128,16 @@ beforeEach(() => { mockDynamoDB.on(QueryCommand).resolves({ Items: [] }); mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); + // Empty packument queue + mockSQS.on(SendMessageBatchCommand).resolves({ Successful: [], Failed: [] }); + mockSQS.on(GetQueueAttributesCommand).resolves({ + Attributes: { + ApproximateNumberOfMessages: '0', + ApproximateNumberOfMessagesNotVisible: '0', + ApproximateNumberOfMessagesDelayed: '0', + }, + }); + mockS3.on(PutObjectCommand).resolves({}); mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 }); }); @@ -124,6 +146,7 @@ afterEach(() => { delete process.env.BUCKET_NAME; delete process.env.FUNCTION_NAME; delete process.env[ENV_KNOWN_VERSIONS_TABLE_NAME]; + delete process.env[ENV_PACKUMENT_QUEUE_URL]; delete process.env[ENV_DENY_LIST_BUCKET_NAME]; delete process.env[ENV_DENY_LIST_OBJECT_KEY]; delete process.env[LicenseListEnv.BUCKET_NAME]; @@ -328,7 +351,17 @@ test('discovers and stages late-inserted changes', async () => { expect(saved.newestCheckpointSeq()).toBe(1_000); }); -test('laggy packument: processes served versions and records the expectation', async () => { +/** The packument requests sent to the packument queue. */ +function queuedRequests(): any[] { + return mockSQS.commandCalls(SendMessageBatchCommand).flatMap((call) => + call.args[0].input.Entries!.map((entry) => ({ + ...JSON.parse(entry.MessageBody!), + delaySeconds: entry.DelaySeconds, + })) + ); +} + +test('laggy packument: processes served versions and sends it to the packument queue', async () => { givenHead(2_010); givenNoStateFile(); givenLegacyMarker(2_000); @@ -351,101 +384,164 @@ test('laggy packument: processes served versions and records the expectation', a nock(REGISTRY) .get('/laggy-package') .reply(200, packageDoc('laggy-package', '3-ffffff', '1.0.0')); + mockSQS.on(GetQueueAttributesCommand).resolves({ + Attributes: { + ApproximateNumberOfMessages: '0', + ApproximateNumberOfMessagesNotVisible: '0', + ApproximateNumberOfMessagesDelayed: '1', + }, + }); + const before = Date.now(); await handler(event, context); // The versions the registry did serve are processed anyway. expect(stagedPackages()).toEqual([ { name: 'laggy-package', version: '1.0.0' }, ]); - // ... and the expectation of rev 5 is recorded for later re-checks. The - // gauge is emitted at the end of the run and includes it. + // ... and a request to check for rev 5 is queued, delayed. + const [request] = queuedRequests(); + expect(request).toMatchObject({ + name: 'laggy-package', + expectedRev: 5, + seq: 2_005, + reason: 'laggy', + delaySeconds: PACKUMENT_INITIAL_DELAY_SECONDS, + }); + expect(request.firstSeen).toBeGreaterThanOrEqual(before); + expect(mockSQS).toHaveReceivedCommandWith(SendMessageBatchCommand, { + QueueUrl: MOCK_PACKUMENT_QUEUE_URL, + }); + // The gauge reports the size of the packument queue. expect(mockPutMetric).toHaveBeenCalledWith( MetricName.LAGGY_PACKUMENTS, 1, Unit.Count ); - const state = savedState(); - expect(state.laggyPackuments()).toEqual([ - expect.objectContaining({ - name: 'laggy-package', - expectedRev: 5, - seq: 2_005, - }), - ]); + // The receipt is recorded. + expect(savedState().has(2_005)).toBe(true); }); -test('laggy packument: recovered once the registry catches up', async () => { - const now = Date.now(); - givenHead(3_000); +test('laggy packument: a failure to queue it leaves the entry without a receipt', async () => { + givenHead(2_010); + givenNoStateFile(); + givenLegacyMarker(2_000); + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '2000' }) + .reply(200, { + results: [ + { + seq: 2_005, + id: 'laggy-package', + changes: [{ rev: '5-eeeeee' }], + deleted: false, + }, + ], + last_seq: 2_005, + }); + nock(REGISTRY) + .get('/laggy-package') + .reply(200, packageDoc('laggy-package', '3-ffffff', '1.0.0')); + mockSQS.on(SendMessageBatchCommand).rejects(new Error('network')); - const state = new FollowerState(); - state.addCheckpoint(3_000, now - 5 * 60_000); - state.completeDeepScan(now - HOUR); - state.recordLaggyPackument('recovered-package', 4, 1_500, now - HOUR); - givenStateFile(state); + await expect(handler(event, context)).rejects.toThrow(/network/); - // The registry has now caught up (rev 4 >= expected 4), serving a version - // we don't know yet. - nock(REGISTRY) - .get('/recovered-package') - .reply(200, packageDoc('recovered-package', '4-gggggg', '2.0.0')); - // The scan itself is empty. + // The next run processes the entry again. + expect(savedState().has(2_005)).toBe(false); +}); + +test('missing packument: a change entry the registry has no packument for yet is sent to the packument queue', async () => { + givenHead(2_010); + givenNoStateFile(); + givenLegacyMarker(2_000); nock(REPLICA) .get('/registry/_changes') - .query({ limit: '10000', since: '3000' }) - .reply(200, { results: [], last_seq: 3_000 }); + .query({ limit: '10000', since: '2000' }) + .reply(200, { + results: [ + { + seq: 2_005, + id: 'brand-new-package', + changes: [{ rev: '1-aaaaaa' }], + deleted: false, + }, + ], + last_seq: 2_005, + }); + nock(REGISTRY).get('/brand-new-package').reply(404, 'not found'); await handler(event, context); - expect(stagedPackages()).toEqual([ - { name: 'recovered-package', version: '2.0.0' }, + expect(stagedPackages()).toEqual([]); + expect(queuedRequests()).toEqual([ + expect.objectContaining({ + name: 'brand-new-package', + expectedRev: 1, + seq: 2_005, + reason: 'missing', + }), ]); expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENTS_RECOVERED, + MetricName.MISSING_PACKUMENTS, 1, Unit.Count ); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENTS, - 0, - Unit.Count - ); - expect(savedState().laggyPackumentCount).toBe(0); + expect(savedState().has(2_005)).toBe(true); }); -test('laggy packument: gives up after the maximum age, still processing what is served', async () => { - const now = Date.now(); - givenHead(3_000); +test('missing packument: an entry that announces no revision expects any packument', async () => { + givenHead(2_010); + givenNoStateFile(); + givenLegacyMarker(2_000); + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '2000' }) + .reply(200, { + results: [ + { seq: 2_005, id: 'no-rev-package', changes: [], deleted: false }, + ], + last_seq: 2_005, + }); + nock(REGISTRY).get('/no-rev-package').reply(404, 'not found'); - const state = new FollowerState(); - state.addCheckpoint(3_000, now - 5 * 60_000); - state.completeDeepScan(now - HOUR); - // First seen 25 hours ago: beyond the give-up age. - state.recordLaggyPackument('stuck-package', 9, 1_600, now - 25 * HOUR); - givenStateFile(state); + await handler(event, context); - // The registry still serves rev 7 < expected 9 - but with a version we - // don't know, which is processed regardless. - nock(REGISTRY) - .get('/stuck-package') - .reply(200, packageDoc('stuck-package', '7-hhhhhh', '3.0.0')); + expect(queuedRequests()).toEqual([ + expect.objectContaining({ name: 'no-rev-package', expectedRev: 0 }), + ]); +}); + +test('ignores change entries for deleted packages, as if they were not in the feed', async () => { + givenHead(2_010); + givenNoStateFile(); + givenLegacyMarker(2_000); nock(REPLICA) .get('/registry/_changes') - .query({ limit: '10000', since: '3000' }) - .reply(200, { results: [], last_seq: 3_000 }); + .query({ limit: '10000', since: '2000' }) + .reply(200, { + results: [ + { + seq: 2_005, + id: 'deleted-package', + changes: [{ rev: '3-aaaaaa' }], + deleted: true, + }, + ], + last_seq: 2_005, + }); await handler(event, context); - expect(stagedPackages()).toEqual([ - { name: 'stuck-package', version: '3.0.0' }, - ]); + // No packument fetch (nock would fail on an unexpected request), nothing + // queued, nothing counted, and no receipt. + expect(queuedRequests()).toEqual([]); expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENT_GIVE_UPS, - 1, + MetricName.CHANGE_COUNT, + 0, Unit.Count ); - expect(savedState().laggyPackumentCount).toBe(0); + expect(savedState().has(2_005)).toBe(false); }); test('seeds at the beginning of the feed on first deployment (automatic backfill)', async () => { diff --git a/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts b/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts new file mode 100644 index 000000000..74d05e558 --- /dev/null +++ b/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts @@ -0,0 +1,475 @@ +import { + BatchWriteItemCommand, + DynamoDBClient, + QueryCommand, +} from '@aws-sdk/client-dynamodb'; +import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { ChangeMessageVisibilityCommand, SQSClient } from '@aws-sdk/client-sqs'; +import type { metricScope, MetricsLogger } from 'aws-embedded-metrics'; +import { Unit } from 'aws-embedded-metrics'; +import type { Context, SQSEvent, SQSRecord } from 'aws-lambda'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import nock from 'nock'; +import { + ENV_DENY_LIST_BUCKET_NAME, + ENV_DENY_LIST_OBJECT_KEY, +} from '../../../backend/deny-list/constants'; +import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_QUEUE_URL, + MetricName, + PACKUMENT_MAX_ATTEMPTS, + PACKUMENT_RETRY_BACKOFF_SECONDS, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import type { PackumentRequest } from '../../../package-sources/npmjs/packument-processing.lambda-shared'; +import { handler } from '../../../package-sources/npmjs/packument-processor.lambda'; +import { stringToStream } from '../../streams'; + +jest.mock('aws-embedded-metrics'); + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const mockMetricScope = require('aws-embedded-metrics') + .metricScope as jest.MockedFunction; +const mockPutMetric = jest.fn() as jest.MockedFunction< + MetricsLogger['putMetric'] +>; +const mockMetrics: MetricsLogger = { + putMetric: mockPutMetric, + setDimensions: jest.fn(), + setProperty: jest.fn(), +} as any; +mockMetricScope.mockImplementation((cb) => { + const impl = cb(mockMetrics); + return async (...args) => impl(...args); +}); + +const REGISTRY = 'https://registry.npmjs.org'; +const QUEUE_URL = + 'https://sqs.us-east-1.amazonaws.com/123456789012/mock-packument-queue'; +const TABLE = 'mock-known-versions-table'; +const STAGING_FUNCTION = 'mock-staging-function'; +const DENY_LIST_BUCKET = 'deny-list-bucket-name'; +const DENY_LIST_OBJECT = 'deny-list.json'; +const LICENSE_LIST_BUCKET = 'license-list-bucket-name'; +const LICENSE_LIST_OBJECT = 'license-list.json'; + +const mockS3 = mockClient(S3Client); +const mockLambda = mockClient(LambdaClient); +const mockDynamoDB = mockClient(DynamoDBClient); +const mockSQS = mockClient(SQSClient); + +const context: Context = {} as any; + +beforeEach(() => { + mockS3.reset(); + mockLambda.reset(); + mockDynamoDB.reset(); + mockSQS.reset(); + mockPutMetric.mockClear(); + + process.env.FUNCTION_NAME = STAGING_FUNCTION; + process.env[ENV_PACKUMENT_QUEUE_URL] = QUEUE_URL; + process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = TABLE; + process.env[ENV_DENY_LIST_BUCKET_NAME] = DENY_LIST_BUCKET; + process.env[ENV_DENY_LIST_OBJECT_KEY] = DENY_LIST_OBJECT; + process.env[LicenseListEnv.BUCKET_NAME] = LICENSE_LIST_BUCKET; + process.env[LicenseListEnv.OBJECT_KEY] = LICENSE_LIST_OBJECT; + + mockS3 + .on(GetObjectCommand, { Bucket: DENY_LIST_BUCKET, Key: DENY_LIST_OBJECT }) + .resolves({ Body: stringToStream(JSON.stringify({})) }); + mockS3 + .on(GetObjectCommand, { + Bucket: LICENSE_LIST_BUCKET, + Key: LICENSE_LIST_OBJECT, + }) + .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) }); + mockDynamoDB.on(QueryCommand).resolves({ Items: [] }); + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); + mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 }); + mockSQS.on(ChangeMessageVisibilityCommand).resolves({}); +}); + +afterEach(() => { + for (const name of [ + 'FUNCTION_NAME', + ENV_PACKUMENT_QUEUE_URL, + ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_DENY_LIST_BUCKET_NAME, + ENV_DENY_LIST_OBJECT_KEY, + LicenseListEnv.BUCKET_NAME, + LicenseListEnv.OBJECT_KEY, + ]) { + delete process.env[name]; + } + nock.cleanAll(); +}); + +function record( + messageId: string, + request: Partial | string, + attempt = 1 +): SQSRecord { + return { + messageId, + receiptHandle: `receipt-${messageId}`, + body: + typeof request === 'string' + ? request + : JSON.stringify({ + seq: 1_000, + firstSeen: 1_700_000_000_000, + ...request, + }), + attributes: { ApproximateReceiveCount: attempt.toString() }, + } as any; +} + +function packageDoc(name: string, rev: string, ...versions: string[]) { + const time: Record = { + created: '2023-01-01T00:00:00.000Z', + modified: '2023-06-01T00:00:00.000Z', + }; + const versionInfos: Record = {}; + for (const [i, version] of versions.entries()) { + time[version] = `2023-06-01T00:0${i}:00.000Z`; + versionInfos[version] = { + dependencies: { constructs: '^10.0.0' }, + dist: { + shasum: 'mock-shasum', + tarball: `${REGISTRY}/${name}/-/${name}-${version}.tgz`, + }, + jsii: {}, + keywords: ['cdk'], + license: 'Apache-2.0', + name, + version, + }; + } + return { + _id: name, + _rev: rev, + 'dist-tags': { latest: versions[versions.length - 1] }, + name, + time, + versions: versionInfos, + }; +} + +function stagedPackages(): Array<{ name: string; version: string }> { + return mockLambda.commandCalls(InvokeCommand).map((call) => { + const payload = JSON.parse( + Buffer.from(call.args[0].input.Payload! as Uint8Array).toString('utf-8') + ); + return { name: payload.name, version: payload.version }; + }); +} + +function event(...records: SQSRecord[]): SQSEvent { + return { Records: records }; +} + +test('completes a request once the registry serves the expected revision', async () => { + nock(REGISTRY) + .get('/caught-up') + .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0', '1.1.0')); + + const result = await handler( + event(record('m1', { name: 'caught-up', expectedRev: 5 })), + context + ); + + expect(result.batchItemFailures).toEqual([]); + expect(stagedPackages()).toEqual([ + { name: 'caught-up', version: '1.0.0' }, + { name: 'caught-up', version: '1.1.0' }, + ]); + expect(mockDynamoDB).toHaveReceivedCommand(BatchWriteItemCommand); + expect(mockSQS).not.toHaveReceivedCommand(ChangeMessageVisibilityCommand); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + 1, + Unit.Count + ); +}); + +test('processes what is served, and retries later with back-off while the registry is behind', async () => { + nock(REGISTRY) + .get('/still-laggy') + .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0')); + + const result = await handler( + event(record('m1', { name: 'still-laggy', expectedRev: 5 }, 3)), + context + ); + + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); + expect(stagedPackages()).toEqual([{ name: 'still-laggy', version: '1.0.0' }]); + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + QueueUrl: QUEUE_URL, + ReceiptHandle: 'receipt-m1', + VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[2], + }); +}); + +test('gives up on the last attempt, so the request moves to the dead-letter queue', async () => { + nock(REGISTRY) + .get('/stuck') + .reply(200, packageDoc('stuck', '3-aaaaaa', '1.0.0')); + + const result = await handler( + event( + record('m1', { name: 'stuck', expectedRev: 5 }, PACKUMENT_MAX_ATTEMPTS) + ), + context + ); + + // Failing the message on its last receive moves it to the dead-letter + // queue; it becomes visible right away so that happens promptly. + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + VisibilityTimeout: 0, + }); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); + // What the registry served was still processed. + expect(stagedPackages()).toEqual([{ name: 'stuck', version: '1.0.0' }]); +}); + +test('retries a request while the registry has no packument for the package yet', async () => { + nock(REGISTRY).get('/not-yet').reply(404, 'not found'); + + const result = await handler( + event(record('m1', { name: 'not-yet', expectedRev: 1, reason: 'missing' })), + context + ); + + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[0], + }); + expect(stagedPackages()).toEqual([]); +}); + +test('counts missing packuments separately from laggy ones', async () => { + nock(REGISTRY) + .get('/arrived') + .reply(200, packageDoc('arrived', '1-aaaaaa', '1.0.0')); + nock(REGISTRY).get('/never-arrived').reply(404, 'not found'); + + const result = await handler( + event( + record('m1', { name: 'arrived', expectedRev: 1, reason: 'missing' }), + record( + 'm2', + { name: 'never-arrived', expectedRev: 1, reason: 'missing' }, + PACKUMENT_MAX_ATTEMPTS + ) + ), + context + ); + + // Both kinds of give-up go to the dead-letter queue. + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm2' }]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.MISSING_PACKUMENTS_RECOVERED, + 1, + Unit.Count + ); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.MISSING_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + 0, + Unit.Count + ); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + 0, + Unit.Count + ); +}); + +test('retries a request that failed with an error, without failing the others', async () => { + nock(REGISTRY).get('/broken').reply(403, 'forbidden'); + nock(REGISTRY) + .get('/fine') + .reply(200, packageDoc('fine', '5-aaaaaa', '1.0.0')); + + const result = await handler( + event( + record('m1', { name: 'broken', expectedRev: 5 }), + record('m2', { name: 'fine', expectedRev: 5 }), + record('m3', 'not json') + ), + context + ); + + expect(result.batchItemFailures).toEqual([ + { itemIdentifier: 'm1' }, + { itemIdentifier: 'm3' }, + ]); + expect(stagedPackages()).toEqual([{ name: 'fine', version: '1.0.0' }]); +}); + +test('does not stage versions that are already known', async () => { + nock(REGISTRY) + .get('/known') + .reply(200, packageDoc('known', '5-aaaaaa', '1.0.0', '1.1.0')); + mockDynamoDB + .on(QueryCommand) + .resolves({ Items: [{ version: { S: '1.0.0' } }] }); + + await handler( + event(record('m1', { name: 'known', expectedRev: 5 })), + context + ); + + expect(stagedPackages()).toEqual([{ name: 'known', version: '1.1.0' }]); +}); + +test('keeps the back-off when the whole batch fails to set up', async () => { + mockS3 + .on(GetObjectCommand, { Bucket: DENY_LIST_BUCKET, Key: DENY_LIST_OBJECT }) + .rejects(new Error('s3 is down')); + + await expect( + handler( + event( + record('m1', { name: 'one', expectedRev: 5 }, 2), + record('m2', { name: 'two', expectedRev: 5 }, 4) + ), + context + ) + ).rejects.toThrow(/s3 is down/); + + // Both messages get their normal next attempt, instead of returning after + // the queue's visibility timeout and using up their attempts quickly. + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + ReceiptHandle: 'receipt-m1', + VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[1], + }); + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + ReceiptHandle: 'receipt-m2', + VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[3], + }); +}); + +test('counts an error on the last attempt as a give-up', async () => { + nock(REGISTRY).get('/broken').reply(403, 'forbidden'); + + const result = await handler( + event( + record('m1', { name: 'broken', expectedRev: 5 }, PACKUMENT_MAX_ATTEMPTS) + ), + context + ); + + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); +}); + +test('checks caught-up packages again when their versions could not be recorded', async () => { + nock(REGISTRY) + .get('/caught-up') + .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0')); + mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('dynamodb is down')); + + const result = await handler( + event(record('m1', { name: 'caught-up', expectedRev: 5 })), + context + ); + + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[0], + }); +}); + +test('does not dead-letter a caught-up request when recording fails on its last attempt', async () => { + nock(REGISTRY) + .get('/caught-up') + .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0')); + mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('dynamodb is down')); + + const result = await handler( + event( + record( + 'm1', + { name: 'caught-up', expectedRev: 5 }, + PACKUMENT_MAX_ATTEMPTS + ) + ), + context + ); + + expect(result.batchItemFailures).toEqual([]); +}); + +test('dead-letters a missing request whose last check failed', async () => { + nock(REGISTRY).get('/unknown').reply(403, 'forbidden'); + + const result = await handler( + event( + record( + 'm1', + { name: 'unknown', expectedRev: 1, reason: 'missing' }, + PACKUMENT_MAX_ATTEMPTS + ) + ), + context + ); + + expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + VisibilityTimeout: 0, + }); +}); + +test('reports how long laggy packuments took to catch up', async () => { + nock(REGISTRY) + .get('/caught-up') + .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0')); + const firstSeen = Date.now() - 2 * 60 * 60 * 1_000; + + await handler( + event(record('m1', { name: 'caught-up', expectedRev: 5, firstSeen })), + context + ); + + const lag = mockPutMetric.mock.calls.find( + ([name]) => name === MetricName.LAGGY_PACKUMENT_LAG + ); + expect(lag?.[1]).toBeGreaterThanOrEqual(2 * 60 * 60 * 1_000); + expect(lag?.[2]).toBe(Unit.Milliseconds); +}); + +test('does not report a lag for missing packuments', async () => { + nock(REGISTRY) + .get('/arrived') + .reply(200, packageDoc('arrived', '1-aaaaaa', '1.0.0')); + + await handler( + event(record('m1', { name: 'arrived', expectedRev: 1, reason: 'missing' })), + context + ); + + expect(mockPutMetric).not.toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_LAG, + expect.anything(), + expect.anything() + ); +}); diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index 5ae6a3b33..ccecf4ed1 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -21,7 +21,7 @@ import { } from 'aws-cdk-lib/aws-dynamodb'; import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets'; -import { Tracing } from 'aws-cdk-lib/aws-lambda'; +import { IFunction, Tracing } from 'aws-cdk-lib/aws-lambda'; import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; import { BlockPublicAccess, IBucket } from 'aws-cdk-lib/aws-s3'; import { Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs'; @@ -57,6 +57,7 @@ import { } from './npmjs/constants.lambda-shared'; import { KnownVersionsTableMigration } from './npmjs/known-versions-table-migration'; import { NpmJsFollower } from './npmjs/npm-js-follower'; +import { PackumentQueueProcessor } from './npmjs/packument-queue-processor'; import { StageAndNotify } from './npmjs/stage-and-notify'; import { S3StorageFactory } from '../s3/storage'; import { ReStagePackageVersion } from './npmjs/re-stage-package-version'; @@ -82,6 +83,19 @@ const FOLLOWER_RUN_RATE = Duration.minutes(5); */ const NO_CHANGES_ALARM_DURATION = Duration.hours(24); +/** + * Alarm if more change entries than this had no packument in the registry + * within 3 hours. In October 2026, a 3 hour window usually saw 170 of them, + * and at most about 500. Well above that means a registry outage. + */ +const MISSING_PACKUMENTS_ALARM_THRESHOLD = 1_500; + +/** + * Alarm if 90% of the laggy packuments that caught up (or were given up on) + * within 3 hours took longer than this. + */ +const LAGGY_PACKUMENT_LAG_ALARM_THRESHOLD = Duration.hours(6); + export interface NpmJsProps { /** * The bucket to use for staging npm packages. @@ -283,6 +297,16 @@ export class NpmJs implements IPackageSource { licenseList.grantRead(follower); stager.grantInvoke(follower); + // Requests to process a package from the registry again, because the + // registry served an older revision than the changes feed announced, or + // no packument at all yet. + const packuments = new PackumentQueueProcessor( + scope, + 'PackumentProcessor', + { knownVersions, stager, denyList, licenseList } + ); + packuments.grantSendRequests(follower); + const restager = new ReStagePackageVersion(scope, 'ReStagePackageVersion', { description: `Manually re-stage a package version`, environment: { @@ -300,7 +324,14 @@ export class NpmJs implements IPackageSource { targets: [new LambdaFunction(follower)], }); - this.registerAlarms(scope, follower, stager, monitoring, rule); + this.registerAlarms( + scope, + follower, + stager, + packuments.function, + monitoring, + rule + ); stager.deadLetterQueue && overviewDashboard.addDLQMetricToDashboard( @@ -312,6 +343,10 @@ export class NpmJs implements IPackageSource { 'NPM JS Follower DLQ', follower.deadLetterQueue ); + overviewDashboard.addDLQMetricToDashboard( + 'NPM JS Packument DLQ', + packuments.deadLetterQueue + ); overviewDashboard.addConcurrentExecutionMetricToDashboard( follower, 'NpmJsLambda' @@ -343,6 +378,15 @@ export class NpmJs implements IPackageSource { }, { name: 'Stager', url: lambdaFunctionUrl(stager) }, { name: 'Stager DLQ', url: sqsQueueUrl(stager.deadLetterQueue!) }, + { name: 'Packument Queue', url: sqsQueueUrl(packuments.queue) }, + { + name: 'Packument DLQ', + url: sqsQueueUrl(packuments.deadLetterQueue), + }, + { + name: 'Packument Processor', + url: lambdaFunctionUrl(packuments.function), + }, { name: 'Pipeline Trace (Log Analytics)', url: logAnalyticsUrl( @@ -453,6 +497,22 @@ export class NpmJs implements IPackageSource { }), 0 ), + fillMetric( + this.metricMissingPackuments({ label: 'Missing Packuments' }), + 0 + ), + fillMetric( + this.metricMissingPackumentsRecovered({ + label: 'Missing Packuments Recovered', + }), + 0 + ), + fillMetric( + this.metricMissingPackumentGiveUps({ + label: 'Missing Packument Give-Ups', + }), + 0 + ), fillMetric( this.metricMetadataFetchFailures({ label: 'Metadata Fetch Failures', @@ -474,6 +534,39 @@ export class NpmJs implements IPackageSource { rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false }, period: Duration.minutes(5), }), + new GraphWidget({ + height: 6, + width: 12, + title: 'Packument Queue', + // SQS reports these every 5 minutes, so no gaps need filling. + left: [ + packuments.queue.metricApproximateNumberOfMessagesVisible({ + label: 'Waiting', + }), + packuments.queue.metricApproximateNumberOfMessagesNotVisible({ + label: 'Waiting for next attempt', + }), + packuments.queue.metricApproximateNumberOfMessagesDelayed({ + label: 'Delayed', + }), + packuments.deadLetterQueue.metricApproximateNumberOfMessagesVisible( + { label: 'Given up (DLQ)' } + ), + ], + leftYAxis: { min: 0 }, + right: [ + this.metricLaggyPackumentLag({ label: 'Laggy Packument Lag (p90)' }), + ], + rightAnnotations: [ + { + color: '#ff0000', + label: 'Alarm', + value: LAGGY_PACKUMENT_LAG_ALARM_THRESHOLD.toMilliseconds(), + }, + ], + rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false }, + period: Duration.minutes(5), + }), ], [ new GraphWidget({ @@ -605,10 +698,9 @@ export class NpmJs implements IPackageSource { } /** - * The number of packages for which the registry packument is still behind - * the revision announced by the `_changes` feed. The versions served so far - * have been processed; the follower keeps re-checking for the announced - * revision. + * The number of requests in the packument queue, for laggy and missing + * packuments. A package that lagged in several change entries has several + * requests. */ public metricLaggyPackuments(opts?: MetricOptions): Metric { return new Metric({ @@ -635,7 +727,7 @@ export class NpmJs implements IPackageSource { } /** - * The number of laggy packuments the follower gave up on: the registry + * The number of laggy packuments given up on: the registry * never served the revision announced by the `_changes` feed within the * maximum retry age. A version announced by the feed may be missing until * the affected package publishes again. @@ -650,6 +742,63 @@ export class NpmJs implements IPackageSource { }); } + /** + * How long laggy packuments took to catch up (or until they were given up + * on), measured from when the follower first saw them. + */ + public metricLaggyPackumentLag(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.p(90), + ...opts, + metricName: MetricName.LAGGY_PACKUMENT_LAG, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of change entries for which the registry had no packument at + * all (HTTP 404). Usually a new package that has not replicated to the + * registry yet; the packument processor keeps re-checking for it. + */ + public metricMissingPackuments(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.MISSING_PACKUMENTS, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of missing packuments the registry has since served. + */ + public metricMissingPackumentsRecovered(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.MISSING_PACKUMENTS_RECOVERED, + namespace: METRICS_NAMESPACE, + }); + } + + /** + * The number of missing packuments given up on: the registry never served a + * packument for the package within the maximum retry age. Usually an old + * change entry for a package that no longer exists. + */ + public metricMissingPackumentGiveUps(opts?: MetricOptions): Metric { + return new Metric({ + period: Duration.minutes(5), + statistic: Stats.SUM, + ...opts, + metricName: MetricName.MISSING_PACKUMENT_GIVE_UPS, + namespace: METRICS_NAMESPACE, + }); + } + public metricNpmJsChangeAge(opts?: MetricOptions): Metric { return new Metric({ period: Duration.minutes(1), @@ -731,6 +880,7 @@ export class NpmJs implements IPackageSource { scope: Construct, follower: NpmJsFollower, stager: StageAndNotify, + packumentProcessor: IFunction, monitoring: IMonitoring, schedule: Rule ) { @@ -848,27 +998,72 @@ export class NpmJs implements IPackageSource { lateChangeLagAlarm ); - const laggyPackumentGiveUpsAlarm = this.metricLaggyPackumentGiveUps({ - period: Duration.hours(1), - }).createAlarm(scope, 'NpmJs/Follower/LaggyPackumentGiveUps', { - alarmName: `${scope.node.path}/NpmJs/Follower/LaggyPackumentGiveUps`, + const missingPackumentsAlarm = this.metricMissingPackuments({ + period: Duration.hours(3), + }).createAlarm(scope, 'NpmJs/Follower/MissingPackumentsHigh', { + alarmName: `${scope.node.path}/NpmJs/Follower/MissingPackumentsHigh`, alarmDescription: [ - 'The npm registry never served the packument revision announced by the CouchDB changes feed', - 'for one or more packages, even after repeated re-checks. All versions the registry did serve', - 'have been processed, but a version announced by the feed may be missing until the affected', - 'package publishes again. The follower logs identify the affected packages, which can be', - 'manually ingested using the ReStagePackageVersion function.', + 'The npm registry had no packument for an unusually high number of packages announced by', + 'the CouchDB changes feed. This usually means the registry has an outage or is far behind', + 'the feed. The packument processor keeps re-checking these packages.', '', `Runbook: ${RUNBOOK_URL}`, ].join('\n'), - comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD, evaluationPeriods: 1, - threshold: 1, + threshold: MISSING_PACKUMENTS_ALARM_THRESHOLD, treatMissingData: TreatMissingData.NOT_BREACHING, }); monitoring.addLowSeverityAlarm( - 'NpmJs/Follower gave up on laggy packuments', - laggyPackumentGiveUpsAlarm + 'NpmJs/Follower Missing Packuments High', + missingPackumentsAlarm + ); + + const laggyPackumentLagAlarm = this.metricLaggyPackumentLag({ + period: Duration.hours(3), + statistic: Stats.p(90), + }).createAlarm(scope, 'NpmJs/PackumentProcessor/LaggyPackumentLagHigh', { + alarmName: `${scope.node.path}/NpmJs/PackumentProcessor/LaggyPackumentLagHigh`, + alarmDescription: [ + 'Laggy packuments take unusually long to catch up: the npm registry serves older revisions', + 'than the CouchDB changes feed announced, for hours. This usually means the registry has an', + 'outage or is far behind the feed. The packument processor keeps re-checking these packages.', + '', + `Runbook: ${RUNBOOK_URL}`, + ].join('\n'), + comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD, + evaluationPeriods: 1, + threshold: LAGGY_PACKUMENT_LAG_ALARM_THRESHOLD.toMilliseconds(), + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + monitoring.addLowSeverityAlarm( + 'NpmJs/PackumentProcessor Laggy Packument Lag High', + laggyPackumentLagAlarm + ); + + const packumentProcessorFailuresAlarm = packumentProcessor + .metricErrors({ period: Duration.minutes(15) }) + .createAlarm(scope, 'NpmJs/PackumentProcessor/Failures', { + alarmName: `${scope.node.path}/NpmJs/PackumentProcessor/Failures`, + alarmDescription: [ + 'The packument processor function is failing (errors or time-outs). Requests in the', + 'packument queue use up their attempts while it fails, and may be given up on early.', + '', + `Link to the lambda function: ${lambdaFunctionUrl( + packumentProcessor + )}`, + '', + `Runbook: ${RUNBOOK_URL}`, + ].join('\n'), + comparisonOperator: + ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + evaluationPeriods: 2, + threshold: 1, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + monitoring.addLowSeverityAlarm( + 'NpmJs/PackumentProcessor Failures', + packumentProcessorFailuresAlarm ); // Finally - the "not running" alarm depends on the schedule (it won't run until the schedule diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index c32054769..9c57a0222 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -54,13 +54,39 @@ export const enum MetricName { LAGGY_PACKUMENTS_RECOVERED = 'LaggyPackumentsRecovered', /** - * Number of laggy packuments the follower gave up on: the registry never + * Number of laggy packuments given up on: the registry never * served the revision announced by the `_changes` feed within the maximum * retry age. Any versions the registry did serve have been processed, but a * version announced by the feed may be missing until the package publishes * again. */ LAGGY_PACKUMENT_GIVE_UPS = 'LaggyPackumentGiveUps', + + /** + * How long a laggy packument took to catch up (or until it was given up + * on), measured from when the follower first saw it. + */ + LAGGY_PACKUMENT_LAG = 'LaggyPackumentLag', + + /** + * Number of change entries for which the registry had no packument at all + * (HTTP 404) when the follower processed them. Usually a new package that + * has not replicated to the registry yet; they are re-checked like laggy + * packuments. + */ + MISSING_PACKUMENTS = 'MissingPackuments', + + /** + * Number of missing packuments the registry has since served. + */ + MISSING_PACKUMENTS_RECOVERED = 'MissingPackumentsRecovered', + + /** + * Number of missing packuments given up on: the registry never served a + * packument for the package within the maximum retry age. Usually an old + * change entry for a deleted package. + */ + MISSING_PACKUMENT_GIVE_UPS = 'MissingPackumentGiveUps', } export const enum S3KeyPrefix { @@ -133,8 +159,46 @@ export const DEEP_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1_000; // 24 hours export const STATE_RETENTION_MS = 24 * 60 * 60 * 1_000; // 24 hours /** - * How long the follower keeps re-checking a laggy packument (a package whose - * registry packument is behind the revision announced by the `_changes` feed) - * before giving up on the announced revision. + * The npm registry replica serving the `_changes` feed. + */ +export const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/'; + +/** + * The environment variable holding the URL of the packument queue: requests + * to process a package from the registry, retried with back-off until the + * registry serves the revision announced by the `_changes` feed. + */ +export const ENV_PACKUMENT_QUEUE_URL = 'PACKUMENT_QUEUE_URL'; + +/** + * How long a new packument request waits before it is first processed. + */ +export const PACKUMENT_INITIAL_DELAY_SECONDS = 5 * 60; // 5 minutes + +/** + * How long a packument request waits before the next attempt, after an + * attempt that did not find the announced revision. Entry `n` is the wait + * after attempt `n + 1`. SQS caps a message's visibility timeout at 12 hours + * from when it was received, so no wait may be that long. + * + * Together with the initial delay, the last attempt happens about 27 hours + * after the request was made. A request that still finds an old revision + * then is given up on, and moves to the dead-letter queue. + */ +export const PACKUMENT_RETRY_BACKOFF_SECONDS: readonly number[] = [ + 5 * 60, // 5 minutes + 15 * 60, // 15 minutes + 30 * 60, // 30 minutes + 60 * 60, // 1 hour + 2 * 60 * 60, // 2 hours + 4 * 60 * 60, // 4 hours + 8 * 60 * 60, // 8 hours + 11 * 60 * 60, // 11 hours +]; + +/** + * The number of attempts made for a packument request before it moves to the + * dead-letter queue (the queue's `maxReceiveCount`). */ -export const LAGGY_PACKUMENT_GIVE_UP_MS = 24 * 60 * 60 * 1_000; // 24 hours +export const PACKUMENT_MAX_ATTEMPTS = + PACKUMENT_RETRY_BACKOFF_SECONDS.length + 1; diff --git a/src/package-sources/npmjs/couch-changes.lambda-shared.ts b/src/package-sources/npmjs/couch-changes.lambda-shared.ts index 243ba742b..235e5d658 100644 --- a/src/package-sources/npmjs/couch-changes.lambda-shared.ts +++ b/src/package-sources/npmjs/couch-changes.lambda-shared.ts @@ -109,26 +109,32 @@ export class CouchChanges extends EventEmitter { /** * Fetches the registry metadata for each of the provided change entries and - * attaches it to the entry. Entries for deleted or unreachable packages are - * dropped. + * attaches it to the entry. Entries for deleted packages are dropped. * * The registry is a separate system from the changes feed and can serve a * packument revision older than the revision the feed announced (a "laggy * packument"). Such entries are returned with `laggy: true`: the caller * should process the versions that were served, and separately track the * expectation that the announced revision will eventually appear. + * + * The registry can also have no packument at all yet (HTTP 404), usually + * because a new package has not replicated to it yet. Such entries are + * returned in `missing`, and should be tracked the same way. */ public async attachAllMetadata( changes: readonly DatabaseChange[] ): Promise { const ok = new Array(); + const missing = new Array(); const failedSeqs = new Array(); await Promise.all( changes.map(async (change) => { try { const outcome = await this.attachMetadata(change); - if (outcome !== undefined) { - ok.push(outcome); + if (outcome?.attached) { + ok.push(outcome.attached); + } else if (outcome?.missing) { + missing.push(outcome.missing); } } catch (error) { // One unreachable packument must not fail the whole batch. The @@ -144,12 +150,16 @@ export class CouchChanges extends EventEmitter { } }) ); - return { ok, failedSeqs }; + return { ok, missing, failedSeqs }; } private async attachMetadata( change: DatabaseChange - ): Promise { + ): Promise< + | { attached: AttachedChange; missing?: undefined } + | { attached?: undefined; missing: MissingPackument } + | undefined + > { // Filter out deleted packages or null ids if (change.deleted || !change.id) { console.log(`Skipping ${change.id}: deleted or null id`); @@ -168,9 +178,11 @@ export class CouchChanges extends EventEmitter { } catch (e: any) { if (e.message?.includes('HTTP 404')) { console.log( - `Skipping ${change.id} because of HTTP 404 (Not Found) error` + `${change.id}: the registry has no packument for it yet (HTTP 404)` ); - return undefined; + return { + missing: { change, announcedRev: getMaxSequentialRevision(change) }, + }; } throw e; } @@ -191,10 +203,12 @@ export class CouchChanges extends EventEmitter { change.doc = meta; // add metadata to the change object return { - change, - announcedRev, - servedRev, - laggy: servedRev < announcedRev, + attached: { + change, + announcedRev, + servedRev, + laggy: servedRev < announcedRev, + }, }; } @@ -344,7 +358,9 @@ export function parseSequentialRevision(rev: string): number { * The highest sequential revision announced by a change entry. */ export function getMaxSequentialRevision(change: DatabaseChange): number { + // 0 when the entry announces no revision, so any packument satisfies it. return Math.max( + 0, ...change.changes .map((c) => parseSequentialRevision(c.rev)) .filter((num) => !isNaN(num)) @@ -385,6 +401,11 @@ export interface AttachedMetadata { */ readonly ok: AttachedChange[]; + /** + * The change entries for which the registry has no packument (yet). + */ + readonly missing: MissingPackument[]; + /** * The sequence numbers of change entries whose metadata fetch failed. No * receipt must be recorded for these, so that a later scan retries them. @@ -392,6 +413,22 @@ export interface AttachedMetadata { readonly failedSeqs: number[]; } +/** + * A change entry for which the registry has no packument (HTTP 404). Usually + * a new package that has not replicated to the registry yet; rarely an old + * change entry for a package that no longer exists. + */ +export interface MissingPackument { + /** The change entry. */ + readonly change: DatabaseChange; + + /** + * The highest sequential revision announced by the changes feed for this + * entry. + */ + readonly announcedRev: number; +} + /** * A change entry with its registry metadata attached. */ diff --git a/src/package-sources/npmjs/follower-state.lambda-shared.ts b/src/package-sources/npmjs/follower-state.lambda-shared.ts index fdf9af06d..2ab987a52 100644 --- a/src/package-sources/npmjs/follower-state.lambda-shared.ts +++ b/src/package-sources/npmjs/follower-state.lambda-shared.ts @@ -20,35 +20,6 @@ export interface Checkpoint { readonly seq: number; } -/** - * A package whose registry packument is behind the revision announced by the - * `_changes` feed. The versions the registry did serve have been processed; - * this expectation tracks the revision we are still waiting for. - */ -export interface LaggyPackument { - /** - * The package name. - */ - readonly name: string; - - /** - * The revision (sequential prefix) announced by the changes feed. - */ - readonly expectedRev: number; - - /** - * The sequence number of the change entry that announced the revision, if - * known. - */ - readonly seq?: number; - - /** - * Epoch milliseconds at which the expectation was first recorded. Drives - * the give-up policy. - */ - readonly firstSeen: number; -} - /** * The persisted state of the *NpmJs Follower*, in a newline-delimited * plain-text format so it can be scanned cheaply (e.g. `grep '^s 131711133$'`) @@ -59,7 +30,6 @@ export interface LaggyPackument { * #chfollower/1 <- header (format magic + version) * w <- deep scan bookkeeping ("-" when unset) * c <- checkpoint (repeated, ascending by time) - * l <- laggy packument (repeated; seq is "-" when unknown) * s <- received sequence number (repeated, ascending) * ``` * @@ -73,7 +43,7 @@ const STATE_FORMAT_HEADER = '#chfollower/1'; /** * Tracks which `_changes` feed entries (by sequence number) the follower has * already received, rolling time/seq checkpoints mapping times to feed - * positions, and the set of laggy packuments awaiting a registry catch-up. + * positions. * * The npm `_changes` feed does not guarantee that entries only ever become * visible above previously returned `last_seq` values. The receipt set @@ -107,12 +77,7 @@ export class FollowerState { }); break; case 'l': - state.laggy.set(parts.slice(4).join(' '), { - name: parts.slice(4).join(' '), - firstSeen: Number(parts[1]), - expectedRev: Number(parts[2]), - seq: parts[3] === '-' ? undefined : Number(parts[3]), - }); + // Laggy packument, recorded by earlier versions: no longer used. break; case 'w': state.lastDeepScanAt = @@ -129,7 +94,6 @@ export class FollowerState { private checkpoints: Checkpoint[] = []; private readonly seqs = new Set(); - private readonly laggy = new Map(); // The threshold below which receipts have already been pruned; avoids // re-iterating the (potentially large) receipt set on every checkpoint. private prunedBelow = 0; @@ -228,8 +192,7 @@ export class FollowerState { /** * Discards all position state (receipts and checkpoints). Used when the * feed's sequence space is no longer compatible with the recorded state - * (e.g. the feed head regressed below our newest checkpoint). Laggy - * packument expectations are package-keyed and remain valid. + * (e.g. the feed head regressed below our newest checkpoint). */ public resetPosition(): void { this.checkpoints = []; @@ -257,53 +220,6 @@ export class FollowerState { } //#endregion - //#region Laggy packuments - /** - * Records (or extends) the expectation that the registry packument for the - * provided package should reach the provided revision. The earliest - * `firstSeen` is retained (so the give-up policy measures from the first - * occurrence), and the highest announced revision wins. - */ - public recordLaggyPackument( - name: string, - expectedRev: number, - seq: number | undefined, - now: number = Date.now() - ): void { - const existing = this.laggy.get(name); - this.laggy.set(name, { - name, - expectedRev: Math.max(expectedRev, existing?.expectedRev ?? 0), - seq: existing?.seq ?? seq, - firstSeen: existing?.firstSeen ?? now, - }); - } - - /** - * Removes the expectation for the provided package (recovered, gone, or - * given up). - */ - public removeLaggyPackument(name: string): void { - this.laggy.delete(name); - } - - /** - * All laggy packument expectations, oldest first. - */ - public laggyPackuments(): LaggyPackument[] { - return Array.from(this.laggy.values()).sort( - (a, b) => a.firstSeen - b.firstSeen - ); - } - - /** - * The number of laggy packument expectations currently tracked. - */ - public get laggyPackumentCount(): number { - return this.laggy.size; - } - //#endregion - public toText(): string { const lines = new Array(); lines.push(STATE_FORMAT_HEADER); @@ -311,13 +227,6 @@ export class FollowerState { for (const { at, seq } of this.checkpoints) { lines.push(`c ${at} ${seq}`); } - for (const laggy of this.laggyPackuments()) { - lines.push( - `l ${laggy.firstSeen} ${laggy.expectedRev} ${laggy.seq ?? '-'} ${ - laggy.name - }` - ); - } for (const seq of Array.from(this.seqs).sort((a, b) => a - b)) { lines.push(`s ${seq}`); } diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index 7fc296ab1..a3d025078 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -1,5 +1,4 @@ import * as console from 'console'; -import { InvokeCommand } from '@aws-sdk/client-lambda'; import { metricScope, Configuration, @@ -17,17 +16,26 @@ import { FOLLOWER_STATE_FILE_NAME, SCAN_WINDOW_MS, STATE_RETENTION_MS, - LAGGY_PACKUMENT_GIVE_UP_MS, + ENV_PACKUMENT_QUEUE_URL, + NPM_REPLICA_REGISTRY_URL, } from './constants.lambda-shared'; import { CouchChanges, DatabaseChange, DatabaseChanges, - parseSequentialRevision, } from './couch-changes.lambda-shared'; import { FollowerState } from './follower-state.lambda-shared'; import { KnownVersions } from './known-versions.lambda-shared'; -import { PackageVersion } from './stage-and-notify.lambda'; +import { + PackumentProcessingContext, + PackumentRequest, + PackumentRequestReason, + processPackuments, +} from './packument-processing.lambda-shared'; +import { + enqueuePackumentRequests, + queuedPackumentRequests, +} from './packument-queue.lambda-shared'; import { loadContentFromS3, MarkerFileSchema, @@ -35,20 +43,8 @@ import { } from './staging-bucket.lambda-shared'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; -import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared'; import { compressContent } from '../../backend/shared/compress-content.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; -// eslint-disable-next-line @typescript-eslint/no-require-imports -const normalizeNPMMetadata = require('normalize-registry-metadata'); - -const CONSTRUCT_KEYWORDS: ReadonlySet = new Set([ - 'cdk', - 'aws-cdk', - 'awscdk', - 'cdk8s', - 'cdktf', -]); -const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/'; /** * The release date of `aws-cdk@0.8.0`. Anything earlier than this basically is @@ -73,17 +69,6 @@ const SCAN_BATCH_SIZE = 10_000; */ const SCAN_TIME_BUDGET_MS = 60_000; -/** - * Laggy packuments are only re-checked while at least this much time remains - * in the Lambda execution, so that re-checks cannot starve the scan. - */ -const LAGGY_RETRY_TIME_BUDGET_MS = 240_000; - -/** - * The maximum number of laggy packuments re-checked per run. - */ -const MAX_LAGGY_RETRIES_PER_RUN = 50; - // Configure embedded metrics format Configuration.namespace = METRICS_NAMESPACE; @@ -113,8 +98,9 @@ captureHTTPsGlobal(require('http')); * version it contains is processed (deny list, license, and known-versions * checks apply). If the packument is behind the revision announced by the * feed (a "laggy packument"), the served versions are still processed, and - * the follower records the expectation and keeps re-checking on subsequent - * runs until the announced revision appears or the expectation ages out. + * a request to check the package again is sent to the packument queue, + * where it is retried until the announced revision appears or it is given + * up on. * * npm registry API docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md * @@ -124,13 +110,16 @@ export async function handler(event: ScheduledEvent, context: Context) { console.log(`Event: ${JSON.stringify(event, null, 2)}`); const stagingBucket = requireEnv('BUCKET_NAME'); - const stagingFunction = requireEnv('FUNCTION_NAME'); + const queueUrl = requireEnv(ENV_PACKUMENT_QUEUE_URL); const knownVersions = new KnownVersions( requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME) ); - - const denyList = await DenyListClient.newClient(); - const licenseList = await LicenseListClient.newClient(); + const processing: PackumentProcessingContext = { + stagingFunction: requireEnv('FUNCTION_NAME'), + denyList: await DenyListClient.newClient(), + licenseList: await LicenseListClient.newClient(), + knownVersions, + }; const npm = new CouchChanges(NPM_REPLICA_REGISTRY_URL, 'registry/_changes'); @@ -150,44 +139,20 @@ export async function handler(event: ScheduledEvent, context: Context) { } try { - // Re-check laggy packuments (packages whose registry packument was behind - // the revision announced by the feed when we first saw them). - await retryLaggyPackuments( - context, - npm, - state, - stagingFunction, - denyList, - licenseList, - knownVersions - ); - // Scan the trailing window of the feed. Progress is persisted after // every processed chunk (see PersistState), so even a run that is hard // killed by the Lambda timeout keeps its completed work. const persist: PersistState = () => persistProgress(context, stagingBucket, state, knownVersions); - await runScan( - context, - npm, - state, - head, - stagingFunction, - denyList, - licenseList, - knownVersions, - persist - ); + await runScan(context, npm, state, head, processing, queueUrl, persist); - // The laggy packument gauge is emitted at the end of the run, so it - // reflects the expectations recorded by this run's scan (emitting it during - // the retry phase would always read the trough: after recoveries, before - // new expectations are recorded). + // Requests waiting in the packument queue, including those this run sent. + const laggyPackuments = await queuedPackumentRequests(queueUrl); await metricScope((metrics) => async () => { metrics.setDimensions({}); metrics.putMetric( MetricName.LAGGY_PACKUMENTS, - state.laggyPackumentCount, + laggyPackuments, Unit.Count ); })(); @@ -247,10 +212,8 @@ async function runScan( npm: CouchChanges, state: FollowerState, head: number, - stagingFunction: string, - denyList: DenyListClient, - licenseList: LicenseListClient, - knownVersions: KnownVersions, + processing: PackumentProcessingContext, + queueUrl: string, persist: PersistState ): Promise { const now = Date.now(); @@ -302,10 +265,8 @@ async function runScan( state, lateBoundary, head, - stagingFunction, - denyList, - licenseList, - knownVersions, + processing, + queueUrl, persist ); staged += outcome.staged; @@ -375,18 +336,19 @@ async function processBatch( state: FollowerState, lateBoundary: number, head: number, - stagingFunction: string, - denyList: DenyListClient, - licenseList: LicenseListClient, - knownVersions: KnownVersions, + processing: PackumentProcessingContext, + queueUrl: string, persist: PersistState ): Promise<{ staged: number; fullyProcessed: boolean }> { const startTime = Date.now(); let staged = 0; let fullyProcessed = true; try { + // Entries for deleted packages are ignored, as if they were not in the + // feed: there is nothing to process, and no receipt is recorded. const fresh = batch.results.filter( - (change) => change.seq != null && !state.has(Number(change.seq)) + (change) => + !change.deleted && change.seq != null && !state.has(Number(change.seq)) ); console.log( `Received ${batch.totalCount} change entr(ies), ${fresh.length} without a receipt` @@ -424,7 +386,11 @@ async function processBatch( ); } - const { ok: attached, failedSeqs } = await npm.attachAllMetadata(chunk); + const { + ok: attached, + missing, + failedSeqs, + } = await npm.attachAllMetadata(chunk); metrics.putMetric( MetricName.METADATA_FETCH_FAILURES, failedSeqs.length, @@ -435,7 +401,7 @@ async function processBatch( let lastModified: Date | undefined; // Emit npm.js replication lag for (const { change } of attached) { - const doc = change.doc as Change['doc'] | undefined; + const doc = change.doc as { time?: { modified?: string } } | undefined; if (doc?.time?.modified) { const modified = new Date(doc.time.modified); metrics.putMetric( @@ -455,46 +421,60 @@ async function processBatch( `Skipping chunk as the latest modification is ${lastModified}, which is pre-Constructs` ); } else { + chunkStaged = await processPackuments( + attached.map((a) => a.change), + metrics, + processing + ); + staged += chunkStaged; + // Laggy packuments: the registry served an older revision than the - // feed announced. Process the versions we did get, and record the - // expectation so the announced revision is re-checked on later runs. - for (const laggy of attached.filter((a) => a.laggy)) { + // feed announced (the versions we did get were processed above), or + // no packument at all yet. The packument queue checks the package + // again until the announced revision appears. This happens before the + // receipts are recorded, so a failure here makes the next run process + // the chunk again. + const laggy = attached.filter((a) => a.laggy); + for (const { change, servedRev, announcedRev } of laggy) { console.log( - `${laggy.change.id}: registry packument rev ${laggy.servedRev} is behind announced rev ${laggy.announcedRev}, recording laggy packument` + `${change.id}: registry packument rev ${servedRev} is behind announced rev ${announcedRev}, sending it to the packument queue` ); - state.recordLaggyPackument( - laggy.change.id, - laggy.announcedRev, - isNaN(Number(laggy.change.seq)) - ? undefined - : Number(laggy.change.seq) + } + for (const { change } of missing) { + console.log( + `${change.id}: the registry has no packument for it yet, sending it to the packument queue` ); } - - const versionInfos = await getRelevantVersionInfos( - attached.map((a) => a.change) as unknown as readonly Change[], - metrics, - denyList, - licenseList, - knownVersions - ); - console.log( - `Identified ${versionInfos.length} relevant package version update(s)` - ); metrics.putMetric( - MetricName.RELEVANT_PACKAGE_VERSIONS, - versionInfos.length, + MetricName.MISSING_PACKUMENTS, + missing.length, Unit.Count ); - await stageVersions(versionInfos, stagingFunction, knownVersions); - chunkStaged = versionInfos.length; - staged += chunkStaged; + const now = Date.now(); + const request = ( + change: DatabaseChange, + expectedRev: number, + reason: PackumentRequestReason + ): PackumentRequest => ({ + name: change.id, + expectedRev, + seq: isNaN(Number(change.seq)) ? undefined : Number(change.seq), + firstSeen: now, + reason, + }); + await enqueuePackumentRequests(queueUrl, [ + ...laggy.map(({ change, announcedRev }) => + request(change, announcedRev, 'laggy') + ), + ...missing.map(({ change, announcedRev }) => + request(change, announcedRev, 'missing') + ), + ]); } - // Record receipts for every entry of the chunk (including deleted or - // unpublished packages: they were received and deliberately skipped) - - // except entries whose metadata fetch failed, so a later scan retries - // them. Then persist, so a timeout cannot undo this chunk. + // Record a receipt for every entry of the chunk, except those whose + // metadata fetch failed (a later scan retries them). Then persist, so a + // timeout cannot undo this chunk. const failed = new Set(failedSeqs); state.addSeqs( chunk @@ -520,130 +500,6 @@ async function processBatch( } //#endregion -//#region Laggy packuments -/** - * Re-checks laggy packuments: packages whose registry packument was behind - * the revision announced by the changes feed. Each re-check processes - * whatever the registry serves now (new versions are staged), and the - * expectation is cleared when the announced revision appears - or aged out - * after `LAGGY_PACKUMENT_GIVE_UP_MS`, at which point a version announced by - * the feed may be missing until the package publishes again. - * - * @returns the number of package versions sent for staging. - */ -async function retryLaggyPackuments( - context: Context, - npm: CouchChanges, - state: FollowerState, - stagingFunction: string, - denyList: DenyListClient, - licenseList: LicenseListClient, - knownVersions: KnownVersions -): Promise { - let staged = 0; - await metricScope((metrics) => async () => { - metrics.setDimensions({}); - let recovered = 0; - let gaveUp = 0; - const now = Date.now(); - for (const laggy of state - .laggyPackuments() - .slice(0, MAX_LAGGY_RETRIES_PER_RUN)) { - if (context.getRemainingTimeInMillis() < LAGGY_RETRY_TIME_BUDGET_MS) { - break; - } - const doc = await npm.getPackageDoc(laggy.name); - if (doc == null || doc._rev == null) { - console.log( - `Laggy packument for ${laggy.name} is no longer available in the registry, dropping the expectation` - ); - state.removeLaggyPackument(laggy.name); - continue; - } - - // Process whatever the registry serves now; knownVersions dedupes. - const change: DatabaseChange = { - changes: [{ rev: doc._rev as string }], - deleted: false, - id: laggy.name, - seq: laggy.seq, - doc, - }; - const versionInfos = await getRelevantVersionInfos( - [change] as unknown as readonly Change[], - metrics, - denyList, - licenseList, - knownVersions - ); - await stageVersions(versionInfos, stagingFunction, knownVersions); - staged += versionInfos.length; - - if (parseSequentialRevision(doc._rev as string) >= laggy.expectedRev) { - console.log( - `Laggy packument for ${laggy.name} caught up (rev ${doc._rev} >= ${laggy.expectedRev})` - ); - state.removeLaggyPackument(laggy.name); - recovered += 1; - } else if (now - laggy.firstSeen > LAGGY_PACKUMENT_GIVE_UP_MS) { - console.warn( - `Giving up on laggy packument for ${ - laggy.name - }: the registry never served rev ${ - laggy.expectedRev - } (first seen ${new Date( - laggy.firstSeen - ).toISOString()}). A version announced by the changes feed may be missing until the package publishes again.` - ); - state.removeLaggyPackument(laggy.name); - gaveUp += 1; - } - } - metrics.putMetric( - MetricName.LAGGY_PACKUMENTS_RECOVERED, - recovered, - Unit.Count - ); - metrics.putMetric(MetricName.LAGGY_PACKUMENT_GIVE_UPS, gaveUp, Unit.Count); - })(); - return staged; -} -//#endregion - -/** - * Sends the provided package version updates to the staging function - * ("fire-and-forget"), and records them as known versions. - */ -async function stageVersions( - versionInfos: readonly UpdatedVersion[], - stagingFunction: string, - knownVersions: KnownVersions -): Promise { - await Promise.all( - versionInfos.map(async ({ packageName, infos, modified, seq }) => { - const invokeArgs: PackageVersion = { - integrity: infos.dist.shasum, - modified: modified.toISOString(), - name: infos.name, - seq: seq?.toString(), - tarballUrl: infos.dist.tarball, - version: infos.version, - }; - // "Fire-and-forget" invocation here. - console.log(`Sending ${invokeArgs.tarballUrl} for staging`); - await LAMBDA_CLIENT.send( - new InvokeCommand({ - FunctionName: stagingFunction, - InvocationType: 'Event', - Payload: Buffer.from(JSON.stringify(invokeArgs)), - }) - ); - // Record that this is now a "known" version (no need to re-discover) - knownVersions.add(packageName, infos.version); - }) - ); -} - //#region State /** * Loads the follower state from S3. When the state file does not exist (or @@ -665,7 +521,7 @@ async function loadFollowerState( try { const state = FollowerState.fromText(content); console.log( - `Loaded follower state: ${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s)` + `Loaded follower state: ${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s)` ); return state; } catch (error) { @@ -728,7 +584,7 @@ async function saveFollowerState( Buffer.from(state.toText(), 'utf-8') ); console.log( - `Updating follower state (${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s))` + `Updating follower state (${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s))` ); await putObject(context, stagingBucket, FOLLOWER_STATE_FILE_NAME, buffer, { ContentType: 'text/plain', @@ -738,283 +594,3 @@ async function saveFollowerState( } //#endregion - -/** - * Obtains the `VersionInfo` corresponding to the modified version(s) in the - * provided `Change` objects, ensures they are relevant (construct libraries), - * and returns those only. - * - * Known versions are only looked up for packages that have at least one - * construct library version: only those versions are ever recorded, so every - * other package has no known versions by definition. Changes are processed - * concurrently; the result keeps their order. - * - * @param changes the changes to be processed. - * @param metrics the metrics logger to use. - * @param denyList deny list client - * - * @returns a list of `VersionInfo` objects - */ -async function getRelevantVersionInfos( - changes: readonly Change[], - metrics: MetricsLogger, - denyList: DenyListClient, - licenseList: LicenseListClient, - knownVersions: KnownVersions -): Promise { - const perChange = await Promise.all( - changes.map((change) => relevantVersionInfosOf(change)) - ); - return perChange.flat(); - - async function relevantVersionInfosOf( - change: Change - ): Promise { - // Filter out all elements that don't have a "name" in the document, as - // these are schemas, which are not relevant to our business here. - if (change.doc.name === undefined) { - console.error( - `[${change.seq}] Changed document contains no 'name': ${change.id}` - ); - metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - return []; - } - - // The normalize function change the object in place, if the doc object is invalid it will return undefined - if (normalizeNPMMetadata(change.doc) === undefined) { - console.error( - `[${change.seq}] Changed document invalid, npm normalize returned undefined: ${change.id}` - ); - metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - return []; - } - - // Sometimes, there are no versions in the document. We skip those. - if (change.doc.versions == null) { - console.error( - `[${change.seq}] Changed document contains no 'versions': ${change.id}` - ); - metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - return []; - } - - // Sometimes, there is no 'time' entry in the document. We skip those. - if (change.doc.time == null) { - console.error( - `[${change.seq}] Changed document contains no 'time': ${change.id}` - ); - metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - return []; - } - - const name = change.doc.name; - const isRelevant = Object.values(change.doc.versions).some( - (infos) => infos != null && isConstructLibrary(infos) - ); - - // Get the last modification date from the change - const packageVersionUpdates = Object.entries(change.doc.time) - // Ignore the "created" and "modified" keys here - .filter(([key]) => key !== 'created' && key !== 'modified') - // Parse all the dates to ensure they are comparable - .map(([version, isoDate]) => [version, new Date(isoDate)] as const); - metrics.putMetric( - MetricName.PACKAGE_VERSION_COUNT, - packageVersionUpdates.length, - Unit.Count - ); - - const result = new Array(); - const unpublishedVersions: string[] = []; - for (const [version, modified] of packageVersionUpdates) { - if (isRelevant && (await knownVersions.has(name, version))) { - continue; - } - const infos = change.doc.versions[version]; - if (infos == null) { - // Could be the version in question was un-published. - unpublishedVersions.push(`${name}@${version}`); - } else if (isConstructLibrary(infos)) { - // skip if this package is denied - const denied = denyList.lookup(infos.name, infos.version); - if (denied) { - console.log( - `[${change.seq}] Package denied: ${JSON.stringify(denied)}` - ); - knownVersions.add(name, version); - metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count); - continue; - } - - metrics.putMetric( - MetricName.PACKAGE_VERSION_AGE, - Date.now() - modified.getTime(), - Unit.Milliseconds - ); - const isEligible = - licenseList.lookup(infos.license ?? 'UNLICENSED') != null; - metrics.putMetric( - MetricName.INELIGIBLE_LICENSE, - isEligible ? 0 : 1, - Unit.Count - ); - if (isEligible) { - result.push({ - packageName: name, - infos, - modified, - seq: change.seq, - }); - } else { - console.log( - `[${ - change.seq - }] Package "${name}@${version}" does not use allow-listed license: ${ - infos.license ?? 'UNLICENSED' - }` - ); - knownVersions.add(name, version); - } - } - // Else this is not a construct library, so we'll just ignore it... - } - - if (unpublishedVersions.length > 0) { - console.log( - `[${ - change.seq - }] Could not find info for the following versions. Were they un-published?\n${unpublishedVersions.join( - ',\n' - )}` - ); - } - return result; - } - - /** - * This determines whether a package is "interesting" to ConstructHub or not. This is related but - * not necessarily identical to the logic in the ingestion process that annotates package metadata - * with a construct framework name + version (those could ultimately be re-factored to share more - * of the logic/heuristics, though). - * - * Concretely, it checks for a list of known "official" packages for various construct frameworks, - * and packages that have a dependency on such a package. It also has a keywords allow-list as a - * fall-back (the current dependency-based logic does not consider transitive dependencies and - * might hence miss certain rare use-cases, which keywords would rescue). - */ - function isConstructLibrary(infos: VersionInfo): boolean { - if (infos.jsii == null) { - return false; - } - // The "constructs" package is a sign of a constructs library - return ( - isConstructFrameworkPackage(infos.name) || - // Recursively apply on dependencies - Object.keys(infos.dependencies ?? {}).some(isConstructFrameworkPackage) || - Object.keys(infos.devDependencies ?? {}).some( - isConstructFrameworkPackage - ) || - Object.keys(infos.peerDependencies ?? {}).some( - isConstructFrameworkPackage - ) || - // Keyword-based fallback - infos.keywords?.some((kw) => CONSTRUCT_KEYWORDS.has(kw)) - ); - } - - /** - * Package is one of the known construct framework's first party packages: - * - @aws-cdk/* - * - @cdktf/* - * - cdk8s or cdk8s-plus - */ - function isConstructFrameworkPackage(name: string): boolean { - // IMPORTANT NOTE: Prefix matching should only be used for @scope/ names. - - // The low-level constructs package - return ( - name === 'constructs' || - // AWS CDK Packages - name === 'aws-cdk-lib' || - name === 'monocdk' || - name.startsWith('@aws-cdk/') || - // CDK8s packages - name === 'cdk8s' || - /^cdk8s-plus(?:-(?:17|20|21|22))?$/.test(name) || - // CDKTf packages - name === 'cdktf' || - name.startsWith('@cdktf/') - ); - } -} - -/** - * The scheme of a package version in the update. Includes the package.json keys, as well as some additional npm metadata - * @see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version - */ -export interface VersionInfo { - readonly dependencies?: { readonly [name: string]: string }; - readonly devDependencies?: { readonly [name: string]: string }; - readonly peerDependencies?: { readonly [name: string]: string }; - readonly jsii: unknown; - readonly license?: string; - readonly name: string; - readonly [key: string]: unknown; - readonly keywords: string[]; - readonly dist: { - readonly shasum: string; - readonly tarball: string; - }; - readonly version: string; -} - -interface UpdatedVersion { - /** - * The name of the package document the version was found in (the key under - * which known versions are recorded). - */ - readonly packageName: string; - - /** - * The `VersionInfo` for the modified package version. - */ - readonly infos: VersionInfo; - - /** - * The time at which the `VersionInfo` was last modified. - */ - readonly modified: Date; - - /** - * The CouchDB transaction number for the update. - */ - readonly seq?: string | number; -} - -interface Document { - /** - * a List of all Version objects for the package - */ - readonly versions: { [key: string]: VersionInfo | undefined }; - - /** - * The package's name. - */ - readonly name: string; - - /** - * Timestamps associated with this document. The values are ISO-8601 encoded - * timestamps. - */ - readonly time: { - readonly created: string; - readonly modified: string; - readonly [version: string]: string; - }; - - readonly [key: string]: unknown; -} - -interface Change extends DatabaseChange { - readonly doc: Document; -} diff --git a/src/package-sources/npmjs/packument-processing.lambda-shared.ts b/src/package-sources/npmjs/packument-processing.lambda-shared.ts new file mode 100644 index 000000000..5fc54f0de --- /dev/null +++ b/src/package-sources/npmjs/packument-processing.lambda-shared.ts @@ -0,0 +1,449 @@ +import { InvokeCommand } from '@aws-sdk/client-lambda'; +import { MetricsLogger, Unit } from 'aws-embedded-metrics'; +import { MetricName } from './constants.lambda-shared'; +import { + CouchChanges, + DatabaseChange, + parseSequentialRevision, +} from './couch-changes.lambda-shared'; +import { KnownVersions } from './known-versions.lambda-shared'; +import type { PackageVersion } from './stage-and-notify.lambda'; +import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; +import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; +import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared'; +// eslint-disable-next-line @typescript-eslint/no-require-imports +const normalizeNPMMetadata = require('normalize-registry-metadata'); + +const CONSTRUCT_KEYWORDS: ReadonlySet = new Set([ + 'cdk', + 'aws-cdk', + 'awscdk', + 'cdk8s', + 'cdktf', +]); + +/** + * What processing a packument needs: where to send relevant versions, which + * versions to skip, and where to record the processed ones. + */ +export interface PackumentProcessingContext { + /** The name of the staging function. */ + readonly stagingFunction: string; + readonly denyList: DenyListClient; + readonly licenseList: LicenseListClient; + readonly knownVersions: KnownVersions; +} + +/** + * Processes packuments: identifies the relevant package versions they contain + * (construct libraries, not denied, with an eligible license, not processed + * before), sends them for staging, and records them as known versions. The + * caller persists the known versions (`KnownVersions.flush`). + * + * @param changes change entries with their packument attached as `doc`. + * + * @returns the number of package versions sent for staging. + */ +export async function processPackuments( + changes: readonly DatabaseChange[], + metrics: MetricsLogger, + context: PackumentProcessingContext +): Promise { + const versionInfos = await getRelevantVersionInfos( + changes as unknown as readonly Change[], + metrics, + context.denyList, + context.licenseList, + context.knownVersions + ); + console.log( + `Identified ${versionInfos.length} relevant package version update(s)` + ); + metrics.putMetric( + MetricName.RELEVANT_PACKAGE_VERSIONS, + versionInfos.length, + Unit.Count + ); + await stageVersions( + versionInfos, + context.stagingFunction, + context.knownVersions + ); + return versionInfos.length; +} + +/** + * A request to process a package from the registry, and to keep checking it + * until the registry serves at least the announced revision. + */ +export interface PackumentRequest { + /** The package name. */ + readonly name: string; + /** The sequential revision announced by the changes feed. */ + readonly expectedRev: number; + /** The changes feed position that announced it, if known. */ + readonly seq?: number; + /** When the request was first made (epoch milliseconds). */ + readonly firstSeen: number; + /** + * Why the package is checked again: the registry served an older revision + * than announced (`laggy`), or no packument at all (`missing`). + * + * @default 'laggy' + */ + readonly reason?: PackumentRequestReason; +} + +/** Why a package is checked again. */ +export type PackumentRequestReason = 'laggy' | 'missing'; + +/** The outcome of `checkPackument`. */ +export type PackumentCheckResult = + /** The registry serves the expected revision (or a newer one). */ + | 'caught-up' + /** The registry still serves an older revision. */ + | 'laggy' + /** The registry has no packument for the package (HTTP 404). */ + | 'missing'; + +/** + * Fetches the packument a request is about from the registry, processes the + * versions it contains (whatever revision it is at), and reports whether the + * registry has caught up with the expected revision. + */ +export async function checkPackument( + npm: CouchChanges, + request: PackumentRequest, + metrics: MetricsLogger, + context: PackumentProcessingContext +): Promise { + const doc = await npm.getPackageDoc(request.name); + if (doc == null || doc._rev == null) { + return 'missing'; + } + const change: DatabaseChange = { + changes: [{ rev: doc._rev as string }], + deleted: false, + id: request.name, + seq: request.seq, + doc, + }; + await processPackuments([change], metrics, context); + return parseSequentialRevision(doc._rev as string) >= request.expectedRev + ? 'caught-up' + : 'laggy'; +} + +/** + * Sends the provided package version updates to the staging function + * ("fire-and-forget"), and records them as known versions. + */ +export async function stageVersions( + versionInfos: readonly UpdatedVersion[], + stagingFunction: string, + knownVersions: KnownVersions +): Promise { + await Promise.all( + versionInfos.map(async ({ packageName, infos, modified, seq }) => { + const invokeArgs: PackageVersion = { + integrity: infos.dist.shasum, + modified: modified.toISOString(), + name: infos.name, + seq: seq?.toString(), + tarballUrl: infos.dist.tarball, + version: infos.version, + }; + // "Fire-and-forget" invocation here. + console.log(`Sending ${invokeArgs.tarballUrl} for staging`); + await LAMBDA_CLIENT.send( + new InvokeCommand({ + FunctionName: stagingFunction, + InvocationType: 'Event', + Payload: Buffer.from(JSON.stringify(invokeArgs)), + }) + ); + // Record that this is now a "known" version (no need to re-discover) + knownVersions.add(packageName, infos.version); + }) + ); +} + +/** + * Obtains the `VersionInfo` corresponding to the modified version(s) in the + * provided `Change` objects, ensures they are relevant (construct libraries), + * and returns those only. + * + * Known versions are only looked up for packages that have at least one + * construct library version: only those versions are ever recorded, so every + * other package has no known versions by definition. Changes are processed + * concurrently; the result keeps their order. + * + * @param changes the changes to be processed. + * @param metrics the metrics logger to use. + * @param denyList deny list client + * + * @returns a list of `VersionInfo` objects + */ +export async function getRelevantVersionInfos( + changes: readonly Change[], + metrics: MetricsLogger, + denyList: DenyListClient, + licenseList: LicenseListClient, + knownVersions: KnownVersions +): Promise { + const perChange = await Promise.all( + changes.map((change) => relevantVersionInfosOf(change)) + ); + return perChange.flat(); + + async function relevantVersionInfosOf( + change: Change + ): Promise { + // Filter out all elements that don't have a "name" in the document, as + // these are schemas, which are not relevant to our business here. + if (change.doc.name === undefined) { + console.error( + `[${change.seq}] Changed document contains no 'name': ${change.id}` + ); + metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); + return []; + } + + // The normalize function change the object in place, if the doc object is invalid it will return undefined + if (normalizeNPMMetadata(change.doc) === undefined) { + console.error( + `[${change.seq}] Changed document invalid, npm normalize returned undefined: ${change.id}` + ); + metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); + return []; + } + + // Sometimes, there are no versions in the document. We skip those. + if (change.doc.versions == null) { + console.error( + `[${change.seq}] Changed document contains no 'versions': ${change.id}` + ); + metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); + return []; + } + + // Sometimes, there is no 'time' entry in the document. We skip those. + if (change.doc.time == null) { + console.error( + `[${change.seq}] Changed document contains no 'time': ${change.id}` + ); + metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); + return []; + } + + const name = change.doc.name; + const isRelevant = Object.values(change.doc.versions).some( + (infos) => infos != null && isConstructLibrary(infos) + ); + + // Get the last modification date from the change + const packageVersionUpdates = Object.entries(change.doc.time) + // Ignore the "created" and "modified" keys here + .filter(([key]) => key !== 'created' && key !== 'modified') + // Parse all the dates to ensure they are comparable + .map(([version, isoDate]) => [version, new Date(isoDate)] as const); + metrics.putMetric( + MetricName.PACKAGE_VERSION_COUNT, + packageVersionUpdates.length, + Unit.Count + ); + + const result = new Array(); + const unpublishedVersions: string[] = []; + for (const [version, modified] of packageVersionUpdates) { + if (isRelevant && (await knownVersions.has(name, version))) { + continue; + } + const infos = change.doc.versions[version]; + if (infos == null) { + // Could be the version in question was un-published. + unpublishedVersions.push(`${name}@${version}`); + } else if (isConstructLibrary(infos)) { + // skip if this package is denied + const denied = denyList.lookup(infos.name, infos.version); + if (denied) { + console.log( + `[${change.seq}] Package denied: ${JSON.stringify(denied)}` + ); + knownVersions.add(name, version); + metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count); + continue; + } + + metrics.putMetric( + MetricName.PACKAGE_VERSION_AGE, + Date.now() - modified.getTime(), + Unit.Milliseconds + ); + const isEligible = + licenseList.lookup(infos.license ?? 'UNLICENSED') != null; + metrics.putMetric( + MetricName.INELIGIBLE_LICENSE, + isEligible ? 0 : 1, + Unit.Count + ); + if (isEligible) { + result.push({ + packageName: name, + infos, + modified, + seq: change.seq, + }); + } else { + console.log( + `[${ + change.seq + }] Package "${name}@${version}" does not use allow-listed license: ${ + infos.license ?? 'UNLICENSED' + }` + ); + knownVersions.add(name, version); + } + } + // Else this is not a construct library, so we'll just ignore it... + } + + if (unpublishedVersions.length > 0) { + console.log( + `[${ + change.seq + }] Could not find info for the following versions. Were they un-published?\n${unpublishedVersions.join( + ',\n' + )}` + ); + } + return result; + } + + /** + * This determines whether a package is "interesting" to ConstructHub or not. This is related but + * not necessarily identical to the logic in the ingestion process that annotates package metadata + * with a construct framework name + version (those could ultimately be re-factored to share more + * of the logic/heuristics, though). + * + * Concretely, it checks for a list of known "official" packages for various construct frameworks, + * and packages that have a dependency on such a package. It also has a keywords allow-list as a + * fall-back (the current dependency-based logic does not consider transitive dependencies and + * might hence miss certain rare use-cases, which keywords would rescue). + */ + function isConstructLibrary(infos: VersionInfo): boolean { + if (infos.jsii == null) { + return false; + } + // The "constructs" package is a sign of a constructs library + return ( + isConstructFrameworkPackage(infos.name) || + // Recursively apply on dependencies + Object.keys(infos.dependencies ?? {}).some(isConstructFrameworkPackage) || + Object.keys(infos.devDependencies ?? {}).some( + isConstructFrameworkPackage + ) || + Object.keys(infos.peerDependencies ?? {}).some( + isConstructFrameworkPackage + ) || + // Keyword-based fallback + infos.keywords?.some((kw) => CONSTRUCT_KEYWORDS.has(kw)) + ); + } + + /** + * Package is one of the known construct framework's first party packages: + * - @aws-cdk/* + * - @cdktf/* + * - cdk8s or cdk8s-plus + */ + function isConstructFrameworkPackage(name: string): boolean { + // IMPORTANT NOTE: Prefix matching should only be used for @scope/ names. + + // The low-level constructs package + return ( + name === 'constructs' || + // AWS CDK Packages + name === 'aws-cdk-lib' || + name === 'monocdk' || + name.startsWith('@aws-cdk/') || + // CDK8s packages + name === 'cdk8s' || + /^cdk8s-plus(?:-(?:17|20|21|22))?$/.test(name) || + // CDKTf packages + name === 'cdktf' || + name.startsWith('@cdktf/') + ); + } +} + +/** + * The scheme of a package version in the update. Includes the package.json keys, as well as some additional npm metadata + * @see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version + */ +export interface VersionInfo { + readonly dependencies?: { readonly [name: string]: string }; + readonly devDependencies?: { readonly [name: string]: string }; + readonly peerDependencies?: { readonly [name: string]: string }; + readonly jsii: unknown; + readonly license?: string; + readonly name: string; + readonly [key: string]: unknown; + readonly keywords: string[]; + readonly dist: { + readonly shasum: string; + readonly tarball: string; + }; + readonly version: string; +} + +export interface UpdatedVersion { + /** + * The name of the package document the version was found in (the key under + * which known versions are recorded). + */ + readonly packageName: string; + + /** + * The `VersionInfo` for the modified package version. + */ + readonly infos: VersionInfo; + + /** + * The time at which the `VersionInfo` was last modified. + */ + readonly modified: Date; + + /** + * The CouchDB transaction number for the update. + */ + readonly seq?: string | number; +} + +interface Document { + /** + * a List of all Version objects for the package + */ + readonly versions: { [key: string]: VersionInfo | undefined }; + + /** + * The package's name. + */ + readonly name: string; + + /** + * Timestamps associated with this document. The values are ISO-8601 encoded + * timestamps. + */ + readonly time: { + readonly created: string; + readonly modified: string; + readonly [version: string]: string; + }; + + readonly [key: string]: unknown; +} + +interface Change extends DatabaseChange { + readonly doc: Document; +} diff --git a/src/package-sources/npmjs/packument-processor.lambda.ts b/src/package-sources/npmjs/packument-processor.lambda.ts new file mode 100644 index 000000000..d629b4fad --- /dev/null +++ b/src/package-sources/npmjs/packument-processor.lambda.ts @@ -0,0 +1,255 @@ +import { + metricScope, + Configuration, + MetricsLogger, + Unit, +} from 'aws-embedded-metrics'; +import type { + Context, + SQSBatchResponse, + SQSEvent, + SQSRecord, +} from 'aws-lambda'; +import { captureHTTPsGlobal } from 'aws-xray-sdk-core'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_QUEUE_URL, + METRICS_NAMESPACE, + MetricName, + NPM_REPLICA_REGISTRY_URL, + PACKUMENT_MAX_ATTEMPTS, +} from './constants.lambda-shared'; +import { CouchChanges } from './couch-changes.lambda-shared'; +import { KnownVersions } from './known-versions.lambda-shared'; +import { + checkPackument, + PackumentCheckResult, + PackumentProcessingContext, + PackumentRequestReason, +} from './packument-processing.lambda-shared'; +import { + parsePackumentRequest, + retryDelaySeconds, + setVisibilityTimeout, +} from './packument-queue.lambda-shared'; +import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; +import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; +import { requireEnv } from '../../backend/shared/env.lambda-shared'; + +Configuration.namespace = METRICS_NAMESPACE; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +captureHTTPsGlobal(require('https')); + +/** + * Processes packument requests: fetches each package from the registry and + * processes the versions it serves. A request is done once the registry serves + * the announced revision. Otherwise it is retried with back-off, for about 27 + * hours, and then moves to the dead-letter queue. + * + * Every message that is not done gets its next attempt scheduled, even when + * the whole batch fails. Otherwise a broken processor would use up all + * attempts within a few hours. + */ +export async function handler( + event: SQSEvent, + _context: Context +): Promise { + const queueUrl = requireEnv(ENV_PACKUMENT_QUEUE_URL); + + let processing: PackumentProcessingContext; + try { + processing = { + stagingFunction: requireEnv('FUNCTION_NAME'), + denyList: await DenyListClient.newClient(), + licenseList: await LicenseListClient.newClient(), + knownVersions: new KnownVersions( + requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME) + ), + }; + } catch (error) { + console.error(`Could not set up processing, retrying the batch: ${error}`); + await Promise.all( + event.Records.map((record) => scheduleNextAttempt(record, queueUrl)) + ); + throw error; + } + const npm = new CouchChanges(NPM_REPLICA_REGISTRY_URL, 'registry/_changes'); + + let outcomes = new Array(); + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + outcomes = await Promise.all( + event.Records.map((record) => + processRecord(record, npm, metrics, processing) + ) + ); + const count = (result: RecordResult, reason: PackumentRequestReason) => + outcomes.filter((o) => o.result === result && o.reason === reason).length; + metrics.putMetric( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + count('caught-up', 'laggy'), + Unit.Count + ); + metrics.putMetric( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + count('given-up', 'laggy'), + Unit.Count + ); + metrics.putMetric( + MetricName.MISSING_PACKUMENTS_RECOVERED, + count('caught-up', 'missing'), + Unit.Count + ); + metrics.putMetric( + MetricName.MISSING_PACKUMENT_GIVE_UPS, + count('given-up', 'missing'), + Unit.Count + ); + for (const { lag } of outcomes) { + if (lag != null) { + metrics.putMetric( + MetricName.LAGGY_PACKUMENT_LAG, + lag, + Unit.Milliseconds + ); + } + } + })(); + + try { + await processing.knownVersions.flush(); + } catch (error) { + // The versions were staged but not recorded. Check these packages again + // so they get recorded, unless that was their last attempt. + console.error(`Could not record the known versions: ${error}`); + for (const [i, outcome] of outcomes.entries()) { + if ( + outcome.result === 'caught-up' && + attemptOf(event.Records[i]) < PACKUMENT_MAX_ATTEMPTS + ) { + outcome.done = false; + } + } + } + + const failures = new Array(); + await Promise.all( + event.Records.map(async (record, i) => { + if (!outcomes[i].done) { + failures.push(record.messageId); + await scheduleNextAttempt(record, queueUrl); + } + }) + ); + return { + batchItemFailures: failures.map((itemIdentifier) => ({ itemIdentifier })), + }; +} + +type RecordResult = PackumentCheckResult | 'given-up' | 'error'; + +interface RecordOutcome { + readonly result: RecordResult; + readonly reason?: PackumentRequestReason; + /** For a laggy packument that caught up or was given up on: since when. */ + readonly lag?: number; + /** Whether the message can be deleted from the queue. */ + done: boolean; +} + +/** + * Processes one queue message. + */ +async function processRecord( + record: SQSRecord, + npm: CouchChanges, + metrics: MetricsLogger, + processing: PackumentProcessingContext +): Promise { + const attempt = attemptOf(record); + let result: RecordResult; + let reason: PackumentRequestReason | undefined; + let name = record.messageId; + let problem: string; + let firstSeen: number | undefined; + try { + const request = parsePackumentRequest(record.body); + name = request.name; + reason = request.reason ?? 'laggy'; + firstSeen = request.firstSeen; + result = await checkPackument(npm, request, metrics, processing); + if (result === 'caught-up') { + console.log( + `Packument for ${name} caught up with rev ${request.expectedRev} (attempt ${attempt}, ${reason})` + ); + return { result, reason, lag: lagOf(reason, firstSeen), done: true }; + } + problem = + result === 'missing' + ? 'never served a packument for it' + : `never served rev ${request.expectedRev}`; + } catch (error) { + console.error( + `Failed to process the packument request for ${name}: ${error}` + ); + result = 'error'; + problem = `could not be checked (${error})`; + } + + if (attempt < PACKUMENT_MAX_ATTEMPTS) { + return { result, reason, done: false }; + } + const since = + firstSeen != null + ? ` (first seen ${new Date(firstSeen).toISOString()})` + : ''; + console.warn( + `Giving up on ${ + reason ?? 'unknown' + } packument for ${name}: the registry ${problem}${since}. A version announced by the changes feed may be missing until the package publishes again.` + ); + // Unparseable requests also go to the dead-letter queue, but are not + // counted as given up. + return { + result: reason == null ? result : 'given-up', + reason, + lag: lagOf(reason, firstSeen), + done: false, + }; +} + +/** How long a laggy packument has been waiting. */ +function lagOf( + reason: PackumentRequestReason | undefined, + firstSeen: number | undefined +): number | undefined { + return reason === 'laggy' && firstSeen != null + ? Date.now() - firstSeen + : undefined; +} + +/** + * Makes a message visible again for its next attempt. After the last attempt + * that is right away, so it moves to the dead-letter queue. + */ +async function scheduleNextAttempt( + record: SQSRecord, + queueUrl: string +): Promise { + const attempt = attemptOf(record); + const delay = + attempt >= PACKUMENT_MAX_ATTEMPTS ? 0 : retryDelaySeconds(attempt); + try { + await setVisibilityTimeout(queueUrl, record.receiptHandle, delay); + } catch (error) { + // The message returns after the queue's visibility timeout instead. + console.error( + `Could not schedule the next attempt for ${record.messageId}: ${error}` + ); + } +} + +function attemptOf(record: SQSRecord): number { + return Number(record.attributes.ApproximateReceiveCount); +} diff --git a/src/package-sources/npmjs/packument-processor.ts b/src/package-sources/npmjs/packument-processor.ts new file mode 100644 index 000000000..81125362a --- /dev/null +++ b/src/package-sources/npmjs/packument-processor.ts @@ -0,0 +1,20 @@ +// ~~ Generated by projen. To modify, edit .projenrc.ts and run "yarn projen". +import * as path from 'path'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { Construct } from 'constructs'; + +export interface PackumentProcessorProps extends lambda.FunctionOptions { +} + +export class PackumentProcessor extends lambda.Function { + constructor(scope: Construct, id: string, props?: PackumentProcessorProps) { + super(scope, id, { + description: 'package-sources/npmjs/packument-processor.lambda.ts', + ...props, + architecture: lambda.Architecture.ARM_64, + runtime: lambda.Runtime.NODEJS_22_X, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '/packument-processor.lambda.bundle')), + }); + } +} \ No newline at end of file diff --git a/src/package-sources/npmjs/packument-queue-processor.ts b/src/package-sources/npmjs/packument-queue-processor.ts new file mode 100644 index 000000000..6e683697a --- /dev/null +++ b/src/package-sources/npmjs/packument-queue-processor.ts @@ -0,0 +1,100 @@ +import { Duration } from 'aws-cdk-lib'; +import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb'; +import { IFunction, Function, Tracing } from 'aws-cdk-lib/aws-lambda'; +import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; +import { IQueue, Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs'; +import { Construct } from 'constructs'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_QUEUE_URL, + PACKUMENT_MAX_ATTEMPTS, +} from './constants.lambda-shared'; +import { PackumentProcessor } from './packument-processor'; +import { IDenyList } from '../../backend/deny-list/api'; +import { ILicenseList } from '../../backend/license-list/api'; + +export interface PackumentQueueProcessorProps { + /** The table recording the package versions that were processed. */ + readonly knownVersions: ITableV2; + + /** The function that stages relevant package versions. */ + readonly stager: IFunction; + + readonly denyList?: IDenyList; + readonly licenseList: ILicenseList; +} + +/** + * A queue of requests to process a package from the npm registry, and the + * function that works through it. A request is retried with back-off until the + * registry serves the revision the changes feed announced; after about a day + * it is given up on, and moves to the dead-letter queue. + */ +export class PackumentQueueProcessor extends Construct { + /** The packument requests. */ + public readonly queue: IQueue; + + /** The packument requests that were given up on. */ + public readonly deadLetterQueue: IQueue; + + /** The function processing the requests. */ + public readonly function: Function; + + public constructor( + scope: Construct, + id: string, + props: PackumentQueueProcessorProps + ) { + super(scope, id); + + this.deadLetterQueue = new Queue(this, 'DLQ', { + encryption: QueueEncryption.KMS_MANAGED, + retentionPeriod: Duration.days(14), + }); + this.queue = new Queue(this, 'Queue', { + encryption: QueueEncryption.KMS_MANAGED, + retentionPeriod: Duration.days(4), + // Only used when the processor cannot schedule the next attempt itself, + // for example on a timeout. AWS recommends 6x the function timeout. + visibilityTimeout: Duration.minutes(30), + deadLetterQueue: { + queue: this.deadLetterQueue, + maxReceiveCount: PACKUMENT_MAX_ATTEMPTS, + }, + }); + + this.function = new PackumentProcessor(this, 'Function', { + description: `[${this.node.path}] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed`, + environment: { + AWS_EMF_ENVIRONMENT: 'Local', + FUNCTION_NAME: props.stager.functionName, + [ENV_KNOWN_VERSIONS_TABLE_NAME]: props.knownVersions.tableName, + [ENV_PACKUMENT_QUEUE_URL]: this.queue.queueUrl, + }, + memorySize: 2_048, + timeout: Duration.minutes(5), + tracing: Tracing.ACTIVE, + }); + this.function.addEventSource( + new SqsEventSource(this.queue, { + batchSize: 10, + reportBatchItemFailures: true, + // Be gentle with the npm registry. + maxConcurrency: 2, + }) + ); + props.knownVersions.grantReadWriteData(this.function); + props.denyList?.grantRead(this.function); + props.licenseList.grantRead(this.function); + props.stager.grantInvoke(this.function); + } + + /** + * Allows a function to send requests to the queue, and tells it where the + * queue is. + */ + public grantSendRequests(fn: Function): void { + fn.addEnvironment(ENV_PACKUMENT_QUEUE_URL, this.queue.queueUrl); + this.queue.grantSendMessages(fn); + } +} diff --git a/src/package-sources/npmjs/packument-queue.lambda-shared.ts b/src/package-sources/npmjs/packument-queue.lambda-shared.ts new file mode 100644 index 000000000..738fa4aa9 --- /dev/null +++ b/src/package-sources/npmjs/packument-queue.lambda-shared.ts @@ -0,0 +1,138 @@ +import { + ChangeMessageVisibilityCommand, + GetQueueAttributesCommand, + SendMessageBatchCommand, + SendMessageBatchRequestEntry, +} from '@aws-sdk/client-sqs'; +import { + PACKUMENT_INITIAL_DELAY_SECONDS, + PACKUMENT_RETRY_BACKOFF_SECONDS, +} from './constants.lambda-shared'; +import type { PackumentRequest } from './packument-processing.lambda-shared'; +import { SQS_CLIENT } from '../../backend/shared/aws.lambda-shared'; + +/** The maximum number of messages in a single `SendMessageBatch` request. */ +const SEND_BATCH_SIZE = 10; + +/** How often the failed entries of a batch send are retried. */ +const MAX_SEND_ATTEMPTS = 3; + +/** + * Sends packument requests to the packument queue. The first attempt happens + * after `PACKUMENT_INITIAL_DELAY_SECONDS`. + * + * @throws if any request could not be sent. + */ +export async function enqueuePackumentRequests( + queueUrl: string, + requests: readonly PackumentRequest[] +): Promise { + const batches = new Array(); + for (let i = 0; i < requests.length; i += SEND_BATCH_SIZE) { + batches.push(requests.slice(i, i + SEND_BATCH_SIZE)); + } + await Promise.all(batches.map((batch) => sendBatch(queueUrl, batch))); +} + +async function sendBatch( + queueUrl: string, + batch: readonly PackumentRequest[] +): Promise { + let entries: SendMessageBatchRequestEntry[] = batch.map((request, i) => ({ + Id: i.toString(), + MessageBody: JSON.stringify(request), + DelaySeconds: PACKUMENT_INITIAL_DELAY_SECONDS, + })); + for (let attempt = 1; ; attempt++) { + const response = await SQS_CLIENT.send( + new SendMessageBatchCommand({ QueueUrl: queueUrl, Entries: entries }) + ); + const failed = new Set((response.Failed ?? []).map((f) => f.Id)); + if (failed.size === 0) { + return; + } + if (attempt >= MAX_SEND_ATTEMPTS) { + const reasons = (response.Failed ?? []) + .map((f) => `${f.Code}: ${f.Message}`) + .join('; '); + throw new Error( + `Could not send ${failed.size} packument request(s) to ${queueUrl}: ${reasons}` + ); + } + entries = entries.filter((entry) => failed.has(entry.Id)); + } +} + +/** + * Parses the body of a packument queue message. + * + * @throws if the body is not a packument request. + */ +export function parsePackumentRequest(body: string): PackumentRequest { + const request = JSON.parse(body) as Partial; + if ( + typeof request.name !== 'string' || + typeof request.expectedRev !== 'number' || + typeof request.firstSeen !== 'number' || + (request.reason != null && + request.reason !== 'laggy' && + request.reason !== 'missing') + ) { + throw new Error(`Not a packument request: ${body}`); + } + return request as PackumentRequest; +} + +/** + * How long to wait before the next attempt, after the given attempt + * (1-based) did not succeed. + */ +export function retryDelaySeconds(attempt: number): number { + const index = Math.min( + Math.max(attempt, 1) - 1, + PACKUMENT_RETRY_BACKOFF_SECONDS.length - 1 + ); + return PACKUMENT_RETRY_BACKOFF_SECONDS[index]; +} + +/** + * Makes a message the queue already delivered visible again after the given + * number of seconds, instead of the queue's visibility timeout. + */ +export async function setVisibilityTimeout( + queueUrl: string, + receiptHandle: string, + seconds: number +): Promise { + await SQS_CLIENT.send( + new ChangeMessageVisibilityCommand({ + QueueUrl: queueUrl, + ReceiptHandle: receiptHandle, + VisibilityTimeout: seconds, + }) + ); +} + +/** + * The approximate number of requests in the packument queue: waiting, being + * processed, and waiting for their next attempt. + */ +export async function queuedPackumentRequests( + queueUrl: string +): Promise { + const { Attributes: attributes = {} } = await SQS_CLIENT.send( + new GetQueueAttributesCommand({ + QueueUrl: queueUrl, + AttributeNames: [ + 'ApproximateNumberOfMessages', + 'ApproximateNumberOfMessagesNotVisible', + 'ApproximateNumberOfMessagesDelayed', + ], + }) + ); + return ( + Number(attributes.ApproximateNumberOfMessages ?? 0) + + Number(attributes.ApproximateNumberOfMessagesNotVisible ?? 0) + + Number(attributes.ApproximateNumberOfMessagesDelayed ?? 0) + ); +} diff --git a/src/package-sources/npmjs/re-stage-package-version.lambda.ts b/src/package-sources/npmjs/re-stage-package-version.lambda.ts index d2c943c41..dd30754c8 100644 --- a/src/package-sources/npmjs/re-stage-package-version.lambda.ts +++ b/src/package-sources/npmjs/re-stage-package-version.lambda.ts @@ -1,5 +1,5 @@ import { InvokeCommand } from '@aws-sdk/client-lambda'; -import { VersionInfo } from './npm-js-follower.lambda'; +import type { VersionInfo } from './packument-processing.lambda-shared'; import { PackageVersion } from './stage-and-notify.lambda'; import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; From 8a466da4367d3088a520a4e4f15fbf345b8b1ee9 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Tue, 13 Oct 2026 16:14:19 +0100 Subject: [PATCH 2/3] fixup --- docs/npm-sync-protocol.md | 12 +- docs/operator-runbook.md | 35 +- .../__snapshots__/construct-hub.test.ts.snap | 500 ++++++++++-------- .../__snapshots__/snapshot.test.ts.snap | 100 ++-- src/__tests__/package-sources/npmjs.test.ts | 15 +- .../npmjs/packument-processor.lambda.test.ts | 295 ++++++----- src/package-sources/npmjs.ts | 15 +- .../npmjs/constants.lambda-shared.ts | 44 +- .../npmjs/packument-processor.lambda.ts | 207 ++++---- .../npmjs/packument-queue-processor.ts | 17 +- .../npmjs/packument-queue.lambda-shared.ts | 27 +- 11 files changed, 690 insertions(+), 577 deletions(-) diff --git a/docs/npm-sync-protocol.md b/docs/npm-sync-protocol.md index 923ef5e20..2350d460a 100644 --- a/docs/npm-sync-protocol.md +++ b/docs/npm-sync-protocol.md @@ -122,12 +122,12 @@ processing any entry it has no receipt for. Rolling checkpoints map times to feed positions, and derive the window's starting position. On top of that baseline, each gap has a dedicated mitigation: -| Gap | Mitigation | Detection | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm | -| Laggy packuments | Process every packument the registry serves immediately; queue a re-check of the announced revision (also when the registry has no packument yet), retried with back-off for about a day, then moved to a dead-letter queue | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` / `LaggyPackumentLag` and `MissingPackuments` / `MissingPackumentsRecovered` / `MissingPackumentGiveUps` metrics; `LaggyPackumentLagHigh` and `MissingPackumentsHigh` alarms | -| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package | -| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric | +| Gap | Mitigation | Detection | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm | +| Laggy packuments | Process every packument the registry serves immediately; queue a re-check of the announced revision (also when the registry has no packument yet), retried with back-off for about 29 hours, then given up on | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` / `LaggyPackumentLag` and `MissingPackuments` / `MissingPackumentsRecovered` / `MissingPackumentGiveUps` metrics; `LaggyPackumentLagHigh` and `MissingPackumentsHigh` alarms | +| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package | +| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric | **The design idea behind all of this is that the feed data are just hints without real guarantees.** The follower treats every change entry as "this package probably changed around here", diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 703c575fd..69f1590ad 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -651,9 +651,10 @@ ticket. Log in to , visit the #### Resolution The packument processor processes the packages once the registry serves them. -Packages it gives up on after about a day move to the *Packument DLQ*. Once the -registry is healthy again, redrive the *Packument DLQ* to the *Packument Queue* -(from the SQS console) to check them again. +Packages it gives up on are logged (`Giving up on missing packument`) and then +dropped. They show up again when the package next publishes. To ingest one +right away, invoke the *ReStagePackageVersion* Lambda function with the package +name and version. -------------------------------------------------------------------------------- @@ -675,12 +676,13 @@ the registry catches up. The registry occasionally serves an older revision of a package than the changes feed announced (a *laggy* packument). The follower processes the versions that were served, and sends the package to the *Packument Queue*. The -*Packument Processor* checks it again with growing waits, for about a day, and -reports how long each laggy packument took to catch up. Most catch up within +*Packument Processor* checks it again with growing waits, for about 29 hours, +and reports how long each laggy packument took to catch up. Most catch up within minutes, and nearly all within 6 hours. -Check the *Packument Queue* graph of the backend dashboard: how many requests -are waiting, and how many were given up on (*Packument DLQ*). The logs of the +Check the *Packument Queue* graph of the backend dashboard for how many +requests are waiting, and the `Laggy Packument Give-Ups` line on the +*Feed Reliability* graph for how many were given up on. The logs of the *Packument Processor* show `caught up` and `Giving up on laggy packument` entries. Verify what the registry serves for a few affected packages: @@ -694,13 +696,10 @@ with upstream npmjs.com (see the previous alarm for how to reach them). #### Resolution The packument processor processes the missed versions once the registry serves -them. Packages it gives up on move to the *Packument DLQ*. Once the registry is -healthy again, redrive the *Packument DLQ* to the *Packument Queue* (from the -SQS console). To ingest a missed version immediately, invoke the -*ReStagePackageVersion* Lambda function with the package name and version. - -The *Packument DLQ* has no alarm of its own. Given-up requests stay in it for -14 days, and it is graphed on the backend dashboard. +them. Packages it gives up on are logged and then dropped; there is nothing to +redrive. The missed versions show up again when the package next publishes. To +ingest one right away, invoke the *ReStagePackageVersion* Lambda function with +the package name and version. -------------------------------------------------------------------------------- @@ -862,8 +861,7 @@ When documentation for a package is not available, it could be due to several re This alarm is only provisioned when the `NpmJs` package source is configured. It triggers when the *Packument Processor* function fails or times out. While it fails, the requests in the *Packument Queue* keep using up their attempts, so -laggy packuments can be given up on (and move to the *Packument DLQ*) earlier -than the usual day. +they can be given up on earlier than the usual 29 hours. #### Investigation @@ -874,8 +872,9 @@ that request. Time-outs usually mean the npm registry is slow to respond. #### Resolution -Fix the cause. Requests that were given up on in the meantime can be redriven -from the *Packument DLQ* to the *Packument Queue* (from the SQS console). +Fix the cause. Requests that were given up on in the meantime are lost. Their +versions show up again when the packages next publish, or can be ingested +right away with the *ReStagePackageVersion* Lambda function. -------------------------------------------------------------------------------- diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 4d8fcf9e9..960e8dfdf 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -786,21 +786,6 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", - { - "Ref": "AWS::Region", - }, - ".amazonaws.com%2F", - { - "Ref": "AWS::AccountId", - }, - "%2F", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ")\\n[button:Packument Processor](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", @@ -878,14 +863,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -9843,20 +9821,6 @@ Request a service quota increase for lambda functions", "QueueName", ], }, - ""},"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "":{"name":"NPM JS Packument DLQ","queueName":"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -10204,7 +10168,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", + "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -10786,7 +10750,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "Event", "ServiceToken": { @@ -10810,7 +10774,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -10824,7 +10788,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -10851,7 +10815,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", + "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -11387,7 +11351,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", + "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -11737,7 +11701,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -11757,6 +11721,12 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_DEAD_LETTER_QUEUE_ARN": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, "PACKUMENT_QUEUE_URL": { "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", }, @@ -11845,6 +11815,22 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo ], }, }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + }, { "Action": [ "dynamodb:BatchGetItem", @@ -11966,6 +11952,28 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", @@ -11995,7 +12003,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "DeletionPolicy": "Delete", "Properties": { "KmsMasterKeyId": "alias/aws/sqs", - "MessageRetentionPeriod": 345600, + "MessageRetentionPeriod": 1209600, "RedrivePolicy": { "deadLetterTargetArn": { "Fn::GetAtt": [ @@ -12003,7 +12011,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Arn", ], }, - "maxReceiveCount": 9, + "maxReceiveCount": 10, }, "VisibilityTimeout": 1800, }, @@ -15690,21 +15698,6 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", - { - "Ref": "AWS::Region", - }, - ".amazonaws.com%2F", - { - "Ref": "AWS::AccountId", - }, - "%2F", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ")\\n[button:Packument Processor](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", @@ -15782,14 +15775,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -25121,20 +25107,6 @@ Request a service quota increase for lambda functions", "QueueName", ], }, - ""},"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "":{"name":"NPM JS Packument DLQ","queueName":"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -25482,7 +25454,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", + "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -26064,7 +26036,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "Event", "ServiceToken": { @@ -26088,7 +26060,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -26102,7 +26074,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -26129,7 +26101,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", + "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -26665,7 +26637,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", + "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -27015,7 +26987,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -27035,6 +27007,12 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_DEAD_LETTER_QUEUE_ARN": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, "PACKUMENT_QUEUE_URL": { "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", }, @@ -27123,6 +27101,22 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo ], }, }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + }, { "Action": [ "dynamodb:BatchGetItem", @@ -27244,6 +27238,28 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", @@ -27273,7 +27289,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "DeletionPolicy": "Delete", "Properties": { "KmsMasterKeyId": "alias/aws/sqs", - "MessageRetentionPeriod": 345600, + "MessageRetentionPeriod": 1209600, "RedrivePolicy": { "deadLetterTargetArn": { "Fn::GetAtt": [ @@ -27281,7 +27297,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Arn", ], }, - "maxReceiveCount": 9, + "maxReceiveCount": 10, }, "VisibilityTimeout": 1800, }, @@ -30906,21 +30922,6 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", - { - "Ref": "AWS::Region", - }, - ".amazonaws.com%2F", - { - "Ref": "AWS::AccountId", - }, - "%2F", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ")\\n[button:Packument Processor](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", @@ -30998,14 +30999,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -39963,20 +39957,6 @@ Request a service quota increase for lambda functions", "QueueName", ], }, - ""},"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "":{"name":"NPM JS Packument DLQ","queueName":"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -40324,7 +40304,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", + "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -40906,7 +40886,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "Event", "ServiceToken": { @@ -40930,7 +40910,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -40944,7 +40924,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -40971,7 +40951,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", + "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -41507,7 +41487,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", + "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -41857,7 +41837,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -41877,6 +41857,12 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_DEAD_LETTER_QUEUE_ARN": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, "PACKUMENT_QUEUE_URL": { "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", }, @@ -41965,6 +41951,22 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo ], }, }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + }, { "Action": [ "dynamodb:BatchGetItem", @@ -42086,6 +42088,28 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", @@ -42115,7 +42139,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "DeletionPolicy": "Delete", "Properties": { "KmsMasterKeyId": "alias/aws/sqs", - "MessageRetentionPeriod": 345600, + "MessageRetentionPeriod": 1209600, "RedrivePolicy": { "deadLetterTargetArn": { "Fn::GetAtt": [ @@ -42123,7 +42147,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Arn", ], }, - "maxReceiveCount": 9, + "maxReceiveCount": 10, }, "VisibilityTimeout": 1800, }, @@ -45896,21 +45920,6 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", - { - "Ref": "AWS::Region", - }, - ".amazonaws.com%2F", - { - "Ref": "AWS::AccountId", - }, - "%2F", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ")\\n[button:Packument Processor](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", @@ -45988,14 +45997,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -54962,20 +54964,6 @@ Request a service quota increase for lambda functions", "QueueName", ], }, - ""},"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "":{"name":"NPM JS Packument DLQ","queueName":"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -55323,7 +55311,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", + "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -55892,7 +55880,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "Event", "ServiceToken": { @@ -55916,7 +55904,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -55930,7 +55918,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -55957,7 +55945,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", + "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -56493,7 +56481,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", + "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -56843,7 +56831,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -56863,6 +56851,12 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_DEAD_LETTER_QUEUE_ARN": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, "PACKUMENT_QUEUE_URL": { "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", }, @@ -56951,6 +56945,22 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo ], }, }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + }, { "Action": [ "dynamodb:BatchGetItem", @@ -57072,6 +57082,28 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", @@ -57101,7 +57133,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "DeletionPolicy": "Delete", "Properties": { "KmsMasterKeyId": "alias/aws/sqs", - "MessageRetentionPeriod": 345600, + "MessageRetentionPeriod": 1209600, "RedrivePolicy": { "deadLetterTargetArn": { "Fn::GetAtt": [ @@ -57109,7 +57141,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Arn", ], }, - "maxReceiveCount": 9, + "maxReceiveCount": 10, }, "VisibilityTimeout": 1800, }, @@ -61071,21 +61103,6 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", - { - "Ref": "AWS::Region", - }, - ".amazonaws.com%2F", - { - "Ref": "AWS::AccountId", - }, - "%2F", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ")\\n[button:Packument Processor](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", @@ -61163,14 +61180,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -69882,20 +69892,6 @@ Request a service quota increase for lambda functions", "QueueName", ], }, - ""},"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "":{"name":"NPM JS Packument DLQ","queueName":"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -70305,7 +70301,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", + "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -70887,7 +70883,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "Event", "ServiceToken": { @@ -70911,7 +70907,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -70925,7 +70921,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a0d20a76f1667369c2df8ea828954659": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -70952,7 +70948,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", + "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -71488,7 +71484,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", + "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -71838,7 +71834,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -71858,6 +71854,12 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_DEAD_LETTER_QUEUE_ARN": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, "PACKUMENT_QUEUE_URL": { "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", }, @@ -71946,6 +71948,22 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo ], }, }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + }, { "Action": [ "dynamodb:BatchGetItem", @@ -72067,6 +72085,28 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", @@ -72096,7 +72136,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "DeletionPolicy": "Delete", "Properties": { "KmsMasterKeyId": "alias/aws/sqs", - "MessageRetentionPeriod": 345600, + "MessageRetentionPeriod": 1209600, "RedrivePolicy": { "deadLetterTargetArn": { "Fn::GetAtt": [ @@ -72104,7 +72144,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Arn", ], }, - "maxReceiveCount": 9, + "maxReceiveCount": 10, }, "VisibilityTimeout": 1800, }, diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 268453366..61fd7dbf9 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -848,21 +848,6 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - ")\\n[button:Packument DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.", - { - "Ref": "AWS::Region", - }, - ".amazonaws.com%2F", - { - "Ref": "AWS::AccountId", - }, - "%2F", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ")\\n[button:Packument Processor](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", @@ -940,14 +925,7 @@ def sort_filter_rules(json_obj): "QueueName", ], }, - "",{"label":"Delayed","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "",{"label":"Given up (DLQ)","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", + "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"", { "Ref": "AWS::Region", }, @@ -10187,20 +10165,6 @@ Request a service quota increase for lambda functions", "QueueName", ], }, - ""},"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, - "":{"name":"NPM JS Packument DLQ","queueName":"", - { - "Fn::GetAtt": [ - "ConstructHubSourcesPackumentProcessorDLQF401BD77", - "QueueName", - ], - }, ""}}},"title":"SQS DLQ","updateOn":{"refresh":true,"resize":false,"timeRange":false}}},{"type":"metric","width":12,"height":8,"x":0,"y":16,"properties":{"view":"timeSeries","title":"CloudFront Metrics","region":"", { "Ref": "AWS::Region", @@ -11479,7 +11443,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "16560ea01eb63ebefcc78973f4010e525b5a2f285f135606e85b26416e184355.zip", + "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -12055,7 +12019,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701bce79df81d41d50a0ac4ad780930bcd7", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7018914ee7ebabff12cadb11438c85d3bec", }, "InvocationType": "Event", "ServiceToken": { @@ -12079,7 +12043,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Properties": { "ExecuteOnHandlerChange": false, "HandlerArn": { - "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701bce79df81d41d50a0ac4ad780930bcd7", + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7018914ee7ebabff12cadb11438c85d3bec", }, "InvocationType": "RequestResponse", "ServiceToken": { @@ -12093,7 +12057,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Type": "Custom::Trigger", "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701bce79df81d41d50a0ac4ad780930bcd7": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7018914ee7ebabff12cadb11438c85d3bec": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -12120,7 +12084,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "525b426ddca6cf79b665e7449a4c94b34681473c42770cefbebb66ae0fbbb858.zip", + "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", "Environment": { @@ -12656,7 +12620,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "58ee8f1c32ce8092a5841887c87511f6db9ab848cb58f12f36f15742ab364e22.zip", + "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -13006,7 +12970,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "20063444ab6ef95601a7f75425e4c45d656341ff748be8d7a203b3cf8915ff28.zip", + "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", }, "Description": "[dev/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -13026,6 +12990,12 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Ref": "ConstructHubLicenseListBucket9334047F", }, "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json", + "PACKUMENT_DEAD_LETTER_QUEUE_ARN": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, "PACKUMENT_QUEUE_URL": { "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578", }, @@ -13114,6 +13084,22 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo ], }, }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + }, { "Action": [ "dynamodb:BatchGetItem", @@ -13235,6 +13221,28 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::IAM::Policy", }, + "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourcedevConstructHubSourcesPackumentProcessorDLQBB0F158EFDD9B731": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "BatchSize": 10, + "EventSourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesPackumentProcessorDLQF401BD77", + "Arn", + ], + }, + "FunctionName": { + "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177", + }, + "FunctionResponseTypes": [ + "ReportBatchItemFailures", + ], + }, + "Type": "AWS::Lambda::EventSourceMapping", + }, "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourcedevConstructHubSourcesPackumentProcessorQueueB1811D0E3C6BE98A": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", @@ -13264,7 +13272,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "DeletionPolicy": "Delete", "Properties": { "KmsMasterKeyId": "alias/aws/sqs", - "MessageRetentionPeriod": 345600, + "MessageRetentionPeriod": 1209600, "RedrivePolicy": { "deadLetterTargetArn": { "Fn::GetAtt": [ @@ -13272,7 +13280,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "Arn", ], }, - "maxReceiveCount": 9, + "maxReceiveCount": 10, }, "VisibilityTimeout": 1800, }, diff --git a/src/__tests__/package-sources/npmjs.test.ts b/src/__tests__/package-sources/npmjs.test.ts index 033d58a93..c6d948b01 100644 --- a/src/__tests__/package-sources/npmjs.test.ts +++ b/src/__tests__/package-sources/npmjs.test.ts @@ -117,13 +117,13 @@ test('retries laggy packuments through a queue that moves give-ups to a dead-let Properties: { MessageRetentionPeriod: 14 * 24 * 60 * 60 }, }); template.hasResourceProperties('AWS::SQS::Queue', { - MessageRetentionPeriod: 4 * 24 * 60 * 60, + MessageRetentionPeriod: 14 * 24 * 60 * 60, VisibilityTimeout: 30 * 60, RedrivePolicy: { deadLetterTargetArn: { 'Fn::GetAtt': [Match.stringLikeRegexp('PackumentProcessorDLQ'), 'Arn'], }, - maxReceiveCount: 9, + maxReceiveCount: 10, }, }); expect( @@ -145,6 +145,17 @@ test('retries laggy packuments through a queue that moves give-ups to a dead-let ScalingConfig: { MaximumConcurrency: 2 }, }); + // The same function reads the dead-letter queue, to log give-ups. + const mappings = Object.values( + template.findResources('AWS::Lambda::EventSourceMapping', { + Properties: { FunctionName: { Ref: Object.keys(processors)[0] } }, + }) + ).map((mapping: any) => JSON.stringify(mapping.Properties.EventSourceArn)); + expect(mappings).toHaveLength(2); + expect(mappings.some((arn) => arn.includes('PackumentProcessorDLQ'))).toBe( + true + ); + // There is no alarm on the dead-letter queue (it is only graphed), and no // alarm on the give-up metric. const alarmNames = Object.values( diff --git a/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts b/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts index 74d05e558..4ec778c11 100644 --- a/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts +++ b/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts @@ -19,10 +19,11 @@ import { import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; import { ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN, ENV_PACKUMENT_QUEUE_URL, MetricName, - PACKUMENT_MAX_ATTEMPTS, - PACKUMENT_RETRY_BACKOFF_SECONDS, + PACKUMENT_MAX_RETRY_DELAY_SECONDS, + PACKUMENT_MIN_RETRY_DELAY_SECONDS, } from '../../../package-sources/npmjs/constants.lambda-shared'; import type { PackumentRequest } from '../../../package-sources/npmjs/packument-processing.lambda-shared'; import { handler } from '../../../package-sources/npmjs/packument-processor.lambda'; @@ -49,6 +50,10 @@ mockMetricScope.mockImplementation((cb) => { const REGISTRY = 'https://registry.npmjs.org'; const QUEUE_URL = 'https://sqs.us-east-1.amazonaws.com/123456789012/mock-packument-queue'; +const QUEUE_ARN = 'arn:aws:sqs:us-east-1:123456789012:mock-packument-queue'; +const DLQ_ARN = 'arn:aws:sqs:us-east-1:123456789012:mock-packument-dlq'; +const MINUTE = 60 * 1_000; +const HOUR = 60 * MINUTE; const TABLE = 'mock-known-versions-table'; const STAGING_FUNCTION = 'mock-staging-function'; const DENY_LIST_BUCKET = 'deny-list-bucket-name'; @@ -72,6 +77,9 @@ beforeEach(() => { process.env.FUNCTION_NAME = STAGING_FUNCTION; process.env[ENV_PACKUMENT_QUEUE_URL] = QUEUE_URL; + process.env[ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN] = DLQ_ARN; + // No jitter, unless a test says otherwise. + jest.spyOn(Math, 'random').mockReturnValue(0.5); process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = TABLE; process.env[ENV_DENY_LIST_BUCKET_NAME] = DENY_LIST_BUCKET; process.env[ENV_DENY_LIST_OBJECT_KEY] = DENY_LIST_OBJECT; @@ -97,6 +105,7 @@ afterEach(() => { for (const name of [ 'FUNCTION_NAME', ENV_PACKUMENT_QUEUE_URL, + ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN, ENV_KNOWN_VERSIONS_TABLE_NAME, ENV_DENY_LIST_BUCKET_NAME, ENV_DENY_LIST_OBJECT_KEY, @@ -106,25 +115,27 @@ afterEach(() => { delete process.env[name]; } nock.cleanAll(); + jest.restoreAllMocks(); }); function record( messageId: string, request: Partial | string, - attempt = 1 + { source = QUEUE_ARN, age = 0 }: { source?: string; age?: number } = {} ): SQSRecord { return { messageId, receiptHandle: `receipt-${messageId}`, + eventSourceARN: source, body: typeof request === 'string' ? request : JSON.stringify({ seq: 1_000, - firstSeen: 1_700_000_000_000, + firstSeen: Date.now() - age, ...request, }), - attributes: { ApproximateReceiveCount: attempt.toString() }, + attributes: { ApproximateReceiveCount: '1' }, } as any; } @@ -196,13 +207,19 @@ test('completes a request once the registry serves the expected revision', async ); }); -test('processes what is served, and retries later with back-off while the registry is behind', async () => { +test('processes what is served, and waits as long as the request is old before the next attempt', async () => { nock(REGISTRY) .get('/still-laggy') .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0')); const result = await handler( - event(record('m1', { name: 'still-laggy', expectedRev: 5 }, 3)), + event( + record( + 'm1', + { name: 'still-laggy', expectedRev: 5 }, + { age: 40 * MINUTE } + ) + ), context ); @@ -211,35 +228,54 @@ test('processes what is served, and retries later with back-off while the regist expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { QueueUrl: QUEUE_URL, ReceiptHandle: 'receipt-m1', - VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[2], + VisibilityTimeout: 40 * 60, }); }); -test('gives up on the last attempt, so the request moves to the dead-letter queue', async () => { +test('keeps the wait within the minimum and maximum delay', async () => { nock(REGISTRY) - .get('/stuck') - .reply(200, packageDoc('stuck', '3-aaaaaa', '1.0.0')); + .persist() + .get('/still-laggy') + .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0')); - const result = await handler( + await handler( event( - record('m1', { name: 'stuck', expectedRev: 5 }, PACKUMENT_MAX_ATTEMPTS) + record('young', { name: 'still-laggy', expectedRev: 5 }, { age: 0 }), + record('old', { name: 'still-laggy', expectedRev: 5 }, { age: 20 * HOUR }) ), context ); - // Failing the message on its last receive moves it to the dead-letter - // queue; it becomes visible right away so that happens promptly. - expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { - VisibilityTimeout: 0, + ReceiptHandle: 'receipt-young', + VisibilityTimeout: PACKUMENT_MIN_RETRY_DELAY_SECONDS, }); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENT_GIVE_UPS, - 1, - Unit.Count + expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { + ReceiptHandle: 'receipt-old', + VisibilityTimeout: PACKUMENT_MAX_RETRY_DELAY_SECONDS, + }); +}); + +test('adds jitter to the wait', async () => { + nock(REGISTRY) + .persist() + .get('/still-laggy') + .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0')); + jest.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(1); + + await handler( + event( + record('m1', { name: 'still-laggy', expectedRev: 5 }, { age: HOUR }), + record('m2', { name: 'still-laggy', expectedRev: 5 }, { age: HOUR }) + ), + context ); - // What the registry served was still processed. - expect(stagedPackages()).toEqual([{ name: 'stuck', version: '1.0.0' }]); + + const timeouts = mockSQS + .commandCalls(ChangeMessageVisibilityCommand) + .map((call) => call.args[0].input.VisibilityTimeout) + .sort(); + expect(timeouts).toEqual([0.8 * 3_600, 1.2 * 3_600]); }); test('retries a request while the registry has no packument for the package yet', async () => { @@ -252,53 +288,11 @@ test('retries a request while the registry has no packument for the package yet' expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { - VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[0], + VisibilityTimeout: PACKUMENT_MIN_RETRY_DELAY_SECONDS, }); expect(stagedPackages()).toEqual([]); }); -test('counts missing packuments separately from laggy ones', async () => { - nock(REGISTRY) - .get('/arrived') - .reply(200, packageDoc('arrived', '1-aaaaaa', '1.0.0')); - nock(REGISTRY).get('/never-arrived').reply(404, 'not found'); - - const result = await handler( - event( - record('m1', { name: 'arrived', expectedRev: 1, reason: 'missing' }), - record( - 'm2', - { name: 'never-arrived', expectedRev: 1, reason: 'missing' }, - PACKUMENT_MAX_ATTEMPTS - ) - ), - context - ); - - // Both kinds of give-up go to the dead-letter queue. - expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm2' }]); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.MISSING_PACKUMENTS_RECOVERED, - 1, - Unit.Count - ); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.MISSING_PACKUMENT_GIVE_UPS, - 1, - Unit.Count - ); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENTS_RECOVERED, - 0, - Unit.Count - ); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENT_GIVE_UPS, - 0, - Unit.Count - ); -}); - test('retries a request that failed with an error, without failing the others', async () => { nock(REGISTRY).get('/broken').reply(403, 'forbidden'); nock(REGISTRY) @@ -337,7 +331,7 @@ test('does not stage versions that are already known', async () => { expect(stagedPackages()).toEqual([{ name: 'known', version: '1.1.0' }]); }); -test('keeps the back-off when the whole batch fails to set up', async () => { +test('keeps the usual waits when the whole batch fails to set up', async () => { mockS3 .on(GetObjectCommand, { Bucket: DENY_LIST_BUCKET, Key: DENY_LIST_OBJECT }) .rejects(new Error('s3 is down')); @@ -345,43 +339,23 @@ test('keeps the back-off when the whole batch fails to set up', async () => { await expect( handler( event( - record('m1', { name: 'one', expectedRev: 5 }, 2), - record('m2', { name: 'two', expectedRev: 5 }, 4) + record('m1', { name: 'one', expectedRev: 5 }, { age: 20 * MINUTE }), + record('m2', { name: 'two', expectedRev: 5 }, { age: 2 * HOUR }) ), context ) ).rejects.toThrow(/s3 is down/); - // Both messages get their normal next attempt, instead of returning after - // the queue's visibility timeout and using up their attempts quickly. expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { ReceiptHandle: 'receipt-m1', - VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[1], + VisibilityTimeout: 20 * 60, }); expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { ReceiptHandle: 'receipt-m2', - VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[3], + VisibilityTimeout: 2 * 3_600, }); }); -test('counts an error on the last attempt as a give-up', async () => { - nock(REGISTRY).get('/broken').reply(403, 'forbidden'); - - const result = await handler( - event( - record('m1', { name: 'broken', expectedRev: 5 }, PACKUMENT_MAX_ATTEMPTS) - ), - context - ); - - expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); - expect(mockPutMetric).toHaveBeenCalledWith( - MetricName.LAGGY_PACKUMENT_GIVE_UPS, - 1, - Unit.Count - ); -}); - test('checks caught-up packages again when their versions could not be recorded', async () => { nock(REGISTRY) .get('/caught-up') @@ -395,47 +369,7 @@ test('checks caught-up packages again when their versions could not be recorded' expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { - VisibilityTimeout: PACKUMENT_RETRY_BACKOFF_SECONDS[0], - }); -}); - -test('does not dead-letter a caught-up request when recording fails on its last attempt', async () => { - nock(REGISTRY) - .get('/caught-up') - .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0')); - mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('dynamodb is down')); - - const result = await handler( - event( - record( - 'm1', - { name: 'caught-up', expectedRev: 5 }, - PACKUMENT_MAX_ATTEMPTS - ) - ), - context - ); - - expect(result.batchItemFailures).toEqual([]); -}); - -test('dead-letters a missing request whose last check failed', async () => { - nock(REGISTRY).get('/unknown').reply(403, 'forbidden'); - - const result = await handler( - event( - record( - 'm1', - { name: 'unknown', expectedRev: 1, reason: 'missing' }, - PACKUMENT_MAX_ATTEMPTS - ) - ), - context - ); - - expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]); - expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, { - VisibilityTimeout: 0, + VisibilityTimeout: PACKUMENT_MIN_RETRY_DELAY_SECONDS, }); }); @@ -443,21 +377,22 @@ test('reports how long laggy packuments took to catch up', async () => { nock(REGISTRY) .get('/caught-up') .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0')); - const firstSeen = Date.now() - 2 * 60 * 60 * 1_000; await handler( - event(record('m1', { name: 'caught-up', expectedRev: 5, firstSeen })), + event( + record('m1', { name: 'caught-up', expectedRev: 5 }, { age: 2 * HOUR }) + ), context ); const lag = mockPutMetric.mock.calls.find( ([name]) => name === MetricName.LAGGY_PACKUMENT_LAG ); - expect(lag?.[1]).toBeGreaterThanOrEqual(2 * 60 * 60 * 1_000); + expect(lag?.[1]).toBeGreaterThanOrEqual(2 * HOUR); expect(lag?.[2]).toBe(Unit.Milliseconds); }); -test('does not report a lag for missing packuments', async () => { +test('counts missing packuments that arrived separately from laggy ones, without a lag', async () => { nock(REGISTRY) .get('/arrived') .reply(200, packageDoc('arrived', '1-aaaaaa', '1.0.0')); @@ -467,9 +402,97 @@ test('does not report a lag for missing packuments', async () => { context ); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.MISSING_PACKUMENTS_RECOVERED, + 1, + Unit.Count + ); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENTS_RECOVERED, + 0, + Unit.Count + ); expect(mockPutMetric).not.toHaveBeenCalledWith( MetricName.LAGGY_PACKUMENT_LAG, expect.anything(), expect.anything() ); }); + +describe('dead-letter queue', () => { + test('logs, counts and deletes requests that were given up on, without checking them', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + + const result = await handler( + event( + record( + 'm1', + { name: 'stuck', expectedRev: 5 }, + { source: DLQ_ARN, age: 29 * HOUR } + ), + record( + 'm2', + { name: 'never-arrived', expectedRev: 1, reason: 'missing' }, + { source: DLQ_ARN } + ) + ), + context + ); + + // Deleted (no fetch: nock would reject one). + expect(result.batchItemFailures).toEqual([]); + expect(mockSQS).not.toHaveReceivedCommand(ChangeMessageVisibilityCommand); + expect(stagedPackages()).toEqual([]); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); + expect(mockPutMetric).toHaveBeenCalledWith( + MetricName.MISSING_PACKUMENT_GIVE_UPS, + 1, + Unit.Count + ); + const lag = mockPutMetric.mock.calls.find( + ([name]) => name === MetricName.LAGGY_PACKUMENT_LAG + ); + expect(lag?.[1]).toBeGreaterThanOrEqual(29 * HOUR); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('Giving up on laggy packument for stuck') + ); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining( + 'Giving up on missing packument for never-arrived' + ) + ); + }); + + test('deletes an invalid request it was given up on', async () => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + + const result = await handler( + event(record('m1', 'not json', { source: DLQ_ARN })), + context + ); + + expect(result.batchItemFailures).toEqual([]); + }); + + test('handles a batch from either queue on its own', async () => { + nock(REGISTRY) + .get('/caught-up') + .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0')); + jest.spyOn(console, 'warn').mockImplementation(() => {}); + + const result = await handler( + event( + record('m1', { name: 'caught-up', expectedRev: 5 }), + record('m2', { name: 'stuck', expectedRev: 5 }, { source: DLQ_ARN }) + ), + context + ); + + expect(result.batchItemFailures).toEqual([]); + expect(stagedPackages()).toEqual([{ name: 'caught-up', version: '1.0.0' }]); + }); +}); diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index ccecf4ed1..4cec8ec7c 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -343,10 +343,6 @@ export class NpmJs implements IPackageSource { 'NPM JS Follower DLQ', follower.deadLetterQueue ); - overviewDashboard.addDLQMetricToDashboard( - 'NPM JS Packument DLQ', - packuments.deadLetterQueue - ); overviewDashboard.addConcurrentExecutionMetricToDashboard( follower, 'NpmJsLambda' @@ -379,10 +375,6 @@ export class NpmJs implements IPackageSource { { name: 'Stager', url: lambdaFunctionUrl(stager) }, { name: 'Stager DLQ', url: sqsQueueUrl(stager.deadLetterQueue!) }, { name: 'Packument Queue', url: sqsQueueUrl(packuments.queue) }, - { - name: 'Packument DLQ', - url: sqsQueueUrl(packuments.deadLetterQueue), - }, { name: 'Packument Processor', url: lambdaFunctionUrl(packuments.function), @@ -549,13 +541,12 @@ export class NpmJs implements IPackageSource { packuments.queue.metricApproximateNumberOfMessagesDelayed({ label: 'Delayed', }), - packuments.deadLetterQueue.metricApproximateNumberOfMessagesVisible( - { label: 'Given up (DLQ)' } - ), ], leftYAxis: { min: 0 }, right: [ - this.metricLaggyPackumentLag({ label: 'Laggy Packument Lag (p90)' }), + this.metricLaggyPackumentLag({ + label: 'Laggy Packument Lag (p90)', + }), ], rightAnnotations: [ { diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index 9c57a0222..7d8972c83 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -170,35 +170,35 @@ export const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/'; */ export const ENV_PACKUMENT_QUEUE_URL = 'PACKUMENT_QUEUE_URL'; +/** + * The environment variable holding the ARN of the packument dead-letter queue. + * The packument processor reads both queues, and tells them apart by this. + */ +export const ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN = + 'PACKUMENT_DEAD_LETTER_QUEUE_ARN'; + /** * How long a new packument request waits before it is first processed. */ export const PACKUMENT_INITIAL_DELAY_SECONDS = 5 * 60; // 5 minutes /** - * How long a packument request waits before the next attempt, after an - * attempt that did not find the announced revision. Entry `n` is the wait - * after attempt `n + 1`. SQS caps a message's visibility timeout at 12 hours - * from when it was received, so no wait may be that long. - * - * Together with the initial delay, the last attempt happens about 27 hours - * after the request was made. A request that still finds an old revision - * then is given up on, and moves to the dead-letter queue. + * The shortest and longest wait before the next attempt. In between, a request + * waits as long as it is already old, so waits double with each attempt. SQS + * caps a visibility timeout at 12 hours. + */ +export const PACKUMENT_MIN_RETRY_DELAY_SECONDS = 5 * 60; // 5 minutes +export const PACKUMENT_MAX_RETRY_DELAY_SECONDS = 6 * 60 * 60; // 6 hours + +/** + * Waits are randomly shortened or lengthened by up to this fraction, so that + * requests made together do not keep coming back together. */ -export const PACKUMENT_RETRY_BACKOFF_SECONDS: readonly number[] = [ - 5 * 60, // 5 minutes - 15 * 60, // 15 minutes - 30 * 60, // 30 minutes - 60 * 60, // 1 hour - 2 * 60 * 60, // 2 hours - 4 * 60 * 60, // 4 hours - 8 * 60 * 60, // 8 hours - 11 * 60 * 60, // 11 hours -]; +export const PACKUMENT_RETRY_JITTER = 0.2; /** - * The number of attempts made for a packument request before it moves to the - * dead-letter queue (the queue's `maxReceiveCount`). + * The number of attempts before a request moves to the dead-letter queue (the + * queue's `maxReceiveCount`). With the waits above, that is after about 29 + * hours. */ -export const PACKUMENT_MAX_ATTEMPTS = - PACKUMENT_RETRY_BACKOFF_SECONDS.length + 1; +export const PACKUMENT_MAX_ATTEMPTS = 10; diff --git a/src/package-sources/npmjs/packument-processor.lambda.ts b/src/package-sources/npmjs/packument-processor.lambda.ts index d629b4fad..df9026539 100644 --- a/src/package-sources/npmjs/packument-processor.lambda.ts +++ b/src/package-sources/npmjs/packument-processor.lambda.ts @@ -13,18 +13,18 @@ import type { import { captureHTTPsGlobal } from 'aws-xray-sdk-core'; import { ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN, ENV_PACKUMENT_QUEUE_URL, METRICS_NAMESPACE, MetricName, NPM_REPLICA_REGISTRY_URL, - PACKUMENT_MAX_ATTEMPTS, } from './constants.lambda-shared'; import { CouchChanges } from './couch-changes.lambda-shared'; import { KnownVersions } from './known-versions.lambda-shared'; import { checkPackument, - PackumentCheckResult, PackumentProcessingContext, + PackumentRequest, PackumentRequestReason, } from './packument-processing.lambda-shared'; import { @@ -44,19 +44,31 @@ captureHTTPsGlobal(require('https')); /** * Processes packument requests: fetches each package from the registry and * processes the versions it serves. A request is done once the registry serves - * the announced revision. Otherwise it is retried with back-off, for about 27 - * hours, and then moves to the dead-letter queue. + * the announced revision. Otherwise it is checked again later, with growing + * waits. After about 29 hours the queue moves it to the dead-letter queue. * - * Every message that is not done gets its next attempt scheduled, even when - * the whole batch fails. Otherwise a broken processor would use up all - * attempts within a few hours. + * This function also reads the dead-letter queue. A request that arrives from + * there was given up on: it is logged, counted, and deleted. */ export async function handler( event: SQSEvent, _context: Context ): Promise { - const queueUrl = requireEnv(ENV_PACKUMENT_QUEUE_URL); + const deadLetterQueueArn = requireEnv(ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN); + const givenUp = event.Records.filter( + (record) => record.eventSourceARN === deadLetterQueueArn + ); + const pending = event.Records.filter( + (record) => record.eventSourceARN !== deadLetterQueueArn + ); + if (givenUp.length > 0) { + await reportGiveUps(givenUp); + } + if (pending.length === 0) { + return { batchItemFailures: [] }; + } + const queueUrl = requireEnv(ENV_PACKUMENT_QUEUE_URL); let processing: PackumentProcessingContext; try { processing = { @@ -68,9 +80,11 @@ export async function handler( ), }; } catch (error) { + // Keep the usual waits; otherwise a broken processor would use up all + // attempts at the queue's visibility timeout. console.error(`Could not set up processing, retrying the batch: ${error}`); await Promise.all( - event.Records.map((record) => scheduleNextAttempt(record, queueUrl)) + pending.map((record) => scheduleNextAttempt(record, queueUrl)) ); throw error; } @@ -80,30 +94,18 @@ export async function handler( await metricScope((metrics) => async () => { metrics.setDimensions({}); outcomes = await Promise.all( - event.Records.map((record) => - processRecord(record, npm, metrics, processing) - ) + pending.map((record) => processRecord(record, npm, metrics, processing)) ); - const count = (result: RecordResult, reason: PackumentRequestReason) => - outcomes.filter((o) => o.result === result && o.reason === reason).length; + const recovered = (reason: PackumentRequestReason) => + outcomes.filter((o) => o.done && o.reason === reason).length; metrics.putMetric( MetricName.LAGGY_PACKUMENTS_RECOVERED, - count('caught-up', 'laggy'), - Unit.Count - ); - metrics.putMetric( - MetricName.LAGGY_PACKUMENT_GIVE_UPS, - count('given-up', 'laggy'), + recovered('laggy'), Unit.Count ); metrics.putMetric( MetricName.MISSING_PACKUMENTS_RECOVERED, - count('caught-up', 'missing'), - Unit.Count - ); - metrics.putMetric( - MetricName.MISSING_PACKUMENT_GIVE_UPS, - count('given-up', 'missing'), + recovered('missing'), Unit.Count ); for (const { lag } of outcomes) { @@ -121,21 +123,16 @@ export async function handler( await processing.knownVersions.flush(); } catch (error) { // The versions were staged but not recorded. Check these packages again - // so they get recorded, unless that was their last attempt. + // so they get recorded. console.error(`Could not record the known versions: ${error}`); - for (const [i, outcome] of outcomes.entries()) { - if ( - outcome.result === 'caught-up' && - attemptOf(event.Records[i]) < PACKUMENT_MAX_ATTEMPTS - ) { - outcome.done = false; - } + for (const outcome of outcomes) { + outcome.done = false; } } const failures = new Array(); await Promise.all( - event.Records.map(async (record, i) => { + pending.map(async (record, i) => { if (!outcomes[i].done) { failures.push(record.messageId); await scheduleNextAttempt(record, queueUrl); @@ -147,12 +144,9 @@ export async function handler( }; } -type RecordResult = PackumentCheckResult | 'given-up' | 'error'; - interface RecordOutcome { - readonly result: RecordResult; readonly reason?: PackumentRequestReason; - /** For a laggy packument that caught up or was given up on: since when. */ + /** For a laggy packument that caught up: how long it took. */ readonly lag?: number; /** Whether the message can be deleted from the queue. */ done: boolean; @@ -167,81 +161,110 @@ async function processRecord( metrics: MetricsLogger, processing: PackumentProcessingContext ): Promise { - const attempt = attemptOf(record); - let result: RecordResult; - let reason: PackumentRequestReason | undefined; let name = record.messageId; - let problem: string; - let firstSeen: number | undefined; + let reason: PackumentRequestReason | undefined; try { const request = parsePackumentRequest(record.body); name = request.name; reason = request.reason ?? 'laggy'; - firstSeen = request.firstSeen; - result = await checkPackument(npm, request, metrics, processing); - if (result === 'caught-up') { - console.log( - `Packument for ${name} caught up with rev ${request.expectedRev} (attempt ${attempt}, ${reason})` - ); - return { result, reason, lag: lagOf(reason, firstSeen), done: true }; + const result = await checkPackument(npm, request, metrics, processing); + if (result !== 'caught-up') { + return { reason, done: false }; } - problem = - result === 'missing' - ? 'never served a packument for it' - : `never served rev ${request.expectedRev}`; + console.log( + `Packument for ${name} caught up with rev ${request.expectedRev} (${reason})` + ); + return { reason, lag: lagOf(request), done: true }; } catch (error) { console.error( `Failed to process the packument request for ${name}: ${error}` ); - result = 'error'; - problem = `could not be checked (${error})`; + return { reason, done: false }; } +} - if (attempt < PACKUMENT_MAX_ATTEMPTS) { - return { result, reason, done: false }; - } - const since = - firstSeen != null - ? ` (first seen ${new Date(firstSeen).toISOString()})` - : ''; - console.warn( - `Giving up on ${ - reason ?? 'unknown' - } packument for ${name}: the registry ${problem}${since}. A version announced by the changes feed may be missing until the package publishes again.` - ); - // Unparseable requests also go to the dead-letter queue, but are not - // counted as given up. - return { - result: reason == null ? result : 'given-up', - reason, - lag: lagOf(reason, firstSeen), - done: false, - }; +/** + * Logs and counts requests that were given up on. They are deleted from the + * dead-letter queue afterwards: nothing would redrive them. + */ +async function reportGiveUps(records: readonly SQSRecord[]): Promise { + await metricScope((metrics) => async () => { + metrics.setDimensions({}); + const counts: Record = { + laggy: 0, + missing: 0, + }; + for (const record of records) { + let request: PackumentRequest; + try { + request = parsePackumentRequest(record.body); + } catch { + console.warn( + `Giving up on an invalid packument request: ${record.body}` + ); + continue; + } + const reason = request.reason ?? 'laggy'; + counts[reason] += 1; + const problem = + reason === 'missing' + ? 'never served a packument for it' + : `never served rev ${request.expectedRev}`; + console.warn( + `Giving up on ${reason} packument for ${ + request.name + }: the registry ${problem} (first seen ${new Date( + request.firstSeen + ).toISOString()}). A version announced by the changes feed may be missing until the package publishes again.` + ); + const lag = lagOf(request); + if (lag != null) { + metrics.putMetric( + MetricName.LAGGY_PACKUMENT_LAG, + lag, + Unit.Milliseconds + ); + } + } + metrics.putMetric( + MetricName.LAGGY_PACKUMENT_GIVE_UPS, + counts.laggy, + Unit.Count + ); + metrics.putMetric( + MetricName.MISSING_PACKUMENT_GIVE_UPS, + counts.missing, + Unit.Count + ); + })(); } -/** How long a laggy packument has been waiting. */ -function lagOf( - reason: PackumentRequestReason | undefined, - firstSeen: number | undefined -): number | undefined { - return reason === 'laggy' && firstSeen != null - ? Date.now() - firstSeen +/** How long a request has been waiting. */ +function lagOf(request: PackumentRequest): number | undefined { + return (request.reason ?? 'laggy') === 'laggy' + ? Date.now() - request.firstSeen : undefined; } /** - * Makes a message visible again for its next attempt. After the last attempt - * that is right away, so it moves to the dead-letter queue. + * Makes a message visible again for its next attempt. */ async function scheduleNextAttempt( record: SQSRecord, queueUrl: string ): Promise { - const attempt = attemptOf(record); - const delay = - attempt >= PACKUMENT_MAX_ATTEMPTS ? 0 : retryDelaySeconds(attempt); + let firstSeen: number | undefined; + try { + firstSeen = parsePackumentRequest(record.body).firstSeen; + } catch { + // An invalid request gets the shortest wait, and ends up given up on. + } try { - await setVisibilityTimeout(queueUrl, record.receiptHandle, delay); + await setVisibilityTimeout( + queueUrl, + record.receiptHandle, + retryDelaySeconds(firstSeen) + ); } catch (error) { // The message returns after the queue's visibility timeout instead. console.error( @@ -249,7 +272,3 @@ async function scheduleNextAttempt( ); } } - -function attemptOf(record: SQSRecord): number { - return Number(record.attributes.ApproximateReceiveCount); -} diff --git a/src/package-sources/npmjs/packument-queue-processor.ts b/src/package-sources/npmjs/packument-queue-processor.ts index 6e683697a..def1da211 100644 --- a/src/package-sources/npmjs/packument-queue-processor.ts +++ b/src/package-sources/npmjs/packument-queue-processor.ts @@ -6,6 +6,7 @@ import { IQueue, Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import { ENV_KNOWN_VERSIONS_TABLE_NAME, + ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN, ENV_PACKUMENT_QUEUE_URL, PACKUMENT_MAX_ATTEMPTS, } from './constants.lambda-shared'; @@ -27,8 +28,9 @@ export interface PackumentQueueProcessorProps { /** * A queue of requests to process a package from the npm registry, and the * function that works through it. A request is retried with back-off until the - * registry serves the revision the changes feed announced; after about a day - * it is given up on, and moves to the dead-letter queue. + * registry serves the revision the changes feed announced. After about 29 + * hours it moves to the dead-letter queue, where the same function logs it as + * given up on. */ export class PackumentQueueProcessor extends Construct { /** The packument requests. */ @@ -47,13 +49,15 @@ export class PackumentQueueProcessor extends Construct { ) { super(scope, id); + // Given-up requests. The function logs and deletes them right away; they + // are only kept while it is broken. this.deadLetterQueue = new Queue(this, 'DLQ', { encryption: QueueEncryption.KMS_MANAGED, retentionPeriod: Duration.days(14), }); this.queue = new Queue(this, 'Queue', { encryption: QueueEncryption.KMS_MANAGED, - retentionPeriod: Duration.days(4), + retentionPeriod: Duration.days(14), // Only used when the processor cannot schedule the next attempt itself, // for example on a timeout. AWS recommends 6x the function timeout. visibilityTimeout: Duration.minutes(30), @@ -70,6 +74,7 @@ export class PackumentQueueProcessor extends Construct { FUNCTION_NAME: props.stager.functionName, [ENV_KNOWN_VERSIONS_TABLE_NAME]: props.knownVersions.tableName, [ENV_PACKUMENT_QUEUE_URL]: this.queue.queueUrl, + [ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN]: this.deadLetterQueue.queueArn, }, memorySize: 2_048, timeout: Duration.minutes(5), @@ -83,6 +88,12 @@ export class PackumentQueueProcessor extends Construct { maxConcurrency: 2, }) ); + this.function.addEventSource( + new SqsEventSource(this.deadLetterQueue, { + batchSize: 10, + reportBatchItemFailures: true, + }) + ); props.knownVersions.grantReadWriteData(this.function); props.denyList?.grantRead(this.function); props.licenseList.grantRead(this.function); diff --git a/src/package-sources/npmjs/packument-queue.lambda-shared.ts b/src/package-sources/npmjs/packument-queue.lambda-shared.ts index 738fa4aa9..bd2af94c4 100644 --- a/src/package-sources/npmjs/packument-queue.lambda-shared.ts +++ b/src/package-sources/npmjs/packument-queue.lambda-shared.ts @@ -6,7 +6,9 @@ import { } from '@aws-sdk/client-sqs'; import { PACKUMENT_INITIAL_DELAY_SECONDS, - PACKUMENT_RETRY_BACKOFF_SECONDS, + PACKUMENT_MAX_RETRY_DELAY_SECONDS, + PACKUMENT_MIN_RETRY_DELAY_SECONDS, + PACKUMENT_RETRY_JITTER, } from './constants.lambda-shared'; import type { PackumentRequest } from './packument-processing.lambda-shared'; import { SQS_CLIENT } from '../../backend/shared/aws.lambda-shared'; @@ -84,15 +86,24 @@ export function parsePackumentRequest(body: string): PackumentRequest { } /** - * How long to wait before the next attempt, after the given attempt - * (1-based) did not succeed. + * How long to wait before the next attempt. A request waits as long as it is + * already old, within the minimum and maximum delay, plus or minus some jitter. + * This doubles the wait with each attempt, without counting attempts. + * + * @param firstSeen when the request was made (epoch milliseconds), if known. */ -export function retryDelaySeconds(attempt: number): number { - const index = Math.min( - Math.max(attempt, 1) - 1, - PACKUMENT_RETRY_BACKOFF_SECONDS.length - 1 +export function retryDelaySeconds( + firstSeen: number | undefined, + now: number = Date.now(), + random: () => number = Math.random +): number { + const ageSeconds = firstSeen == null ? 0 : (now - firstSeen) / 1_000; + const delay = Math.min( + Math.max(ageSeconds, PACKUMENT_MIN_RETRY_DELAY_SECONDS), + PACKUMENT_MAX_RETRY_DELAY_SECONDS ); - return PACKUMENT_RETRY_BACKOFF_SECONDS[index]; + const jitter = 1 + PACKUMENT_RETRY_JITTER * (2 * random() - 1); + return Math.round(delay * jitter); } /** From cc70b0a7d9c9ba558c24885939e3329ecf6687d5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:26:19 +0000 Subject: [PATCH 3/3] chore: self mutation Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/__tests__/__snapshots__/construct-hub.test.ts.snap | 10 +++++----- .../devapp/__snapshots__/snapshot.test.ts.snap | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 960e8dfdf..2dd5303ba 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -11701,7 +11701,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", + "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -26987,7 +26987,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", + "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -41837,7 +41837,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", + "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -56831,7 +56831,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", + "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { @@ -71834,7 +71834,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", + "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip", }, "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 61fd7dbf9..930aad7ae 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -12970,7 +12970,7 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "000d20d8d7ff39f223c25a1804c1a3a71cd97426b5c45c2cc1e045fd83d9c718.zip", + "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip", }, "Description": "[dev/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed", "Environment": {