diff --git a/API.md b/API.md
index 9c61d1136..c734c9425 100644
--- a/API.md
+++ b/API.md
@@ -2881,11 +2881,15 @@ new sources.NpmJs(props?: NpmJsProps)
| metricBatchProcessingTime | The average time it took to process a changes batch. |
| metricChangeCount | The total count of changes that were processed. |
| metricLastSeq | The last sequence number that was processed. |
+| metricLateChangeCount | The number of changes discovered by the overlap sweep that were missed by the regular head-of-feed pass (i.e. rows inserted into the CouchDB `_changes` feed behind the follower's cursor). |
+| metricLateChangeLag | For each late change discovered by the overlap sweep, the time elapsed between the moment the follower first read past the change's sequence number and the moment the change was discovered. |
| metricNpmJsChangeAge | *No description.* |
| metricPackageVersionAge | The age of the oldest package version that was processed. |
| metricPackageVersionCount | The total count of package versions that were inspected. |
| metricRelevantPackageVersions | The total count of package versions that were deemed relevant. |
| metricRemainingTime | The amount of time that was remaining when the lambda returned in order to avoid hitting a timeout. |
+| metricStaleMetadataDeferred | The number of changes that were deferred to the stale-metadata retry queue because the npm registry metadata was still behind the revision announced by the `_changes` feed. |
+| metricStaleMetadataRecovered | The number of previously deferred changes that were successfully recovered from the stale-metadata retry queue. |
| metricUnprocessableEntity | The amount of changes that were not processed due to having an invalid format. |
---
@@ -2955,6 +2959,37 @@ discover when a sequence reset has happened in the CouchDB instance.
---
+##### `metricLateChangeCount`
+
+```typescript
+public metricLateChangeCount(opts?: MetricOptions): Metric
+```
+
+The number of changes discovered by the overlap sweep that were missed by the regular head-of-feed pass (i.e. rows inserted into the CouchDB `_changes` feed behind the follower's cursor).
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
+##### `metricLateChangeLag`
+
+```typescript
+public metricLateChangeLag(opts?: MetricOptions): Metric
+```
+
+For each late change discovered by the overlap sweep, the time elapsed between the moment the follower first read past the change's sequence number and the moment the change was discovered.
+
+This is a lower bound on
+the feed's insertion lag, and can be used to tune the sweep margin.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
##### `metricNpmJsChangeAge`
```typescript
@@ -3023,6 +3058,34 @@ The amount of time that was remaining when the lambda returned in order to avoid
---
+##### `metricStaleMetadataDeferred`
+
+```typescript
+public metricStaleMetadataDeferred(opts?: MetricOptions): Metric
+```
+
+The number of changes that were deferred to the stale-metadata retry queue because the npm registry metadata was still behind the revision announced by the `_changes` feed.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
+##### `metricStaleMetadataRecovered`
+
+```typescript
+public metricStaleMetadataRecovered(opts?: MetricOptions): Metric
+```
+
+The number of previously deferred changes that were successfully recovered from the stale-metadata retry queue.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
##### `metricUnprocessableEntity`
```typescript
diff --git a/docs/errors-encountered.md b/docs/errors-encountered.md
index dea1482f9..3828be2b6 100644
--- a/docs/errors-encountered.md
+++ b/docs/errors-encountered.md
@@ -18,7 +18,7 @@ resources if Construct Hub is running in a VPC, etc.).
### ``
-A `SIGKILL` error is likely a memory issue with a Custom Resource AWS Lambda Function
+A `SIGKILL` error is likely a memory issue with a Custom Resource AWS Lambda Function
that is running during deployment. One such example is the Web App Deploy construct
that copies the Web App into an S3 Bucket using a Custom Resource. This is an AWS Lambda
Function that is subject to memory constraints.
diff --git a/docs/npm-sync-protocol.md b/docs/npm-sync-protocol.md
new file mode 100644
index 000000000..5c61e2ab8
--- /dev/null
+++ b/docs/npm-sync-protocol.md
@@ -0,0 +1,169 @@
+# The npm sync protocol: what it does and does not guarantee
+
+This document describes the protocol ConstructHub's `NpmJs` package source uses to discover new package versions,
+the guarantees npm gives (and explicitly does not give) for that protocol,
+and how ConstructHub compensates for each gap.
+Claims cite public npm statements where they exist;
+anything marked *(our observation)* we established ourselves through incident investigation.
+
+## The protocol
+
+Syncing from npm uses two endpoints:
+
+1. **The changes feed** at `https://replicate.npmjs.com/registry/_changes`.
+ A paginated stream of change entries, each `{ seq, id, changes: [{ rev }] }`, read with `?since=&limit=`.
+ A consumer pages forward by passing the previous response's `last_seq` as the next `since`.
+2. **The registry** at `https://registry.npmjs.org/`.
+ Change entries carry no package content,
+ so a consumer fetches the full metadata document (the "packument") from the registry,
+ and derives new versions from its `time` map.
+
+Since May 2025 the feed is no longer a real CouchDB.
+npm replaced it with a purpose-built service (the root endpoint reports `engine: "npm-replicate"`)
+that keeps the CouchDB API shape but supports only a subset of it
+([migration guide][152515], [deprecation notice][changelog]).
+Several of the non-guarantees below date from this migration.
+
+## What npm guarantees
+
+- **Pagination by sequence number.**
+ `_changes?since=&limit=` returns change entries with `seq > since` in ascending order,
+ plus a `last_seq` to continue from.
+ `limit` defaults to 1,000 and caps at 10,000 ([migration guide][152515]).
+- **A head-of-feed pointer.**
+ The root endpoint (`/`) reports the current `update_seq`;
+ npm recommends it as the starting point for a new consumer ([npm engineering response][152515-updateseq]).
+- **One entry per package.**
+ A package has at most one entry in the feed:
+ a new change replaces the package's previous entry, at a new sequence number.
+ npm confirmed this is intended ([npm: "That is indeed normal feed behavior"][152515-late]).
+- **Metadata lives in the registry, not the feed.**
+ `include_docs` is not supported;
+ npm's documented guidance is to fetch packuments from `registry.npmjs.org` separately ([migration guide][152515]).
+
+That is the entire public contract.
+Notably absent: any statement about *when* a change becomes visible in the feed,
+or that it becomes visible at all.
+
+## What npm does not guarantee
+
+Each heading below states a property a consumer might reasonably expect, but that npm does not guarantee.
+For ease of use, each gap also gets a name;
+the code and the operator runbook reference the same names.
+
+### Changes become visible in order
+
+Change entries can appear in the feed *behind* a previously returned `last_seq`;
+we name this gap *late insertion*.
+A consumer that reads strictly forward will never see such an entry.
+Community measurement found ~4 late-inserted entries per 1,024-entry window over two hours;
+npm's response was that entries appearing behind `last_seq` is expected
+([report][152515-late-report], [npm confirmation][152515-late]).
+
+*(our observation)* The delay can be far larger than the minutes suggested in that thread.
+On 2026-09-21 we read past a probe package's sequence range within ~3 minutes of its entry's sequence number
+being assigned, and the entry was not served;
+a manual re-read of the same range two hours later returned it.
+Five consecutive versions of the package were missed this way over ~15 hours.
+See the [appendix](#appendix-a-late-insertion-reconstructed) for a reconstructed timeline.
+
+### The registry is up-to-date with the changes feed
+
+The registry can serve a packument revision *older* than the revision the feed announced for the same package,
+sometimes for extended periods;
+we name this gap *stale metadata*.
+The feed and the registry are separate systems and are not consistent with each other
+([community description][152515-stale], [metadata out of sync][128412]).
+
+A consumer can detect this:
+the packument carries no sequence number,
+but the feed entry announces the revision(s) it covers and the packument carries its current revision (`_rev`).
+Comparing the numeric prefix of the two tells a consumer whether the registry has caught up
+with the change it is processing.
+
+*(our observation)* This is not rare.
+E.g. on 2026-09-21 the follower logged ~66,000 revision-mismatch retries across unrelated packages in a single day.
+
+### Every change becomes visible
+
+Occasionally a change never becomes visible in the feed at any sequence number;
+we name this gap *lost events*.
+Because a package has at most one entry, there is nothing to find by re-reading the feed:
+the package's only entry still describes its previous publish.
+The change only surfaces when the package publishes again and a fresh entry replaces the old one.
+This was reported for `patch-package@7.0.0` (missing for a month, recovered only by the next publish)
+and for `zod-validation-error@1.5.0`;
+deletions have also been observed not to propagate, in the thousands
+([lost events][128412], [version skipped a revision][128412-lost], [missing deletions][128412-deletions]).
+
+### Sequence numbers are dense
+
+Sequence numbers are opaque cursors, not counters.
+The sequence space has gaps,
+and values have jumped by tens of millions within weeks during the 2025 migration ([seq jumps][152515-jumps]).
+A consumer must not derive meaning from the distance between two sequence numbers.
+
+### The feed head only moves forward
+
+Change entries can disappear from the end of the feed,
+in effect moving the head (`update_seq`) *backwards*;
+we name this gap *head regression*.
+During the 2025 migration the head regressed by ~20 million,
+and previously returned entries above the new head no longer existed ([seq regression][152515-regression]).
+The pre-migration feed also once froze its `update_seq` entirely for hours ([stuck update_seq][128412-stuck]).
+
+## How ConstructHub compensates
+
+The *NpmJs Follower* runs on a regular schedule,
+reads the feed forward from a marker stored in S3,
+fetches packuments from the registry to identify new versions,
+and retries failed requests with back-off, resuming from the marker after any outage.
+On top of that baseline, each gap has a dedicated mitigation:
+
+| Gap | Mitigation | Detection |
+| --------------- | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- |
+| Late insertion | Receipt log of received seqs; regular overlap sweep of a trailing feed window; periodic deep sweep of the full receipts retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` and `SweepNotRunning` alarms |
+| Stale metadata | Wait briefly for the registry to catch up, then defer the change to the stale-metadata retry queue and re-check on later runs | `StaleMetadataDeferred` / `StaleMetadataRecovered` metrics; `StaleMetadataDLQNotEmpty` alarm |
+| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package |
+| Head regression | The follower resets its marker to 0 (triggering a re-scan) when the feed's `update_seq` drops below the stored marker | `LastSeq` metric |
+
+**The design idea behind all of this is that the feed data are just hints without real guarantees.**
+The follower treats every change entry as "this package probably changed around here",
+and keeps enough state (marker, receipts, known versions) to re-derive correctness from re-reads.
+
+## Appendix: a late insertion, reconstructed
+
+The timeline below reconstructs the 2026-09-21 incident from log evidence and manual feed reads
+*(our observation)*.
+Times are UTC; "seq X" is the sequence number assigned to the probe package's change entry.
+
+```text
+time consumer feed
+───── ─────────────────────────────────── ──────────────────────────────────────
+15:09 package version is published entry assigned seq X, NOT yet visible
+15:12 reads past seq X, entry not served entry still not visible
+15:?? entry becomes visible at seq X, which
+ is behind the consumer's position
+17:10 manually re-reads the range at seq X entry is served ("late insertion")
+```
+
+A consumer that never re-reads previously covered ranges has no way to notice that anything happened.
+
+## References
+
+- Operator runbook sections for the alarms named above: [operator-runbook.md](./operator-runbook.md)
+- Follower implementation: `src/package-sources/npmjs/npm-js-follower.lambda.ts`
+- Incident that motivated the sweep and retry queue: internal reference V2376798198 *(our observation)*
+
+[152515]: https://github.com/orgs/community/discussions/152515
+[changelog]: https://github.blog/changelog/2025-02-27-changes-and-deprecation-notice-for-npm-replication-apis
+[152515-updateseq]: https://github.com/orgs/community/discussions/152515#discussioncomment-12713712
+[152515-late-report]: https://github.com/orgs/community/discussions/152515#discussioncomment-13561875
+[152515-late]: https://github.com/orgs/community/discussions/152515#discussioncomment-13563633
+[152515-stale]: https://github.com/orgs/community/discussions/152515#discussioncomment-15298851
+[128412-lost]: https://github.com/orgs/community/discussions/128412#discussioncomment-7445623
+[152515-jumps]: https://github.com/orgs/community/discussions/152515#discussioncomment-12731335
+[152515-regression]: https://github.com/orgs/community/discussions/152515#discussioncomment-12754432
+[128412]: https://github.com/orgs/community/discussions/128412
+[128412-deletions]: https://github.com/orgs/community/discussions/128412#discussioncomment-6755605
+[128412-stuck]: https://github.com/orgs/community/discussions/128412#discussioncomment-7653871
diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md
index 794fb44f1..25b1a8169 100644
--- a/docs/operator-runbook.md
+++ b/docs/operator-runbook.md
@@ -582,6 +582,115 @@ but is disabled by default.
Once all messages have cleared from the dead-letter queue, do not forget to
disable the SQS Trigger again.
+### `ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty`
+
+#### Description
+
+This alarm is only provisioned when the `NpmJs` package source is configured. It
+triggers when a change from the npmjs.com registry could not be processed after
+repeated retries, because the registry kept serving package metadata older than
+the revision announced by the changes feed. The message was moved to the
+dead-letter queue.
+
+The package versions that were targeted by those messages have hence not been
+ingested into ConstructHub, and will only be discovered once the affected
+packages publish their next version.
+
+#### Investigation
+
+The *NpmJs Follower* fetches package metadata from `registry.npmjs.org` for
+each change announced by the CouchDB replica's `_changes` feed. The registry
+occasionally lags behind the feed. When it does, the change is placed on a
+retry queue and re-checked on subsequent runs, until the maximum receive count
+is exhausted.
+
+Each message in the dead-letter queue identifies the affected package name and
+the expected document revision. Verify what the registry currently serves for
+the package:
+
+```console
+$ curl https://registry.npmjs.org/ | jq ._rev
+```
+
+If the served revision is still older than the expected one, the problem is
+with upstream npmjs.com. Look at `npmjs.com` status updates and announcements.
+
+If you suspect the problem is with upstream npmjs.com, you can cut them a ticket. Log
+in to , visit the [Support Page](https://www.npmjs.com/support) and
+select *There is a problem with the npm registry*.
+
+#### Resolution
+
+The missed package versions will automatically be discovered once the affected
+packages publish their next version. To ingest a missed version immediately,
+invoke the *ReStagePackageVersion* Lambda function with the package name and
+version from the dead-letter queue message.
+
+Once all messages have been handled, purge them from the dead-letter queue so
+the alarm goes back to green.
+
+### `ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh`
+
+#### Description
+
+This alarm is only provisioned when the `NpmJs` package source is configured. It
+triggers when the *NpmJs Follower*'s overlap sweep discovers changes that were
+inserted into the CouchDB `_changes` feed with a delay approaching the sweep
+margin. Should the delay exceed the margin, the hourly sweep will miss those
+changes, and only the daily deep sweep would still discover them.
+
+#### Investigation
+
+The `_changes` feed occasionally inserts rows behind the follower's cursor
+(npm considers this normal feed behavior). The overlap sweep compensates by
+re-reading a trailing window of the feed, and the `LateChangeLag` metric
+records how far behind the cursor each late row was discovered.
+
+Review the *Feed Reliability (Overlap Sweep)* widget on the backend dashboard.
+A short burst of high-delay insertions usually indicates turbulence in the
+npmjs.com replication pipelines, and resolves on its own. Look at `npmjs.com`
+status updates and announcements.
+
+#### Resolution
+
+The alarm will automatically go back to green once the insertion delays
+subside. If elevated delays persist over multiple days, the sweep margin
+should be increased: adjust `SWEEP_MARGIN_MS` in
+`src/package-sources/npmjs/constants.lambda-shared.ts` (keeping it below
+`RECEIPTS_RETENTION_MS`) and deploy the updated construct.
+
+### `ConstructHub/Sources/NpmJs/Follower/SweepNotRunning`
+
+#### Description
+
+This alarm is only provisioned when the `NpmJs` package source is configured. It
+triggers when the *NpmJs Follower*'s overlap sweep has not reported in several
+hours. While the sweep is not running, changes inserted into the CouchDB
+`_changes` feed behind the follower's cursor will not be discovered.
+
+#### Investigation
+
+The overlap sweep runs at the tail of the *NpmJs Follower*'s scheduled
+executions, in the time remaining after the regular changes processing. The
+most common cause for sweeps not running is the follower being in extended
+catch-up (for example, after an npmjs.com outage), in which case the regular
+changes processing consumes the entire execution time budget.
+
+Review the logs of the *NpmJs Follower* for `sweep` entries, and verify the
+`LastSeq` metric is steadily progressing towards the feed head.
+
+For additional recommendations for diving into CloudWatch Logs, refer to the
+[Diving into Lambda Function logs in CloudWatch Logs][#lambda-log-dive] section.
+
+#### Resolution
+
+The alarm will automatically go back to green once the follower has caught up
+with the feed head and sweeps resume. If the follower is not catching up,
+investigate that instead, following the
+[`ConstructHub/Sources/NpmJs/Follower/NoChanges`](#constructhubsourcesnpmjsfollowernochanges)
+instructions.
+
+
--------------------------------------------------------------------------------
## :repeat: Bulk Re-processing
diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap
index 624d4e5cc..5a1d8a01f 100644
--- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap
+++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap
@@ -572,19 +572,19 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Received Bytes","expression":"FILL(m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
+ "","metrics":[[{"label":"Received Bytes","expression":"FILL(mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0"}],[{"label":"Transmitted Bytes","expression":"FILL(md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
+ "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1"}],[{"label":"Transmitted Bytes","expression":"FILL(mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1"}],[{"label":"Task Count","expression":"FILL(mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
+ "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70"}],[{"label":"Task Count","expression":"FILL(md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Task Count","stat":"Sum","visible":false,"id":"mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
+ "",{"label":"Task Count","stat":"Sum","visible":false,"id":"md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
{
"Ref": "AWS::Region",
},
@@ -640,7 +640,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
},
- "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/",
+ "?prefix=couchdb-known-versions.2)\\n[button:Receipts Object](/s3/object/",
+ {
+ "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ },
+ "?prefix=couchdb-received-seqs)\\n[button:Stager](/lambda/home#/functions/",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -659,6 +663,36 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Stale Metadata Retry Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Stale Metadata Retry DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -671,27 +705,27 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["AWS/Lambda","Duration","FunctionName","",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["AWS/Lambda","Duration","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -699,40 +733,69 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Change Count","expression":"FILL(mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121"}],[{"label":"Unprocessable","expression":"FILL(m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a"}],[{"label":"Lag to npmjs.com","expression":"FILL(m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b"}],[{"label":"Package Version Age","expression":"FILL(mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ "","metrics":[[{"label":"Change Count","expression":"FILL(m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5"}],[{"label":"Unprocessable","expression":"FILL(mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62"}],[{"label":"Lag to npmjs.com","expression":"FILL(m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7"}],[{"label":"Package Version Age","expression":"FILL(m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability (Overlap Sweep)","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e"}],[{"label":"Stale Metadata Deferred","expression":"FILL(m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataDeferred",{"label":"Stale Metadata Deferred","stat":"Sum","visible":false,"id":"m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610"}],[{"label":"Stale Metadata Recovered","expression":"FILL(m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataRecovered",{"label":"Stale Metadata Recovered","stat":"Sum","visible":false,"id":"m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","period":3600,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Sweep Margin","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":3600}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Stale Metadata Retry Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b"}],[{"label":"DLQ Visible Messages","expression":"FILL(m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "",{"label":"DLQ Visible Messages","stat":"Maximum","visible":false,"id":"m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Visible Messages","expression":"FILL(m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400"}],[{"label":"Invisible Messages","expression":"FILL(mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5"}],[{"label":"Invisible Messages","expression":"FILL(m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
+ "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
{
"Ref": "AWS::Region",
},
@@ -740,7 +803,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsCanary5558FC45",
},
- "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
+ "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
{
"Ref": "ConstructHubIngestion407909CE",
},
@@ -763,19 +826,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -800,11 +863,11 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
+ "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
{
"Ref": "AWS::Region",
},
- "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
+ "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47"}],[{"label":"Invalid Tarball","expression":"FILL(mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2"}],[{"label":"Ineligible License","expression":"FILL(mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a"}],[{"label":"Mismatched Identity","expression":"FILL(m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189"}],[{"label":"Found License file","expression":"FILL(m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
{
"Ref": "AWS::Region",
},
@@ -829,7 +892,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubOrchestration39161A46",
},
@@ -860,43 +923,43 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Started","expression":"FILL(m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
+ "","metrics":[[{"label":"Started","expression":"FILL(md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Started","stat":"Sum","visible":false,"id":"m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c"}],[{"label":"Succeeded","expression":"FILL(md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
+ "",{"label":"Started","stat":"Sum","visible":false,"id":"md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b"}],[{"label":"Succeeded","expression":"FILL(mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1"}],["AWS/States","ExecutionTime","StateMachineArn","",
+ "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4"}],["AWS/States","ExecutionTime","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
+ "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Aborted","expression":"FILL(m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
+ "","metrics":[[{"label":"Aborted","expression":"FILL(m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08"}],[{"label":"Failed","expression":"FILL(m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
+ "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5"}],[{"label":"Failed","expression":"FILL(mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Failed","stat":"Sum","visible":false,"id":"m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597"}],[{"label":"Throttled","expression":"FILL(m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
+ "",{"label":"Failed","stat":"Sum","visible":false,"id":"mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3"}],[{"label":"Throttled","expression":"FILL(m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f"}],[{"label":"Timed Out","expression":"FILL(m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
+ "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe"}],[{"label":"Timed Out","expression":"FILL(m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
+ "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -921,7 +984,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
{
"Ref": "ConstructHubDenyListBucket1B3C2C2E",
},
@@ -952,30 +1015,30 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Rules","expression":"FILL(m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
+ "","metrics":[[{"label":"Rules","expression":"FILL(m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubDenyListPruneDeleteQueueBBF60185",
"QueueName",
],
},
- "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
+ "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -995,11 +1058,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsAggregator3345DB4E",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Packages with stats","expression":"FILL(mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -1007,7 +1070,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsStateMachine1A1DEC97",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -1019,11 +1082,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Package versions recorded","expression":"FILL(mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -9875,6 +9938,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -9958,7 +10029,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip",
+ "S3Key": "a0cc00d9c18e07c65112616e2da5da6fdc8cf2155dad9e4d76ca4f05fd7a2229.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -9978,6 +10049,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "RETRY_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ },
},
},
"Handler": "index.handler",
@@ -10412,6 +10486,26 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep is discovering changes that were inserted into the CouchDB changes feed
+with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep
+will miss those changes (only the daily deep sweep would catch them). Consider increasing
+the sweep margin (SWEEP_MARGIN_MS).
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "LateChangeLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Maximum",
+ "Threshold": 17280000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -10485,6 +10579,82 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerStaleMetadataDLQNotEmpty68864A28": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The npm registry never caught up with the revision announced by the CouchDB changes feed
+for one or more packages, even after repeated retries. The affected package versions were
+NOT ingested, and will only be discovered when the package publishes its next version.
+The messages in the dead-letter queue identify the affected packages, which can be
+manually ingested using the ReStagePackageVersion function.
+
+Link to the dead letter queue: /sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "QueueName",
+ "Value": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ },
+ ],
+ "EvaluationPeriods": 1,
+ "MetricName": "ApproximateNumberOfMessagesVisible",
+ "Namespace": "AWS/SQS",
+ "Period": 300,
+ "Statistic": "Maximum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsFollowerSweepNotRunningC7CA155E": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep of the NpmJs follower has not reported in several hours. The follower
+is currently blind to changes inserted into the CouchDB changes feed behind its cursor.
+This usually means the follower is in extended catch-up and the head-of-feed pass is
+consuming the entire execution time budget.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/SweepNotRunning",
+ "ComparisonOperator": "LessThanThreshold",
+ "EvaluationPeriods": 6,
+ "MetricName": "LateChangeCount",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Sum",
+ "Threshold": 0,
+ "TreatMissingData": "breaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -10654,6 +10824,43 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ "s3:DeleteObject*",
+ "s3:PutObject",
+ "s3:PutObjectLegalHold",
+ "s3:PutObjectRetention",
+ "s3:PutObjectTagging",
+ "s3:PutObjectVersionTagging",
+ "s3:Abort*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ "/couchdb-received-seqs",
+ ],
+ ],
+ },
+ ],
+ },
{
"Action": [
"s3:GetObject*",
@@ -10740,6 +10947,36 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -10765,7 +11002,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip",
+ "S3Key": "e34dcef0fa9dde1269f8a4cf5e401b4b8be2ffda10905a4279111e1477a65bfc.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -11021,6 +11258,14 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -11083,6 +11328,34 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 8,
+ },
+ "VisibilityTimeout": 900,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -14549,19 +14822,19 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Received Bytes","expression":"FILL(m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
+ "","metrics":[[{"label":"Received Bytes","expression":"FILL(mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0"}],[{"label":"Transmitted Bytes","expression":"FILL(md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
+ "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1"}],[{"label":"Transmitted Bytes","expression":"FILL(mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1"}],[{"label":"Task Count","expression":"FILL(mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
+ "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70"}],[{"label":"Task Count","expression":"FILL(md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Task Count","stat":"Sum","visible":false,"id":"mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
+ "",{"label":"Task Count","stat":"Sum","visible":false,"id":"md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
{
"Ref": "AWS::Region",
},
@@ -14617,7 +14890,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
},
- "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/",
+ "?prefix=couchdb-known-versions.2)\\n[button:Receipts Object](/s3/object/",
+ {
+ "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ },
+ "?prefix=couchdb-received-seqs)\\n[button:Stager](/lambda/home#/functions/",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -14636,6 +14913,36 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Stale Metadata Retry Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Stale Metadata Retry DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -14648,27 +14955,27 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["AWS/Lambda","Duration","FunctionName","",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["AWS/Lambda","Duration","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -14676,40 +14983,69 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Change Count","expression":"FILL(mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121"}],[{"label":"Unprocessable","expression":"FILL(m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a"}],[{"label":"Lag to npmjs.com","expression":"FILL(m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b"}],[{"label":"Package Version Age","expression":"FILL(mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ "","metrics":[[{"label":"Change Count","expression":"FILL(m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5"}],[{"label":"Unprocessable","expression":"FILL(mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62"}],[{"label":"Lag to npmjs.com","expression":"FILL(m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7"}],[{"label":"Package Version Age","expression":"FILL(m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability (Overlap Sweep)","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e"}],[{"label":"Stale Metadata Deferred","expression":"FILL(m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataDeferred",{"label":"Stale Metadata Deferred","stat":"Sum","visible":false,"id":"m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610"}],[{"label":"Stale Metadata Recovered","expression":"FILL(m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataRecovered",{"label":"Stale Metadata Recovered","stat":"Sum","visible":false,"id":"m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","period":3600,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Sweep Margin","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":3600}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Stale Metadata Retry Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b"}],[{"label":"DLQ Visible Messages","expression":"FILL(m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "",{"label":"DLQ Visible Messages","stat":"Maximum","visible":false,"id":"m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Visible Messages","expression":"FILL(m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400"}],[{"label":"Invisible Messages","expression":"FILL(mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5"}],[{"label":"Invisible Messages","expression":"FILL(m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
+ "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
{
"Ref": "AWS::Region",
},
@@ -14717,7 +15053,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsCanary5558FC45",
},
- "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
+ "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
{
"Ref": "ConstructHubIngestion407909CE",
},
@@ -14740,19 +15076,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -14777,11 +15113,11 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
+ "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
{
"Ref": "AWS::Region",
},
- "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
+ "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47"}],[{"label":"Invalid Tarball","expression":"FILL(mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2"}],[{"label":"Ineligible License","expression":"FILL(mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a"}],[{"label":"Mismatched Identity","expression":"FILL(m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189"}],[{"label":"Found License file","expression":"FILL(m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
{
"Ref": "AWS::Region",
},
@@ -14806,7 +15142,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubOrchestration39161A46",
},
@@ -14837,43 +15173,43 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Started","expression":"FILL(m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
+ "","metrics":[[{"label":"Started","expression":"FILL(md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Started","stat":"Sum","visible":false,"id":"m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c"}],[{"label":"Succeeded","expression":"FILL(md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
+ "",{"label":"Started","stat":"Sum","visible":false,"id":"md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b"}],[{"label":"Succeeded","expression":"FILL(mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1"}],["AWS/States","ExecutionTime","StateMachineArn","",
+ "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4"}],["AWS/States","ExecutionTime","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
+ "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Aborted","expression":"FILL(m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
+ "","metrics":[[{"label":"Aborted","expression":"FILL(m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08"}],[{"label":"Failed","expression":"FILL(m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
+ "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5"}],[{"label":"Failed","expression":"FILL(mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Failed","stat":"Sum","visible":false,"id":"m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597"}],[{"label":"Throttled","expression":"FILL(m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
+ "",{"label":"Failed","stat":"Sum","visible":false,"id":"mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3"}],[{"label":"Throttled","expression":"FILL(m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f"}],[{"label":"Timed Out","expression":"FILL(m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
+ "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe"}],[{"label":"Timed Out","expression":"FILL(m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
+ "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -14898,7 +15234,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
{
"Ref": "ConstructHubDenyListBucket1B3C2C2E",
},
@@ -14929,30 +15265,30 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Rules","expression":"FILL(m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
+ "","metrics":[[{"label":"Rules","expression":"FILL(m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubDenyListPruneDeleteQueueBBF60185",
"QueueName",
],
},
- "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
+ "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -14972,11 +15308,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsAggregator3345DB4E",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Packages with stats","expression":"FILL(mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -14984,7 +15320,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsStateMachine1A1DEC97",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -14996,11 +15332,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Package versions recorded","expression":"FILL(mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -24226,6 +24562,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -24309,7 +24653,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip",
+ "S3Key": "a0cc00d9c18e07c65112616e2da5da6fdc8cf2155dad9e4d76ca4f05fd7a2229.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -24329,6 +24673,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "RETRY_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ },
},
},
"Handler": "index.handler",
@@ -24763,6 +25110,26 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep is discovering changes that were inserted into the CouchDB changes feed
+with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep
+will miss those changes (only the daily deep sweep would catch them). Consider increasing
+the sweep margin (SWEEP_MARGIN_MS).
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "LateChangeLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Maximum",
+ "Threshold": 17280000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -24836,6 +25203,82 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerStaleMetadataDLQNotEmpty68864A28": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The npm registry never caught up with the revision announced by the CouchDB changes feed
+for one or more packages, even after repeated retries. The affected package versions were
+NOT ingested, and will only be discovered when the package publishes its next version.
+The messages in the dead-letter queue identify the affected packages, which can be
+manually ingested using the ReStagePackageVersion function.
+
+Link to the dead letter queue: /sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "QueueName",
+ "Value": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ },
+ ],
+ "EvaluationPeriods": 1,
+ "MetricName": "ApproximateNumberOfMessagesVisible",
+ "Namespace": "AWS/SQS",
+ "Period": 300,
+ "Statistic": "Maximum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsFollowerSweepNotRunningC7CA155E": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep of the NpmJs follower has not reported in several hours. The follower
+is currently blind to changes inserted into the CouchDB changes feed behind its cursor.
+This usually means the follower is in extended catch-up and the head-of-feed pass is
+consuming the entire execution time budget.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/SweepNotRunning",
+ "ComparisonOperator": "LessThanThreshold",
+ "EvaluationPeriods": 6,
+ "MetricName": "LateChangeCount",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Sum",
+ "Threshold": 0,
+ "TreatMissingData": "breaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -25005,6 +25448,43 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ "s3:DeleteObject*",
+ "s3:PutObject",
+ "s3:PutObjectLegalHold",
+ "s3:PutObjectRetention",
+ "s3:PutObjectTagging",
+ "s3:PutObjectVersionTagging",
+ "s3:Abort*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ "/couchdb-received-seqs",
+ ],
+ ],
+ },
+ ],
+ },
{
"Action": [
"s3:GetObject*",
@@ -25091,6 +25571,36 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -25116,7 +25626,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip",
+ "S3Key": "e34dcef0fa9dde1269f8a4cf5e401b4b8be2ffda10905a4279111e1477a65bfc.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -25372,6 +25882,14 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -25434,6 +25952,34 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 8,
+ },
+ "VisibilityTimeout": 900,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -28838,19 +29384,19 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Received Bytes","expression":"FILL(m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
+ "","metrics":[[{"label":"Received Bytes","expression":"FILL(mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0"}],[{"label":"Transmitted Bytes","expression":"FILL(md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
+ "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1"}],[{"label":"Transmitted Bytes","expression":"FILL(mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1"}],[{"label":"Task Count","expression":"FILL(mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
+ "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70"}],[{"label":"Task Count","expression":"FILL(md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Task Count","stat":"Sum","visible":false,"id":"mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
+ "",{"label":"Task Count","stat":"Sum","visible":false,"id":"md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
{
"Ref": "AWS::Region",
},
@@ -28906,7 +29452,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
},
- "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/",
+ "?prefix=couchdb-known-versions.2)\\n[button:Receipts Object](/s3/object/",
+ {
+ "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ },
+ "?prefix=couchdb-received-seqs)\\n[button:Stager](/lambda/home#/functions/",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -28925,6 +29475,36 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Stale Metadata Retry Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Stale Metadata Retry DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -28937,27 +29517,27 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["AWS/Lambda","Duration","FunctionName","",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["AWS/Lambda","Duration","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -28965,40 +29545,69 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Change Count","expression":"FILL(mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121"}],[{"label":"Unprocessable","expression":"FILL(m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a"}],[{"label":"Lag to npmjs.com","expression":"FILL(m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b"}],[{"label":"Package Version Age","expression":"FILL(mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ "","metrics":[[{"label":"Change Count","expression":"FILL(m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5"}],[{"label":"Unprocessable","expression":"FILL(mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62"}],[{"label":"Lag to npmjs.com","expression":"FILL(m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7"}],[{"label":"Package Version Age","expression":"FILL(m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability (Overlap Sweep)","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Visible Messages","expression":"FILL(m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e"}],[{"label":"Stale Metadata Deferred","expression":"FILL(m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataDeferred",{"label":"Stale Metadata Deferred","stat":"Sum","visible":false,"id":"m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610"}],[{"label":"Stale Metadata Recovered","expression":"FILL(m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataRecovered",{"label":"Stale Metadata Recovered","stat":"Sum","visible":false,"id":"m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","period":3600,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Sweep Margin","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":3600}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Stale Metadata Retry Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b"}],[{"label":"DLQ Visible Messages","expression":"FILL(m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "",{"label":"DLQ Visible Messages","stat":"Maximum","visible":false,"id":"m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400"}],[{"label":"Invisible Messages","expression":"FILL(mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5"}],[{"label":"Invisible Messages","expression":"FILL(m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
+ "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
{
"Ref": "AWS::Region",
},
@@ -29006,7 +29615,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsCanary5558FC45",
},
- "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
+ "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
{
"Ref": "ConstructHubIngestion407909CE",
},
@@ -29029,19 +29638,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -29066,11 +29675,11 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
+ "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
{
"Ref": "AWS::Region",
},
- "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
+ "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47"}],[{"label":"Invalid Tarball","expression":"FILL(mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2"}],[{"label":"Ineligible License","expression":"FILL(mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a"}],[{"label":"Mismatched Identity","expression":"FILL(m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189"}],[{"label":"Found License file","expression":"FILL(m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
{
"Ref": "AWS::Region",
},
@@ -29095,7 +29704,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubOrchestration39161A46",
},
@@ -29126,43 +29735,43 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Started","expression":"FILL(m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
+ "","metrics":[[{"label":"Started","expression":"FILL(md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Started","stat":"Sum","visible":false,"id":"m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c"}],[{"label":"Succeeded","expression":"FILL(md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
+ "",{"label":"Started","stat":"Sum","visible":false,"id":"md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b"}],[{"label":"Succeeded","expression":"FILL(mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1"}],["AWS/States","ExecutionTime","StateMachineArn","",
+ "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4"}],["AWS/States","ExecutionTime","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
+ "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Aborted","expression":"FILL(m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
+ "","metrics":[[{"label":"Aborted","expression":"FILL(m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08"}],[{"label":"Failed","expression":"FILL(m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
+ "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5"}],[{"label":"Failed","expression":"FILL(mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Failed","stat":"Sum","visible":false,"id":"m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597"}],[{"label":"Throttled","expression":"FILL(m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
+ "",{"label":"Failed","stat":"Sum","visible":false,"id":"mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3"}],[{"label":"Throttled","expression":"FILL(m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f"}],[{"label":"Timed Out","expression":"FILL(m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
+ "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe"}],[{"label":"Timed Out","expression":"FILL(m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
+ "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -29187,7 +29796,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
{
"Ref": "ConstructHubDenyListBucket1B3C2C2E",
},
@@ -29218,30 +29827,30 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Rules","expression":"FILL(m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
+ "","metrics":[[{"label":"Rules","expression":"FILL(m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubDenyListPruneDeleteQueueBBF60185",
"QueueName",
],
},
- "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
+ "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -29261,11 +29870,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsAggregator3345DB4E",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Packages with stats","expression":"FILL(mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -29273,7 +29882,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsStateMachine1A1DEC97",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -29285,11 +29894,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Package versions recorded","expression":"FILL(mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -38141,6 +38750,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -38224,7 +38841,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip",
+ "S3Key": "a0cc00d9c18e07c65112616e2da5da6fdc8cf2155dad9e4d76ca4f05fd7a2229.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -38244,6 +38861,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "RETRY_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ },
},
},
"Handler": "index.handler",
@@ -38678,6 +39298,26 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep is discovering changes that were inserted into the CouchDB changes feed
+with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep
+will miss those changes (only the daily deep sweep would catch them). Consider increasing
+the sweep margin (SWEEP_MARGIN_MS).
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "LateChangeLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Maximum",
+ "Threshold": 17280000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -38751,6 +39391,82 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerStaleMetadataDLQNotEmpty68864A28": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The npm registry never caught up with the revision announced by the CouchDB changes feed
+for one or more packages, even after repeated retries. The affected package versions were
+NOT ingested, and will only be discovered when the package publishes its next version.
+The messages in the dead-letter queue identify the affected packages, which can be
+manually ingested using the ReStagePackageVersion function.
+
+Link to the dead letter queue: /sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "QueueName",
+ "Value": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ },
+ ],
+ "EvaluationPeriods": 1,
+ "MetricName": "ApproximateNumberOfMessagesVisible",
+ "Namespace": "AWS/SQS",
+ "Period": 300,
+ "Statistic": "Maximum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsFollowerSweepNotRunningC7CA155E": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep of the NpmJs follower has not reported in several hours. The follower
+is currently blind to changes inserted into the CouchDB changes feed behind its cursor.
+This usually means the follower is in extended catch-up and the head-of-feed pass is
+consuming the entire execution time budget.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/SweepNotRunning",
+ "ComparisonOperator": "LessThanThreshold",
+ "EvaluationPeriods": 6,
+ "MetricName": "LateChangeCount",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Sum",
+ "Threshold": 0,
+ "TreatMissingData": "breaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -38920,6 +39636,43 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ "s3:DeleteObject*",
+ "s3:PutObject",
+ "s3:PutObjectLegalHold",
+ "s3:PutObjectRetention",
+ "s3:PutObjectTagging",
+ "s3:PutObjectVersionTagging",
+ "s3:Abort*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ "/couchdb-received-seqs",
+ ],
+ ],
+ },
+ ],
+ },
{
"Action": [
"s3:GetObject*",
@@ -39006,6 +39759,36 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -39031,7 +39814,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip",
+ "S3Key": "e34dcef0fa9dde1269f8a4cf5e401b4b8be2ffda10905a4279111e1477a65bfc.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -39287,6 +40070,14 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -39349,6 +40140,34 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 8,
+ },
+ "VisibilityTimeout": 900,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -42901,19 +43720,19 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Received Bytes","expression":"FILL(m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
+ "","metrics":[[{"label":"Received Bytes","expression":"FILL(mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0"}],[{"label":"Transmitted Bytes","expression":"FILL(md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
+ "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1"}],[{"label":"Transmitted Bytes","expression":"FILL(mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1"}],[{"label":"Task Count","expression":"FILL(mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
+ "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70"}],[{"label":"Task Count","expression":"FILL(md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Task Count","stat":"Sum","visible":false,"id":"mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
+ "",{"label":"Task Count","stat":"Sum","visible":false,"id":"md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
{
"Ref": "AWS::Region",
},
@@ -42969,7 +43788,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
},
- "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/",
+ "?prefix=couchdb-known-versions.2)\\n[button:Receipts Object](/s3/object/",
+ {
+ "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ },
+ "?prefix=couchdb-received-seqs)\\n[button:Stager](/lambda/home#/functions/",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -42988,6 +43811,36 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Stale Metadata Retry Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Stale Metadata Retry DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -43000,27 +43853,27 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["AWS/Lambda","Duration","FunctionName","",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["AWS/Lambda","Duration","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -43028,40 +43881,69 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Change Count","expression":"FILL(mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121"}],[{"label":"Unprocessable","expression":"FILL(m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a"}],[{"label":"Lag to npmjs.com","expression":"FILL(m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b"}],[{"label":"Package Version Age","expression":"FILL(mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ "","metrics":[[{"label":"Change Count","expression":"FILL(m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5"}],[{"label":"Unprocessable","expression":"FILL(mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62"}],[{"label":"Lag to npmjs.com","expression":"FILL(m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7"}],[{"label":"Package Version Age","expression":"FILL(m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability (Overlap Sweep)","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e"}],[{"label":"Stale Metadata Deferred","expression":"FILL(m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataDeferred",{"label":"Stale Metadata Deferred","stat":"Sum","visible":false,"id":"m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610"}],[{"label":"Stale Metadata Recovered","expression":"FILL(m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataRecovered",{"label":"Stale Metadata Recovered","stat":"Sum","visible":false,"id":"m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","period":3600,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Sweep Margin","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":3600}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Stale Metadata Retry Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Visible Messages","expression":"FILL(m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b"}],[{"label":"DLQ Visible Messages","expression":"FILL(m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "",{"label":"DLQ Visible Messages","stat":"Maximum","visible":false,"id":"m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400"}],[{"label":"Invisible Messages","expression":"FILL(mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5"}],[{"label":"Invisible Messages","expression":"FILL(m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
+ "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
{
"Ref": "AWS::Region",
},
@@ -43069,7 +43951,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsCanary5558FC45",
},
- "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
+ "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
{
"Ref": "ConstructHubIngestion407909CE",
},
@@ -43092,19 +43974,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -43129,11 +44011,11 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
+ "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
{
"Ref": "AWS::Region",
},
- "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
+ "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47"}],[{"label":"Invalid Tarball","expression":"FILL(mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2"}],[{"label":"Ineligible License","expression":"FILL(mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a"}],[{"label":"Mismatched Identity","expression":"FILL(m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189"}],[{"label":"Found License file","expression":"FILL(m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
{
"Ref": "AWS::Region",
},
@@ -43158,7 +44040,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubOrchestration39161A46",
},
@@ -43189,43 +44071,43 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Started","expression":"FILL(m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
+ "","metrics":[[{"label":"Started","expression":"FILL(md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Started","stat":"Sum","visible":false,"id":"m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c"}],[{"label":"Succeeded","expression":"FILL(md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
+ "",{"label":"Started","stat":"Sum","visible":false,"id":"md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b"}],[{"label":"Succeeded","expression":"FILL(mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1"}],["AWS/States","ExecutionTime","StateMachineArn","",
+ "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4"}],["AWS/States","ExecutionTime","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
+ "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Aborted","expression":"FILL(m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
+ "","metrics":[[{"label":"Aborted","expression":"FILL(m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08"}],[{"label":"Failed","expression":"FILL(m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
+ "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5"}],[{"label":"Failed","expression":"FILL(mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Failed","stat":"Sum","visible":false,"id":"m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597"}],[{"label":"Throttled","expression":"FILL(m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
+ "",{"label":"Failed","stat":"Sum","visible":false,"id":"mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3"}],[{"label":"Throttled","expression":"FILL(m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f"}],[{"label":"Timed Out","expression":"FILL(m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
+ "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe"}],[{"label":"Timed Out","expression":"FILL(m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
+ "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -43250,7 +44132,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
{
"Ref": "ConstructHubDenyListBucket1B3C2C2E",
},
@@ -43281,30 +44163,30 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Rules","expression":"FILL(m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
+ "","metrics":[[{"label":"Rules","expression":"FILL(m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubDenyListPruneDeleteQueueBBF60185",
"QueueName",
],
},
- "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
+ "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -43324,11 +44206,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsAggregator3345DB4E",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Packages with stats","expression":"FILL(mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -43336,7 +44218,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsStateMachine1A1DEC97",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -43348,11 +44230,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Package versions recorded","expression":"FILL(mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -52213,6 +53095,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -52296,7 +53186,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip",
+ "S3Key": "a0cc00d9c18e07c65112616e2da5da6fdc8cf2155dad9e4d76ca4f05fd7a2229.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -52316,6 +53206,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "RETRY_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ },
},
},
"Handler": "index.handler",
@@ -52737,6 +53630,26 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep is discovering changes that were inserted into the CouchDB changes feed
+with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep
+will miss those changes (only the daily deep sweep would catch them). Consider increasing
+the sweep margin (SWEEP_MARGIN_MS).
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "LateChangeLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Maximum",
+ "Threshold": 17280000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -52810,6 +53723,82 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerStaleMetadataDLQNotEmpty68864A28": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The npm registry never caught up with the revision announced by the CouchDB changes feed
+for one or more packages, even after repeated retries. The affected package versions were
+NOT ingested, and will only be discovered when the package publishes its next version.
+The messages in the dead-letter queue identify the affected packages, which can be
+manually ingested using the ReStagePackageVersion function.
+
+Link to the dead letter queue: /sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "QueueName",
+ "Value": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ },
+ ],
+ "EvaluationPeriods": 1,
+ "MetricName": "ApproximateNumberOfMessagesVisible",
+ "Namespace": "AWS/SQS",
+ "Period": 300,
+ "Statistic": "Maximum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsFollowerSweepNotRunningC7CA155E": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep of the NpmJs follower has not reported in several hours. The follower
+is currently blind to changes inserted into the CouchDB changes feed behind its cursor.
+This usually means the follower is in extended catch-up and the head-of-feed pass is
+consuming the entire execution time budget.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/SweepNotRunning",
+ "ComparisonOperator": "LessThanThreshold",
+ "EvaluationPeriods": 6,
+ "MetricName": "LateChangeCount",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Sum",
+ "Threshold": 0,
+ "TreatMissingData": "breaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -52979,6 +53968,43 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ "s3:DeleteObject*",
+ "s3:PutObject",
+ "s3:PutObjectLegalHold",
+ "s3:PutObjectRetention",
+ "s3:PutObjectTagging",
+ "s3:PutObjectVersionTagging",
+ "s3:Abort*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ "/couchdb-received-seqs",
+ ],
+ ],
+ },
+ ],
+ },
{
"Action": [
"s3:GetObject*",
@@ -53065,6 +54091,36 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -53090,7 +54146,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip",
+ "S3Key": "e34dcef0fa9dde1269f8a4cf5e401b4b8be2ffda10905a4279111e1477a65bfc.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -53346,6 +54402,14 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -53408,6 +54472,34 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 8,
+ },
+ "VisibilityTimeout": 900,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -57149,19 +58241,19 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Received Bytes","expression":"FILL(m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
+ "","metrics":[[{"label":"Received Bytes","expression":"FILL(mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0"}],[{"label":"Transmitted Bytes","expression":"FILL(md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
+ "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1"}],[{"label":"Transmitted Bytes","expression":"FILL(mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1"}],[{"label":"Task Count","expression":"FILL(mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
+ "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70"}],[{"label":"Task Count","expression":"FILL(md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Task Count","stat":"Sum","visible":false,"id":"mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
+ "",{"label":"Task Count","stat":"Sum","visible":false,"id":"md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
{
"Ref": "AWS::Region",
},
@@ -57217,7 +58309,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
},
- "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/",
+ "?prefix=couchdb-known-versions.2)\\n[button:Receipts Object](/s3/object/",
+ {
+ "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ },
+ "?prefix=couchdb-received-seqs)\\n[button:Stager](/lambda/home#/functions/",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -57236,6 +58332,36 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Stale Metadata Retry Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Stale Metadata Retry DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -57248,27 +58374,27 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["AWS/Lambda","Duration","FunctionName","",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["AWS/Lambda","Duration","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -57276,40 +58402,69 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Change Count","expression":"FILL(mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121"}],[{"label":"Unprocessable","expression":"FILL(m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a"}],[{"label":"Lag to npmjs.com","expression":"FILL(m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b"}],[{"label":"Package Version Age","expression":"FILL(mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ "","metrics":[[{"label":"Change Count","expression":"FILL(m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5"}],[{"label":"Unprocessable","expression":"FILL(mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62"}],[{"label":"Lag to npmjs.com","expression":"FILL(m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7"}],[{"label":"Package Version Age","expression":"FILL(m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability (Overlap Sweep)","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e"}],[{"label":"Stale Metadata Deferred","expression":"FILL(m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataDeferred",{"label":"Stale Metadata Deferred","stat":"Sum","visible":false,"id":"m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610"}],[{"label":"Stale Metadata Recovered","expression":"FILL(m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataRecovered",{"label":"Stale Metadata Recovered","stat":"Sum","visible":false,"id":"m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","period":3600,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Sweep Margin","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":3600}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Stale Metadata Retry Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b"}],[{"label":"DLQ Visible Messages","expression":"FILL(m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "",{"label":"DLQ Visible Messages","stat":"Maximum","visible":false,"id":"m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Visible Messages","expression":"FILL(m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400"}],[{"label":"Invisible Messages","expression":"FILL(mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5"}],[{"label":"Invisible Messages","expression":"FILL(m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
+ "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
{
"Ref": "AWS::Region",
},
@@ -57317,7 +58472,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsCanary5558FC45",
},
- "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
+ "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
{
"Ref": "ConstructHubIngestion407909CE",
},
@@ -57340,19 +58495,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -57377,11 +58532,11 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
+ "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
{
"Ref": "AWS::Region",
},
- "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
+ "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47"}],[{"label":"Invalid Tarball","expression":"FILL(mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2"}],[{"label":"Ineligible License","expression":"FILL(mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a"}],[{"label":"Mismatched Identity","expression":"FILL(m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189"}],[{"label":"Found License file","expression":"FILL(m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
{
"Ref": "AWS::Region",
},
@@ -57406,7 +58561,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubOrchestration39161A46",
},
@@ -57437,43 +58592,43 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Started","expression":"FILL(m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
+ "","metrics":[[{"label":"Started","expression":"FILL(md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Started","stat":"Sum","visible":false,"id":"m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c"}],[{"label":"Succeeded","expression":"FILL(md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
+ "",{"label":"Started","stat":"Sum","visible":false,"id":"md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b"}],[{"label":"Succeeded","expression":"FILL(mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1"}],["AWS/States","ExecutionTime","StateMachineArn","",
+ "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4"}],["AWS/States","ExecutionTime","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
+ "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Aborted","expression":"FILL(m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
+ "","metrics":[[{"label":"Aborted","expression":"FILL(m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08"}],[{"label":"Failed","expression":"FILL(m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
+ "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5"}],[{"label":"Failed","expression":"FILL(mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Failed","stat":"Sum","visible":false,"id":"m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597"}],[{"label":"Throttled","expression":"FILL(m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
+ "",{"label":"Failed","stat":"Sum","visible":false,"id":"mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3"}],[{"label":"Throttled","expression":"FILL(m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f"}],[{"label":"Timed Out","expression":"FILL(m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
+ "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe"}],[{"label":"Timed Out","expression":"FILL(m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
+ "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -57498,7 +58653,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
{
"Ref": "ConstructHubDenyListBucket1B3C2C2E",
},
@@ -57529,30 +58684,30 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Rules","expression":"FILL(m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
+ "","metrics":[[{"label":"Rules","expression":"FILL(m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubDenyListPruneDeleteQueueBBF60185",
"QueueName",
],
},
- "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
+ "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -57572,11 +58727,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsAggregator3345DB4E",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Packages with stats","expression":"FILL(mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -57584,7 +58739,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsStateMachine1A1DEC97",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -57596,11 +58751,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Package versions recorded","expression":"FILL(mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -66268,6 +67423,14 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -66351,7 +67514,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip",
+ "S3Key": "a0cc00d9c18e07c65112616e2da5da6fdc8cf2155dad9e4d76ca4f05fd7a2229.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -66371,6 +67534,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "RETRY_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ },
},
},
"Handler": "index.handler",
@@ -66805,6 +67971,26 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep is discovering changes that were inserted into the CouchDB changes feed
+with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep
+will miss those changes (only the daily deep sweep would catch them). Consider increasing
+the sweep margin (SWEEP_MARGIN_MS).
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "LateChangeLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Maximum",
+ "Threshold": 17280000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -66878,6 +68064,82 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerStaleMetadataDLQNotEmpty68864A28": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The npm registry never caught up with the revision announced by the CouchDB changes feed
+for one or more packages, even after repeated retries. The affected package versions were
+NOT ingested, and will only be discovered when the package publishes its next version.
+The messages in the dead-letter queue identify the affected packages, which can be
+manually ingested using the ReStagePackageVersion function.
+
+Link to the dead letter queue: /sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "QueueName",
+ "Value": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ },
+ ],
+ "EvaluationPeriods": 1,
+ "MetricName": "ApproximateNumberOfMessagesVisible",
+ "Namespace": "AWS/SQS",
+ "Period": 300,
+ "Statistic": "Maximum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsFollowerSweepNotRunningC7CA155E": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep of the NpmJs follower has not reported in several hours. The follower
+is currently blind to changes inserted into the CouchDB changes feed behind its cursor.
+This usually means the follower is in extended catch-up and the head-of-feed pass is
+consuming the entire execution time budget.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/SweepNotRunning",
+ "ComparisonOperator": "LessThanThreshold",
+ "EvaluationPeriods": 6,
+ "MetricName": "LateChangeCount",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Sum",
+ "Threshold": 0,
+ "TreatMissingData": "breaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -67047,6 +68309,43 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ "s3:DeleteObject*",
+ "s3:PutObject",
+ "s3:PutObjectLegalHold",
+ "s3:PutObjectRetention",
+ "s3:PutObjectTagging",
+ "s3:PutObjectVersionTagging",
+ "s3:Abort*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ "/couchdb-received-seqs",
+ ],
+ ],
+ },
+ ],
+ },
{
"Action": [
"s3:GetObject*",
@@ -67133,6 +68432,36 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -67158,7 +68487,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip",
+ "S3Key": "e34dcef0fa9dde1269f8a4cf5e401b4b8be2ffda10905a4279111e1477a65bfc.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -67414,6 +68743,14 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -67476,6 +68813,34 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 8,
+ },
+ "VisibilityTimeout": 900,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
index bf55cb6b3..ce413818c 100644
--- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
+++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
@@ -634,19 +634,19 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Received Bytes","expression":"FILL(m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
+ "","metrics":[[{"label":"Received Bytes","expression":"FILL(mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1, 0)"}],["ECS/ContainerInsights","NetworkRxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"m67cc14db3275d4a2af71847df2ebb376a00b015f62c4ff9d1aedb9bc615a6cf0"}],[{"label":"Transmitted Bytes","expression":"FILL(md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
+ "",{"label":"Received Bytes","stat":"Maximum","visible":false,"id":"mfc29e97ddb123822420c2c4832576b9b5a122defc55f65894354053a927177d1"}],[{"label":"Transmitted Bytes","expression":"FILL(mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70, 0)"}],["ECS/ContainerInsights","NetworkTxBytes","ClusterName","",
{
"Ref": "ConstructHubOrchestrationCluster3D6F0081",
},
- "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"md6781afc8bf73de5587d36039d8226d1c0fe0b50a1c949759d3d8d8cd2e3f6d1"}],[{"label":"Task Count","expression":"FILL(mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
+ "",{"label":"Transmitted Bytes","stat":"Maximum","visible":false,"id":"mef6349ff005c3fc912ee40b47726809d8007e2f6315c55bb831e4e0f526e0c70"}],[{"label":"Task Count","expression":"FILL(md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c, 0)","yAxis":"right"}],["ECS/ContainerInsights","TaskCount","ExecutionMetrics","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Task Count","stat":"Sum","visible":false,"id":"mbb258976f5c025a7dff5289417a6b07619e000e4af1a427fecc57b31586fd6dd"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
+ "",{"label":"Task Count","stat":"Sum","visible":false,"id":"md923e869a0044de432ace532a4e4f8accf176ba0869e72d3322dd4cbd39e7c0c"}]],"yAxis":{"left":{"min":0},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":0,"y":16,"properties":{"view":"timeSeries","title":"ECS Task Monitor","region":"",
{
"Ref": "AWS::Region",
},
@@ -702,7 +702,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
},
- "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/",
+ "?prefix=couchdb-known-versions.2)\\n[button:Receipts Object](/s3/object/",
+ {
+ "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ },
+ "?prefix=couchdb-received-seqs)\\n[button:Stager](/lambda/home#/functions/",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -721,6 +725,36 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Stale Metadata Retry Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Stale Metadata Retry DLQ](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -733,27 +767,27 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["ConstructHub/PackageSource/NpmJs/Follower","RemainingTime",{"label":"Remaining Time","stat":"Minimum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":59,"properties":{"view":"timeSeries","title":"Stager Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}],["AWS/Lambda","Duration","FunctionName","",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}],["AWS/Lambda","Duration","FunctionName","",
{
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
@@ -761,40 +795,69 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Change Count","expression":"FILL(mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"mf6a938229aec623f5cf21e3ae3401901e7f7c6ebd5774ab6ac6e74af83efd121"}],[{"label":"Unprocessable","expression":"FILL(m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"m5b235d813bd8ded65d7de9a1345c9b49a8cf2af40e7e98ddbb8cb73e00783a8a"}],[{"label":"Lag to npmjs.com","expression":"FILL(m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m4d6c513faac55be4bca785e8587a9a145b65f5fe675318c9c5c52f3ada197b7b"}],[{"label":"Package Version Age","expression":"FILL(mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"mec485aa83505431ed7a356970082cd4815c310504ff176dd3eeb241ae7bbf0c4"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ "","metrics":[[{"label":"Change Count","expression":"FILL(m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","ChangeCount",{"label":"Change Count","stat":"Sum","visible":false,"id":"m7dd96da53b843cbea52ec982e51e6f10adf5c3450276be550dc1a37fd306d5c5"}],[{"label":"Unprocessable","expression":"FILL(mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","UnprocessableEntity",{"label":"Unprocessable","stat":"Sum","visible":false,"id":"mc0b8edffe5268f2643cee6ab0dc182e9701a2c1acb2022fec1ad73ab6d04bc62"}],[{"label":"Lag to npmjs.com","expression":"FILL(m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","NpmJsChangeAge",{"label":"Lag to npmjs.com","stat":"Minimum","visible":false,"id":"m071ea3a4182a8bb74be01c558e5f2b73374219944a0a21c421a2674d9aa6e6d7"}],[{"label":"Package Version Age","expression":"FILL(m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a, REPEAT)","yAxis":"right"}],["ConstructHub/PackageSource/NpmJs/Follower","PackageVersionAge",{"label":"Package Version Age","stat":"Maximum","visible":false,"id":"m0964a38e961dedc0f8b3001db100c5e1d7274ca8abb122720a24dce627e9747a"}]],"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":65,"properties":{"view":"timeSeries","title":"CouchDB Changes","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mfd9b322a892b6ebd4710f9bc5a30b1fbd118e20e47f9c8eaeee194d77322b72a"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Feed Reliability (Overlap Sweep)","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"me5a01511901821ea0f462a1ed69195258d22bcdcf2e3401f0f50f5eaf57aeb1e"}],[{"label":"Stale Metadata Deferred","expression":"FILL(m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataDeferred",{"label":"Stale Metadata Deferred","stat":"Sum","visible":false,"id":"m484b96e9d8d4c74b70b5c05d0088abd203b88ce18b760b2c892a9f5c29b01610"}],[{"label":"Stale Metadata Recovered","expression":"FILL(m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","StaleMetadataRecovered",{"label":"Stale Metadata Recovered","stat":"Sum","visible":false,"id":"m03268eb1a78863792e9896c19cd19564761e6d8e79d770dc73b39c5b3132b30f"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","period":3600,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Sweep Margin","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":3600}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Stale Metadata Retry Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Last Sequence Number","expression":"FILL(mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500, REPEAT)"}],["ConstructHub/PackageSource/NpmJs/Follower","LastSeq",{"label":"Last Sequence Number","stat":"Maximum","visible":false,"id":"mcc89a801dd7db60ea52e74b9851bd34a3899e7a26323ac98feb632cdf70c2500"}]],"yAxis":{},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":71,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m850e080ca0cd99796703c3f041dcb994f126e8f9249f902d2d691c04dc3a977b"}],[{"label":"DLQ Visible Messages","expression":"FILL(m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "",{"label":"DLQ Visible Messages","stat":"Maximum","visible":false,"id":"m3bbc7927730cd5004ea9959c2786578dac68358d4d70f17f1c2645b5f778382b"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Visible Messages","expression":"FILL(m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ "","metrics":[[{"label":"Visible Messages","expression":"FILL(mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"m96c7bb4160011a049b6b07c0feab8fb02e5a8d4aa466939592829f864dde6400"}],[{"label":"Invisible Messages","expression":"FILL(mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ "",{"label":"Visible Messages","stat":"Maximum","visible":false,"id":"mf966fd33b8a743d601b8e64794dee25b745762e3add6ac1cff04ab82493507b5"}],[{"label":"Invisible Messages","expression":"FILL(m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d, 0)"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"mf74066283a392fc98a0cd7a405f2aa9b5af544d3a87542f06fff7fd069fd8578"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
+ "",{"label":"Invisible Messages","stat":"Maximum","visible":false,"id":"m4a2489d3e1658e50053af6351a1b6de6eedbf25cff1270c0c89ff187dad2876d"}],["AWS/SQS","ApproximateAgeOfOldestMessage","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubSourcesStagerDLQ80BD2600",
"QueueName",
],
},
- "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
+ "",{"label":"Oldest Message","stat":"Maximum","yAxis":"right"}]],"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":77,"properties":{"view":"timeSeries","title":"Package Canary","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":77,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
+ "","metrics":[["ConstructHub/PackageCanary","DwellTime",{"label":"Dwell Time","stat":"Maximum"}],["ConstructHub/PackageCanary","TimeToCatalog",{"label":"Time to Catalog","stat":"Maximum"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"SLA (5 minutes)","value":300,"yAxis":"left"}]},"yAxis":{"left":{"min":0}}}},{"type":"log","width":12,"height":6,"x":0,"y":83,"properties":{"view":"table","title":"Stuck Versions (still not visible in ConstructHub)","region":"",
{
"Ref": "AWS::Region",
},
@@ -802,7 +865,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubSourcesNpmJsCanary5558FC45",
},
- "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":83,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
+ "' | fields @timestamp, @message\\n| filter @message like /\\"DwellTime\\"/\\n| filter toMillis(@timestamp) > (now() - 900) * 1000\\n| parse @message '\\"PackageVersion\\":\\"*\\"' as version\\n| parse @message '\\"DwellTime\\":*,' as dwellTimeSec\\n| stats max(dwellTimeSec) as maxDwellTimeSec by version\\n| sort maxDwellTimeSec desc"}},{"type":"text","width":24,"height":2,"x":0,"y":89,"properties":{"markdown":"# Ingestion Function\\n\\n[button:Ingestion Function](/lambda/home#/functions/",
{
"Ref": "ConstructHubIngestion407909CE",
},
@@ -825,19 +888,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":85,"properties":{"view":"timeSeries","title":"Function Health","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubIngestion407909CE",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":85,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Input Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -862,11 +925,11 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":91,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
+ "",{"label":"Oldest Message Age","period":60,"stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"10 Minutes","value":600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"metric","width":12,"height":6,"x":0,"y":97,"properties":{"view":"timeSeries","title":"Input Quality","region":"",
{
"Ref": "AWS::Region",
},
- "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m0c5f10aa73687a21aa44b8985d19311246bca6c2cd1b2256b9f3e78c71e94fa3"}],[{"label":"Invalid Tarball","expression":"FILL(m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"m3a39d9b0bf974255f27dc95e44c8574bdfce2b97e49a7504346a77c5c9f6a3e1"}],[{"label":"Ineligible License","expression":"FILL(me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"me19ae9c3f20ae715e3462f9402684a3817282218f36e69494a60bc251e3435a6"}],[{"label":"Mismatched Identity","expression":"FILL(m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m7c1d407f17f8d2452a0bf87ac06fdd032de385a2758254f4192d6e17ed40d44b"}],[{"label":"Found License file","expression":"FILL(m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m7084dfec3c5bab86694d7ae438460f471c174a94bc67c9c9738e3ab60246b661"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":91,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
+ "","stacked":true,"metrics":[[{"label":"Invalid Assemblies","expression":"FILL(m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47, 0)"}],["ConstructHub/Ingestion","InvalidAssembly",{"label":"Invalid Assemblies","stat":"Sum","visible":false,"id":"m722588e4526cf1b3f2a6e10a1b7c739c7ba78de7c2e3f632e81443bb12f0ae47"}],[{"label":"Invalid Tarball","expression":"FILL(mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2, 0)"}],["ConstructHub/Ingestion","InvalidTarball",{"label":"Invalid Tarball","stat":"Sum","visible":false,"id":"mdb9b4024d1d49a57800b806b32dac9a0b6af98dda0e792ba1df7c7a0b5d431e2"}],[{"label":"Ineligible License","expression":"FILL(mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a, 0)"}],["ConstructHub/Ingestion","IneligibleLicense",{"label":"Ineligible License","stat":"Sum","visible":false,"id":"mb891ed2a7ce11d9e6a61621191ec41ae1b60c4f5d09df5974b3f47ebb7b1192a"}],[{"label":"Mismatched Identity","expression":"FILL(m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189, 0)"}],["ConstructHub/Ingestion","MismatchedIdentityRejections",{"label":"Mismatched Identity","stat":"Sum","visible":false,"id":"m1264778e17e711d8c749484339ef7aea45da3c760f873cef8e2587ed717a1189"}],[{"label":"Found License file","expression":"FILL(m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95, 0)"}],["ConstructHub/Ingestion","FoundLicenseFile",{"label":"Found License file","stat":"Sum","visible":false,"id":"m0c4ef260dab72cfb5e163a1751cef2c9631aac2abafbc5ae964480e1befb5c95"}]],"yAxis":{"left":{"label":"Count","min":0,"showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":12,"y":97,"properties":{"view":"timeSeries","title":"Dead Letters","region":"",
{
"Ref": "AWS::Region",
},
@@ -891,7 +954,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":97,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":103,"properties":{"markdown":"# Orchestration\\n\\n[button:primary:State Machine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubOrchestration39161A46",
},
@@ -922,43 +985,43 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubOrchestrationRetryUninstallablePackagesA408F9B3",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":99,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"State Machine Executions","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Started","expression":"FILL(m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
+ "","metrics":[[{"label":"Started","expression":"FILL(md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b, 0)"}],["AWS/States","ExecutionsStarted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Started","stat":"Sum","visible":false,"id":"m392141ff4cfa3bbe1889b2db42bcf20599513e199a0b9fb8dc736cde893c1d7c"}],[{"label":"Succeeded","expression":"FILL(md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
+ "",{"label":"Started","stat":"Sum","visible":false,"id":"md743bc8e2d09dc13a3dcbcf31845985ef87f5821d2a8229c2132c0bf67d2d09b"}],[{"label":"Succeeded","expression":"FILL(mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4, 0)"}],["AWS/States","ExecutionsSucceeded","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"md1b80634be6e2f057c84f44fab13979a6445395ea2dc357cd12d0ba9f504e6d1"}],["AWS/States","ExecutionTime","StateMachineArn","",
+ "",{"label":"Succeeded","stat":"Sum","visible":false,"id":"mac3fed2e8a56aee0460c9720a50bd361cdbbc4e39be589fcd2eb6101f99607f4"}],["AWS/States","ExecutionTime","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":99,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
+ "",{"label":"Duration","yAxis":"right"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false},"right":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":105,"properties":{"view":"timeSeries","title":"State Machine Failures","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Aborted","expression":"FILL(m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
+ "","metrics":[[{"label":"Aborted","expression":"FILL(m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5, 0)"}],["AWS/States","ExecutionsAborted","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m7c13255adf2b6d90baaa9e4e952e72ce260730c93b662336d01e3342f6255e08"}],[{"label":"Failed","expression":"FILL(m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
+ "",{"label":"Aborted","stat":"Sum","visible":false,"id":"m5028722e11cc328c5056c293e01f101d5b0b645d7609cdb5342add75b70e81b5"}],[{"label":"Failed","expression":"FILL(mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3, 0)"}],["AWS/States","ExecutionsFailed","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Failed","stat":"Sum","visible":false,"id":"m503cb5a7568675222eb2ac827982f88ad300c986505dabfafe1707080946e597"}],[{"label":"Throttled","expression":"FILL(m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
+ "",{"label":"Failed","stat":"Sum","visible":false,"id":"mbccfe4119610b7cd90761ff728a29536ce8db96bb9796434e1b8ae1f6abf3ea3"}],[{"label":"Throttled","expression":"FILL(m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe, 0)"}],["AWS/States","ExecutionThrottled","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m278670a6ffa3ce32fec8fcb73c1cfdc4c965389617e2387ab5852ebb00b2a81f"}],[{"label":"Timed Out","expression":"FILL(m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
+ "",{"label":"Throttled","stat":"Sum","visible":false,"id":"m07bd2f11d05861ff62e3b67f34f4f3fb5484b6c3bcbd1e0ebaa30465c4809ffe"}],[{"label":"Timed Out","expression":"FILL(m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3, 0)"}],["AWS/States","ExecutionsTimedOut","StateMachineArn","",
{
"Ref": "ConstructHubOrchestration39161A46",
},
- "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m106d91118c893ec9037d16448722184f1ebdcca08a617e1f13758151a8c85a21"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":105,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
+ "",{"label":"Timed Out","stat":"Sum","visible":false,"id":"m9f26430a0433776283742de97841340a090adcb446f52bfb0cb37ce1271995a3"}]],"yAxis":{"left":{"min":0,"label":"Count","showUnits":false}}}},{"type":"metric","width":12,"height":6,"x":0,"y":111,"properties":{"view":"timeSeries","title":"Dead Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -983,7 +1046,7 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":111,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
+ "",{"label":"Oldest Message Age","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff7f0e","label":"10 days","value":864000,"yAxis":"right"},{"color":"#ff0000","label":"14 days (DLQ Retention)","value":1209600,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"min":0}},"period":60}},{"type":"text","width":24,"height":2,"x":0,"y":117,"properties":{"markdown":"# Deny List\\n\\n[button:primary:Deny List Object](/s3/object/",
{
"Ref": "ConstructHubDenyListBucket1B3C2C2E",
},
@@ -1014,30 +1077,30 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubDenyListPrunePruneQueueHandlerF7EB599B",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":113,"properties":{"view":"timeSeries","title":"Deny List","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":119,"properties":{"view":"timeSeries","title":"Deny List","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Rules","expression":"FILL(m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m41327b0edbbef1ca627d9503d1b9b9fab401700118519537b58e0557adee7e4f"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
+ "","metrics":[[{"label":"Rules","expression":"FILL(m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6, REPEAT)"}],["ConstructHub/DenyList","DenyListRuleCount",{"label":"Rules","stat":"Maximum","visible":false,"id":"m2fa15e0382effe051e052f96fdcb4b1528cdcfd05aba5663829d1c5e99c112d6"}],["AWS/SQS","NumberOfMessagesDeleted","QueueName","",
{
"Fn::GetAtt": [
"ConstructHubDenyListPruneDeleteQueueBBF60185",
"QueueName",
],
},
- "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":113,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
+ "",{"label":"Deleted Files","stat":"Sum"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":119,"properties":{"view":"timeSeries","title":"Prune Function Health","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Invocations","expression":"FILL(m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
+ "","metrics":[[{"label":"Invocations","expression":"FILL(m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec, 0)"}],["AWS/Lambda","Invocations","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m9ff955bd33652ecefb4dd9402064988aa5e418fcf59360156ff8c9e38dbde5e2"}],[{"label":"Errors","expression":"FILL(m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55, 0)"}],["AWS/Lambda","Errors","FunctionName","",
+ "",{"label":"Invocations","stat":"Sum","visible":false,"id":"m0ce71c87a23b5c835573bc90ecad01da1ebd4c248ce94f9c257d658bc6069aec"}],[{"label":"Errors","expression":"FILL(m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e, 0)"}],["AWS/Lambda","Errors","FunctionName","",
{
"Ref": "ConstructHubDenyListPrunePruneHandler30B33551",
},
- "",{"label":"Errors","stat":"Sum","visible":false,"id":"m3aa18789f406dc22d5fd69d6a8b1ae08cbc04aabfde21bee51e16796b37a2e55"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":119,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
+ "",{"label":"Errors","stat":"Sum","visible":false,"id":"m60b1a9f536a27f3aaa3c2385cff8fac75b927f8690a6e416afafc4d63108c63e"}]],"yAxis":{"left":{"min":0}},"period":300}},{"type":"text","width":24,"height":2,"x":0,"y":125,"properties":{"markdown":"# Package Stats\\n\\n[button:primary:Package Stats Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -1057,11 +1120,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsAggregator3345DB4E",
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":121,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":127,"properties":{"view":"timeSeries","title":"Number of Package Stats Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with stats","expression":"FILL(m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"m47411943a6e8f8e16a1dd905ff43cab11425b802c4d32615e081cab30c15f29f"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":121,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Packages with stats","expression":"FILL(mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87, REPEAT)"}],["ConstructHub/PackageStats","RegisteredPackagesWithStats",{"label":"Packages with stats","stat":"Maximum","visible":false,"id":"mcfaa85bc3bba4c6a19d9471c142866ca1b3e14e191b036a0e57f05f7dc775b87"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":127,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -1069,7 +1132,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubStatsStateMachine1A1DEC97",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":127,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"6 hours (State Machine timeout)","value":21600,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":133,"properties":{"markdown":"# Version Tracker\\n\\n[button:primary:Versions Object](/s3/object/",
{
"Ref": "ConstructHubPackageDataDC5EF35E",
},
@@ -1081,11 +1144,11 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":129,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
+ "/log-events)"}},{"type":"metric","width":12,"height":6,"x":0,"y":135,"properties":{"view":"timeSeries","title":"Number of Package Versions Recorded","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Package versions recorded","expression":"FILL(m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"m343ebb2e5802fe223e549d8262d8d0c364846e0276426bc0a2d872e83bd8e63d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":129,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
+ "","metrics":[[{"label":"Package versions recorded","expression":"FILL(mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7, REPEAT)"}],["ConstructHub/VersionTracker","TrackedVersionsCount",{"label":"Package versions recorded","stat":"Maximum","visible":false,"id":"mafd748e1416b5c8063b187f4cdd8765c1a74c29a0e0ea4ab3c4d67aa2de263d7"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":135,"properties":{"view":"timeSeries","title":"Invocation Duration","region":"",
{
"Ref": "AWS::Region",
},
@@ -1093,7 +1156,7 @@ def sort_filter_rules(json_obj):
{
"Ref": "ConstructHubVersionTrackerD5E8AEAE",
},
- "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"1 minutes (Lambda timeout)","value":60,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":135,"properties":{"markdown":"# Release Notes\\n\\n[button:primary:StateMachine](/states/home#/statemachines/view/",
+ "",{"label":"Duration"}]],"annotations":{"horizontal":[{"color":"#ffa500","label":"1 minutes (Lambda timeout)","value":60,"yAxis":"right"}]},"yAxis":{"left":{"min":0}}}},{"type":"text","width":24,"height":2,"x":0,"y":141,"properties":{"markdown":"# Release Notes\\n\\n[button:primary:StateMachine](/states/home#/statemachines/view/",
{
"Ref": "ConstructHubReleaseNotesStateMachine8C711CC7",
},
@@ -1154,19 +1217,19 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
- ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":137,"properties":{"view":"timeSeries","title":"Number of release Notes","region":"",
+ ")"}},{"type":"metric","width":12,"height":6,"x":0,"y":143,"properties":{"view":"timeSeries","title":"Number of release Notes","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Packages with release notes","expression":"FILL(m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d, REPEAT)"}],["ConstructHub/ReleaseNotes","PackageWithChangeLog",{"label":"Packages with release notes","stat":"Sum","visible":false,"id":"m7a7a3037972a088d52187e29766f681a9cb9642d00a12e818823d134184ff48d"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":137,"properties":{"view":"timeSeries","title":"Release notes generation Errors","region":"",
+ "","metrics":[[{"label":"Packages with release notes","expression":"FILL(m2b837d3d9541bb2129b8500390287020ea25ecc79f811ffa09e55e57cac85cce, REPEAT)"}],["ConstructHub/ReleaseNotes","PackageWithChangeLog",{"label":"Packages with release notes","stat":"Sum","visible":false,"id":"m2b837d3d9541bb2129b8500390287020ea25ecc79f811ffa09e55e57cac85cce"}]],"yAxis":{"left":{"min":0}}}},{"type":"metric","width":12,"height":6,"x":12,"y":143,"properties":{"view":"timeSeries","title":"Release notes generation Errors","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[["ConstructHub/ReleaseNotes","AllErrors",{"label":"All Errors","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnknownError",{"label":"UnknownError","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidCredentials",{"label":"InvalidCredentials","stat":"Maximum"}],["ConstructHub/ReleaseNotes","RequestQuotaExhausted",{"label":"RequestQuotaExhausted","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnSupportedRepo",{"label":"UnSupportedRepo","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidPackageJson",{"label":"InvalidPackageJson","stat":"Maximum"}],["ConstructHub/ReleaseNotes","ChangelogFetchError",{"label":"ChangeLogFetchError","stat":"Maximum"}]],"yAxis":{}}},{"type":"metric","width":12,"height":6,"x":0,"y":143,"properties":{"view":"timeSeries","title":"GitHub API Rate Limits","region":"",
+ "","metrics":[["ConstructHub/ReleaseNotes","AllErrors",{"label":"All Errors","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnknownError",{"label":"UnknownError","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidCredentials",{"label":"InvalidCredentials","stat":"Maximum"}],["ConstructHub/ReleaseNotes","RequestQuotaExhausted",{"label":"RequestQuotaExhausted","stat":"Maximum"}],["ConstructHub/ReleaseNotes","UnSupportedRepo",{"label":"UnSupportedRepo","stat":"Maximum"}],["ConstructHub/ReleaseNotes","InvalidPackageJson",{"label":"InvalidPackageJson","stat":"Maximum"}],["ConstructHub/ReleaseNotes","ChangelogFetchError",{"label":"ChangeLogFetchError","stat":"Maximum"}]],"yAxis":{}}},{"type":"metric","width":12,"height":6,"x":0,"y":149,"properties":{"view":"timeSeries","title":"GitHub API Rate Limits","region":"",
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Limit","expression":"FILL(mcf810098fa7b0359cfa64484e165026013fa6611de36e48a68984a9615f7599d, REPEAT)"}],["ConstructHub/ReleaseNotes","GhLimitsLimit",{"label":"Limit","stat":"Maximum","visible":false,"id":"mcf810098fa7b0359cfa64484e165026013fa6611de36e48a68984a9615f7599d"}],[{"label":"Used","expression":"FILL(mc54991399581cec857a3c1773e2649cbfb65cef7ea4e459f2ada3ee31c60782e, REPEAT)"}],["ConstructHub/ReleaseNotes","GhLimitsUsed",{"label":"Used","stat":"Maximum","visible":false,"id":"mc54991399581cec857a3c1773e2649cbfb65cef7ea4e459f2ada3ee31c60782e"}],[{"label":"Remaining","expression":"FILL(m3c4202dbd25b40c6524935352f9d34858b3374ed5079dbd6aea164767195d54a, REPEAT)"}],["ConstructHub/ReleaseNotes","GhRateLimitsRemaining",{"label":"Remaining","stat":"Minimum","visible":false,"id":"m3c4202dbd25b40c6524935352f9d34858b3374ed5079dbd6aea164767195d54a"}]],"yAxis":{}}}]}",
+ "","metrics":[[{"label":"Limit","expression":"FILL(m1a9dcabdab0ed918e0c045f7f467f637ea3dd3a90d362ab329916a09516c8da9, REPEAT)"}],["ConstructHub/ReleaseNotes","GhLimitsLimit",{"label":"Limit","stat":"Maximum","visible":false,"id":"m1a9dcabdab0ed918e0c045f7f467f637ea3dd3a90d362ab329916a09516c8da9"}],[{"label":"Used","expression":"FILL(mf77a43d3843c077bbce01ef69848120e4600993b32b8680e5e558436cef53415, REPEAT)"}],["ConstructHub/ReleaseNotes","GhLimitsUsed",{"label":"Used","stat":"Maximum","visible":false,"id":"mf77a43d3843c077bbce01ef69848120e4600993b32b8680e5e558436cef53415"}],[{"label":"Remaining","expression":"FILL(m5d3e24e985f04a097522cd3f1dd10067eebce2da0b207f99e090b9717600957f, REPEAT)"}],["ConstructHub/ReleaseNotes","GhRateLimitsRemaining",{"label":"Remaining","stat":"Minimum","visible":false,"id":"m5d3e24e985f04a097522cd3f1dd10067eebce2da0b207f99e090b9717600957f"}]],"yAxis":{}}}]}",
],
],
},
@@ -11150,6 +11213,14 @@ Direct link to the function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -11233,7 +11304,7 @@ Direct link to the function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "70bbd5637676be1a0dd5cdb9e37372b27d89fd58f0e04ffbe4f442988717a4c2.zip",
+ "S3Key": "a0cc00d9c18e07c65112616e2da5da6fdc8cf2155dad9e4d76ca4f05fd7a2229.zip",
},
"Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -11253,6 +11324,9 @@ Direct link to the function: /lambda/home#/functions/",
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "RETRY_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ },
},
},
"Handler": "index.handler",
@@ -11684,6 +11758,26 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep is discovering changes that were inserted into the CouchDB changes feed
+with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep
+will miss those changes (only the daily deep sweep would catch them). Consider increasing
+the sweep margin (SWEEP_MARGIN_MS).
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LateChangeLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "LateChangeLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Maximum",
+ "Threshold": 17280000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -11754,6 +11848,82 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerStaleMetadataDLQNotEmpty68864A28": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The npm registry never caught up with the revision announced by the CouchDB changes feed
+for one or more packages, even after repeated retries. The affected package versions were
+NOT ingested, and will only be discovered when the package publishes its next version.
+The messages in the dead-letter queue identify the affected packages, which can be
+manually ingested using the ReStagePackageVersion function.
+
+Link to the dead letter queue: /sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/StaleMetadataDLQNotEmpty",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "QueueName",
+ "Value": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "QueueName",
+ ],
+ },
+ },
+ ],
+ "EvaluationPeriods": 1,
+ "MetricName": "ApproximateNumberOfMessagesVisible",
+ "Namespace": "AWS/SQS",
+ "Period": 300,
+ "Statistic": "Maximum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsFollowerSweepNotRunningC7CA155E": {
+ "Properties": {
+ "AlarmDescription": "The overlap sweep of the NpmJs follower has not reported in several hours. The follower
+is currently blind to changes inserted into the CouchDB changes feed behind its cursor.
+This usually means the follower is in extended catch-up and the head-of-feed pass is
+consuming the entire execution time budget.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/SweepNotRunning",
+ "ComparisonOperator": "LessThanThreshold",
+ "EvaluationPeriods": 6,
+ "MetricName": "LateChangeCount",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 3600,
+ "Statistic": "Sum",
+ "Threshold": 0,
+ "TreatMissingData": "breaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -11923,6 +12093,43 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ "s3:DeleteObject*",
+ "s3:PutObject",
+ "s3:PutObjectLegalHold",
+ "s3:PutObjectRetention",
+ "s3:PutObjectTagging",
+ "s3:PutObjectVersionTagging",
+ "s3:Abort*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStagingBucketB286F0E6",
+ "Arn",
+ ],
+ },
+ "/couchdb-received-seqs",
+ ],
+ ],
+ },
+ ],
+ },
{
"Action": [
"s3:GetObject*",
@@ -12009,6 +12216,36 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -12034,7 +12271,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "846ba6dee713ab358e7132db46279fd65eb27e9e342e7e8eaee23d296d37ef30.zip",
+ "S3Key": "e34dcef0fa9dde1269f8a4cf5e401b4b8be2ffda10905a4279111e1477a65bfc.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -12290,6 +12527,14 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Prefix": "couchdb-known-versions.2",
"Status": "Enabled",
},
+ {
+ "ExpiredObjectDeleteMarker": true,
+ "NoncurrentVersionExpiration": {
+ "NoncurrentDays": 1,
+ },
+ "Prefix": "couchdb-received-seqs",
+ "Status": "Enabled",
+ },
],
},
"PublicAccessBlockConfiguration": {
@@ -12352,6 +12597,34 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesNpmJsStaleMetadataRetryQueue2125C09B": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStaleMetadataRetryDLQ0E99D546",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 8,
+ },
+ "VisibilityTimeout": 900,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
diff --git a/src/__tests__/package-sources/npmjs/follower-receipts.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/follower-receipts.lambda-shared.test.ts
new file mode 100644
index 000000000..6fcf01274
--- /dev/null
+++ b/src/__tests__/package-sources/npmjs/follower-receipts.lambda-shared.test.ts
@@ -0,0 +1,248 @@
+import {
+ RECEIPTS_RETENTION_MS,
+ SWEEP_INTERVAL_MS,
+ SWEEP_MARGIN_MS,
+} from '../../../package-sources/npmjs/constants.lambda-shared';
+import { FollowerReceipts } from '../../../package-sources/npmjs/follower-receipts.lambda-shared';
+
+const T0 = 1_700_000_000_000;
+const HOUR = 60 * 60 * 1_000;
+
+test('round-trips through the text format', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(90, T0);
+ receipts.addSeqs([100, 105, 106, 250]);
+ receipts.addCheckpoint(300, T0 + HOUR);
+ receipts.lastSweepAt = T0 + HOUR;
+ receipts.sweepCursor = { seq: 123, deep: false };
+
+ const restored = FollowerReceipts.fromText(receipts.toText());
+
+ for (const seq of [100, 105, 106, 250]) {
+ expect(restored.has(seq)).toBe(true);
+ }
+ expect(restored.has(101)).toBe(false);
+ expect(restored.receiptCount).toBe(4);
+ expect(restored.checkpointCount).toBe(2);
+ expect(restored.lastSweepAt).toBe(T0 + HOUR);
+ expect(restored.sweepCursor).toEqual({ seq: 123, deep: false });
+ expect(restored.seqAtOrBefore(T0)).toBe(90);
+ expect(restored.seqAtOrBefore(T0 + 2 * HOUR)).toBe(300);
+});
+
+test('serializes to a scannable line-based format', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(90, T0);
+ receipts.addSeqs([250, 100]);
+ const text = receipts.toText();
+
+ const lines = text.split('\n');
+ expect(lines[0]).toBe('#chreceipts/1');
+ // One record per line, greppable, seqs sorted ascending.
+ expect(text).toContain(`\nc ${T0} 90\n`);
+ expect(text.indexOf('\ns 100\n')).toBeGreaterThan(-1);
+ expect(text.indexOf('\ns 100\n')).toBeLessThan(text.indexOf('\ns 250\n'));
+
+ // Garbage input is rejected (so the follower can fall back to a fresh state).
+ expect(() => FollowerReceipts.fromText('garbage')).toThrow(
+ /Unexpected receipts file header/
+ );
+});
+
+describe('text format', () => {
+ test('produces the exact documented golden output', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(90, T0);
+ receipts.addCheckpoint(300, T0 + HOUR);
+ receipts.addSeqs([100, 250]);
+ receipts.lastSweepAt = T0 + HOUR;
+ receipts.lastDeepSweepAt = T0;
+ receipts.sweepCursor = { seq: 123, deep: true };
+
+ expect(receipts.toText()).toBe(
+ [
+ '#chreceipts/1',
+ `w ${T0 + HOUR} ${T0}`,
+ 'u 123 deep',
+ `c ${T0} 90`,
+ `c ${T0 + HOUR} 300`,
+ 's 100',
+ 's 250',
+ '', // trailing newline
+ ].join('\n')
+ );
+ });
+
+ test('a fresh state round-trips', () => {
+ const restored = FollowerReceipts.fromText(new FollowerReceipts().toText());
+ expect(restored.receiptCount).toBe(0);
+ expect(restored.checkpointCount).toBe(0);
+ expect(restored.lastSweepAt).toBeUndefined();
+ expect(restored.lastDeepSweepAt).toBeUndefined();
+ expect(restored.sweepCursor).toBeUndefined();
+ });
+
+ test('serialization is stable (parse . print is identity)', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(90, T0);
+ receipts.addSeqs([100, 250, 251]);
+ receipts.lastSweepAt = T0;
+ receipts.sweepCursor = { seq: 42, deep: false };
+
+ const once = receipts.toText();
+ const twice = FollowerReceipts.fromText(once).toText();
+ expect(twice).toBe(once);
+ });
+
+ test('unset sweep timestamps are encoded as "-"', () => {
+ const receipts = new FollowerReceipts();
+ receipts.lastSweepAt = T0;
+ // lastDeepSweepAt remains unset
+ expect(receipts.toText()).toContain(`\nw ${T0} -\n`);
+
+ const restored = FollowerReceipts.fromText(receipts.toText());
+ expect(restored.lastSweepAt).toBe(T0);
+ expect(restored.lastDeepSweepAt).toBeUndefined();
+ });
+
+ test('regular and deep sweep cursors round-trip distinctly', () => {
+ for (const deep of [true, false]) {
+ const receipts = new FollowerReceipts();
+ receipts.sweepCursor = { seq: 1_234, deep };
+ const restored = FollowerReceipts.fromText(receipts.toText());
+ expect(restored.sweepCursor).toEqual({ seq: 1_234, deep });
+ }
+ });
+
+ test('handles large sequence numbers (beyond 2^31)', () => {
+ const bigSeq = 131_711_133_000; // larger than a 32-bit integer
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(bigSeq - 10, T0);
+ receipts.addSeqs([bigSeq]);
+ const restored = FollowerReceipts.fromText(receipts.toText());
+ expect(restored.has(bigSeq)).toBe(true);
+ expect(restored.seqAtOrBefore(T0)).toBe(bigSeq - 10);
+ });
+
+ test('round-trips a large receipt set intact', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(0, T0);
+ const seqs = Array.from({ length: 10_000 }, (_, i) => 131_000_000 + i * 3);
+ receipts.addSeqs(seqs);
+
+ const restored = FollowerReceipts.fromText(receipts.toText());
+ expect(restored.receiptCount).toBe(10_000);
+ expect(restored.has(131_000_000)).toBe(true);
+ expect(restored.has(131_000_000 + 9_999 * 3)).toBe(true);
+ expect(restored.has(131_000_001)).toBe(false);
+ });
+
+ test('tolerates blank lines, surrounding whitespace, and CRLF line endings', () => {
+ const text = [
+ '#chreceipts/1',
+ `w ${T0} -`,
+ '',
+ `c ${T0} 90 `,
+ ' s 100',
+ 's 250\r',
+ '',
+ ].join('\n');
+
+ const restored = FollowerReceipts.fromText(text.replace(/\n/g, '\r\n'));
+ expect(restored.lastSweepAt).toBe(T0);
+ expect(restored.checkpointCount).toBe(1);
+ expect(restored.has(100)).toBe(true);
+ expect(restored.has(250)).toBe(true);
+ });
+
+ test.each([
+ ['empty input', ''],
+ ['missing header', `w ${T0} -\ns 100\n`],
+ ['JSON input (previous format)', '{"seqDeltas":[100]}'],
+ ['wrong version', '#chreceipts/2\ns 100\n'],
+ ])('rejects %s', (_name, text) => {
+ expect(() => FollowerReceipts.fromText(text)).toThrow(
+ /Unexpected receipts file header/
+ );
+ });
+
+ test('rejects unknown record types', () => {
+ expect(() =>
+ FollowerReceipts.fromText('#chreceipts/1\nx something\n')
+ ).toThrow(/Unexpected receipts file line: x something/);
+ });
+});
+
+test('seqAtOrBefore falls back to earliest checkpoint', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(150, T0);
+ // Requesting a time before any checkpoint exists yields the earliest one.
+ expect(receipts.seqAtOrBefore(T0 - HOUR)).toBe(150);
+ // No checkpoints at all yields undefined.
+ expect(new FollowerReceipts().seqAtOrBefore(T0)).toBeUndefined();
+});
+
+test('timeCrossed reports when the head pass first passed a seq', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(100, T0);
+ receipts.addCheckpoint(200, T0 + HOUR);
+ receipts.addCheckpoint(300, T0 + 2 * HOUR);
+
+ expect(receipts.timeCrossed(150)).toBe(T0 + HOUR);
+ expect(receipts.timeCrossed(100)).toBe(T0);
+ expect(receipts.timeCrossed(999)).toBeUndefined();
+});
+
+test('prunes checkpoints and receipts outside the retention window', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addSeqs([10, 20, 500, 600]);
+ receipts.addCheckpoint(50, T0 - RECEIPTS_RETENTION_MS - 2 * HOUR);
+ receipts.addCheckpoint(450, T0 - RECEIPTS_RETENTION_MS - HOUR);
+ receipts.addCheckpoint(700, T0);
+
+ // The checkpoint at seq=450 is the last one at/before the cut-off and is
+ // retained (so the full retention window remains mappable); the older one
+ // is dropped, along with receipts below seq=450.
+ expect(receipts.checkpointCount).toBe(2);
+ expect(receipts.timeCrossed(500)).toBe(T0);
+ expect(receipts.has(10)).toBe(false);
+ expect(receipts.has(20)).toBe(false);
+ expect(receipts.has(500)).toBe(true);
+ expect(receipts.has(600)).toBe(true);
+});
+
+describe('sweep scheduling', () => {
+ test('first ever sweep is a deep sweep', () => {
+ const receipts = new FollowerReceipts();
+ expect(receipts.dueSweep(T0)).toBe('deep');
+ });
+
+ test('regular sweep is due after the sweep interval', () => {
+ const receipts = new FollowerReceipts();
+ receipts.completeSweep('deep', T0);
+ expect(receipts.dueSweep(T0 + SWEEP_INTERVAL_MS - 1)).toBeUndefined();
+ expect(receipts.dueSweep(T0 + SWEEP_INTERVAL_MS)).toBe('regular');
+ });
+
+ test('deep sweep is due after the deep sweep interval', () => {
+ const receipts = new FollowerReceipts();
+ receipts.completeSweep('deep', T0);
+ expect(receipts.dueSweep(T0 + 25 * HOUR)).toBe('deep');
+ });
+
+ test('an interrupted sweep is resumed first', () => {
+ const receipts = new FollowerReceipts();
+ receipts.completeSweep('deep', T0);
+ receipts.sweepCursor = { seq: 42, deep: false };
+ expect(receipts.dueSweep(T0 + 1)).toBe('regular');
+ expect(receipts.sweepStartSeq('regular', T0 + 1)).toBe(42);
+ });
+
+ test('sweep start seq maps the margin through checkpoints', () => {
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(100, T0 - SWEEP_MARGIN_MS - HOUR);
+ receipts.addCheckpoint(200, T0 - SWEEP_MARGIN_MS);
+ receipts.addCheckpoint(900, T0);
+ expect(receipts.sweepStartSeq('regular', T0)).toBe(200);
+ });
+});
diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts
new file mode 100644
index 000000000..e9b7556fa
--- /dev/null
+++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts
@@ -0,0 +1,448 @@
+import { gunzipSync } from 'zlib';
+import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda';
+import {
+ GetObjectCommand,
+ PutObjectCommand,
+ S3Client,
+} from '@aws-sdk/client-s3';
+import {
+ DeleteMessageCommand,
+ ReceiveMessageCommand,
+ SendMessageCommand,
+ SQSClient,
+} from '@aws-sdk/client-sqs';
+import type { metricScope, MetricsLogger } from 'aws-embedded-metrics';
+import { Unit } from 'aws-embedded-metrics';
+import type { Context, ScheduledEvent } from 'aws-lambda';
+import { mockClient } from 'aws-sdk-client-mock';
+import 'aws-sdk-client-mock-jest';
+import nock from 'nock';
+import {
+ ENV_DENY_LIST_BUCKET_NAME,
+ ENV_DENY_LIST_OBJECT_KEY,
+} from '../../../backend/deny-list/constants';
+import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants';
+import {
+ KNOWN_VERSIONS_FILE_NAME,
+ MARKER_FILE_NAME,
+ MetricName,
+ RECEIPTS_FILE_NAME,
+} from '../../../package-sources/npmjs/constants.lambda-shared';
+import { FollowerReceipts } from '../../../package-sources/npmjs/follower-receipts.lambda-shared';
+import { handler } from '../../../package-sources/npmjs/npm-js-follower.lambda';
+import { stringToStream } from '../../streams';
+
+jest.setTimeout(30_000);
+
+jest.mock('aws-embedded-metrics');
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const mockMetricScope = require('aws-embedded-metrics')
+ .metricScope as jest.MockedFunction;
+const mockPutMetric = jest
+ .fn()
+ .mockName('MetricsLogger.putMetric') as jest.MockedFunction<
+ MetricsLogger['putMetric']
+>;
+const mockMetrics: MetricsLogger = {
+ putMetric: mockPutMetric,
+ setDimensions: (...args: any[]) => expect(args).toEqual([{}]),
+ setProperty: jest.fn(),
+} as any;
+mockMetricScope.mockImplementation((cb) => {
+ const impl = cb(mockMetrics);
+ return async (...args) => impl(...args);
+});
+
+const REPLICA = 'https://replicate.npmjs.com';
+const REGISTRY = 'https://registry.npmjs.org';
+
+const MOCK_BUCKET = 'mock-staging-bucket';
+const MOCK_STAGING_FUNCTION = 'mock-staging-function';
+const MOCK_RETRY_QUEUE_URL =
+ 'https://sqs.test.amazonaws.com/123456789012/retry-queue';
+const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name';
+const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json';
+const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name';
+const MOCK_LICENSE_LIST_OBJECT = 'license-list.json';
+
+const HOUR = 60 * 60 * 1_000;
+
+const mockS3 = mockClient(S3Client);
+const mockLambda = mockClient(LambdaClient);
+const mockSQS = mockClient(SQSClient);
+
+const context: Context = {
+ awsRequestId: 'mock-request-id',
+ logGroupName: 'mock-log-group',
+ logStreamName: 'mock-log-stream',
+ getRemainingTimeInMillis: () => 300_000,
+} as any;
+
+const event: ScheduledEvent = {} as any;
+
+beforeEach(() => {
+ mockS3.reset();
+ mockLambda.reset();
+ mockSQS.reset();
+ mockPutMetric.mockClear();
+
+ process.env.BUCKET_NAME = MOCK_BUCKET;
+ process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION;
+ process.env.RETRY_QUEUE_URL = MOCK_RETRY_QUEUE_URL;
+ process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET;
+ process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT;
+ process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET;
+ process.env[LicenseListEnv.OBJECT_KEY] = MOCK_LICENSE_LIST_OBJECT;
+ // Do not wait for the registry to catch up in tests.
+ process.env.MAX_PACKAGE_SERVER_LAG_MS = '0';
+
+ // Empty deny list
+ mockS3
+ .on(GetObjectCommand, {
+ Bucket: MOCK_DENY_LIST_BUCKET,
+ Key: MOCK_DENY_LIST_OBJECT,
+ })
+ .resolves({ Body: stringToStream(JSON.stringify({})) });
+ // License list allowing Apache-2.0
+ mockS3
+ .on(GetObjectCommand, {
+ Bucket: MOCK_LICENSE_LIST_BUCKET,
+ Key: MOCK_LICENSE_LIST_OBJECT,
+ })
+ .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) });
+ // Empty known versions
+ mockS3
+ .on(GetObjectCommand, {
+ Bucket: MOCK_BUCKET,
+ Key: KNOWN_VERSIONS_FILE_NAME,
+ })
+ .resolves({ Body: stringToStream(JSON.stringify({ knownVersions: {} })) });
+
+ mockS3.on(PutObjectCommand).resolves({});
+ mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 });
+ mockSQS.on(ReceiveMessageCommand).resolves({});
+ mockSQS.on(SendMessageCommand).resolves({ MessageId: 'mock-message-id' });
+ mockSQS.on(DeleteMessageCommand).resolves({});
+});
+
+afterEach(() => {
+ delete process.env.BUCKET_NAME;
+ delete process.env.FUNCTION_NAME;
+ delete process.env.RETRY_QUEUE_URL;
+ delete process.env[ENV_DENY_LIST_BUCKET_NAME];
+ delete process.env[ENV_DENY_LIST_OBJECT_KEY];
+ delete process.env[LicenseListEnv.BUCKET_NAME];
+ delete process.env[LicenseListEnv.OBJECT_KEY];
+ delete process.env.MAX_PACKAGE_SERVER_LAG_MS;
+ nock.cleanAll();
+});
+
+function givenMarker(marker: number, updateSeq: number) {
+ mockS3
+ .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME })
+ .resolves({ Body: stringToStream(JSON.stringify({ marker })) });
+ // The follower verifies the marker against the current DB update_seq.
+ nock(REPLICA).get('/').reply(200, {
+ db_name: 'registry',
+ engine: 'npm-replicate',
+ doc_count: 1,
+ update_seq: updateSeq,
+ });
+}
+
+function givenNoReceiptsFile() {
+ mockS3
+ .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: RECEIPTS_FILE_NAME })
+ .rejects(Object.assign(new Error('no such key'), { name: 'NoSuchKey' }));
+}
+
+function givenReceiptsFile(receipts: FollowerReceipts) {
+ mockS3
+ .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: RECEIPTS_FILE_NAME })
+ .resolves({ Body: stringToStream(receipts.toText()) });
+}
+
+function savedReceipts(): FollowerReceipts {
+ const calls = mockS3
+ .commandCalls(PutObjectCommand)
+ .filter((call) => call.args[0].input.Key === RECEIPTS_FILE_NAME);
+ expect(calls.length).toBeGreaterThan(0);
+ const input = calls[calls.length - 1].args[0].input;
+ let body = input.Body as Buffer;
+ if (input.ContentEncoding === 'gzip') {
+ body = gunzipSync(body);
+ }
+ return FollowerReceipts.fromText(body.toString('utf-8'));
+}
+
+function packageDoc(name: string, rev: string, version: string) {
+ return {
+ _id: name,
+ _rev: rev,
+ 'dist-tags': { latest: version },
+ name,
+ time: {
+ created: '2023-01-01T00:00:00.000Z',
+ modified: '2023-06-01T00:00:00.000Z',
+ [version]: '2023-06-01T00:00:00.000Z',
+ },
+ versions: {
+ [version]: {
+ dependencies: { constructs: '^10.0.0' },
+ dist: {
+ shasum: 'mock-shasum',
+ tarball: `${REGISTRY}/${name}/-/${name}-${version}.tgz`,
+ },
+ jsii: {},
+ keywords: ['cdk'],
+ license: 'Apache-2.0',
+ name,
+ version,
+ },
+ },
+ };
+}
+
+function stagedPackages(): Array<{ name: string; version: string }> {
+ return mockLambda.commandCalls(InvokeCommand).map((call) => {
+ const payload = JSON.parse(
+ Buffer.from(call.args[0].input.Payload! as Uint8Array).toString('utf-8')
+ );
+ return { name: payload.name, version: payload.version };
+ });
+}
+
+test('defers changes with stale registry metadata to the retry queue', async () => {
+ givenMarker(1_000, 2_000);
+ givenNoReceiptsFile();
+
+ // Head pass: one batch with a change announcing rev 5...
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '100', since: '1000' })
+ .reply(200, {
+ results: [
+ {
+ seq: 1_005,
+ id: 'stale-package',
+ changes: [{ rev: '5-aaaaaa' }],
+ deleted: false,
+ },
+ ],
+ last_seq: 1_010,
+ });
+ // ... but the registry still serves rev 3.
+ nock(REGISTRY)
+ .get('/stale-package')
+ .reply(200, packageDoc('stale-package', '3-bbbbbb', '1.0.0'));
+ // Second head batch: caught up.
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '100', since: '1010' })
+ .reply(200, { results: [], last_seq: 1_010 });
+ // The (first ever) sweep starts from the only checkpoint, and is empty.
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '10000', since: '1010' })
+ .reply(200, { results: [], last_seq: 1_010 });
+
+ await handler(event, context);
+
+ // The change was deferred to the retry queue, not processed.
+ expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand);
+ expect(mockSQS).toHaveReceivedCommandTimes(SendMessageCommand, 1);
+ const sent = JSON.parse(
+ mockSQS.commandCalls(SendMessageCommand)[0].args[0].input.MessageBody!
+ );
+ expect(sent).toEqual(
+ expect.objectContaining({
+ name: 'stale-package',
+ expectedRev: 5,
+ seq: 1_005,
+ })
+ );
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.STALE_METADATA_DEFERRED,
+ 1,
+ Unit.Count
+ );
+
+ // The follower's position was checkpointed, and the first (deep) sweep
+ // completed. Note the received row's receipt is pruned away because it is
+ // below the earliest checkpoint (no sweep can ever reach it) - the row's
+ // safety is guaranteed by the retry queue message instead.
+ const receipts = savedReceipts();
+ expect(receipts.checkpointCount).toBe(1);
+ expect(receipts.lastDeepSweepAt).toBeDefined();
+});
+
+test('overlap sweep discovers and stages late-inserted changes', async () => {
+ const now = Date.now();
+ givenMarker(1_000, 2_000);
+
+ // Receipts: the head pass passed seq 500 seven hours ago and seq 1000 five
+ // minutes ago. Seq 550 was received; seq 600 was NOT (it was inserted into
+ // the feed late). Sweeps have run recently enough that only a regular
+ // sweep is due.
+ const receipts = new FollowerReceipts();
+ receipts.addCheckpoint(500, now - 7 * HOUR);
+ receipts.addCheckpoint(1_000, now - 5 * 60_000);
+ receipts.addSeqs([550]);
+ receipts.lastSweepAt = now - 2 * HOUR;
+ receipts.lastDeepSweepAt = now - 2 * HOUR;
+ givenReceiptsFile(receipts);
+
+ // Head pass: already caught up.
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '100', since: '1000' })
+ .reply(200, { results: [], last_seq: 1_000 });
+ // Regular sweep: starts from the checkpoint at/bordering (now - margin),
+ // i.e. seq 500. It returns one already-received row (550) and one late row
+ // (600). Only the late row's metadata may be fetched.
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '10000', since: '500' })
+ .reply(200, {
+ results: [
+ {
+ seq: 550,
+ id: 'already-seen-package',
+ changes: [{ rev: '1-cccccc' }],
+ deleted: false,
+ },
+ {
+ seq: 600,
+ id: 'late-package',
+ changes: [{ rev: '2-dddddd' }],
+ deleted: false,
+ },
+ ],
+ last_seq: 1_000,
+ });
+ nock(REGISTRY)
+ .get('/late-package')
+ .reply(200, packageDoc('late-package', '2-dddddd', '1.2.3'));
+
+ await handler(event, context);
+
+ // The late package version was sent for staging.
+ expect(stagedPackages()).toEqual([
+ { name: 'late-package', version: '1.2.3' },
+ ]);
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.LATE_CHANGE_COUNT,
+ 1,
+ Unit.Count
+ );
+ // The lag is measured against the checkpoint that first crossed seq 600
+ // (the one taken 5 minutes ago).
+ const lagCalls = mockPutMetric.mock.calls.filter(
+ ([name]) => name === MetricName.LATE_CHANGE_LAG
+ );
+ expect(lagCalls).toHaveLength(1);
+ expect(lagCalls[0][1]).toBeGreaterThanOrEqual(5 * 60_000);
+ expect(lagCalls[0][1]).toBeLessThan(HOUR);
+
+ // The sweep completed and recorded receipts for the swept rows.
+ const saved = savedReceipts();
+ expect(saved.has(600)).toBe(true);
+ expect(saved.lastSweepAt).toBeGreaterThanOrEqual(now);
+ expect(saved.sweepCursor).toBeUndefined();
+});
+
+test('recovers deferred changes from the retry queue once the registry catches up', async () => {
+ givenMarker(2_000, 3_000);
+ givenNoReceiptsFile();
+
+ // One deferred message is waiting on the retry queue.
+ mockSQS
+ .on(ReceiveMessageCommand)
+ .resolvesOnce({
+ Messages: [
+ {
+ Body: JSON.stringify({
+ name: 'recovered-package',
+ expectedRev: 4,
+ seq: 1_500,
+ firstSeenAt: '2023-06-01T00:00:00.000Z',
+ }),
+ MessageId: 'mock-message-id',
+ ReceiptHandle: 'mock-receipt-handle',
+ },
+ ],
+ })
+ .resolves({});
+ // The registry has now caught up with the announced revision.
+ nock(REGISTRY)
+ .get('/recovered-package')
+ .reply(200, packageDoc('recovered-package', '4-eeeeee', '2.0.0'));
+
+ // Head pass: caught up; sweep: empty.
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '100', since: '2000' })
+ .reply(200, { results: [], last_seq: 2_000 });
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '10000', since: '2000' })
+ .reply(200, { results: [], last_seq: 2_000 });
+
+ await handler(event, context);
+
+ // The deferred package version was staged and the message deleted.
+ expect(stagedPackages()).toEqual([
+ { name: 'recovered-package', version: '2.0.0' },
+ ]);
+ expect(mockSQS).toHaveReceivedCommandWith(DeleteMessageCommand, {
+ QueueUrl: MOCK_RETRY_QUEUE_URL,
+ ReceiptHandle: 'mock-receipt-handle',
+ });
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.STALE_METADATA_RECOVERED,
+ 1,
+ Unit.Count
+ );
+});
+
+test('leaves still-stale messages on the retry queue', async () => {
+ givenMarker(2_000, 3_000);
+ givenNoReceiptsFile();
+
+ mockSQS
+ .on(ReceiveMessageCommand)
+ .resolvesOnce({
+ Messages: [
+ {
+ Body: JSON.stringify({
+ name: 'still-stale-package',
+ expectedRev: 9,
+ seq: 1_600,
+ }),
+ MessageId: 'mock-message-id',
+ ReceiptHandle: 'mock-receipt-handle',
+ },
+ ],
+ })
+ .resolves({});
+ // The registry is still behind the announced revision.
+ nock(REGISTRY)
+ .get('/still-stale-package')
+ .reply(200, packageDoc('still-stale-package', '7-ffffff', '3.0.0'));
+
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '100', since: '2000' })
+ .reply(200, { results: [], last_seq: 2_000 });
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '10000', since: '2000' })
+ .reply(200, { results: [], last_seq: 2_000 });
+
+ await handler(event, context);
+
+ // Nothing staged, message NOT deleted (it will be re-delivered).
+ expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand);
+ expect(mockSQS).not.toHaveReceivedCommand(DeleteMessageCommand);
+});
diff --git a/src/metric-utils.ts b/src/metric-utils.ts
index 08a9cd282..f73811bed 100644
--- a/src/metric-utils.ts
+++ b/src/metric-utils.ts
@@ -4,18 +4,29 @@ import { MathExpression, Metric } from 'aws-cdk-lib/aws-cloudwatch';
/**
* Creates a MathExpression on the current metric, applying the `FILL` function
* to it. This is useful to turn a sparse metric into a continuous one.
+ *
+ * @param metric the metric to be filled.
+ * @param value the value to fill gaps with.
+ * @param id a discriminator, required when several metrics sharing the
+ * same namespace and name (e.g: the same metric for two
+ * different resources) are rendered in the same graph or alarm.
*/
export function fillMetric(
metric: Metric,
- value: number | 'REPEAT' = 0
+ value: number | 'REPEAT' = 0,
+ id?: string
): MathExpression {
- // We assume namespace + name is enough to uniquely identify a metric here.
- // This is true locally at this time, but in case this ever changes, consider
- // also processing dimensions and period.
+ // We assume namespace + name (+ the optional discriminator) is enough to
+ // uniquely identify a metric here. This is true locally at this time, but
+ // in case this ever changes, consider also processing dimensions and
+ // period. Note: dimensions usually contain unresolved CDK tokens, which
+ // CANNOT be hashed deterministically!
const h = createHash('sha256')
.update(metric.namespace)
.update('\0')
.update(metric.metricName)
+ .update('\0')
+ .update(id ?? '')
.digest('hex');
const metricName = `m${h}`;
diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts
index f7c3bbd1a..39cb9af96 100644
--- a/src/package-sources/npmjs.ts
+++ b/src/package-sources/npmjs.ts
@@ -10,6 +10,7 @@ import {
Metric,
MetricOptions,
Statistic,
+ Stats,
TreatMissingData,
} from 'aws-cdk-lib/aws-cloudwatch';
import { Rule, Schedule } from 'aws-cdk-lib/aws-events';
@@ -34,7 +35,9 @@ import {
MARKER_FILE_NAME,
METRICS_NAMESPACE,
MetricName,
+ RECEIPTS_FILE_NAME,
S3KeyPrefix,
+ SWEEP_MARGIN_MS,
} from './npmjs/constants.lambda-shared';
import { NpmJsFollower } from './npmjs/npm-js-follower';
import { StageAndNotify } from './npmjs/stage-and-notify';
@@ -158,6 +161,12 @@ export class NpmJs implements IPackageSource {
noncurrentVersionExpiration: Duration.days(30),
expiredObjectDeleteMarker: true,
},
+ // Permanently delete receipts file updates after 1 day (we don't need these)
+ {
+ prefix: RECEIPTS_FILE_NAME,
+ noncurrentVersionExpiration: Duration.days(1),
+ expiredObjectDeleteMarker: true,
+ },
],
});
bucket.grantRead(ingestion);
@@ -193,12 +202,36 @@ export class NpmJs implements IPackageSource {
})
);
+ // Queue for changes whose npm registry metadata was still stale (behind
+ // the revision announced by the CouchDB changes feed) when the follower
+ // first saw them. The follower re-checks these on subsequent runs, and
+ // messages that never recover end up in the dead-letter queue.
+ const staleMetadataRetryQueue = new Queue(
+ scope,
+ 'NpmJs/StaleMetadataRetryQueue',
+ {
+ encryption: QueueEncryption.KMS_MANAGED,
+ retentionPeriod: Duration.days(14),
+ // The follower runs every 5 minutes; a 15 minute visibility timeout
+ // paces retries of still-stale messages.
+ visibilityTimeout: Duration.minutes(15),
+ deadLetterQueue: {
+ maxReceiveCount: 8,
+ queue: new Queue(scope, 'NpmJs/StaleMetadataRetryDLQ', {
+ encryption: QueueEncryption.KMS_MANAGED,
+ retentionPeriod: Duration.days(14),
+ }),
+ },
+ }
+ );
+
const follower = new NpmJsFollower(scope, 'NpmJs', {
description: `[${scope.node.path}/NpmJs] Periodically query npmjs.com index for new packages`,
environment: {
AWS_EMF_ENVIRONMENT: 'Local',
BUCKET_NAME: bucket.bucketName,
FUNCTION_NAME: stager.functionName,
+ RETRY_QUEUE_URL: staleMetadataRetryQueue.queueUrl,
},
memorySize: 10_024, // 10 GiB
reservedConcurrentExecutions: 1, // Only one execution at a time, to avoid race conditions on the S3 marker object
@@ -208,9 +241,12 @@ export class NpmJs implements IPackageSource {
bucket.grantReadWrite(follower, MARKER_FILE_NAME);
bucket.grantReadWrite(follower, KNOWN_VERSIONS_FILE_NAME);
+ bucket.grantReadWrite(follower, RECEIPTS_FILE_NAME);
denyList?.grantRead(follower);
licenseList.grantRead(follower);
stager.grantInvoke(follower);
+ staleMetadataRetryQueue.grantSendMessages(follower);
+ staleMetadataRetryQueue.grantConsumeMessages(follower);
const restager = new ReStagePackageVersion(scope, 'ReStagePackageVersion', {
description: `Manually re-stage a package version`,
@@ -229,7 +265,14 @@ export class NpmJs implements IPackageSource {
targets: [new LambdaFunction(follower)],
});
- this.registerAlarms(scope, follower, stager, monitoring, rule);
+ this.registerAlarms(
+ scope,
+ follower,
+ stager,
+ monitoring,
+ rule,
+ staleMetadataRetryQueue
+ );
stager.deadLetterQueue &&
overviewDashboard.addDLQMetricToDashboard(
@@ -263,8 +306,20 @@ export class NpmJs implements IPackageSource {
name: 'Known Versions',
url: s3ObjectUrl(bucket, KNOWN_VERSIONS_FILE_NAME),
},
+ {
+ name: 'Receipts Object',
+ url: s3ObjectUrl(bucket, RECEIPTS_FILE_NAME),
+ },
{ name: 'Stager', url: lambdaFunctionUrl(stager) },
{ name: 'Stager DLQ', url: sqsQueueUrl(stager.deadLetterQueue!) },
+ {
+ name: 'Stale Metadata Retry Queue',
+ url: sqsQueueUrl(staleMetadataRetryQueue),
+ },
+ {
+ name: 'Stale Metadata Retry DLQ',
+ url: sqsQueueUrl(staleMetadataRetryQueue.deadLetterQueue!.queue),
+ },
{
name: 'Pipeline Trace (Log Analytics)',
url: logAnalyticsUrl(
@@ -349,6 +404,73 @@ export class NpmJs implements IPackageSource {
period: Duration.minutes(5),
}),
],
+ [
+ new GraphWidget({
+ height: 6,
+ width: 12,
+ title: 'Feed Reliability (Overlap Sweep)',
+ left: [
+ fillMetric(
+ this.metricLateChangeCount({ label: 'Late Changes' }),
+ 0
+ ),
+ fillMetric(
+ this.metricStaleMetadataDeferred({
+ label: 'Stale Metadata Deferred',
+ }),
+ 0
+ ),
+ fillMetric(
+ this.metricStaleMetadataRecovered({
+ label: 'Stale Metadata Recovered',
+ }),
+ 0
+ ),
+ ],
+ leftYAxis: { min: 0 },
+ right: [
+ this.metricLateChangeLag({ label: 'Late Change Lag (max)' }),
+ ],
+ rightAnnotations: [
+ {
+ color: '#ff0000',
+ label: 'Sweep Margin',
+ value: SWEEP_MARGIN_MS,
+ },
+ ],
+ rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false },
+ period: Duration.hours(1),
+ }),
+ new GraphWidget({
+ height: 6,
+ width: 12,
+ title: 'Stale Metadata Retry Queue',
+ left: [
+ fillMetric(
+ staleMetadataRetryQueue.metricApproximateNumberOfMessagesVisible(
+ { label: 'Visible Messages' }
+ ),
+ 0,
+ 'retryQueue'
+ ),
+ fillMetric(
+ staleMetadataRetryQueue.deadLetterQueue!.queue.metricApproximateNumberOfMessagesVisible(
+ { label: 'DLQ Visible Messages' }
+ ),
+ 0,
+ 'retryDlq'
+ ),
+ ],
+ leftYAxis: { min: 0 },
+ right: [
+ staleMetadataRetryQueue.metricApproximateAgeOfOldestMessage({
+ label: 'Oldest Message',
+ }),
+ ],
+ rightYAxis: { min: 0 },
+ period: Duration.minutes(5),
+ }),
+ ],
[
new GraphWidget({
height: 6,
@@ -433,6 +555,66 @@ export class NpmJs implements IPackageSource {
});
}
+ /**
+ * The number of changes discovered by the overlap sweep that were missed by
+ * the regular head-of-feed pass (i.e. rows inserted into the CouchDB
+ * `_changes` feed behind the follower's cursor).
+ */
+ public metricLateChangeCount(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.hours(1),
+ statistic: Stats.SUM,
+ ...opts,
+ metricName: MetricName.LATE_CHANGE_COUNT,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
+ /**
+ * For each late change discovered by the overlap sweep, the time elapsed
+ * between the moment the follower first read past the change's sequence
+ * number and the moment the change was discovered. This is a lower bound on
+ * the feed's insertion lag, and can be used to tune the sweep margin.
+ */
+ public metricLateChangeLag(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.hours(1),
+ statistic: Stats.MAXIMUM,
+ ...opts,
+ metricName: MetricName.LATE_CHANGE_LAG,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
+ /**
+ * The number of changes that were deferred to the stale-metadata retry
+ * queue because the npm registry metadata was still behind the revision
+ * announced by the `_changes` feed.
+ */
+ public metricStaleMetadataDeferred(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.minutes(5),
+ statistic: Stats.SUM,
+ ...opts,
+ metricName: MetricName.STALE_METADATA_DEFERRED,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
+ /**
+ * The number of previously deferred changes that were successfully
+ * recovered from the stale-metadata retry queue.
+ */
+ public metricStaleMetadataRecovered(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.minutes(5),
+ statistic: Stats.SUM,
+ ...opts,
+ metricName: MetricName.STALE_METADATA_RECOVERED,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
public metricNpmJsChangeAge(opts?: MetricOptions): Metric {
return new Metric({
period: Duration.minutes(1),
@@ -515,7 +697,8 @@ export class NpmJs implements IPackageSource {
follower: NpmJsFollower,
stager: StageAndNotify,
monitoring: IMonitoring,
- schedule: Rule
+ schedule: Rule,
+ staleMetadataRetryQueue: Queue
) {
const failureAlarm = follower
.metricErrors()
@@ -609,6 +792,85 @@ export class NpmJs implements IPackageSource {
dlqNotEmptyAlarm
);
+ const staleMetadataDlqNotEmptyAlarm = staleMetadataRetryQueue
+ .deadLetterQueue!.queue.metricApproximateNumberOfMessagesVisible({
+ period: Duration.minutes(5),
+ })
+ .createAlarm(scope, 'NpmJs/Follower/StaleMetadataDLQNotEmpty', {
+ alarmName: `${scope.node.path}/NpmJs/Follower/StaleMetadataDLQNotEmpty`,
+ alarmDescription: [
+ 'The npm registry never caught up with the revision announced by the CouchDB changes feed',
+ 'for one or more packages, even after repeated retries. The affected package versions were',
+ 'NOT ingested, and will only be discovered when the package publishes its next version.',
+ 'The messages in the dead-letter queue identify the affected packages, which can be',
+ 'manually ingested using the ReStagePackageVersion function.',
+ '',
+ `Link to the dead letter queue: ${sqsQueueUrl(
+ staleMetadataRetryQueue.deadLetterQueue!.queue
+ )}`,
+ '',
+ `Runbook: ${RUNBOOK_URL}`,
+ ].join('\n'),
+ comparisonOperator:
+ ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+ evaluationPeriods: 1,
+ threshold: 1,
+ treatMissingData: TreatMissingData.NOT_BREACHING,
+ });
+ monitoring.addLowSeverityAlarm(
+ 'NpmJs/Follower Stale Metadata DLQ Not Empty',
+ staleMetadataDlqNotEmptyAlarm
+ );
+
+ const lateChangeLagAlarm = this.metricLateChangeLag({
+ period: Duration.hours(1),
+ }).createAlarm(scope, 'NpmJs/Follower/LateChangeLagHigh', {
+ alarmName: `${scope.node.path}/NpmJs/Follower/LateChangeLagHigh`,
+ alarmDescription: [
+ 'The overlap sweep is discovering changes that were inserted into the CouchDB changes feed',
+ 'with a lag approaching the sweep margin. If the lag exceeds the margin, the regular sweep',
+ 'will miss those changes (only the daily deep sweep would catch them). Consider increasing',
+ 'the sweep margin (SWEEP_MARGIN_MS).',
+ '',
+ `Runbook: ${RUNBOOK_URL}`,
+ ].join('\n'),
+ comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
+ evaluationPeriods: 1,
+ threshold: 0.8 * SWEEP_MARGIN_MS,
+ treatMissingData: TreatMissingData.NOT_BREACHING,
+ });
+ monitoring.addLowSeverityAlarm(
+ 'NpmJs/Follower Late Change Lag approaching sweep margin',
+ lateChangeLagAlarm
+ );
+
+ // The sweep emits LateChangeCount on every batch (including zero-hit
+ // ones), so a prolonged absence of datapoints means sweeps have stopped
+ // running (e.g. the follower is in extended catch-up and the head-of-feed
+ // pass starves the sweep), leaving late-insertion protection blind.
+ const sweepNotRunningAlarm = this.metricLateChangeCount({
+ period: Duration.hours(1),
+ }).createAlarm(scope, 'NpmJs/Follower/SweepNotRunning', {
+ alarmName: `${scope.node.path}/NpmJs/Follower/SweepNotRunning`,
+ alarmDescription: [
+ 'The overlap sweep of the NpmJs follower has not reported in several hours. The follower',
+ 'is currently blind to changes inserted into the CouchDB changes feed behind its cursor.',
+ 'This usually means the follower is in extended catch-up and the head-of-feed pass is',
+ 'consuming the entire execution time budget.',
+ '',
+ `Runbook: ${RUNBOOK_URL}`,
+ ].join('\n'),
+ // Any emitted value (including 0) is healthy; only missing data breaches.
+ comparisonOperator: ComparisonOperator.LESS_THAN_THRESHOLD,
+ evaluationPeriods: 6,
+ threshold: 0,
+ treatMissingData: TreatMissingData.BREACHING,
+ });
+ monitoring.addLowSeverityAlarm(
+ 'NpmJs/Follower overlap sweep is not running',
+ sweepNotRunningAlarm
+ );
+
// Finally - the "not running" alarm depends on the schedule (it won't run until the schedule
// exists!), and the schedule depends on the failure alarm existing (we don't want it to run
// before we can know it is failing). This means the returned `IDependable` effectively ensures
diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts
index 560e23ac0..bcf8d148c 100644
--- a/src/package-sources/npmjs/constants.lambda-shared.ts
+++ b/src/package-sources/npmjs/constants.lambda-shared.ts
@@ -16,6 +16,37 @@ export const enum MetricName {
* Number of package versions that were skipped since they were in the deny list.
*/
DENY_LISTED_COUNT = 'DenyListedCount',
+
+ /**
+ * Number of changes discovered by the overlap sweep that were missed by the
+ * regular head-of-feed pass (i.e. rows that were inserted into the CouchDB
+ * `_changes` feed behind our cursor).
+ */
+ LATE_CHANGE_COUNT = 'LateChangeCount',
+
+ /**
+ * For each late change discovered by the overlap sweep, the time elapsed
+ * between the moment the follower first read past the change's sequence
+ * number and the moment the change was discovered. This is a lower bound on
+ * the feed's insertion lag, and is the quantity the sweep margin must
+ * exceed.
+ */
+ LATE_CHANGE_LAG = 'LateChangeLag',
+
+ /**
+ * Number of changes that could not be processed because the metadata
+ * returned by the npm registry was still stale (its revision was older than
+ * the revision announced by the `_changes` feed) after the maximum wait.
+ * These changes are deferred to the stale-metadata retry queue.
+ */
+ STALE_METADATA_DEFERRED = 'StaleMetadataDeferred',
+
+ /**
+ * Number of previously deferred changes that were successfully recovered
+ * from the stale-metadata retry queue (the registry caught up with the
+ * revision announced by the `_changes` feed).
+ */
+ STALE_METADATA_RECOVERED = 'StaleMetadataRecovered',
}
export const enum S3KeyPrefix {
@@ -39,3 +70,38 @@ export const MARKER_FILE_NAME = 'couchdb-last-transaction-id.2';
* The name of the object that contains the list of known versions.
*/
export const KNOWN_VERSIONS_FILE_NAME = 'couchdb-known-versions.2';
+
+/**
+ * The name of the object that contains the follower receipts: the sequence
+ * numbers received from the CouchDB `_changes` feed (within the retention
+ * window), the rolling time/sequence checkpoints, and the sweep state.
+ */
+export const RECEIPTS_FILE_NAME = 'couchdb-received-seqs';
+
+/**
+ * How far behind the head of the `_changes` feed the overlap sweep re-reads.
+ * The npm `_changes` feed occasionally inserts rows *behind* the current
+ * `last_seq` (confirmed "normal feed behavior" by npm), so a follower that
+ * only ever reads forward will silently miss those changes. The sweep
+ * re-reads this margin and processes any row it has no receipt for.
+ */
+export const SWEEP_MARGIN_MS = 6 * 60 * 60 * 1_000; // 6 hours
+
+/**
+ * How often the overlap sweep runs.
+ */
+export const SWEEP_INTERVAL_MS = 60 * 60 * 1_000; // 1 hour
+
+/**
+ * How often a "deep" sweep runs. The deep sweep covers the entire receipts
+ * retention window (instead of just `SWEEP_MARGIN_MS`), so that insertion
+ * lag *beyond* the regular sweep margin is still observed (via the
+ * `LateChangeLag` metric) and can inform tuning of the margin.
+ */
+export const DEEP_SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1_000; // 24 hours
+
+/**
+ * How long sequence number receipts and checkpoints are retained. This is
+ * the upper bound of what any sweep can cover.
+ */
+export const RECEIPTS_RETENTION_MS = 24 * 60 * 60 * 1_000; // 24 hours
diff --git a/src/package-sources/npmjs/couch-changes.lambda-shared.ts b/src/package-sources/npmjs/couch-changes.lambda-shared.ts
index da9f36744..f1a2a2af2 100644
--- a/src/package-sources/npmjs/couch-changes.lambda-shared.ts
+++ b/src/package-sources/npmjs/couch-changes.lambda-shared.ts
@@ -16,7 +16,16 @@ const DEFAULT_BATCH_SIZE = 100;
const MAX_CONNS_PER_HOST = 100;
-const MAX_PACKAGE_SERVER_LAG_MS = 30_000; // 30 seconds
+/**
+ * How long to wait for the npm registry metadata to catch up with the
+ * revision announced by the `_changes` feed, before giving up on the change
+ * (it is then deferred to the stale-metadata retry queue by the caller).
+ * Can be overridden through the environment for testing purposes.
+ */
+function maxPackageServerLagMs(): number {
+ const fromEnv = process.env.MAX_PACKAGE_SERVER_LAG_MS;
+ return fromEnv ? Number(fromEnv) : 30_000; // 30 seconds
+}
/**
* A utility class that helps with traversing CouchDB database changes streams
@@ -66,6 +75,32 @@ export class CouchChanges extends EventEmitter {
since: string | number,
opts?: { readonly batchSize?: number }
): Promise {
+ const raw = await this.rawChanges(since, opts);
+ const { ok, stale } = await this.fetchAndFilterAllMetadata(raw.results);
+
+ return {
+ last_seq: raw.last_seq,
+ actionableResults: ok,
+ staleResults: stale,
+ seqs: raw.seqs,
+ totalCount: raw.totalCount,
+ };
+ }
+
+ /**
+ * Obtains a batch of changes from the database, without fetching the
+ * associated package metadata. This is used by the overlap sweep, which
+ * discards already-processed rows before paying for any metadata fetch.
+ *
+ * @param since the sequence value since when history should be fetched.
+ * @param batchSize the maximum amount of changes to return in a single page.
+ *
+ * @returns a page of raw changes.
+ */
+ public async rawChanges(
+ since: string | number,
+ opts?: { readonly batchSize?: number }
+ ): Promise {
const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE;
const changesUrl = new URL(this.database, this.baseUrl);
@@ -74,23 +109,49 @@ export class CouchChanges extends EventEmitter {
const result = (await this.https('get', changesUrl)) as any;
- const last_seq = result.last_seq;
- const results = await this.fetchAndFilterAllMetadata(result.results);
-
+ const results: DatabaseChange[] = result.results ?? [];
return {
- last_seq,
- actionableResults: results,
- totalCount: result.results.length,
+ last_seq: result.last_seq,
+ results,
+ seqs: results
+ .map((change) => Number(change.seq))
+ .filter((seq) => !isNaN(seq)),
+ totalCount: results.length,
};
}
+ /**
+ * Fetches the current metadata document for the provided package name from
+ * the npm registry.
+ *
+ * @returns the metadata document, or `undefined` if the package does not
+ * exist (HTTP 404).
+ */
+ public async getPackageDoc(
+ packageName: string
+ ): Promise<{ readonly [key: string]: unknown } | undefined> {
+ const metadataUrl = new URL(packageName, NPM_REGISTRY_URL);
+ try {
+ return await this.https('get', metadataUrl);
+ } catch (e: any) {
+ if (e.message?.includes('HTTP 404')) {
+ return undefined;
+ }
+ throw e;
+ }
+ }
+
/**
* Fetch the metadata associated with a change. The change comes associated with a revision number,
* which can be compared to the revision number of the metadata to determine if the replica is
* lagging behind the changes stream. If so, we retry until the replica is up-to-date or until
- * 30 seconds elapsed after which we return the potentially stale metadata.
+ * 30 seconds elapsed, after which the change is reported as stale so the caller can defer it
+ * for a later retry (the registry occasionally takes much longer to serve the new revision, and
+ * processing the stale document would silently miss the new versions).
*/
- private async fetchAndFilterMetadata(change: DatabaseChange) {
+ private async fetchAndFilterMetadata(
+ change: DatabaseChange
+ ): Promise<{ change: DatabaseChange; stale: boolean } | undefined> {
// Filter out deleted packages or null ids
if (change.deleted || !change.id) {
console.log(`Skipping ${change.id}: deleted or null id`);
@@ -110,9 +171,9 @@ export class CouchChanges extends EventEmitter {
// note: this function should not throw validation errors as
// it may cause a poison pill - whereby a single corrupt package will
// fail the entire lambda execution and prevent us from ingesting any package.
- // instead, log the violation and return undefined.
+ // instead, log the violation and return undefined.
- while (Date.now() - startTime < MAX_PACKAGE_SERVER_LAG_MS) {
+ do {
try {
const meta = await this.https('get', metadataUrl);
if (!meta) {
@@ -122,22 +183,27 @@ export class CouchChanges extends EventEmitter {
}
if (!meta._rev) {
// can happen if a package was removed from npm
- console.log(`Skipping ${change.id} because no _rev found in metadata`);
+ console.log(
+ `Skipping ${change.id} because no _rev found in metadata`
+ );
return;
}
const latestReplicaRev = parseSequentialRevision(meta._rev as string);
- change.doc = meta; // add metadata to the change object
-
// Happy path: replica is up-to-date
if (latestReplicaRev >= latestChangesRev) {
- return change;
+ change.doc = meta; // add metadata to the change object
+ return { change, stale: false };
}
// Unhappy path: replica is behind. Calculate delay and retry
- const delay = Math.floor(Math.random() * Math.min(baseDelay * Math.pow(2, attempt), maxDelay));
- console.log(`${change.id}: package _rev ${latestReplicaRev} < expected replication rev ${latestChangesRev}, retrying in ${delay} ms`);
- await new Promise(resolve => setTimeout(resolve, delay));
+ const delay = Math.floor(
+ Math.random() * Math.min(baseDelay * Math.pow(2, attempt), maxDelay)
+ );
+ console.log(
+ `${change.id}: package _rev ${latestReplicaRev} < expected replication rev ${latestChangesRev}, retrying in ${delay} ms`
+ );
+ await new Promise((resolve) => setTimeout(resolve, delay));
attempt++;
} catch (e: any) {
if (e.message?.includes('HTTP 404')) {
@@ -148,21 +214,37 @@ export class CouchChanges extends EventEmitter {
}
throw e;
}
- }
-
- // Timeout reached, proceed with stale data
- console.log(`Timeout reached for ${change.id}, replica may be stale`);
- return change;
+ } while (Date.now() - startTime < maxPackageServerLagMs());
+
+ // Timeout reached. Do NOT process the stale document (it may be missing
+ // the very versions this change is about) - report it for deferral.
+ console.log(
+ `Timeout reached for ${change.id}, metadata is stale (registry lags behind the changes feed), deferring`
+ );
+ return { change, stale: true };
}
- private async fetchAndFilterAllMetadata(
- changes: DatabaseChange[]
- ): Promise {
- return (
- await Promise.all(
- changes.map((change) => this.fetchAndFilterMetadata(change))
- )
- ).filter((change): change is DatabaseChange => change !== undefined);
+ /**
+ * Fetches the metadata for all the provided changes, and splits them into
+ * changes for which up-to-date metadata could be obtained (`ok`) and
+ * changes for which the registry metadata was still stale after the
+ * maximum wait (`stale`).
+ */
+ public async fetchAndFilterAllMetadata(
+ changes: readonly DatabaseChange[]
+ ): Promise<{ ok: DatabaseChange[]; stale: DatabaseChange[] }> {
+ const outcomes = await Promise.all(
+ changes.map((change) => this.fetchAndFilterMetadata(change))
+ );
+ const ok = new Array();
+ const stale = new Array();
+ for (const outcome of outcomes) {
+ if (outcome == null) {
+ continue;
+ }
+ (outcome.stale ? stale : ok).push(outcome.change);
+ }
+ return { ok, stale };
}
/**
@@ -226,7 +308,8 @@ export class CouchChanges extends EventEmitter {
}
console.log(
- `Response: ${method.toUpperCase()} ${url} => HTTP ${res.statusCode
+ `Response: ${method.toUpperCase()} ${url} => HTTP ${
+ res.statusCode
} (${res.statusMessage})`
);
@@ -282,7 +365,7 @@ function readResponseJson(
});
}
-class RetryableError extends Error { }
+class RetryableError extends Error {}
function isRetryableError(e: Error): boolean {
return e instanceof RetryableError || (e as any).code === 'ECONNRESET';
@@ -292,14 +375,22 @@ async function sleep(ms: number) {
return new Promise((ok) => setTimeout(ok, ms));
}
-function parseSequentialRevision(rev: string): number {
+/**
+ * Parses the sequential (numeric) prefix of a CouchDB revision string (e.g.
+ * `42` for `42-0bf6e0fa87ae20bc7245f96216263817`).
+ */
+export function parseSequentialRevision(rev: string): number {
return parseInt(rev.split('-')[0]);
}
-function getMaxSequentialRevision(change: DatabaseChange): number {
- return Math.max(...change.changes
- .map(change => parseSequentialRevision(change.rev))
- .filter(num => !isNaN(num))
+/**
+ * The highest sequential revision announced by a change.
+ */
+export function getMaxSequentialRevision(change: DatabaseChange): number {
+ return Math.max(
+ ...change.changes
+ .map((c) => parseSequentialRevision(c.rev))
+ .filter((num) => !isNaN(num))
);
}
@@ -316,6 +407,19 @@ export interface DatabaseChanges {
*/
readonly actionableResults: readonly DatabaseChange[];
+ /**
+ * Changes for which the registry metadata was still stale (behind the
+ * revision announced by the changes feed) after the maximum wait. These
+ * should be deferred for a later retry.
+ */
+ readonly staleResults: readonly DatabaseChange[];
+
+ /**
+ * The sequence numbers of ALL the changes received in this batch (including
+ * deleted and unreachable packages), for receipt tracking.
+ */
+ readonly seqs: readonly number[];
+
/**
* The total count of changes in this batch. This includes unprocessable changes.
* 0 indicates we are up to date with "now".
@@ -323,6 +427,29 @@ export interface DatabaseChanges {
readonly totalCount: number;
}
+export interface RawDatabaseChanges {
+ /**
+ * The last sequence ID from this change set.
+ */
+ readonly last_seq: string | number;
+
+ /**
+ * The raw changes that are part of this batch (no metadata attached).
+ */
+ readonly results: readonly DatabaseChange[];
+
+ /**
+ * The sequence numbers of all the changes received in this batch.
+ */
+ readonly seqs: readonly number[];
+
+ /**
+ * The total count of changes in this batch. 0 indicates we are up to date
+ * with "now".
+ */
+ readonly totalCount: number;
+}
+
export interface DatabaseChange {
/**
* The set of revisions to the object that were resolved as part of this
diff --git a/src/package-sources/npmjs/follower-receipts.lambda-shared.ts b/src/package-sources/npmjs/follower-receipts.lambda-shared.ts
new file mode 100644
index 000000000..f5801a187
--- /dev/null
+++ b/src/package-sources/npmjs/follower-receipts.lambda-shared.ts
@@ -0,0 +1,285 @@
+import {
+ RECEIPTS_RETENTION_MS,
+ SWEEP_INTERVAL_MS,
+ DEEP_SWEEP_INTERVAL_MS,
+ SWEEP_MARGIN_MS,
+} from './constants.lambda-shared';
+
+/**
+ * A point-in-time record of where the follower's head-of-feed pass was in the
+ * `_changes` feed.
+ */
+export interface Checkpoint {
+ /**
+ * Epoch milliseconds at which the checkpoint was taken.
+ */
+ readonly at: number;
+
+ /**
+ * The `last_seq` value the head-of-feed pass had reached at that time.
+ */
+ readonly seq: number;
+}
+
+/**
+ * The persisted receipts file uses a newline-delimited plain-text format, so
+ * it can be scanned cheaply (e.g. `grep '^s 131711133$'`) and parsed without
+ * materializing a large JSON document. Each line is a record identified by
+ * its first token:
+ *
+ * ```text
+ * #chreceipts/1 <- header (format magic + version)
+ * w <- sweep state ("-" when unset)
+ * u <- sweep cursor (only when a sweep was interrupted)
+ * c <- checkpoint (repeated, ascending by time)
+ * s <- received sequence number (repeated, ascending)
+ * ```
+ */
+const RECEIPTS_FORMAT_HEADER = '#chreceipts/1';
+
+export interface SweepCursor {
+ /**
+ * The sequence number to resume the sweep from.
+ */
+ readonly seq: number;
+
+ /**
+ * Whether the interrupted sweep was a deep sweep.
+ */
+ readonly deep: boolean;
+}
+
+/**
+ * Tracks which `_changes` feed rows (by sequence number) the follower has
+ * already received, together with rolling time/seq checkpoints and sweep
+ * state.
+ *
+ * The npm `_changes` feed does not guarantee that rows only ever appear with
+ * a sequence number greater than previously returned `last_seq` values: rows
+ * are occasionally inserted *behind* the cursor. The receipt set upgrades the
+ * follower's position tracking from "I have processed everything up to X" to
+ * "here is exactly what I received up to X", enabling a cheap overlap sweep
+ * that re-reads a trailing window and only processes rows it has no receipt
+ * for.
+ */
+export class FollowerReceipts {
+ public static fromText(text: string): FollowerReceipts {
+ const receipts = new FollowerReceipts();
+ const lines = text.split('\n');
+ if (lines[0]?.trim() !== RECEIPTS_FORMAT_HEADER) {
+ throw new Error(
+ `Unexpected receipts file header: ${JSON.stringify(lines[0])}`
+ );
+ }
+ for (let i = 1; i < lines.length; i++) {
+ const line = lines[i].trim();
+ if (line === '') {
+ continue;
+ }
+ const parts = line.split(' ');
+ switch (parts[0]) {
+ case 's':
+ receipts.seqs.add(Number(parts[1]));
+ break;
+ case 'c':
+ receipts.checkpoints.push({
+ at: Number(parts[1]),
+ seq: Number(parts[2]),
+ });
+ break;
+ case 'w':
+ receipts.lastSweepAt =
+ parts[1] === '-' ? undefined : Number(parts[1]);
+ receipts.lastDeepSweepAt =
+ parts[2] === '-' ? undefined : Number(parts[2]);
+ break;
+ case 'u':
+ receipts.sweepCursor = {
+ seq: Number(parts[1]),
+ deep: parts[2] === 'deep',
+ };
+ break;
+ default:
+ throw new Error(`Unexpected receipts file line: ${line}`);
+ }
+ }
+ return receipts;
+ }
+
+ public lastSweepAt?: number;
+ public lastDeepSweepAt?: number;
+ public sweepCursor?: SweepCursor;
+
+ private checkpoints: Checkpoint[] = [];
+ private readonly seqs = new Set();
+
+ /**
+ * Whether a receipt exists for the provided sequence number.
+ */
+ public has(seq: number): boolean {
+ return this.seqs.has(seq);
+ }
+
+ /**
+ * Records receipts for the provided sequence numbers.
+ */
+ public addSeqs(seqs: Iterable): void {
+ for (const seq of seqs) {
+ this.seqs.add(seq);
+ }
+ }
+
+ /**
+ * Records a checkpoint at the current time for the provided sequence
+ * number, then prunes checkpoints and receipts that have aged out of the
+ * retention window.
+ */
+ public addCheckpoint(seq: number, now: number = Date.now()): void {
+ this.checkpoints.push({ at: now, seq });
+ this.prune(now);
+ }
+
+ /**
+ * The number of checkpoints currently retained.
+ */
+ public get checkpointCount(): number {
+ return this.checkpoints.length;
+ }
+
+ /**
+ * The number of sequence number receipts currently retained.
+ */
+ public get receiptCount(): number {
+ return this.seqs.size;
+ }
+
+ /**
+ * The sequence number the head-of-feed pass had reached at (or before) the
+ * provided time. If the checkpoint history does not reach that far back,
+ * the earliest known checkpoint's sequence number is returned. Returns
+ * `undefined` if no checkpoints exist at all.
+ */
+ public seqAtOrBefore(timeMs: number): number | undefined {
+ let candidate: Checkpoint | undefined;
+ for (const checkpoint of this.checkpoints) {
+ if (checkpoint.at > timeMs) {
+ break;
+ }
+ candidate = checkpoint;
+ }
+ return (candidate ?? this.checkpoints[0])?.seq;
+ }
+
+ /**
+ * The (approximate) time at which the head-of-feed pass first read past the
+ * provided sequence number: the timestamp of the earliest checkpoint whose
+ * sequence number is `>= seq`. Returns `undefined` if the head-of-feed pass
+ * has not passed that sequence number yet (or no checkpoints exist).
+ */
+ public timeCrossed(seq: number): number | undefined {
+ for (const checkpoint of this.checkpoints) {
+ if (checkpoint.seq >= seq) {
+ return checkpoint.at;
+ }
+ }
+ return undefined;
+ }
+
+ /**
+ * Determines the kind of sweep that is due, if any. Deep sweeps take
+ * precedence over regular sweeps. An interrupted sweep (cursor present) is
+ * always resumed first.
+ */
+ public dueSweep(now: number = Date.now()): 'deep' | 'regular' | undefined {
+ if (this.sweepCursor != null) {
+ return this.sweepCursor.deep ? 'deep' : 'regular';
+ }
+ if (
+ this.lastDeepSweepAt == null ||
+ now - this.lastDeepSweepAt >= DEEP_SWEEP_INTERVAL_MS
+ ) {
+ return 'deep';
+ }
+ if (
+ this.lastSweepAt == null ||
+ now - this.lastSweepAt >= SWEEP_INTERVAL_MS
+ ) {
+ return 'regular';
+ }
+ return undefined;
+ }
+
+ /**
+ * The sequence number a sweep of the given kind should start from, based on
+ * the checkpoint history. Returns `undefined` when there is not any
+ * checkpoint history to sweep over.
+ */
+ public sweepStartSeq(
+ kind: 'deep' | 'regular',
+ now: number = Date.now()
+ ): number | undefined {
+ if (this.sweepCursor != null) {
+ return this.sweepCursor.seq;
+ }
+ const margin = kind === 'deep' ? RECEIPTS_RETENTION_MS : SWEEP_MARGIN_MS;
+ return this.seqAtOrBefore(now - margin);
+ }
+
+ /**
+ * Marks a sweep of the given kind as completed at the provided time.
+ * Completing a deep sweep also completes a regular sweep (it covers a
+ * superset of the regular window).
+ */
+ public completeSweep(kind: 'deep' | 'regular', now: number = Date.now()) {
+ this.sweepCursor = undefined;
+ this.lastSweepAt = now;
+ if (kind === 'deep') {
+ this.lastDeepSweepAt = now;
+ }
+ }
+
+ public toText(): string {
+ const lines = new Array();
+ lines.push(RECEIPTS_FORMAT_HEADER);
+ lines.push(`w ${this.lastSweepAt ?? '-'} ${this.lastDeepSweepAt ?? '-'}`);
+ if (this.sweepCursor != null) {
+ lines.push(
+ `u ${this.sweepCursor.seq} ${
+ this.sweepCursor.deep ? 'deep' : 'regular'
+ }`
+ );
+ }
+ for (const { at, seq } of this.checkpoints) {
+ lines.push(`c ${at} ${seq}`);
+ }
+ for (const seq of Array.from(this.seqs).sort((a, b) => a - b)) {
+ lines.push(`s ${seq}`);
+ }
+ lines.push(''); // trailing newline
+ return lines.join('\n');
+ }
+
+ private prune(now: number): void {
+ const cutOff = now - RECEIPTS_RETENTION_MS;
+ // Always keep at least one checkpoint at or before the cut-off, so that
+ // `seqAtOrBefore` remains well-defined over the whole retention window.
+ let firstRetained = 0;
+ for (let i = 0; i < this.checkpoints.length; i++) {
+ if (this.checkpoints[i].at <= cutOff) {
+ firstRetained = i;
+ } else {
+ break;
+ }
+ }
+ this.checkpoints = this.checkpoints.slice(firstRetained);
+
+ // Receipts older (lower) than the earliest retained checkpoint's sequence
+ // number can no longer be reached by any sweep, so drop them.
+ const minSeq = this.checkpoints[0]?.seq ?? 0;
+ for (const seq of this.seqs) {
+ if (seq < minSeq) {
+ this.seqs.delete(seq);
+ }
+ }
+ }
+}
diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts
index e021c1d53..b1d1aa481 100644
--- a/src/package-sources/npmjs/npm-js-follower.lambda.ts
+++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts
@@ -6,6 +6,11 @@ import {
PutObjectCommand,
PutObjectCommandInput,
} from '@aws-sdk/client-s3';
+import {
+ DeleteMessageCommand,
+ ReceiveMessageCommand,
+ SendMessageCommand,
+} from '@aws-sdk/client-sqs';
import type {
NodeJsRuntimeStreamingBlobPayloadOutputTypes,
StreamingBlobPayloadInputTypes,
@@ -24,16 +29,27 @@ import {
MARKER_FILE_NAME,
METRICS_NAMESPACE,
KNOWN_VERSIONS_FILE_NAME,
+ RECEIPTS_FILE_NAME,
} from './constants.lambda-shared';
-import { CouchChanges, DatabaseChange } from './couch-changes.lambda-shared';
+import {
+ CouchChanges,
+ DatabaseChange,
+ getMaxSequentialRevision,
+ parseSequentialRevision,
+} from './couch-changes.lambda-shared';
+import { FollowerReceipts } from './follower-receipts.lambda-shared';
import { PackageVersion } from './stage-and-notify.lambda';
import { DenyListClient } from '../../backend/deny-list/client.lambda-shared';
import { LicenseListClient } from '../../backend/license-list/client.lambda-shared';
import {
LAMBDA_CLIENT,
S3_CLIENT,
+ SQS_CLIENT,
} from '../../backend/shared/aws.lambda-shared';
-import { decompressContent } from '../../backend/shared/compress-content.lambda-shared';
+import {
+ compressContent,
+ decompressContent,
+} from '../../backend/shared/compress-content.lambda-shared';
import { requireEnv } from '../../backend/shared/env.lambda-shared';
// eslint-disable-next-line @typescript-eslint/no-require-imports
const normalizeNPMMetadata = require('normalize-registry-metadata');
@@ -47,6 +63,33 @@ const CONSTRUCT_KEYWORDS: ReadonlySet = new Set([
]);
const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/';
+/**
+ * The batch size used by the overlap sweep. The sweep discards
+ * already-received rows before fetching any package metadata, so it can
+ * afford much larger pages than the head-of-feed pass (the feed supports up
+ * to 10,000).
+ */
+const SWEEP_BATCH_SIZE = 10_000;
+
+/**
+ * A sweep batch is only started when at least this much time remains in the
+ * Lambda execution. When the budget runs out, the sweep position is persisted
+ * and the sweep resumes on the next scheduled run.
+ */
+const SWEEP_TIME_BUDGET_MS = 60_000;
+
+/**
+ * Stale-metadata retry messages are only received while at least this much
+ * time remains in the Lambda execution, so that retry processing cannot
+ * starve the head-of-feed pass.
+ */
+const RETRY_QUEUE_TIME_BUDGET_MS = 240_000;
+
+/**
+ * The maximum number of stale-metadata retry messages processed per run.
+ */
+const MAX_RETRY_MESSAGES_PER_RUN = 30;
+
/**
* The release date of `aws-cdk@0.8.0`. Anything earlier than this basically is
* not a relevant package, as it cannot possibly be a constructs-based package.
@@ -79,6 +122,7 @@ export async function handler(event: ScheduledEvent, context: Context) {
const stagingBucket = requireEnv('BUCKET_NAME');
const stagingFunction = requireEnv('FUNCTION_NAME');
+ const retryQueueUrl = requireEnv('RETRY_QUEUE_URL');
const denyList = await DenyListClient.newClient();
const licenseList = await LicenseListClient.newClient();
@@ -91,9 +135,23 @@ export async function handler(event: ScheduledEvent, context: Context) {
didLoadVersionsFromLegacy,
} = await loadFollowerPosition(stagingBucket, npm);
+ const receipts = await loadReceipts(stagingBucket, initialMarker);
+
// If we loaded from the old location, write to the new
let writeKnownVersionsFile = didLoadVersionsFromLegacy;
+ // Process previously deferred changes whose registry metadata was stale.
+ const recoveredCount = await processRetryQueue(
+ context,
+ npm,
+ retryQueueUrl,
+ stagingFunction,
+ denyList,
+ licenseList,
+ knownVersions
+ );
+ writeKnownVersionsFile ||= recoveredCount > 0;
+
// The last written marker seq id.
let updatedMarker = initialMarker;
@@ -117,11 +175,26 @@ export async function handler(event: ScheduledEvent, context: Context) {
updatedMarker = Number(changes.last_seq);
metrics.setProperty('EndSeq', updatedMarker);
+ // Record a receipt for every row received in this batch, so the
+ // overlap sweep can cheaply identify rows that were inserted into the
+ // feed behind our cursor.
+ receipts.addSeqs(changes.seqs);
+
const startTime = Date.now();
try {
const batch = changes.actionableResults as readonly Change[];
+ // Changes whose registry metadata was still stale after the maximum
+ // wait are deferred to the retry queue instead of being processed
+ // with a potentially incomplete document.
+ metrics.putMetric(
+ MetricName.STALE_METADATA_DEFERRED,
+ changes.staleResults.length,
+ Unit.Count
+ );
+ await deferStaleChanges(retryQueueUrl, changes.staleResults);
+
// The most recent "modified" timestamp observed in the batch.
let lastModified: Date | undefined;
// Emit npm.js replication lag
@@ -174,29 +247,7 @@ export async function handler(event: ScheduledEvent, context: Context) {
writeKnownVersionsFile ||= versionInfos.length > 0;
// Process all remaining updates
- await Promise.all(
- versionInfos.map(async ({ infos, modified, seq }) => {
- const invokeArgs: PackageVersion = {
- integrity: infos.dist.shasum,
- modified: modified.toISOString(),
- name: infos.name,
- seq: seq?.toString(),
- tarballUrl: infos.dist.tarball,
- version: infos.version,
- };
- // "Fire-and-forget" invocation here.
- console.log(`Sending ${invokeArgs.tarballUrl} for staging`);
- await LAMBDA_CLIENT.send(
- new InvokeCommand({
- FunctionName: stagingFunction,
- InvocationType: 'Event',
- Payload: Buffer.from(JSON.stringify(invokeArgs)),
- })
- );
- // Record that this is now a "known" version (no need to re-discover)
- knownVersions.set(`${infos.name}@${infos.version}`, modified);
- })
- );
+ await stageVersions(versionInfos, stagingFunction, knownVersions);
}
// Updating the S3 stored marker with the new seq id and known versions concurrently
@@ -227,11 +278,349 @@ export async function handler(event: ScheduledEvent, context: Context) {
context.getRemainingTimeInMillis() >= maxBatchProcessingTime
);
+ // Record where the head-of-feed pass got to, so future sweeps can map
+ // times to sequence numbers (and know which ranges we have receipts for).
+ if (updatedMarker > 0) {
+ receipts.addCheckpoint(updatedMarker);
+ }
+
+ // Re-read a trailing window of the feed to discover rows that were
+ // inserted behind our cursor (the feed does not guarantee monotonic
+ // insertion), then persist the receipts.
+ const sweptCount = await runSweep(
+ context,
+ npm,
+ receipts,
+ retryQueueUrl,
+ stagingFunction,
+ denyList,
+ licenseList,
+ knownVersions,
+ updatedMarker
+ );
+
+ await Promise.all([
+ saveReceipts(context, stagingBucket, receipts),
+ ...(sweptCount > 0
+ ? [saveLastKnownVersions(context, stagingBucket, knownVersions)]
+ : []),
+ ]);
+
console.log('All done here, we have success!');
return { initialMarker, updatedMarker };
}
+/**
+ * Sends the provided package version updates to the staging function
+ * ("fire-and-forget"), and records them as known versions.
+ */
+async function stageVersions(
+ versionInfos: readonly UpdatedVersion[],
+ stagingFunction: string,
+ knownVersions: Map
+): Promise {
+ await Promise.all(
+ versionInfos.map(async ({ infos, modified, seq }) => {
+ const invokeArgs: PackageVersion = {
+ integrity: infos.dist.shasum,
+ modified: modified.toISOString(),
+ name: infos.name,
+ seq: seq?.toString(),
+ tarballUrl: infos.dist.tarball,
+ version: infos.version,
+ };
+ // "Fire-and-forget" invocation here.
+ console.log(`Sending ${invokeArgs.tarballUrl} for staging`);
+ await LAMBDA_CLIENT.send(
+ new InvokeCommand({
+ FunctionName: stagingFunction,
+ InvocationType: 'Event',
+ Payload: Buffer.from(JSON.stringify(invokeArgs)),
+ })
+ );
+ // Record that this is now a "known" version (no need to re-discover)
+ knownVersions.set(`${infos.name}@${infos.version}`, modified);
+ })
+ );
+}
+
+//#region Stale metadata retry queue
+/**
+ * The message payload used on the stale-metadata retry queue.
+ */
+interface StaleMetadataMessage {
+ readonly name: string;
+ readonly expectedRev: number;
+ readonly seq?: string | number;
+ readonly firstSeenAt?: string;
+}
+
+/**
+ * Defers the provided changes (whose registry metadata was stale) to the
+ * retry queue, for re-processing on subsequent runs once the registry has
+ * caught up with the revision announced by the changes feed.
+ */
+async function deferStaleChanges(
+ queueUrl: string,
+ changes: readonly DatabaseChange[]
+): Promise {
+ for (const change of changes) {
+ const message: StaleMetadataMessage = {
+ name: change.id,
+ expectedRev: getMaxSequentialRevision(change),
+ seq: change.seq,
+ firstSeenAt: new Date().toISOString(),
+ };
+ console.log(
+ `Deferring change for ${change.id} (expected rev ${message.expectedRev}) to the stale-metadata retry queue`
+ );
+ await SQS_CLIENT.send(
+ new SendMessageCommand({
+ QueueUrl: queueUrl,
+ MessageBody: JSON.stringify(message),
+ })
+ );
+ }
+}
+
+/**
+ * Processes messages from the stale-metadata retry queue: for each deferred
+ * change, the registry metadata is fetched again. If it has caught up with
+ * the revision announced by the changes feed, the change is processed
+ * normally and the message is deleted. Otherwise the message is left on the
+ * queue for a later retry (it moves to the dead-letter queue once the maximum
+ * receive count is exhausted).
+ *
+ * @returns the number of package versions that were sent for staging.
+ */
+async function processRetryQueue(
+ context: Context,
+ npm: CouchChanges,
+ queueUrl: string,
+ stagingFunction: string,
+ denyList: DenyListClient,
+ licenseList: LicenseListClient,
+ knownVersions: Map
+): Promise {
+ let staged = 0;
+ await metricScope((metrics) => async () => {
+ metrics.setDimensions({});
+ let recovered = 0;
+ let received = 0;
+ while (
+ received < MAX_RETRY_MESSAGES_PER_RUN &&
+ context.getRemainingTimeInMillis() > RETRY_QUEUE_TIME_BUDGET_MS
+ ) {
+ const response = await SQS_CLIENT.send(
+ new ReceiveMessageCommand({
+ QueueUrl: queueUrl,
+ MaxNumberOfMessages: 10,
+ })
+ );
+ const messages = response.Messages ?? [];
+ if (messages.length === 0) {
+ break;
+ }
+ received += messages.length;
+ for (const message of messages) {
+ if (message.Body == null) {
+ continue;
+ }
+ const payload: StaleMetadataMessage = JSON.parse(message.Body);
+ const doc = await npm.getPackageDoc(payload.name);
+ if (doc == null || doc._rev == null) {
+ console.log(
+ `Deferred package ${payload.name} is no longer available in the registry, dropping`
+ );
+ await SQS_CLIENT.send(
+ new DeleteMessageCommand({
+ QueueUrl: queueUrl,
+ ReceiptHandle: message.ReceiptHandle,
+ })
+ );
+ continue;
+ }
+ if (parseSequentialRevision(doc._rev as string) < payload.expectedRev) {
+ // Registry is still behind the announced revision: leave the
+ // message on the queue, it will be re-delivered after the
+ // visibility timeout expires.
+ console.log(
+ `Deferred package ${payload.name} is still stale (rev ${doc._rev} < expected ${payload.expectedRev})`
+ );
+ continue;
+ }
+ const change: DatabaseChange = {
+ changes: [{ rev: doc._rev as string }],
+ deleted: false,
+ id: payload.name,
+ seq: payload.seq,
+ doc,
+ };
+ const versionInfos = getRelevantVersionInfos(
+ [change as Change],
+ metrics,
+ denyList,
+ licenseList,
+ knownVersions
+ );
+ console.log(
+ `Recovered deferred change for ${payload.name}: ${versionInfos.length} relevant version(s)`
+ );
+ await stageVersions(versionInfos, stagingFunction, knownVersions);
+ staged += versionInfos.length;
+ recovered += 1;
+ await SQS_CLIENT.send(
+ new DeleteMessageCommand({
+ QueueUrl: queueUrl,
+ ReceiptHandle: message.ReceiptHandle,
+ })
+ );
+ }
+ }
+ metrics.putMetric(
+ MetricName.STALE_METADATA_RECOVERED,
+ recovered,
+ Unit.Count
+ );
+ })();
+ return staged;
+}
+//#endregion
+
+//#region Overlap sweep
+/**
+ * Re-reads a trailing window of the `_changes` feed and processes any row for
+ * which no receipt exists (i.e. rows that were inserted into the feed behind
+ * the follower's cursor). Rows beyond the current head-of-feed marker are
+ * left for the regular head-of-feed pass.
+ *
+ * @returns the number of package versions that were sent for staging.
+ */
+async function runSweep(
+ context: Context,
+ npm: CouchChanges,
+ receipts: FollowerReceipts,
+ retryQueueUrl: string,
+ stagingFunction: string,
+ denyList: DenyListClient,
+ licenseList: LicenseListClient,
+ knownVersions: Map,
+ headMarker: number
+): Promise {
+ const kind = receipts.dueSweep();
+ if (kind == null || headMarker <= 0) {
+ return 0;
+ }
+ const startSeq = receipts.sweepStartSeq(kind);
+ if (startSeq == null) {
+ // No checkpoint history yet (e.g. first run): nothing to sweep over.
+ return 0;
+ }
+
+ console.log(
+ `Starting ${kind} sweep from seq ${startSeq} up to head marker ${headMarker}`
+ );
+ let staged = 0;
+ let sweepSeq = startSeq;
+ let completed = false;
+
+ while (
+ !completed &&
+ context.getRemainingTimeInMillis() > SWEEP_TIME_BUDGET_MS
+ ) {
+ await metricScope((metrics) => async () => {
+ metrics.setDimensions({});
+ metrics.setProperty('SweepKind', kind);
+ metrics.setProperty('SweepStartSeq', sweepSeq);
+
+ const raw = await npm.rawChanges(sweepSeq, {
+ batchSize: SWEEP_BATCH_SIZE,
+ });
+ const nextSeq = Number(raw.last_seq);
+ metrics.setProperty('SweepEndSeq', nextSeq);
+
+ // Rows beyond the head marker will be handled by the next head-of-feed
+ // pass; do not process (or record) them here.
+ const inScope = raw.results.filter(
+ (change) => change.seq != null && Number(change.seq) <= headMarker
+ );
+ const missed = inScope.filter(
+ (change) => !receipts.has(Number(change.seq))
+ );
+
+ metrics.putMetric(
+ MetricName.LATE_CHANGE_COUNT,
+ missed.length,
+ Unit.Count
+ );
+ const now = Date.now();
+ for (const change of missed) {
+ const crossedAt = receipts.timeCrossed(Number(change.seq));
+ if (crossedAt != null) {
+ metrics.putMetric(
+ MetricName.LATE_CHANGE_LAG,
+ now - crossedAt,
+ Unit.Milliseconds
+ );
+ }
+ console.log(
+ `[sweep] Late change discovered at seq ${change.seq}: ${change.id}`
+ );
+ }
+
+ const processable = missed.filter(
+ (change) => !change.deleted && change.id
+ );
+ if (processable.length > 0) {
+ const { ok, stale } = await npm.fetchAndFilterAllMetadata(processable);
+ metrics.putMetric(
+ MetricName.STALE_METADATA_DEFERRED,
+ stale.length,
+ Unit.Count
+ );
+ await deferStaleChanges(retryQueueUrl, stale);
+ const versionInfos = getRelevantVersionInfos(
+ ok as unknown as readonly Change[],
+ metrics,
+ denyList,
+ licenseList,
+ knownVersions
+ );
+ await stageVersions(versionInfos, stagingFunction, knownVersions);
+ staged += versionInfos.length;
+ }
+
+ // Record receipts for everything in scope (processed or deliberately
+ // skipped), so the next sweep does not revisit them.
+ receipts.addSeqs(
+ inScope.map((change) => Number(change.seq)).filter((seq) => !isNaN(seq))
+ );
+
+ if (
+ raw.totalCount === 0 ||
+ nextSeq >= headMarker ||
+ nextSeq <= sweepSeq
+ ) {
+ completed = true;
+ receipts.completeSweep(kind);
+ console.log(`Completed ${kind} sweep at seq ${nextSeq}`);
+ } else {
+ sweepSeq = nextSeq;
+ }
+ })();
+ }
+
+ if (!completed) {
+ receipts.sweepCursor = { seq: sweepSeq, deep: kind === 'deep' };
+ console.log(
+ `Sweep ran out of time at seq ${sweepSeq}; it will resume on the next run`
+ );
+ }
+ return staged;
+}
+//#endregion
+
//#region Last known versions and marker
/**
* Common function to load data from an S3 file with error handling
@@ -423,6 +812,65 @@ async function saveLastTransactionMarker(
});
console.log('Successfully updated marker');
}
+
+/**
+ * Loads the follower receipts (received sequence numbers, checkpoints, and
+ * sweep state) from S3. A fresh state is returned when the file is missing,
+ * cannot be parsed, or when the sequence space was reset (marker is 0), since
+ * receipts from a previous sequence space are meaningless.
+ */
+async function loadReceipts(
+ stagingBucket: string,
+ initialMarker: number
+): Promise {
+ if (initialMarker === 0) {
+ console.log(
+ 'Marker is 0 (fresh start or sequence space reset), starting with fresh receipts'
+ );
+ return new FollowerReceipts();
+ }
+ const warningMessage = `Receipts object (s3://${stagingBucket}/${RECEIPTS_FILE_NAME}) does not exist, starting from scratch`;
+ const content = await loadContentFromS3(
+ stagingBucket,
+ RECEIPTS_FILE_NAME,
+ warningMessage
+ );
+ if (content === null) {
+ return new FollowerReceipts();
+ }
+ try {
+ const receipts = FollowerReceipts.fromText(content);
+ console.log(
+ `Loaded receipts: ${receipts.receiptCount} seq(s), ${receipts.checkpointCount} checkpoint(s)`
+ );
+ return receipts;
+ } catch (error) {
+ console.warn(`Could not parse receipts file, starting fresh: ${error}`);
+ return new FollowerReceipts();
+ }
+}
+
+/**
+ * Persists the follower receipts to S3 (gzip-compressed when large enough to
+ * warrant it).
+ */
+async function saveReceipts(
+ context: Context,
+ stagingBucket: string,
+ receipts: FollowerReceipts
+) {
+ const { buffer, contentEncoding } = compressContent(
+ Buffer.from(receipts.toText(), 'utf-8')
+ );
+ console.log(
+ `Updating receipts (${receipts.receiptCount} seq(s), ${receipts.checkpointCount} checkpoint(s))`
+ );
+ await putObject(context, stagingBucket, RECEIPTS_FILE_NAME, buffer, {
+ ContentType: 'text/plain',
+ ContentEncoding: contentEncoding,
+ });
+ console.log('Successfully updated receipts');
+}
//#endregion
//#region Asynchronous Primitives