diff --git a/.gitattributes b/.gitattributes index 562f4ad06..8def7a98f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -66,6 +66,7 @@ /src/overview-dashboard/sqs-dlq-stats-widget-function.ts linguist-generated /src/package-sources/codeartifact/code-artifact-forwarder.ts linguist-generated /src/package-sources/npmjs/canary/npmjs-package-canary.ts linguist-generated +/src/package-sources/npmjs/known-versions-migration.ts linguist-generated /src/package-sources/npmjs/npm-js-follower.ts linguist-generated /src/package-sources/npmjs/re-stage-package-version.ts linguist-generated /src/package-sources/npmjs/stage-and-notify.ts linguist-generated diff --git a/.gitignore b/.gitignore index 330f0b481..ec6ca9dda 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ lib/__tests__/devapp/cdk.out !/src/package-sources/npmjs/stage-and-notify.ts !/src/package-sources/npmjs/re-stage-package-version.ts !/src/package-sources/npmjs/npm-js-follower.ts +!/src/package-sources/npmjs/known-versions-migration.ts !/src/package-sources/npmjs/canary/npmjs-package-canary.ts !/src/package-sources/codeartifact/code-artifact-forwarder.ts !/src/overview-dashboard/sqs-dlq-stats-widget-function.ts diff --git a/.prettierignore b/.prettierignore index a050ba12f..c9902bb09 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,6 +2,7 @@ src/package-sources/npmjs/stage-and-notify.ts src/package-sources/npmjs/re-stage-package-version.ts src/package-sources/npmjs/npm-js-follower.ts +src/package-sources/npmjs/known-versions-migration.ts src/package-sources/npmjs/canary/npmjs-package-canary.ts src/package-sources/codeartifact/code-artifact-forwarder.ts src/overview-dashboard/sqs-dlq-stats-widget-function.ts diff --git a/.projen/deps.json b/.projen/deps.json index 61587305a..cfe664c0d 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -23,6 +23,10 @@ "name": "@aws-sdk/client-codeartifact", "type": "build" }, + { + "name": "@aws-sdk/client-dynamodb", + "type": "build" + }, { "name": "@aws-sdk/client-ecs", "type": "build" diff --git a/.projen/files.json b/.projen/files.json index 188e1349b..a8dae9bd6 100644 --- a/.projen/files.json +++ b/.projen/files.json @@ -55,6 +55,7 @@ "src/overview-dashboard/sqs-dlq-stats-widget-function.ts", "src/package-sources/codeartifact/code-artifact-forwarder.ts", "src/package-sources/npmjs/canary/npmjs-package-canary.ts", + "src/package-sources/npmjs/known-versions-migration.ts", "src/package-sources/npmjs/npm-js-follower.ts", "src/package-sources/npmjs/re-stage-package-version.ts", "src/package-sources/npmjs/stage-and-notify.ts", diff --git a/.projen/tasks.json b/.projen/tasks.json index fcb1eaec9..b5296b1f6 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -82,6 +82,9 @@ { "spawn": "bundle:npm-js-follower" }, + { + "spawn": "bundle:known-versions-migration" + }, { "spawn": "bundle:npmjs-package-canary" }, @@ -363,12 +366,30 @@ } ] }, + "bundle:known-versions-migration": { + "name": "bundle:known-versions-migration", + "description": "Create an AWS Lambda bundle from src/package-sources/npmjs/known-versions-migration.lambda.ts", + "steps": [ + { + "exec": "esbuild --bundle src/package-sources/npmjs/known-versions-migration.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/known-versions-migration.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json" + } + ] + }, + "bundle:known-versions-migration:watch": { + "name": "bundle:known-versions-migration:watch", + "description": "Continuously update an AWS Lambda bundle from src/package-sources/npmjs/known-versions-migration.lambda.ts", + "steps": [ + { + "exec": "esbuild --bundle src/package-sources/npmjs/known-versions-migration.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/known-versions-migration.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json --watch" + } + ] + }, "bundle:lambda:watch": { "name": "bundle:lambda:watch", "description": "Continuously bundle all AWS Lambda functions", "steps": [ { - "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json" + "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/known-versions-migration.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json" } ] }, @@ -843,6 +864,9 @@ { "spawn": "bundle:npm-js-follower" }, + { + "spawn": "bundle:known-versions-migration" + }, { "spawn": "bundle:npmjs-package-canary" }, @@ -1321,7 +1345,7 @@ "--peer", "--no-deprecated", "--dep=dev", - "--filter=@aws-sdk/client-cloudwatch,@aws-sdk/client-codeartifact,@aws-sdk/client-ecs,@aws-sdk/client-lambda,@aws-sdk/client-s3,@aws-sdk/client-sfn,@aws-sdk/client-sqs,@jsii/spec,@octokit/rest,@smithy/types,@smithy/util-retry,@smithy/util-stream,@types/aws-lambda,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/semver,@types/tough-cookie,@types/uuid,async-sema,aws-embedded-metrics,aws-sdk-client-mock,aws-sdk-client-mock-jest,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,chalk,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,feed,fs-extra,glob,got,jest,jsii-diff,jsii-pacmak,jsii-rosetta,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,prettier,semver,spdx-license-list,streamcount,streamx,tar-stream,ts-node,uuid,yaml" + "--filter=@aws-sdk/client-cloudwatch,@aws-sdk/client-codeartifact,@aws-sdk/client-dynamodb,@aws-sdk/client-ecs,@aws-sdk/client-lambda,@aws-sdk/client-s3,@aws-sdk/client-sfn,@aws-sdk/client-sqs,@jsii/spec,@octokit/rest,@smithy/types,@smithy/util-retry,@smithy/util-stream,@types/aws-lambda,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/semver,@types/tough-cookie,@types/uuid,async-sema,aws-embedded-metrics,aws-sdk-client-mock,aws-sdk-client-mock-jest,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,chalk,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,feed,fs-extra,glob,got,jest,jsii-diff,jsii-pacmak,jsii-rosetta,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,prettier,semver,spdx-license-list,streamcount,streamx,tar-stream,ts-node,uuid,yaml" ] }, { @@ -1337,6 +1361,7 @@ "@aws-cdk/integ-tests-alpha", "@aws-sdk/client-cloudwatch", "@aws-sdk/client-codeartifact", + "@aws-sdk/client-dynamodb", "@aws-sdk/client-ecs", "@aws-sdk/client-lambda", "@aws-sdk/client-s3", diff --git a/.projenrc.ts b/.projenrc.ts index c38f8b4c6..b0b3c4776 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -56,6 +56,7 @@ const project = new CdklabsConstructLibrary({ cdkCli, '@aws-sdk/client-cloudwatch', '@aws-sdk/client-codeartifact', + '@aws-sdk/client-dynamodb', '@aws-sdk/client-lambda', '@aws-sdk/client-s3', '@aws-sdk/client-sfn', diff --git a/API.md b/API.md index 628b9e841..25f5a18fd 100644 --- a/API.md +++ b/API.md @@ -1724,6 +1724,8 @@ const npmJsProps: sources.NpmJsProps = { ... } | canaryPackage | string | The package that is monitored by the package canary, if enabled by `enableCanary`. | | canarySla | aws-cdk-lib.Duration | The maximum amount of time it is supposed to take for packages to become visible in this ConstructHub instance. | | enableCanary | boolean | Registers a package canary, which will track availability of a canary package in ConstructHub, and emit dedicated metrics. | +| enableKnownVersionsMigration | boolean | Copies the known versions of a deployment created before the known versions table existed from the staging bucket to the table, when the table is created. | +| knownVersionsTableBilling | aws-cdk-lib.aws_dynamodb.Billing | The billing mode of the table in which the follower records the package versions it has processed. | | stagingBucket | aws-cdk-lib.aws_s3.IBucket | The bucket to use for staging npm packages. | --- @@ -1787,6 +1789,37 @@ Registers a package canary, which will track availability of a canary package in --- +##### `enableKnownVersionsMigration`Optional + +```typescript +public readonly enableKnownVersionsMigration: boolean; +``` + +- *Type:* boolean +- *Default:* true + +Copies the known versions of a deployment created before the known versions table existed from the staging bucket to the table, when the table is created. + +Without the copy, an existing deployment starts with an empty table and +processes every version of every construct library again, as their +packages change. Turn this off for new deployments, or if the table was +filled some other way. + +--- + +##### `knownVersionsTableBilling`Optional + +```typescript +public readonly knownVersionsTableBilling: Billing; +``` + +- *Type:* aws-cdk-lib.aws_dynamodb.Billing +- *Default:* Billing.onDemand() + +The billing mode of the table in which the follower records the package versions it has processed. + +--- + ##### `stagingBucket`Optional ```typescript diff --git a/docs/application-overview.md b/docs/application-overview.md index 12521402d..319a94322 100644 --- a/docs/application-overview.md +++ b/docs/application-overview.md @@ -44,7 +44,14 @@ ConstructHub provides two package source implementations: `NpmJs` and scheduled to run every `5 minutes`, and stores its state (receipts for the change entries received, time/sequence checkpoints, and laggy packument expectations) in a specific object in the S3 bucket used for staging package - tarballs. See [npm-sync-protocol.md](./npm-sync-protocol.md) for the + tarballs. + The package versions it has processed are recorded in a DynamoDB table, so + each version is only processed once. + Existing deployments kept this list in the S3 bucket; it is copied to the + table by a function that runs when the table is first deployed. + The follower is only updated once the copy completed, and the copy runs once + more afterwards, to pick up what the old follower recorded in the meantime. + See [npm-sync-protocol.md](./npm-sync-protocol.md) for the guarantees the feed does and does not give, and how the follower compensates. > Back-filling is automatic for the `NpmJs` source. Upon initial deployment, @@ -60,6 +67,7 @@ ConstructHub provides two package source implementations: `NpmJs` and > ``` > > The next scan then covers everything from that position to the feed head. + > Versions recorded in the known versions table are not staged again. > The state object is linked from the backend dashboard. - A **high-severity** alarm triggers if the NpmJs Follower is not running at diff --git a/package.json b/package.json index 8f395fa73..0e391f502 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "bundle:http-get-function:watch": "projen bundle:http-get-function:watch", "bundle:ingestion": "projen bundle:ingestion", "bundle:ingestion:watch": "projen bundle:ingestion:watch", + "bundle:known-versions-migration": "projen bundle:known-versions-migration", + "bundle:known-versions-migration:watch": "projen bundle:known-versions-migration:watch", "bundle:lambda:watch": "projen bundle:lambda:watch", "bundle:monitor": "projen bundle:monitor", "bundle:monitor:watch": "projen bundle:monitor:watch", @@ -120,6 +122,7 @@ "@aws-cdk/integ-tests-alpha": "2.194.0-alpha.0", "@aws-sdk/client-cloudwatch": "^3.1141.0", "@aws-sdk/client-codeartifact": "^3.1141.0", + "@aws-sdk/client-dynamodb": "^3.1141.0", "@aws-sdk/client-ecs": "^3.1141.0", "@aws-sdk/client-lambda": "^3.1141.0", "@aws-sdk/client-s3": "^3.1141.0", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 99561bf20..40e4220e0 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -296,6 +296,114 @@ exports[`minimal usage 1`] = ` }, }, "Resources": { + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -640,11 +748,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -9959,10 +10067,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -9974,7 +10124,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -9990,6 +10140,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -10542,6 +10695,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -10642,13 +11038,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -10705,7 +11094,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -10713,40 +11102,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -10859,7 +11234,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -14375,6 +14750,114 @@ exports[`piggy-backing on an existing CodeArtifact domain 1`] = ` }, "Type": "AWS::IAM::Role", }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -14719,11 +15202,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -24412,10 +24895,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -24427,7 +24952,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -24443,6 +24968,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -24995,6 +25523,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -25095,13 +25866,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -25158,7 +25922,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -25166,40 +25930,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -25312,7 +26062,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -28766,6 +29516,114 @@ exports[`with all internet access 1`] = ` }, }, "Resources": { + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -29110,11 +29968,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -38429,10 +39287,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -38444,7 +39344,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -38460,6 +39360,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -39012,51 +39915,111 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsSchedule34031870": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", "DependsOn": [ - "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", ], "Properties": { - "Description": "Test/ConstructHub/Sources/NpmJs/Schedule", - "ScheduleExpression": "rate(5 minutes)", - "State": "ENABLED", - "Targets": [ - { - "Arn": { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJs15A77D2D", - "Arn", - ], - }, - "Id": "Target0", - }, - ], + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", }, - "Type": "AWS::Events::Rule", + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsScheduleAllowEventRuleTestConstructHubSourcesNpmJs5171C2A673CCB598": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", "DependsOn": [ - "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", ], "Properties": { - "Action": "lambda:InvokeFunction", - "FunctionName": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { "Fn::GetAtt": [ - "ConstructHubSourcesNpmJs15A77D2D", + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", "Arn", ], }, - "Principal": "events.amazonaws.com", - "SourceArn": { + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsSchedule34031870", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", "Arn", ], }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, }, - "Type": "AWS::Lambda::Permission", + "Type": "AWS::Lambda::Function", }, - "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -39091,7 +40054,7 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Role", }, - "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -39112,13 +40075,36 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", ], "Effect": "Allow", "Resource": [ @@ -39144,18 +40130,134 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesNpmJsSchedule34031870": { + "DependsOn": [ + "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + ], + "Properties": { + "Description": "Test/ConstructHub/Sources/NpmJs/Schedule", + "ScheduleExpression": "rate(5 minutes)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJs15A77D2D", + "Arn", + ], + }, + "Id": "Target0", + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "ConstructHubSourcesNpmJsScheduleAllowEventRuleTestConstructHubSourcesNpmJs5171C2A673CCB598": { + "DependsOn": [ + "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + ], + "Properties": { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJs15A77D2D", + "Arn", + ], + }, + "Principal": "events.amazonaws.com", + "SourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsSchedule34031870", + "Arn", + ], + }, + }, + "Type": "AWS::Lambda::Permission", + }, + "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -39175,7 +40277,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-last-transaction-id.2", ], ], }, @@ -39218,6 +40320,29 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, { "Action": [ "s3:GetObject*", @@ -39329,7 +40454,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -42791,6 +43916,114 @@ exports[`with domain 1`] = ` }, }, "Resources": { + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -43275,11 +44508,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -52603,10 +53836,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -52618,7 +53893,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -52634,6 +53909,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -53173,6 +54451,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -53273,13 +54794,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -53336,7 +54850,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -53344,40 +54858,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -53490,7 +54990,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -57281,6 +58781,114 @@ exports[`with limited internet access 1`] = ` }, "Type": "AWS::IAM::Role", }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -57625,11 +59233,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -66760,10 +68368,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -66775,7 +68425,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -66791,6 +68441,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -67343,6 +68996,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -67443,13 +69339,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -67506,7 +69395,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -67514,40 +69403,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -67660,7 +69535,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 770f4ace2..59db2a477 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -358,6 +358,114 @@ exports[`golden snapshot 1`] = ` }, "Type": "AWS::IAM::Role", }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -702,11 +810,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -11234,10 +11342,52 @@ Direct link to the function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -11249,7 +11399,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -11265,6 +11415,9 @@ Direct link to the function: /lambda/home#/functions/", "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -11811,6 +11964,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7011fc689a372f2abb95be7d24b67a416cf", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7011fc689a372f2abb95be7d24b67a416cf", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7011fc689a372f2abb95be7d24b67a416cf": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[dev/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -11911,13 +12307,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -11974,7 +12363,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -11982,40 +12371,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -12128,7 +12503,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/package-sources/npmjs.test.ts b/src/__tests__/package-sources/npmjs.test.ts new file mode 100644 index 000000000..a5b6b5edd --- /dev/null +++ b/src/__tests__/package-sources/npmjs.test.ts @@ -0,0 +1,126 @@ +import { App, Stack } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { Billing, Capacity } from 'aws-cdk-lib/aws-dynamodb'; +import { ConstructHub } from '../../construct-hub'; +import { NpmJs } from '../../package-sources'; + +function synth(npmJs: NpmJs): Template { + const app = new App(); + const stack = new Stack(app, 'Test'); + new ConstructHub(stack, 'ConstructHub', { packageSources: [npmJs] }); + return Template.fromStack(stack); +} + +test('creates a retained, backed-up, on-demand known versions table', () => { + const template = synth(new NpmJs()); + + template.hasResource('AWS::DynamoDB::GlobalTable', { + DeletionPolicy: 'Retain', + UpdateReplacePolicy: 'Retain', + Properties: Match.objectLike({ + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'name', KeyType: 'HASH' }, + { AttributeName: 'version', KeyType: 'RANGE' }, + ], + Replicas: [ + Match.objectLike({ + PointInTimeRecoverySpecification: { + PointInTimeRecoveryEnabled: true, + }, + }), + ], + }), + }); +}); + +test('passes the known versions table to the follower', () => { + const template = synth(new NpmJs()); + + template.hasResourceProperties('AWS::Lambda::Function', { + Environment: { + Variables: Match.objectLike({ + KNOWN_VERSIONS_TABLE_NAME: Match.anyValue(), + }), + }, + }); +}); + +test('migrates existing known versions before the follower is updated, and catches up after', () => { + const template = synth(new NpmJs()); + + const migrations = template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp('Copies the known versions'), + }, + }); + expect(Object.keys(migrations)).toHaveLength(1); + const [migrationId] = Object.keys(migrations); + + const versions = template.findResources('AWS::Lambda::Version', { + Properties: { FunctionName: { Ref: migrationId } }, + }); + expect(Object.keys(versions)).toHaveLength(1); + + const triggers = template.findResources('Custom::Trigger', { + Properties: { + HandlerArn: { Ref: Object.keys(versions)[0] }, + InvocationType: 'RequestResponse', + ExecuteOnHandlerChange: false, + }, + }); + expect(Object.keys(triggers)).toHaveLength(1); + const [triggerId] = Object.keys(triggers); + + // The follower function is only deployed after the migration ran. + const followers = template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp('Periodically query npmjs.com'), + }, + }); + expect(Object.keys(followers)).toHaveLength(1); + expect(Object.values(followers)[0].DependsOn).toContain(triggerId); + const [followerId] = Object.keys(followers); + + // A second pass after the follower update picks up what the old follower + // code recorded in the meantime. + const catchUps = template.findResources('Custom::Trigger', { + Properties: { + HandlerArn: { Ref: Object.keys(versions)[0] }, + InvocationType: 'Event', + ExecuteOnHandlerChange: false, + }, + }); + expect(Object.keys(catchUps)).toHaveLength(1); + expect(Object.values(catchUps)[0].DependsOn).toContain(followerId); +}); + +test('the known versions migration can be turned off', () => { + const template = synth(new NpmJs({ enableKnownVersionsMigration: false })); + + template.resourceCountIs('Custom::Trigger', 0); + expect( + Object.keys( + template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp('Copies the known versions'), + }, + }) + ) + ).toHaveLength(0); +}); + +test('the billing mode of the known versions table is configurable', () => { + const template = synth( + new NpmJs({ + knownVersionsTableBilling: Billing.provisioned({ + readCapacity: Capacity.fixed(10), + writeCapacity: Capacity.autoscaled({ maxCapacity: 20 }), + }), + }) + ); + + template.hasResourceProperties('AWS::DynamoDB::GlobalTable', { + BillingMode: 'PROVISIONED', + }); +}); diff --git a/src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts b/src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts new file mode 100644 index 000000000..41190ba0b --- /dev/null +++ b/src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts @@ -0,0 +1,126 @@ +import { + BatchWriteItemCommand, + DynamoDBClient, +} from '@aws-sdk/client-dynamodb'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import { handler } from '../../../package-sources/npmjs/known-versions-migration.lambda'; +import { stringToStream } from '../../streams'; + +const MOCK_BUCKET = 'mock-staging-bucket'; +const MOCK_TABLE = 'mock-known-versions-table'; + +const mockS3 = mockClient(S3Client); +const mockDynamoDB = mockClient(DynamoDBClient); + +beforeEach(() => { + mockS3.reset(); + mockDynamoDB.reset(); + process.env.BUCKET_NAME = MOCK_BUCKET; + process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = MOCK_TABLE; + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); + givenNoObject(KNOWN_VERSIONS_FILE_NAME); + givenNoObject(MARKER_FILE_NAME); +}); + +afterEach(() => { + delete process.env.BUCKET_NAME; + delete process.env[ENV_KNOWN_VERSIONS_TABLE_NAME]; +}); + +function givenNoObject(key: string) { + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: key }) + .rejects(Object.assign(new Error('no such key'), { name: 'NoSuchKey' })); +} + +function givenObject(key: string, content: unknown) { + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: key }) + .resolves({ Body: stringToStream(JSON.stringify(content)) }); +} + +/** The `name@version` keys written to the table, sorted. */ +function writtenKeys(): string[] { + return mockDynamoDB + .commandCalls(BatchWriteItemCommand) + .flatMap((call) => call.args[0].input.RequestItems![MOCK_TABLE]) + .map( + (request) => + `${request.PutRequest!.Item!.name.S}@${ + request.PutRequest!.Item!.version.S + }` + ) + .sort(); +} + +test('copies the known versions file to the table', async () => { + givenObject(KNOWN_VERSIONS_FILE_NAME, { + knownVersions: { + 'some-package@1.0.0': '2023-06-01T00:00:00.000Z', + '@scope/other-package@2.0.0-pre.1': '2023-06-02T00:00:00.000Z', + }, + }); + + const result = await handler(); + + expect(result).toEqual({ written: 2, skipped: [] }); + expect(writtenKeys()).toEqual([ + '@scope/other-package@2.0.0-pre.1', + 'some-package@1.0.0', + ]); +}); + +test('copies the known versions embedded in the legacy marker file', async () => { + givenObject(MARKER_FILE_NAME, { + marker: 1_000, + knownVersions: { 'legacy-package@1.0.0': 1685577600000 }, + }); + + const result = await handler(); + + expect(result).toEqual({ written: 1, skipped: [] }); + expect(writtenKeys()).toEqual(['legacy-package@1.0.0']); +}); + +test('does nothing on a new deployment', async () => { + const result = await handler(); + + expect(result).toEqual({ written: 0, skipped: [] }); + expect(mockDynamoDB).not.toHaveReceivedCommand(BatchWriteItemCommand); +}); + +test('skips records the table cannot store', async () => { + const tooLong = `long-package@${'1'.repeat(1_025)}`; + givenObject(KNOWN_VERSIONS_FILE_NAME, { + knownVersions: { + 'good-package@1.0.0': '2023-06-01T00:00:00.000Z', + '@scope/no-version': '2023-06-01T00:00:00.000Z', + 'no-version@': '2023-06-01T00:00:00.000Z', + [tooLong]: '2023-06-01T00:00:00.000Z', + }, + }); + + const result = await handler(); + + expect(result).toEqual({ + written: 1, + skipped: ['@scope/no-version', 'no-version@', tooLong], + }); + expect(writtenKeys()).toEqual(['good-package@1.0.0']); +}); + +test('fails on other errors, so the invocation is retried', async () => { + givenObject(KNOWN_VERSIONS_FILE_NAME, { + knownVersions: { 'good-package@1.0.0': '2023-06-01T00:00:00.000Z' }, + }); + mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('network')); + + await expect(handler()).rejects.toThrow(/network/); +}); diff --git a/src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts new file mode 100644 index 000000000..d12c8185a --- /dev/null +++ b/src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts @@ -0,0 +1,175 @@ +import { + BatchWriteItemCommand, + DynamoDB, + DynamoDBClient, + QueryCommand, +} from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import { KnownVersions } from '../../../package-sources/npmjs/known-versions.lambda-shared'; + +const TABLE = 'known-versions-table'; +const mockDynamoDB = mockClient(DynamoDBClient); + +function newKnownVersions() { + return new KnownVersions(TABLE, new DynamoDB({})); +} + +/** The `name@version` keys written by `BatchWriteItem`, sorted. */ +function writtenKeys(): string[] { + return mockDynamoDB + .commandCalls(BatchWriteItemCommand) + .flatMap((call) => call.args[0].input.RequestItems![TABLE]) + .map( + (request) => + `${request.PutRequest!.Item!.name.S}@${ + request.PutRequest!.Item!.version.S + }` + ) + .sort(); +} + +beforeEach(() => { + mockDynamoDB.reset(); + mockDynamoDB.on(QueryCommand).resolves({ Items: [] }); + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); +}); + +describe('has', () => { + test('returns whether a version is recorded for the package', async () => { + mockDynamoDB.on(QueryCommand).resolves({ + Items: [{ version: { S: '1.0.0' } }, { version: { S: '1.1.0' } }], + }); + const known = newKnownVersions(); + + expect(await known.has('some-package', '1.0.0')).toBe(true); + expect(await known.has('some-package', '1.1.0')).toBe(true); + expect(await known.has('some-package', '2.0.0')).toBe(false); + expect(mockDynamoDB).toHaveReceivedCommandWith(QueryCommand, { + TableName: TABLE, + ConsistentRead: true, + ExpressionAttributeValues: { ':name': { S: 'some-package' } }, + }); + }); + + test('follows query pagination', async () => { + mockDynamoDB + .on(QueryCommand) + .resolvesOnce({ + Items: [{ version: { S: '1.0.0' } }], + LastEvaluatedKey: { + name: { S: 'big-package' }, + version: { S: '1.0.0' }, + }, + }) + .resolvesOnce({ Items: [{ version: { S: '2.0.0' } }] }); + const known = newKnownVersions(); + + expect(await known.has('big-package', '1.0.0')).toBe(true); + expect(await known.has('big-package', '2.0.0')).toBe(true); + expect(mockDynamoDB).toHaveReceivedCommandTimes(QueryCommand, 2); + }); + + test('queries each package only once, also for concurrent callers', async () => { + const known = newKnownVersions(); + + await Promise.all([ + known.has('some-package', '1.0.0'), + known.has('some-package', '2.0.0'), + ]); + await known.has('some-package', '3.0.0'); + + expect(mockDynamoDB).toHaveReceivedCommandTimes(QueryCommand, 1); + }); + + test('queries again after a failed query', async () => { + mockDynamoDB + .on(QueryCommand) + .rejectsOnce(new Error('network')) + .resolves({ Items: [{ version: { S: '1.0.0' } }] }); + const known = newKnownVersions(); + + await expect(known.has('some-package', '1.0.0')).rejects.toThrow(/network/); + expect(await known.has('some-package', '1.0.0')).toBe(true); + }); + + test('sees versions added locally right away, without a query', async () => { + const known = newKnownVersions(); + + known.add('@scope/package', '1.0.0'); + + expect(await known.has('@scope/package', '1.0.0')).toBe(true); + expect(mockDynamoDB).not.toHaveReceivedCommand(QueryCommand); + }); +}); + +describe('add and flush', () => { + test('flush persists the added versions', async () => { + const known = newKnownVersions(); + + known.add('@scope/package', '1.0.0'); + known.add('other-package', '2.0.0'); + known.add('other-package', '2.0.0'); + expect(mockDynamoDB).not.toHaveReceivedCommand(BatchWriteItemCommand); + expect(known.pendingCount).toBe(2); + + await known.flush(); + + expect(known.pendingCount).toBe(0); + expect(writtenKeys()).toEqual([ + '@scope/package@1.0.0', + 'other-package@2.0.0', + ]); + }); + + test('keeps the records of a failed flush for the next attempt', async () => { + mockDynamoDB + .on(BatchWriteItemCommand) + .rejectsOnce(new Error('network')) + .resolves({ UnprocessedItems: {} }); + const known = newKnownVersions(); + known.add('some-package', '1.0.0'); + + await expect(known.flush()).rejects.toThrow(/network/); + expect(known.pendingCount).toBe(1); + + await known.flush(); + expect(known.pendingCount).toBe(0); + }); +}); + +describe('flush batching', () => { + test('writes the records in batches of 25', async () => { + const known = newKnownVersions(); + for (let i = 0; i < 60; i++) { + known.add(`package-${i}`, `1.0.${i}`); + } + known.add('@scope/package', '2.0.0'); + + await known.flush(); + + expect(mockDynamoDB).toHaveReceivedCommandTimes(BatchWriteItemCommand, 3); + expect(writtenKeys()).toHaveLength(61); + expect(writtenKeys()).toContain('@scope/package@2.0.0'); + }); + + test('retries unprocessed items', async () => { + const unprocessed = { + PutRequest: { + Item: { name: { S: 'some-package' }, version: { S: '1.0.0' } }, + }, + }; + mockDynamoDB + .on(BatchWriteItemCommand) + .resolvesOnce({ UnprocessedItems: { [TABLE]: [unprocessed] } }) + .resolves({ UnprocessedItems: {} }); + const known = newKnownVersions(); + known.add('some-package', '1.0.0'); + + await known.flush(); + + const calls = mockDynamoDB.commandCalls(BatchWriteItemCommand); + expect(calls).toHaveLength(2); + expect(calls[1].args[0].input.RequestItems![TABLE]).toEqual([unprocessed]); + }); +}); diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts index bb92a1c16..e93482311 100644 --- a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts +++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts @@ -1,4 +1,9 @@ import { gunzipSync, gzipSync } from 'zlib'; +import { + BatchWriteItemCommand, + DynamoDBClient, + QueryCommand, +} from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; import { GetObjectCommand, @@ -17,8 +22,8 @@ import { } from '../../../backend/deny-list/constants'; import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; import { + ENV_KNOWN_VERSIONS_TABLE_NAME, FOLLOWER_STATE_FILE_NAME, - KNOWN_VERSIONS_FILE_NAME, MARKER_FILE_NAME, MetricName, } from '../../../package-sources/npmjs/constants.lambda-shared'; @@ -53,6 +58,7 @@ const REGISTRY = 'https://registry.npmjs.org'; const MOCK_BUCKET = 'mock-staging-bucket'; const MOCK_STAGING_FUNCTION = 'mock-staging-function'; +const MOCK_KNOWN_VERSIONS_TABLE = 'mock-known-versions-table'; const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name'; @@ -62,6 +68,7 @@ const HOUR = 60 * 60 * 1_000; const mockS3 = mockClient(S3Client); const mockLambda = mockClient(LambdaClient); +const mockDynamoDB = mockClient(DynamoDBClient); const context: Context = { awsRequestId: 'mock-request-id', @@ -75,10 +82,12 @@ const event: ScheduledEvent = {} as any; beforeEach(() => { mockS3.reset(); mockLambda.reset(); + mockDynamoDB.reset(); mockPutMetric.mockClear(); process.env.BUCKET_NAME = MOCK_BUCKET; process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION; + process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = MOCK_KNOWN_VERSIONS_TABLE; process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET; @@ -98,18 +107,15 @@ beforeEach(() => { Key: MOCK_LICENSE_LIST_OBJECT, }) .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) }); - // Empty known versions - mockS3 - .on(GetObjectCommand, { - Bucket: MOCK_BUCKET, - Key: KNOWN_VERSIONS_FILE_NAME, - }) - .resolves({ Body: stringToStream(JSON.stringify({ knownVersions: {} })) }); // No legacy marker by default mockS3 .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME }) .rejects(noSuchKey()); + // Empty known versions table + mockDynamoDB.on(QueryCommand).resolves({ Items: [] }); + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); + mockS3.on(PutObjectCommand).resolves({}); mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 }); }); @@ -117,6 +123,7 @@ beforeEach(() => { afterEach(() => { delete process.env.BUCKET_NAME; delete process.env.FUNCTION_NAME; + delete process.env[ENV_KNOWN_VERSIONS_TABLE_NAME]; delete process.env[ENV_DENY_LIST_BUCKET_NAME]; delete process.env[ENV_DENY_LIST_OBJECT_KEY]; delete process.env[LicenseListEnv.BUCKET_NAME]; @@ -757,3 +764,93 @@ test('resets its position when the feed head regresses below the newest checkpoi expect(saved.newestCheckpointSeq()).toBe(3_000); expect(saved.has(4_500)).toBe(false); }); + +//#region Known versions table +function givenKnownVersionsInTable(name: string, versions: string[]) { + mockDynamoDB + .on(QueryCommand, { + ExpressionAttributeValues: { ':name': { S: name } }, + }) + .resolves({ + Items: versions.map((version) => ({ version: { S: version } })), + }); +} + +/** The `name@version` keys recorded in the table, sorted. */ +function recordedVersions(): string[] { + return mockDynamoDB + .commandCalls(BatchWriteItemCommand) + .flatMap( + (call) => call.args[0].input.RequestItems![MOCK_KNOWN_VERSIONS_TABLE] + ) + .map( + (request) => + `${request.PutRequest!.Item!.name.S}@${ + request.PutRequest!.Item!.version.S + }` + ) + .sort(); +} + +function givenOneChange(id: string, doc: object) { + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(200, { + results: [ + { seq: 1_005, id, changes: [{ rev: '1-aaaaaa' }], deleted: false }, + ], + last_seq: 1_005, + }); + nock(REGISTRY).get(`/${id}`).reply(200, doc); +} + +test('stages only versions not recorded in the table, and records them', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + givenOneChange( + 'known-package', + packageDoc('known-package', '1-aaaaaa', '1.0.0', '1.1.0') + ); + givenKnownVersionsInTable('known-package', ['1.0.0']); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'known-package', version: '1.1.0' }, + ]); + expect(recordedVersions()).toEqual(['known-package@1.1.0']); +}); + +test('records versions before the follower state, so a failed save never skips a version', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + givenOneChange( + 'fresh-package', + packageDoc('fresh-package', '1-aaaaaa', '1.0.0') + ); + mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('network')); + + await expect(handler(event, context)).rejects.toThrow(/network/); + + // The chunk's receipts were not saved: the next run processes it again. + const statePuts = mockS3 + .commandCalls(PutObjectCommand) + .filter((call) => call.args[0].input.Key === FOLLOWER_STATE_FILE_NAME); + expect(statePuts).toHaveLength(0); +}); + +test('does not query the table for packages without construct library versions', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + givenOneChange('boring-package', nonConstructDoc('boring-package')); + + await handler(event, context); + + expect(mockDynamoDB).not.toHaveReceivedCommand(QueryCommand); + expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand); +}); +//#endregion diff --git a/src/construct-hub.ts b/src/construct-hub.ts index fa78a37fa..faeb9e28e 100644 --- a/src/construct-hub.ts +++ b/src/construct-hub.ts @@ -501,19 +501,18 @@ export class ConstructHub extends Construct implements iam.IGrantable { feedBuilder.setConstructHubUrl(webApp.baseUrl); const sources = new Construct(this, 'Sources'); - const packageSources = ( - props.packageSources ?? [new NpmJs()] - ).map((source) => - source.bind(sources, { - baseUrl: webApp.baseUrl, - denyList, - ingestion: this.ingestion, - licenseList, - monitoring: this.monitoring, - queue: this.ingestion.queue, - repository: codeArtifact, - overviewDashboard: overviewDashboard, - }) + const packageSources = (props.packageSources ?? [new NpmJs()]).map( + (source) => + source.bind(sources, { + baseUrl: webApp.baseUrl, + denyList, + ingestion: this.ingestion, + licenseList, + monitoring: this.monitoring, + queue: this.ingestion.queue, + repository: codeArtifact, + overviewDashboard: overviewDashboard, + }) ); const inventory = new Inventory(this, 'InventoryCanary', { diff --git a/src/deep-link.ts b/src/deep-link.ts index f67a62562..bbaa8f126 100644 --- a/src/deep-link.ts +++ b/src/deep-link.ts @@ -1,5 +1,6 @@ import { Fn, Stack } from 'aws-cdk-lib'; import { CfnRepository } from 'aws-cdk-lib/aws-codeartifact'; +import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb'; import { ICluster } from 'aws-cdk-lib/aws-ecs'; import { IFunction } from 'aws-cdk-lib/aws-lambda'; import { ILogGroup } from 'aws-cdk-lib/aws-logs'; @@ -11,6 +12,10 @@ export function codeArtifactRepositoryUrl(repository: CfnRepository) { return `/codesuite/codeartifact/d/${repository.attrDomainOwner}/${repository.attrDomainName}/r/${repository.attrName}`; } +export function dynamoDbTableUrl(table: ITableV2): string { + return `/dynamodbv2/home#table?name=${table.tableName}`; +} + export function ecsClusterUrl(cluster: ICluster): string { return `/ecs/home#/clusters/${cluster.clusterName}/tasks`; } diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index c11b3aa71..5ae6a3b33 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -1,4 +1,4 @@ -import { Duration } from 'aws-cdk-lib'; +import { Duration, RemovalPolicy } from 'aws-cdk-lib'; import { AlarmRule, ComparisonOperator, @@ -13,6 +13,12 @@ import { Stats, TreatMissingData, } from 'aws-cdk-lib/aws-cloudwatch'; +import { + AttributeType, + Billing, + TableEncryptionV2, + TableV2, +} from 'aws-cdk-lib/aws-dynamodb'; import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets'; import { Tracing } from 'aws-cdk-lib/aws-lambda'; @@ -22,6 +28,7 @@ import { Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import { AlarmSeverity } from '../api'; import { + dynamoDbTableUrl, lambdaFunctionUrl, logAnalyticsUrl, s3ObjectUrl, @@ -29,25 +36,28 @@ import { } from '../deep-link'; import { fillMetric } from '../metric-utils'; import { addAlarm } from '../monitoring'; +import { IMonitoring } from '../monitoring/api'; +import type { + IPackageSource, + PackageSourceBindOptions, + PackageSourceBindResult, +} from '../package-source'; +import { RUNBOOK_URL } from '../runbook-url'; import { NpmJsPackageCanary } from './npmjs/canary'; import { + ENV_KNOWN_VERSIONS_TABLE_NAME, FOLLOWER_STATE_FILE_NAME, KNOWN_VERSIONS_FILE_NAME, + KnownVersionsAttribute, MARKER_FILE_NAME, METRICS_NAMESPACE, MetricName, S3KeyPrefix, SCAN_WINDOW_MS, } from './npmjs/constants.lambda-shared'; +import { KnownVersionsTableMigration } from './npmjs/known-versions-table-migration'; import { NpmJsFollower } from './npmjs/npm-js-follower'; import { StageAndNotify } from './npmjs/stage-and-notify'; -import { IMonitoring } from '../monitoring/api'; -import type { - IPackageSource, - PackageSourceBindOptions, - PackageSourceBindResult, -} from '../package-source'; -import { RUNBOOK_URL } from '../runbook-url'; import { S3StorageFactory } from '../s3/storage'; import { ReStagePackageVersion } from './npmjs/re-stage-package-version'; @@ -114,6 +124,28 @@ export interface NpmJsProps { * @default Duration.days(1) */ readonly canaryMaxStale?: Duration; + + /** + * The billing mode of the table in which the follower records the package + * versions it has processed. + * + * @default Billing.onDemand() + */ + readonly knownVersionsTableBilling?: Billing; + + /** + * Copies the known versions of a deployment created before the known + * versions table existed from the staging bucket to the table, when the + * table is created. + * + * Without the copy, an existing deployment starts with an empty table and + * processes every version of every construct library again, as their + * packages change. Turn this off for new deployments, or if the table was + * filled some other way. + * + * @default true + */ + readonly enableKnownVersionsMigration?: boolean; } /** @@ -202,22 +234,51 @@ export class NpmJs implements IPackageSource { }) ); + // The package versions the follower has processed. Losing this table + // means re-processing every construct library version ever published, so + // it is retained and backed up. + const knownVersions = new TableV2(scope, 'KnownVersions', { + partitionKey: { + name: KnownVersionsAttribute.NAME, + type: AttributeType.STRING, + }, + sortKey: { + name: KnownVersionsAttribute.VERSION, + type: AttributeType.STRING, + }, + billing: this.props.knownVersionsTableBilling ?? Billing.onDemand(), + encryption: TableEncryptionV2.dynamoOwnedKey(), + pointInTimeRecoverySpecification: { pointInTimeRecoveryEnabled: true }, + removalPolicy: RemovalPolicy.RETAIN, + }); + const follower = new NpmJsFollower(scope, 'NpmJs', { description: `[${scope.node.path}/NpmJs] Periodically query npmjs.com index for new packages`, environment: { AWS_EMF_ENVIRONMENT: 'Local', BUCKET_NAME: bucket.bucketName, FUNCTION_NAME: stager.functionName, + [ENV_KNOWN_VERSIONS_TABLE_NAME]: knownVersions.tableName, }, memorySize: 10_024, // 10 GiB - reservedConcurrentExecutions: 1, // Only one execution at a time, to avoid race conditions on the S3 marker object + reservedConcurrentExecutions: 1, // Only one execution at a time, to avoid race conditions on the S3 follower state object timeout: FOLLOWER_RUN_RATE, tracing: Tracing.ACTIVE, }); - bucket.grantReadWrite(follower, MARKER_FILE_NAME); - bucket.grantReadWrite(follower, KNOWN_VERSIONS_FILE_NAME); + // The legacy marker is only read, to seed the follower state of existing + // deployments. + bucket.grantRead(follower, MARKER_FILE_NAME); bucket.grantReadWrite(follower, FOLLOWER_STATE_FILE_NAME); + knownVersions.grantReadWriteData(follower); + + if (this.props.enableKnownVersionsMigration ?? true) { + new KnownVersionsTableMigration(follower, 'KnownVersionsMigration', { + bucket, + table: knownVersions, + follower, + }); + } denyList?.grantRead(follower); licenseList.grantRead(follower); stager.grantInvoke(follower); @@ -278,7 +339,7 @@ export class NpmJs implements IPackageSource { }, { name: 'Known Versions', - url: s3ObjectUrl(bucket, KNOWN_VERSIONS_FILE_NAME), + url: dynamoDbTableUrl(knownVersions), }, { name: 'Stager', url: lambdaFunctionUrl(stager) }, { name: 'Stager DLQ', url: sqsQueueUrl(stager.deadLetterQueue!) }, diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index ea5f09ef6..c32054769 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -81,10 +81,27 @@ export const enum S3KeyPrefix { export const MARKER_FILE_NAME = 'couchdb-last-transaction-id.2'; /** - * The name of the object that contains the list of known versions. + * The name of the object that contains the list of known versions. Known + * versions are now kept in a DynamoDB table; this file is only read once, by + * the migration that seeds the table of an existing deployment. */ export const KNOWN_VERSIONS_FILE_NAME = 'couchdb-known-versions.2'; +/** + * The environment variable holding the name of the known versions table. + */ +export const ENV_KNOWN_VERSIONS_TABLE_NAME = 'KNOWN_VERSIONS_TABLE_NAME'; + +/** + * The attribute names of the known versions table. The table is keyed by + * package name (partition key) and version (sort key); an item exists for + * every package version the follower has processed. + */ +export const enum KnownVersionsAttribute { + NAME = 'name', + VERSION = 'version', +} + /** * The name of the object that contains the follower state: receipts for the * change entries received from the `_changes` feed, rolling time/sequence diff --git a/src/package-sources/npmjs/known-versions-migration.lambda.ts b/src/package-sources/npmjs/known-versions-migration.lambda.ts new file mode 100644 index 000000000..647174106 --- /dev/null +++ b/src/package-sources/npmjs/known-versions-migration.lambda.ts @@ -0,0 +1,140 @@ +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, +} from './constants.lambda-shared'; +import { KnownVersions } from './known-versions.lambda-shared'; +import { + loadContentFromS3, + MarkerFileSchema, +} from './staging-bucket.lambda-shared'; +import { requireEnv } from '../../backend/shared/env.lambda-shared'; + +/** + * Copies the known versions of an existing deployment from the known versions + * file in S3 to the known versions table. Invoked twice during the deployment + * that creates the table: before the follower function is updated (so the new + * follower code only runs once the table is filled; if this fails, the + * deployment fails), and again after it (to pick up the versions the old + * follower code recorded in the file in the meantime). + * + * Writes are idempotent, so running it again only adds what is missing. Records + * that can never be written are skipped and logged. A new deployment has no + * file, and this does nothing: the table fills up during the backfill. + * + * The file is left in place, and is no longer updated: if a deployment is + * rolled back to a version that still reads it, only the versions processed + * since the migration are processed again. + */ +export async function handler(): Promise<{ + readonly written: number; + readonly skipped: readonly string[]; +}> { + const stagingBucket = requireEnv('BUCKET_NAME'); + const knownVersions = new KnownVersions( + requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME) + ); + + const keys = await loadKnownVersionsFile(stagingBucket); + console.log( + `Migrating ${keys.length} known version(s) to the known versions table` + ); + + // Records that can never be written are skipped, so they cannot block the + // others. + const skipped = new Array(); + for (const key of keys) { + const parsed = parseKey(key); + if (parsed == null) { + console.warn(`Skipping invalid known version record: ${key}`); + skipped.push(key); + continue; + } + knownVersions.add(...parsed); + } + const written = knownVersions.pendingCount; + await knownVersions.flush(); + + console.log( + `Known versions migration complete: ${written} migrated, ${skipped.length} skipped` + ); + return { written, skipped }; +} + +/** + * The maximum sizes of the table's key attributes, in bytes. + * @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html#limits-partition-sort-keys + */ +const MAX_PARTITION_KEY_BYTES = 2_048; +const MAX_SORT_KEY_BYTES = 1_024; + +/** + * Splits a `name@version` key. Scoped package names start with `@`, so the + * separator is the last `@`. + * + * @returns `undefined` for a key the table cannot store: no name, no version, + * or a part longer than DynamoDB allows for a key attribute. + */ +function parseKey(key: string): [name: string, version: string] | undefined { + const at = key.lastIndexOf('@'); + if (at <= 0 || at === key.length - 1) { + return undefined; + } + const name = key.slice(0, at); + const version = key.slice(at + 1); + if ( + Buffer.byteLength(name) > MAX_PARTITION_KEY_BYTES || + Buffer.byteLength(version) > MAX_SORT_KEY_BYTES + ) { + return undefined; + } + return [name, version]; +} + +/** + * Loads the known versions from the known versions file in S3. For legacy + * reasons, they may also be embedded in the transaction marker file; when the + * dedicated file does not exist, that location is used. + * + * @returns the `name@version` keys; empty if neither file holds known + * versions (a new deployment). + */ +async function loadKnownVersionsFile(stagingBucket: string): Promise { + const content = await loadContentFromS3( + stagingBucket, + KNOWN_VERSIONS_FILE_NAME, + `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist` + ); + if (content != null) { + const contentsObj: KnownVersionsFileSchema = JSON.parse(content); + console.log('Loaded known versions file'); + return Object.keys(contentsObj.knownVersions); + } + + // Legacy location: embedded in the transaction marker file. + const markerContent = await loadContentFromS3( + stagingBucket, + MARKER_FILE_NAME, + `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` + ); + if (markerContent != null) { + try { + const parsed: MarkerFileSchema = JSON.parse(markerContent); + if (typeof parsed !== 'number' && parsed.knownVersions) { + console.log('Loaded known versions from legacy marker file'); + return Object.keys(parsed.knownVersions); + } + } catch (error) { + console.warn(`Could not parse legacy marker: ${error}`); + } + } + return []; +} + +/** + * The known versions file format: `name@version` keys mapped to the + * `modified` timestamps of the versions (no longer used). + */ +interface KnownVersionsFileSchema { + knownVersions: Record; +} diff --git a/src/package-sources/npmjs/known-versions-migration.ts b/src/package-sources/npmjs/known-versions-migration.ts new file mode 100644 index 000000000..208b36d8f --- /dev/null +++ b/src/package-sources/npmjs/known-versions-migration.ts @@ -0,0 +1,20 @@ +// ~~ Generated by projen. To modify, edit .projenrc.ts and run "yarn projen". +import * as path from 'path'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { Construct } from 'constructs'; + +export interface KnownVersionsMigrationProps extends lambda.FunctionOptions { +} + +export class KnownVersionsMigration extends lambda.Function { + constructor(scope: Construct, id: string, props?: KnownVersionsMigrationProps) { + super(scope, id, { + description: 'package-sources/npmjs/known-versions-migration.lambda.ts', + ...props, + architecture: lambda.Architecture.ARM_64, + runtime: lambda.Runtime.NODEJS_22_X, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '/known-versions-migration.lambda.bundle')), + }); + } +} \ No newline at end of file diff --git a/src/package-sources/npmjs/known-versions-table-migration.ts b/src/package-sources/npmjs/known-versions-table-migration.ts new file mode 100644 index 000000000..9f144fca0 --- /dev/null +++ b/src/package-sources/npmjs/known-versions-table-migration.ts @@ -0,0 +1,84 @@ +import { Duration } from 'aws-cdk-lib'; +import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb'; +import { CfnFunction, Function, Tracing } from 'aws-cdk-lib/aws-lambda'; +import { IBucket } from 'aws-cdk-lib/aws-s3'; +import { InvocationType, Trigger } from 'aws-cdk-lib/triggers'; +import { Construct } from 'constructs'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, +} from './constants.lambda-shared'; +import { KnownVersionsMigration } from './known-versions-migration'; + +export interface KnownVersionsTableMigrationProps { + /** The bucket holding the known versions file. */ + readonly bucket: IBucket; + + /** The table to copy the known versions to. */ + readonly table: ITableV2; + + /** The follower function, which is only updated once the copy completed. */ + readonly follower: Function; +} + +/** + * Copies the known versions of an existing deployment from the staging bucket + * to the known versions table, when the table is created. The copy runs in two + * passes: + * + * - Before the follower function is updated, so the new follower code never + * runs against an empty table. If this copy fails, the deployment fails and + * rolls back. + * - After the follower function is updated, to pick up the versions the old + * follower code recorded in the file while the first copy ran. + * + * Copying is idempotent, so the second pass only adds what is missing. + */ +export class KnownVersionsTableMigration extends Construct { + public constructor( + scope: Construct, + id: string, + props: KnownVersionsTableMigrationProps + ) { + super(scope, id); + + const handler = new KnownVersionsMigration(this, 'Function', { + description: `[${props.follower.node.path}] Copies the known versions from the staging bucket to the known versions table`, + environment: { + BUCKET_NAME: props.bucket.bucketName, + [ENV_KNOWN_VERSIONS_TABLE_NAME]: props.table.tableName, + }, + memorySize: 2_048, + timeout: Duration.minutes(10), + tracing: Tracing.ACTIVE, + }); + props.bucket.grantRead(handler, KNOWN_VERSIONS_FILE_NAME); + props.bucket.grantRead(handler, MARKER_FILE_NAME); + props.table.grantWriteData(handler); + + // The function resource only, not the follower's whole scope (which may + // contain this construct). + const followerFunction = props.follower.node.defaultChild as CfnFunction; + + new Trigger(this, 'BeforeFollowerUpdate', { + handler, + invocationType: InvocationType.REQUEST_RESPONSE, + // Longer than the function's own timeout (the trigger provider allows + // up to 15 minutes). + timeout: Duration.minutes(12), + // Only when the trigger is created, not on later updates. + executeOnHandlerChange: false, + executeAfter: [props.table], + executeBefore: [followerFunction], + }); + + new Trigger(this, 'AfterFollowerUpdate', { + handler, + // Nothing waits for this one; a failure is retried by Lambda. + invocationType: InvocationType.EVENT, + executeOnHandlerChange: false, + executeAfter: [followerFunction], + }); + } +} diff --git a/src/package-sources/npmjs/known-versions.lambda-shared.ts b/src/package-sources/npmjs/known-versions.lambda-shared.ts new file mode 100644 index 000000000..07282eaa6 --- /dev/null +++ b/src/package-sources/npmjs/known-versions.lambda-shared.ts @@ -0,0 +1,201 @@ +import { DynamoDB, WriteRequest } from '@aws-sdk/client-dynamodb'; +import { Sema } from 'async-sema'; +import * as AWSXRay from 'aws-xray-sdk-core'; +import { KnownVersionsAttribute } from './constants.lambda-shared'; + +/** The maximum number of concurrent requests to the table. */ +const MAX_CONCURRENT_REQUESTS = 32; + +/** The maximum number of items in a single `BatchWriteItem` request. */ +const BATCH_WRITE_SIZE = 25; + +/** How often unprocessed items of a batch write are retried. */ +const MAX_BATCH_WRITE_ATTEMPTS = 10; + +/** + * The DynamoDB client. Throttling errors are retried by the SDK; the default + * of 3 attempts is raised, since a burst of writes (for example during a + * backfill) can be throttled for longer than that. + */ +const DYNAMODB = AWSXRay.captureAWSv3Client(new DynamoDB({ maxAttempts: 10 })); + +/** + * The package versions the follower has processed, backed by a DynamoDB + * table keyed by package name and version. + * + * `has` loads all recorded versions of a package on first use and caches them. + * `add` records a version locally right away; `flush` persists the records, + * in batches. + */ +export class KnownVersions { + private readonly loads = new Map>>(); + private readonly added = new Set(); + private readonly pending = new Set(); + private readonly semaphore = new Sema(MAX_CONCURRENT_REQUESTS); + + public constructor( + private readonly tableName: string, + private readonly client: DynamoDB = DYNAMODB + ) {} + + /** + * Whether a package version has been processed. + */ + public async has(name: string, version: string): Promise { + if (this.added.has(keyOf(name, version))) { + return true; + } + return (await this.versionsOf(name)).has(version); + } + + /** + * Records a processed package version. The record is persisted by the next + * `flush`. + */ + public add(name: string, version: string): void { + const key = keyOf(name, version); + this.added.add(key); + this.pending.add(key); + } + + /** The number of records waiting to be persisted. */ + public get pendingCount(): number { + return this.pending.size; + } + + /** + * Persists all records made by `add` since the last flush. + */ + public async flush(): Promise { + const keys = Array.from(this.pending); + this.pending.clear(); + try { + const requests = keys.map((key) => { + const [name, version] = splitKey(key); + const request: WriteRequest = { + PutRequest: { Item: toItem(name, version) }, + }; + return request; + }); + const batches = new Array(); + for (let i = 0; i < requests.length; i += BATCH_WRITE_SIZE) { + batches.push(requests.slice(i, i + BATCH_WRITE_SIZE)); + } + await Promise.all( + batches.map((batch) => this.limited(() => this.batchWrite(batch))) + ); + } catch (error) { + // Keep the records for the next attempt (writes are idempotent). + for (const key of keys) { + this.pending.add(key); + } + throw error; + } + } + + /** + * The recorded versions of a package. Each package is queried at most once; + * concurrent callers share the same query. + */ + private versionsOf(name: string): Promise> { + let load = this.loads.get(name); + if (load == null) { + load = this.query(name); + this.loads.set(name, load); + // A failed query must not be cached. + load.catch(() => this.loads.delete(name)); + } + return load; + } + + private async query(name: string): Promise> { + const versions = new Set(); + let exclusiveStartKey: Parameters< + DynamoDB['query'] + >[0]['ExclusiveStartKey']; + do { + const response = await this.limited(() => + this.client.query({ + TableName: this.tableName, + ConsistentRead: true, + KeyConditionExpression: '#name = :name', + ProjectionExpression: '#version', + ExpressionAttributeNames: { + '#name': KnownVersionsAttribute.NAME, + '#version': KnownVersionsAttribute.VERSION, + }, + ExpressionAttributeValues: { ':name': { S: name } }, + ExclusiveStartKey: exclusiveStartKey, + }) + ); + for (const item of response.Items ?? []) { + const version = item[KnownVersionsAttribute.VERSION]?.S; + if (version != null) { + versions.add(version); + } + } + exclusiveStartKey = response.LastEvaluatedKey; + } while (exclusiveStartKey != null); + return versions; + } + + /** + * Writes a batch, retrying unprocessed items. The SDK retries throttling + * errors, but not unprocessed items (which DynamoDB returns when part of a + * batch was throttled). + */ + private async batchWrite(batch: WriteRequest[]): Promise { + let unprocessed: WriteRequest[] = batch; + for ( + let attempt = 1; + unprocessed.length > 0 && attempt <= MAX_BATCH_WRITE_ATTEMPTS; + attempt++ + ) { + if (attempt > 1) { + await sleep(Math.min(2 ** attempt * 50, 5_000)); + } + const response = await this.client.batchWriteItem({ + RequestItems: { [this.tableName]: unprocessed }, + }); + unprocessed = response.UnprocessedItems?.[this.tableName] ?? []; + } + if (unprocessed.length > 0) { + throw new Error( + `Could not write ${unprocessed.length} known version(s) to ${this.tableName} after ${MAX_BATCH_WRITE_ATTEMPTS} attempts` + ); + } + } + + private async limited(fn: () => Promise): Promise { + await this.semaphore.acquire(); + try { + return await fn(); + } finally { + this.semaphore.release(); + } + } +} + +function toItem(name: string, version: string) { + return { + [KnownVersionsAttribute.NAME]: { S: name }, + [KnownVersionsAttribute.VERSION]: { S: version }, + }; +} + +function keyOf(name: string, version: string): string { + return `${name}@${version}`; +} + +/** + * Splits a `name@version` key made by `keyOf`. Scoped package names start + * with `@`, so the separator is the last `@`. + */ +function splitKey(key: string): [name: string, version: string] { + const at = key.lastIndexOf('@'); + return [key.slice(0, at), key.slice(at + 1)]; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index a7aa872b2..7fc296ab1 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -1,15 +1,5 @@ import * as console from 'console'; import { InvokeCommand } from '@aws-sdk/client-lambda'; -import { - GetObjectCommand, - NoSuchKey, - PutObjectCommand, - PutObjectCommandInput, -} from '@aws-sdk/client-s3'; -import type { - NodeJsRuntimeStreamingBlobPayloadOutputTypes, - StreamingBlobPayloadInputTypes, -} from '@smithy/types'; import { metricScope, Configuration, @@ -23,7 +13,7 @@ import { MetricName, MARKER_FILE_NAME, METRICS_NAMESPACE, - KNOWN_VERSIONS_FILE_NAME, + ENV_KNOWN_VERSIONS_TABLE_NAME, FOLLOWER_STATE_FILE_NAME, SCAN_WINDOW_MS, STATE_RETENTION_MS, @@ -36,17 +26,17 @@ import { parseSequentialRevision, } from './couch-changes.lambda-shared'; import { FollowerState } from './follower-state.lambda-shared'; +import { KnownVersions } from './known-versions.lambda-shared'; import { PackageVersion } from './stage-and-notify.lambda'; +import { + loadContentFromS3, + MarkerFileSchema, + putObject, +} from './staging-bucket.lambda-shared'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; -import { - LAMBDA_CLIENT, - S3_CLIENT, -} from '../../backend/shared/aws.lambda-shared'; -import { - compressContent, - decompressContent, -} from '../../backend/shared/compress-content.lambda-shared'; +import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared'; +import { compressContent } from '../../backend/shared/compress-content.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; // eslint-disable-next-line @typescript-eslint/no-require-imports const normalizeNPMMetadata = require('normalize-registry-metadata'); @@ -135,6 +125,9 @@ export async function handler(event: ScheduledEvent, context: Context) { const stagingBucket = requireEnv('BUCKET_NAME'); const stagingFunction = requireEnv('FUNCTION_NAME'); + const knownVersions = new KnownVersions( + requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME) + ); const denyList = await DenyListClient.newClient(); const licenseList = await LicenseListClient.newClient(); @@ -143,11 +136,6 @@ export async function handler(event: ScheduledEvent, context: Context) { const head = Number((await npm.info()).update_seq); - const { knownVersions, didLoadVersionsFromLegacy } = await loadKnownVersions( - stagingBucket - ); - let writeKnownVersionsFile = didLoadVersionsFromLegacy; - const state = await loadFollowerState(stagingBucket, head); // The feed's sequence space can change (entries can disappear from the end, @@ -164,7 +152,7 @@ export async function handler(event: ScheduledEvent, context: Context) { try { // Re-check laggy packuments (packages whose registry packument was behind // the revision announced by the feed when we first saw them). - const laggyStaged = await retryLaggyPackuments( + await retryLaggyPackuments( context, npm, state, @@ -173,19 +161,13 @@ export async function handler(event: ScheduledEvent, context: Context) { licenseList, knownVersions ); - writeKnownVersionsFile ||= laggyStaged > 0; // Scan the trailing window of the feed. Progress is persisted after // every processed chunk (see PersistState), so even a run that is hard // killed by the Lambda timeout keeps its completed work. - const persist: PersistState = async (saveKnownVersions) => - void (await Promise.all([ - saveFollowerState(context, stagingBucket, state), - ...(saveKnownVersions - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ])); - const scanStaged = await runScan( + const persist: PersistState = () => + persistProgress(context, stagingBucket, state, knownVersions); + await runScan( context, npm, state, @@ -196,7 +178,6 @@ export async function handler(event: ScheduledEvent, context: Context) { knownVersions, persist ); - writeKnownVersionsFile ||= scanStaged > 0; // The laggy packument gauge is emitted at the end of the run, so it // reflects the expectations recorded by this run's scan (emitting it during @@ -214,12 +195,7 @@ export async function handler(event: ScheduledEvent, context: Context) { // Persist the state even when a scan failed part-way: receipts, interim // checkpoints, and staged known versions represent completed work that // the next run should not redo. - await Promise.all([ - saveFollowerState(context, stagingBucket, state), - ...(writeKnownVersionsFile - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ]); + await persistProgress(context, stagingBucket, state, knownVersions); } console.log('All done here, we have success!'); @@ -236,7 +212,27 @@ export async function handler(event: ScheduledEvent, context: Context) { * backlog too large for a single execution would never complete (every run * would restart it from scratch). */ -type PersistState = (saveKnownVersions: boolean) => Promise; +type PersistState = () => Promise; + +/** + * Persists the known versions recorded since the last call, then the follower + * state. Known versions go first: if saving the state fails, the next run + * redoes the work, and the known versions prevent it from staging the same + * versions again. The reverse order could record receipts for entries whose + * staged versions were never recorded as known. + */ +async function persistProgress( + context: Context, + stagingBucket: string, + state: FollowerState, + knownVersions: KnownVersions +): Promise { + if (knownVersions.pendingCount > 0) { + console.log(`Recording ${knownVersions.pendingCount} known version(s)`); + await knownVersions.flush(); + } + await saveFollowerState(context, stagingBucket, state); +} /** * Re-reads the trailing window of the feed and processes every change entry @@ -254,7 +250,7 @@ async function runScan( stagingFunction: string, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map, + knownVersions: KnownVersions, persist: PersistState ): Promise { const now = Date.now(); @@ -328,7 +324,7 @@ async function runScan( // importantly: a backfill) resume from here instead of the floor. if (next > (state.newestCheckpointSeq() ?? 0)) { state.addCheckpoint(next); - await persist(false); + await persist(); } } })(); @@ -382,7 +378,7 @@ async function processBatch( stagingFunction: string, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map, + knownVersions: KnownVersions, persist: PersistState ): Promise<{ staged: number; fullyProcessed: boolean }> { const startTime = Date.now(); @@ -475,7 +471,7 @@ async function processBatch( ); } - const versionInfos = getRelevantVersionInfos( + const versionInfos = await getRelevantVersionInfos( attached.map((a) => a.change) as unknown as readonly Change[], metrics, denyList, @@ -505,7 +501,7 @@ async function processBatch( .map((change) => Number(change.seq)) .filter((seq) => !isNaN(seq) && !failed.has(seq)) ); - await persist(chunkStaged > 0); + await persist(); } } finally { metrics.putMetric(MetricName.LAST_SEQ, head, Unit.None); @@ -542,7 +538,7 @@ async function retryLaggyPackuments( stagingFunction: string, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map + knownVersions: KnownVersions ): Promise { let staged = 0; await metricScope((metrics) => async () => { @@ -573,7 +569,7 @@ async function retryLaggyPackuments( seq: laggy.seq, doc, }; - const versionInfos = getRelevantVersionInfos( + const versionInfos = await getRelevantVersionInfos( [change] as unknown as readonly Change[], metrics, denyList, @@ -621,10 +617,10 @@ async function retryLaggyPackuments( async function stageVersions( versionInfos: readonly UpdatedVersion[], stagingFunction: string, - knownVersions: Map + knownVersions: KnownVersions ): Promise { await Promise.all( - versionInfos.map(async ({ infos, modified, seq }) => { + versionInfos.map(async ({ packageName, infos, modified, seq }) => { const invokeArgs: PackageVersion = { integrity: infos.dist.shasum, modified: modified.toISOString(), @@ -643,49 +639,12 @@ async function stageVersions( }) ); // Record that this is now a "known" version (no need to re-discover) - knownVersions.set(`${infos.name}@${infos.version}`, modified); + knownVersions.add(packageName, infos.version); }) ); } -//#region State and known versions -/** - * Common function to load data from an S3 file with error handling - * - * @param stagingBucket The S3 bucket name - * @param key The file key in the bucket - * @param warningMessage Message to log when file doesn't exist - * @returns The decompressed file content as string, or null if file doesn't exist - */ -async function loadContentFromS3( - stagingBucket: string, - key: string, - warningMessage: string -): Promise { - try { - const response = await S3_CLIENT.send( - new GetObjectCommand({ - Bucket: stagingBucket, - Key: key, - }) - ); - if (!response.Body) { - throw new Error(`Response Body for ${key} is empty`); - } - return await decompressContent( - response.Body as NodeJsRuntimeStreamingBlobPayloadOutputTypes, - response.ContentEncoding - ); - } catch (error: any) { - if (error instanceof NoSuchKey || error.name === 'NoSuchKey') { - console.warn(warningMessage); - return null; - } - // re-throw unexpected errors - throw error; - } -} - +//#region State /** * Loads the follower state from S3. When the state file does not exist (or * cannot be parsed), a fresh state is seeded: from the legacy transaction @@ -778,113 +737,6 @@ async function saveFollowerState( console.log('Successfully updated follower state'); } -/** - * Loads the known versions from S3. For legacy reasons, the map may also be - * embedded in the transaction marker file; when the dedicated file does not - * exist, that location is used and the map is migrated on the next save. - */ -async function loadKnownVersions(stagingBucket: string): Promise<{ - knownVersions: Map; - didLoadVersionsFromLegacy: boolean; -}> { - const warningMessage = `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist, starting from scratch`; - const content = await loadContentFromS3( - stagingBucket, - KNOWN_VERSIONS_FILE_NAME, - warningMessage - ); - if (content != null) { - const contentsObj: KnownVersionsFileSchema = JSON.parse(content); - console.log('Loaded last known versions data'); - return { - knownVersions: dateMapFromObj(contentsObj.knownVersions), - didLoadVersionsFromLegacy: false, - }; - } - - // Legacy location: embedded in the transaction marker file. - const markerContent = await loadContentFromS3( - stagingBucket, - MARKER_FILE_NAME, - `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` - ); - if (markerContent != null) { - try { - const parsed: MarkerFileSchema = JSON.parse(markerContent); - if (typeof parsed !== 'number' && parsed.knownVersions) { - console.log('Loaded known versions from legacy marker file'); - return { - knownVersions: dateMapFromObj(parsed.knownVersions), - didLoadVersionsFromLegacy: true, - }; - } - } catch (error) { - console.warn(`Could not parse legacy marker: ${error}`); - } - } - return { knownVersions: new Map(), didLoadVersionsFromLegacy: true }; -} - -/** - * Updates the last known versions in S3. - * - * @param knownVersions the map of package name + version to last modified timestamp of packages that have been processed. - */ -async function saveLastKnownVersions( - context: Context, - stagingBucket: string, - knownVersions: Map -) { - const contentsObj: KnownVersionsFileSchema = { - knownVersions: objFromDateMap(knownVersions), - }; - - console.log(`Known versions changed, updating...`); - await putObject( - context, - stagingBucket, - KNOWN_VERSIONS_FILE_NAME, - JSON.stringify(contentsObj, undefined, 2), - { - ContentType: 'application/json', - } - ); - console.log('Successfully updated known versions'); -} -//#endregion - -//#region Asynchronous Primitives -/** - * Puts an object in the staging bucket, with standardized object metadata. - * - * @param key the key for the object to be put. - * @param body the body of the object to be put. - * @param opts any other options to use when sending the S3 request. - * - * @returns the result of the S3 request. - */ -function putObject( - context: Context, - bucket: string, - key: string, - body: StreamingBlobPayloadInputTypes, - opts: Omit = {} -) { - return S3_CLIENT.send( - new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - 'Lambda-Log-Group': context.logGroupName, - 'Lambda-Log-Stream': context.logStreamName, - 'Lambda-Run-Id': context.awsRequestId, - ...opts.Metadata, - }, - ...opts, - }) - ); -} //#endregion /** @@ -892,22 +744,32 @@ function putObject( * provided `Change` objects, ensures they are relevant (construct libraries), * and returns those only. * + * Known versions are only looked up for packages that have at least one + * construct library version: only those versions are ever recorded, so every + * other package has no known versions by definition. Changes are processed + * concurrently; the result keeps their order. + * * @param changes the changes to be processed. * @param metrics the metrics logger to use. * @param denyList deny list client * * @returns a list of `VersionInfo` objects */ -function getRelevantVersionInfos( +async function getRelevantVersionInfos( changes: readonly Change[], metrics: MetricsLogger, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map -): readonly UpdatedVersion[] { - const result = new Array(); + knownVersions: KnownVersions +): Promise { + const perChange = await Promise.all( + changes.map((change) => relevantVersionInfosOf(change)) + ); + return perChange.flat(); - for (const change of changes) { + async function relevantVersionInfosOf( + change: Change + ): Promise { // Filter out all elements that don't have a "name" in the document, as // these are schemas, which are not relevant to our business here. if (change.doc.name === undefined) { @@ -915,7 +777,7 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document contains no 'name': ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } // The normalize function change the object in place, if the doc object is invalid it will return undefined @@ -924,7 +786,7 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document invalid, npm normalize returned undefined: ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } // Sometimes, there are no versions in the document. We skip those. @@ -933,7 +795,7 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document contains no 'versions': ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } // Sometimes, there is no 'time' entry in the document. We skip those. @@ -942,9 +804,14 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document contains no 'time': ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } + const name = change.doc.name; + const isRelevant = Object.values(change.doc.versions).some( + (infos) => infos != null && isConstructLibrary(infos) + ); + // Get the last modification date from the change const packageVersionUpdates = Object.entries(change.doc.time) // Ignore the "created" and "modified" keys here @@ -957,54 +824,59 @@ function getRelevantVersionInfos( Unit.Count ); + const result = new Array(); const unpublishedVersions: string[] = []; for (const [version, modified] of packageVersionUpdates) { - const knownKey = `${change.doc.name}@${version}`; - const known = knownVersions.get(knownKey); - if (known == null || known < modified) { - const infos = change.doc.versions[version]; - if (infos == null) { - // Could be the version in question was un-published. - unpublishedVersions.push(knownKey); - } else if (isConstructLibrary(infos)) { - // skip if this package is denied - const denied = denyList.lookup(infos.name, infos.version); - if (denied) { - console.log( - `[${change.seq}] Package denied: ${JSON.stringify(denied)}` - ); - knownVersions.set(knownKey, modified); - metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count); - continue; - } - - metrics.putMetric( - MetricName.PACKAGE_VERSION_AGE, - Date.now() - modified.getTime(), - Unit.Milliseconds + if (isRelevant && (await knownVersions.has(name, version))) { + continue; + } + const infos = change.doc.versions[version]; + if (infos == null) { + // Could be the version in question was un-published. + unpublishedVersions.push(`${name}@${version}`); + } else if (isConstructLibrary(infos)) { + // skip if this package is denied + const denied = denyList.lookup(infos.name, infos.version); + if (denied) { + console.log( + `[${change.seq}] Package denied: ${JSON.stringify(denied)}` ); - const isEligible = - licenseList.lookup(infos.license ?? 'UNLICENSED') != null; - metrics.putMetric( - MetricName.INELIGIBLE_LICENSE, - isEligible ? 0 : 1, - Unit.Count + knownVersions.add(name, version); + metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count); + continue; + } + + metrics.putMetric( + MetricName.PACKAGE_VERSION_AGE, + Date.now() - modified.getTime(), + Unit.Milliseconds + ); + const isEligible = + licenseList.lookup(infos.license ?? 'UNLICENSED') != null; + metrics.putMetric( + MetricName.INELIGIBLE_LICENSE, + isEligible ? 0 : 1, + Unit.Count + ); + if (isEligible) { + result.push({ + packageName: name, + infos, + modified, + seq: change.seq, + }); + } else { + console.log( + `[${ + change.seq + }] Package "${name}@${version}" does not use allow-listed license: ${ + infos.license ?? 'UNLICENSED' + }` ); - if (isEligible) { - result.push({ infos, modified, seq: change.seq }); - } else { - console.log( - `[${change.seq}] Package "${ - change.doc.name - }@${version}" does not use allow-listed license: ${ - infos.license ?? 'UNLICENSED' - }` - ); - knownVersions.set(knownKey, modified); - } + knownVersions.add(name, version); } - // Else this is not a construct library, so we'll just ignore it... } + // Else this is not a construct library, so we'll just ignore it... } if (unpublishedVersions.length > 0) { @@ -1016,8 +888,8 @@ function getRelevantVersionInfos( )}` ); } + return result; } - return result; /** * This determines whether a package is "interesting" to ConstructHub or not. This is related but @@ -1076,18 +948,6 @@ function getRelevantVersionInfos( } } -function objFromDateMap(xs: Map): Record { - return Object.fromEntries( - Array.from(xs).map(([k, v]) => [k, v.toISOString()]) - ); -} - -function dateMapFromObj( - xs: Record -): Map { - return new Map(Object.entries(xs).map(([k, v]) => [k, new Date(v)])); -} - /** * The scheme of a package version in the update. Includes the package.json keys, as well as some additional npm metadata * @see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version @@ -1109,6 +969,12 @@ export interface VersionInfo { } interface UpdatedVersion { + /** + * The name of the package document the version was found in (the key under + * which known versions are recorded). + */ + readonly packageName: string; + /** * The `VersionInfo` for the modified package version. */ @@ -1125,26 +991,6 @@ interface UpdatedVersion { readonly seq?: string | number; } -/** - * The legacy transaction marker file format, kept around to seed the follower - * state (and known versions) of existing deployments. - * - * The file can be a just a number, or a combination of a sequence number (which - * is potentially encoded as a string) and a set of known versions and the dates - * we first saw them. The date can be encoded as an ISO string, or a timestamp - * number. - */ -type MarkerFileSchema = - | number - | { - marker: number | string; - knownVersions?: Record; - }; - -interface KnownVersionsFileSchema { - knownVersions: Record; -} - interface Document { /** * a List of all Version objects for the package diff --git a/src/package-sources/npmjs/staging-bucket.lambda-shared.ts b/src/package-sources/npmjs/staging-bucket.lambda-shared.ts new file mode 100644 index 000000000..c25103f80 --- /dev/null +++ b/src/package-sources/npmjs/staging-bucket.lambda-shared.ts @@ -0,0 +1,98 @@ +import { + GetObjectCommand, + NoSuchKey, + PutObjectCommand, + PutObjectCommandInput, +} from '@aws-sdk/client-s3'; +import type { + NodeJsRuntimeStreamingBlobPayloadOutputTypes, + StreamingBlobPayloadInputTypes, +} from '@smithy/types'; +import type { Context } from 'aws-lambda'; +import { S3_CLIENT } from '../../backend/shared/aws.lambda-shared'; +import { decompressContent } from '../../backend/shared/compress-content.lambda-shared'; + +/** + * The legacy transaction marker file format, kept around to seed the follower + * state (and known versions) of existing deployments. + * + * The file can be a just a number, or a combination of a sequence number (which + * is potentially encoded as a string) and a set of known versions and the dates + * we first saw them. The date can be encoded as an ISO string, or a timestamp + * number. + */ +export type MarkerFileSchema = + | number + | { + marker: number | string; + knownVersions?: Record; + }; + +/** + * Common function to load data from an S3 file with error handling + * + * @param stagingBucket The S3 bucket name + * @param key The file key in the bucket + * @param warningMessage Message to log when file doesn't exist + * @returns The decompressed file content as string, or null if file doesn't exist + */ +export async function loadContentFromS3( + stagingBucket: string, + key: string, + warningMessage: string +): Promise { + try { + const response = await S3_CLIENT.send( + new GetObjectCommand({ + Bucket: stagingBucket, + Key: key, + }) + ); + if (!response.Body) { + throw new Error(`Response Body for ${key} is empty`); + } + return await decompressContent( + response.Body as NodeJsRuntimeStreamingBlobPayloadOutputTypes, + response.ContentEncoding + ); + } catch (error: any) { + if (error instanceof NoSuchKey || error.name === 'NoSuchKey') { + console.warn(warningMessage); + return null; + } + // re-throw unexpected errors + throw error; + } +} + +/** + * Puts an object in the staging bucket, with standardized object metadata. + * + * @param key the key for the object to be put. + * @param body the body of the object to be put. + * @param opts any other options to use when sending the S3 request. + * + * @returns the result of the S3 request. + */ +export function putObject( + context: Context, + bucket: string, + key: string, + body: StreamingBlobPayloadInputTypes, + opts: Omit = {} +) { + return S3_CLIENT.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + 'Lambda-Log-Group': context.logGroupName, + 'Lambda-Log-Stream': context.logStreamName, + 'Lambda-Run-Id': context.awsRequestId, + ...opts.Metadata, + }, + ...opts, + }) + ); +} diff --git a/yarn.lock b/yarn.lock index d1d452e99..7ad137d2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -155,6 +155,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1141.0": + version: 3.1146.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1146.0" + dependencies: + "@aws-sdk/core": "npm:^3.978.1" + "@aws-sdk/credential-provider-node": "npm:^3.972.84" + "@aws-sdk/dynamodb-codec": "npm:^3.973.46" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.31" + "@aws-sdk/types": "npm:^3.974.6" + "@smithy/core": "npm:^3.35.0" + "@smithy/fetch-http-handler": "npm:^5.8.0" + "@smithy/node-http-handler": "npm:^4.12.1" + "@smithy/types": "npm:^4.19.0" + tslib: "npm:^2.6.2" + checksum: 10c0/764f9df99ffbbed62f241cab101bc41ac1874744b27f4825f571206f59ba5ea21293717eeac3f89c56c29c35e30c35f77fb9c4b3a9b4cc53f96e3dd57845f940 + languageName: node + linkType: hard + "@aws-sdk/client-ecs@npm:^3.1141.0": version: 3.1141.0 resolution: "@aws-sdk/client-ecs@npm:3.1141.0" @@ -379,6 +397,41 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/dynamodb-codec@npm:^3.973.46": + version: 3.973.46 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.46" + dependencies: + "@aws-sdk/core": "npm:^3.978.1" + "@smithy/core": "npm:^3.35.0" + "@smithy/types": "npm:^4.19.0" + tslib: "npm:^2.6.2" + checksum: 10c0/34343d4a082807748e6bb0b5bbf411ce6616d07359317a281419f2ad63362ce4fe25972a2e8c93dc0ff5fb6ea66e65f5abb9e9d12607458266cd95e43c1d352c + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.11": + version: 3.972.11 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.11" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/8f3c039fb2dd8e434cd2992bce2a840e6f6470be5ae6d71c762f6cb842149fb60d5566ca470319d5abcf90b115d2c1a0114ed3328afbfc0abed4cbcd52ad03a1 + languageName: node + linkType: hard + +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.31": + version: 3.972.31 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.31" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.11" + "@aws-sdk/types": "npm:^3.974.6" + "@smithy/core": "npm:^3.35.0" + "@smithy/types": "npm:^4.19.0" + tslib: "npm:^2.6.2" + checksum: 10c0/ede83171b14fa392b6b499d6fd8fe5554cd0b4a0468df99362cf82eeb2d907eab9f76973a00e8a321fc55ff167c4774697029121745748b0ca2334dc9c246f36 + languageName: node + linkType: hard + "@aws-sdk/middleware-sdk-s3@npm:^3.972.77": version: 3.972.77 resolution: "@aws-sdk/middleware-sdk-s3@npm:3.972.77" @@ -5246,6 +5299,7 @@ __metadata: "@aws-cdk/integ-tests-alpha": "npm:2.194.0-alpha.0" "@aws-sdk/client-cloudwatch": "npm:^3.1141.0" "@aws-sdk/client-codeartifact": "npm:^3.1141.0" + "@aws-sdk/client-dynamodb": "npm:^3.1141.0" "@aws-sdk/client-ecs": "npm:^3.1141.0" "@aws-sdk/client-lambda": "npm:^3.1141.0" "@aws-sdk/client-s3": "npm:^3.1141.0" @@ -9619,6 +9673,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "modify-values@npm:^1.0.1": version: 1.0.1 resolution: "modify-values@npm:1.0.1" @@ -9906,6 +9969,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "oblivious-set@npm:1.0.0": version: 1.0.0 resolution: "oblivious-set@npm:1.0.0"