From 0909b4eee3b09f9f92c7ff8ac10d27169876b996 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:59:24 +0200 Subject: [PATCH 1/2] Re-vendor 0.7.3-0.7.6 from forge-cli v0.21.0, and pin what the 0.7.6 additions refuse Vendored byte for byte from forge-cli v0.21.0 (91e071e2c2273e2ba06fd0deabfe42d78de82f2e). 0.7.6 gains engine: flink, authz.rowFilters, access: mask with its mask rule, lakeFormation.maskedViews, the lakekeeper and polaris sink catalogs, and sink.catalogAuth. 0.7.3-0.7.6 change only the Lake Formation tag descriptions. Without new cases 0.7.6 mutation coverage fell to 39.9%, under the 40% floor. 36 cases bring it to 43.1%, each invalid one failing with exactly one error. The 'sink catalog outside the recognized set' case used polaris, which 0.7.6 now accepts; it uses hadoop. 0.7.5 rejects engine: flink. check_reference against data-product-forge 0.21.0: 1098 cases, 0 disagree. --- schema-diffs/diff-0.7.2-to-0.7.3.md | 2 +- schema-diffs/diff-0.7.5-to-0.7.6.md | 183 +- schema/fluid-schema-0.7.3.json | 4 +- schema/fluid-schema-0.7.4.json | 4 +- schema/fluid-schema-0.7.5.json | 4 +- schema/fluid-schema-0.7.6.json | 271 +- scripts/schema-versions.json | 4 +- specs/0.7.3/fluid-spec.html | 22 +- specs/0.7.4/fluid-spec.html | 22 +- specs/0.7.5/fluid-spec.html | 22 +- specs/0.7.6/fluid-spec.html | 21111 ++++++++++++++++---------- tests/0.7.5/build-consumes.json | 32 + tests/0.7.6/acquisition.json | 1281 +- tests/0.7.6/binding-governance.json | 142 +- tests/0.7.6/build-consumes.json | 36 + tests/0.7.6/expose-policy.json | 363 +- tests/README.md | 4 +- 17 files changed, 15289 insertions(+), 8218 deletions(-) diff --git a/schema-diffs/diff-0.7.2-to-0.7.3.md b/schema-diffs/diff-0.7.2-to-0.7.3.md index 494aeb7..e7a7d4b 100644 --- a/schema-diffs/diff-0.7.2-to-0.7.3.md +++ b/schema-diffs/diff-0.7.2-to-0.7.3.md @@ -579,7 +579,7 @@ } tags: { type: "object" - description: "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in th..." + description: "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_..." additionalProperties: { type: "string" } diff --git a/schema-diffs/diff-0.7.5-to-0.7.6.md b/schema-diffs/diff-0.7.5-to-0.7.6.md index a89ef4b..1bcfc0e 100644 --- a/schema-diffs/diff-0.7.5-to-0.7.6.md +++ b/schema-diffs/diff-0.7.5-to-0.7.6.md @@ -1,9 +1,9 @@ # Schema Changes: 0.7.5 โ†’ 0.7.6 -**Total changes:** 30 -- โœ… Added: 22 +**Total changes:** 40 +- โœ… Added: 27 - โŒ Removed: 0 -- ๐Ÿ“ Modified: 8 +- ๐Ÿ“ Modified: 13 > **Note โ€” 0.7.6 is a preview.** 0.7.5 is the latest stable version. A contract @@ -19,6 +19,57 @@ ## โœ… Added Properties +### `$defs.acquisitionSink.properties.catalogAuth` +```json +{ + type: "object" + description: "NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg..." + additionalProperties: false + required: ["type"] + properties: { + type: { + type: "string" + enum: ["oauth2", "bearer"] + description: "oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token." + } + credentialEnv: { + type: "string" + pattern: "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])" + not: { + pattern: "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" + } + description: "type oauth2: the NAME of the environment variable holding client_id:client_secret (Iceberg splits it..." + } + oauth2ServerUri: { + type: "string" + pattern: "^https?://[^\\s/?#@]+(?:[/?#]\\S*)?$" + not: { + pattern: "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + } + description: "type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's t..." + } + scope: { + type: "string" + minLength: 1 + not: { + pattern: "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + } + description: "type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity provi..." + } + audience: { + type: "string" + minLength: 1 + not: { + pattern: "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + } + description: "type oauth2, optional: sent as the audience parameter on the token request." + } + ... (2 more) + } + ... (1 more) +} +``` + ### `$defs.binding.allOf` ```json [{"if": {"properties": {"platform": {"const": "gcp"}}, "required": ["platform"]}, "then": {"properties": {"encryption": {"properties": {"kms": {"not": {"pattern": "^(alias/|arn:)"}}}}, "principals": {"additionalProperties": {"anyOf": [{"type": "string", "pattern": "^(user|group|serviceAccount|domain):.+$"}, {"type": "array", "items": {"type": "string", "pattern": "^(user|group|serviceAccount|domain):.+$"}}]}}}}}, {"if": {"properties": {"platform": {"const": "aws"}}, "required": ["platform"]}, "then": {"properties": {"encryption": {"properties": {"kms": {"not": {"pattern": "^projects/"}}}}, "principals": {"additionalProperties": {"anyOf": [{"type": "string", "pattern": "^arn:aws[a-z0-9-]*:iam::"}, {"type": "array", "items": {"type": "string", "pattern": "^arn:aws[a-z0-9-]*:iam::"}}]}}}}}] @@ -73,6 +124,26 @@ } ``` +### `$defs.bindingGovernance.properties.lakeFormation.properties.maskedViews` +```json +{ + type: "object" + additionalProperties: false + required: ["definer", "validationConnection"] + description: "Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6)..." + properties: { + definer: { + type: "string" + description: "ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (..." + } + validationConnection: { + type: "string" + description: "The Glue connection of type VIEW_VALIDATION_ATHENA (WORKGROUP_NAME) Glue validates each view's SQL t..." + } + } +} +``` + ### `$defs.consumeRef.dependentRequired` ```json { @@ -197,6 +268,11 @@ } ``` +### `$defs.exposePolicy.properties.authz.properties.columnRestrictions.items.allOf` +```json +[{"if": {"properties": {"access": {"const": "mask"}}, "required": ["access"]}, "then": {"required": ["mask"]}}, {"if": {"required": ["mask"]}, "then": {"properties": {"access": {"const": "mask"}}, "required": ["access"]}}] +``` + ### `$defs.exposePolicy.properties.authz.properties.columnRestrictions.items.properties.access.description` ```json "deny: the principal may not read the columns. allow: only the principals an allow names may read the..." @@ -207,11 +283,52 @@ "Columns of this expose's schema." ``` +### `$defs.exposePolicy.properties.authz.properties.columnRestrictions.items.properties.mask` +```json +{ + type: "string" + enum: ["last_four", "first_four", "nullify"] + description: "The masking rule an access: mask restriction applies, required with it and refused without it. last_..." +} +``` + ### `$defs.exposePolicy.properties.authz.properties.columnRestrictions.items.properties.principal.description` ```json "Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@compan..." ``` +### `$defs.exposePolicy.properties.authz.properties.rowFilters` +```json +{ + type: "array" + description: "Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its wher..." + items: { + type: "object" + additionalProperties: false + required: ["principal", "where"] + properties: { + principal: { + type: "string" + description: "Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per ..." + } + name: { + type: "string" + pattern: "^[a-z][a-z0-9_]{0,62}$" + description: "The filter's name on the platform (the row access policy or data cells filter); derived from the pri..." + } + where: { + type: "string" + minLength: 1 + description: "One SQL boolean expression over the expose's columns, e.g. consent_data_analytics = true AND is_supp..." + } + description: { + type: "string" + } + } + } +} +``` + ### `$defs.exposePolicy.properties.privacy.properties.masking.description` ```json "Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside t..." @@ -363,6 +480,30 @@ ## ๐Ÿ“ Modified Properties +### `$defs.acquisitionSink.properties.catalog.enum` + +**Before:** +```json +[...6 items...] +``` + +**After:** +```json +[...8 items...] +``` + +### `$defs.bindingLocation.properties.catalog.description` + +**Before:** +```json +"NEW in v0.7.5: Iceberg catalog kind/profile selector โ€” maps to iceberg.catalog.type (e.g. glue, rest..." +``` + +**After:** +```json +"NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nes..." +``` + ### `$defs.bindingLocation.properties.partitionBy.description` **Before:** @@ -375,6 +516,30 @@ "NEW in v0.7.5: Iceberg partition columns โ€” a FLUID abstraction over the Iceberg PARTITIONED BY / Par..." ``` +### `$defs.build.properties.engine.anyOf` + +**Before:** +```json +[{"enum": ["dbt", "sql", "python", "spark", "glue", "custom", "duckdb", "airbyte", "meltano", "dlt", "kafka-connect", "debezium"]}, {"pattern": "^dbt-[a-z0-9]+([_-][a-z0-9]+)*$"}] +``` + +**After:** +```json +[{"enum": ["dbt", "sql", "python", "spark", "glue", "custom", "duckdb", "airbyte", "meltano", "dlt", "kafka-connect", "debezium", "flink"]}, {"pattern": "^dbt-[a-z0-9]+([_-][a-z0-9]+)*$"}] +``` + +### `$defs.build.properties.engine.description` + +**Before:** +```json +"Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-quali..." +``` + +**After:** +```json +"Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-quali..." +``` + ### `$defs.expose.properties.lifecycle.$ref` **Before:** @@ -399,6 +564,18 @@ "Column-level access control. A column named in any restriction is restricted. 'deny': the principal ..." ``` +### `$defs.exposePolicy.properties.authz.properties.columnRestrictions.items.properties.access.enum` + +**Before:** +```json +["allow", "deny"] +``` + +**After:** +```json +["allow", "deny", "mask"] +``` + ### `$id` **Before:** diff --git a/schema/fluid-schema-0.7.3.json b/schema/fluid-schema-0.7.3.json index 7f26de9..25589ee 100644 --- a/schema/fluid-schema-0.7.3.json +++ b/schema/fluid-schema-0.7.3.json @@ -1386,7 +1386,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": {"type": "string"}, @@ -1470,7 +1470,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": {"type": "string"} }, "rowFilter": { diff --git a/schema/fluid-schema-0.7.4.json b/schema/fluid-schema-0.7.4.json index 613ea0a..8e2a455 100644 --- a/schema/fluid-schema-0.7.4.json +++ b/schema/fluid-schema-0.7.4.json @@ -5289,7 +5289,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { @@ -5384,7 +5384,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } diff --git a/schema/fluid-schema-0.7.5.json b/schema/fluid-schema-0.7.5.json index e21ece2..a0557cc 100644 --- a/schema/fluid-schema-0.7.5.json +++ b/schema/fluid-schema-0.7.5.json @@ -5482,7 +5482,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model \u2014 see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model \u2014 see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { @@ -5577,7 +5577,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } diff --git a/schema/fluid-schema-0.7.6.json b/schema/fluid-schema-0.7.6.json index 28ca63e..3168d07 100644 --- a/schema/fluid-schema-0.7.6.json +++ b/schema/fluid-schema-0.7.6.json @@ -1184,7 +1184,7 @@ }, "columnRestrictions": { "type": "array", - "description": "Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): `fluid apply` enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (table_with_columns.excluded_column_names); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. `fluid verify` checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS.", + "description": "Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): `fluid apply` enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (table_with_columns.excluded_column_names); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. `fluid verify` checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.", "items": { "type": "object", "additionalProperties": false, @@ -1204,9 +1204,10 @@ "type": "string", "enum": [ "allow", - "deny" + "deny", + "mask" ], - "description": "deny: the principal may not read the columns. allow: only the principals an allow names may read them." + "description": "deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6)." }, "tags": { "$ref": "#/$defs/tags", @@ -1215,8 +1216,53 @@ "labels": { "$ref": "#/$defs/labels", "description": "Access control rule labels." + }, + "mask": { + "type": "string", + "enum": [ + "last_four", + "first_four", + "nullify" + ], + "description": "The masking rule an access: mask restriction applies, required with it and refused without it. last_four: 'XXXXX' and the value's last four characters; first_four: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATA_MASKING_POLICY, LAST_FOUR_CHARACTERS, FIRST_FOUR_CHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6)." } - } + }, + "allOf": [ + { + "if": { + "properties": { + "access": { + "const": "mask" + } + }, + "required": [ + "access" + ] + }, + "then": { + "required": [ + "mask" + ] + } + }, + { + "if": { + "required": [ + "mask" + ] + }, + "then": { + "properties": { + "access": { + "const": "mask" + } + }, + "required": [ + "access" + ] + } + } + ] } }, "tags": { @@ -1226,6 +1272,37 @@ "labels": { "$ref": "#/$defs/labels", "description": "Authorization policy labels." + }, + "rowFilters": { + "type": "array", + "description": "Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filter_predicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (row_filter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "principal", + "where" + ], + "properties": { + "principal": { + "type": "string", + "description": "Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud." + }, + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,62}$", + "description": "The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted." + }, + "where": { + "type": "string", + "minLength": 1, + "description": "One SQL boolean expression over the expose's columns, e.g. consent_data_analytics = true AND is_suppressed = false." + }, + "description": { + "type": "string" + } + } + } } } }, @@ -1866,7 +1943,7 @@ }, "catalog": { "type": "string", - "description": "NEW in v0.7.5: Iceberg catalog kind/profile selector โ€” maps to iceberg.catalog.type (e.g. glue, rest, nessie, hive). REST-fronted catalogs (polaris / unity / snowflake open catalog) map to 'rest'. Consumed by providers/_iceberg_catalog.py and the streaming-sink validator." + "description": "NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'_', so iceberg_rest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under /catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/_iceberg_catalog.py." }, "warehouse": { "type": "string", @@ -2801,7 +2878,8 @@ "meltano", "dlt", "kafka-connect", - "debezium" + "debezium", + "flink" ] }, { @@ -2809,7 +2887,7 @@ } ], "default": "dbt", - "description": "Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue)." + "description": "Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration." }, "capabilities": { "type": "array", @@ -3514,7 +3592,9 @@ "nessie", "unity", "snowflake-managed", - "hive" + "hive", + "lakekeeper", + "polaris" ] }, "partitionBy": { @@ -3522,6 +3602,158 @@ "items": { "type": "string" } + }, + "catalogAuth": { + "type": "object", + "description": "NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding client_id:client_secret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is /api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.", + "additionalProperties": false, + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "oauth2", + "bearer" + ], + "description": "oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token." + }, + "credentialEnv": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "not": { + "pattern": "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" + }, + "description": "type oauth2: the NAME of the environment variable holding client_id:client_secret (Iceberg splits it on the first ':'). Emitted as credential=env:. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused." + }, + "oauth2ServerUri": { + "type": "string", + "pattern": "^https?://[^\\s/?#@]+(?:[/?#]\\S*)?$", + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's /realms//protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens." + }, + "scope": { + "type": "string", + "minLength": 1, + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope." + }, + "audience": { + "type": "string", + "minLength": 1, + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2, optional: sent as the audience parameter on the token request." + }, + "resource": { + "type": "string", + "minLength": 1, + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2, optional: sent as the resource parameter on the token request." + }, + "tokenEnv": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "not": { + "pattern": "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" + }, + "description": "type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused." + } + }, + "allOf": [ + { + "if": { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "oauth2" + } + }, + "type": "object" + }, + "then": { + "required": [ + "credentialEnv", + "oauth2ServerUri", + "scope" + ], + "allOf": [ + { + "not": { + "required": [ + "tokenEnv" + ] + } + } + ] + } + }, + { + "if": { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "bearer" + } + }, + "type": "object" + }, + "then": { + "required": [ + "tokenEnv" + ], + "allOf": [ + { + "not": { + "required": [ + "credentialEnv" + ] + } + }, + { + "not": { + "required": [ + "oauth2ServerUri" + ] + } + }, + { + "not": { + "required": [ + "scope" + ] + } + }, + { + "not": { + "required": [ + "audience" + ] + } + }, + { + "not": { + "required": [ + "resource" + ] + } + } + ] + } + } + ] } } }, @@ -5817,7 +6049,7 @@ }, "tagDefinitions": { "type": "object", - "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { @@ -5922,7 +6154,7 @@ }, "tags": { "type": "object", - "description": "LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as aws_lakeformation_resource_lf_tags.", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } @@ -5966,6 +6198,25 @@ "name", "rowExpression" ] + }, + "maskedViews": { + "type": "object", + "additionalProperties": false, + "required": [ + "definer", + "validationConnection" + ], + "description": "Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, _masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.", + "properties": { + "definer": { + "type": "string", + "description": "ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole)." + }, + "validationConnection": { + "type": "string", + "description": "The Glue connection of type VIEW_VALIDATION_ATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it." + } + } } } } diff --git a/scripts/schema-versions.json b/scripts/schema-versions.json index f4bc56c..45e5153 100644 --- a/scripts/schema-versions.json +++ b/scripts/schema-versions.json @@ -11,8 +11,8 @@ "repository": "Agenticstiger/forge-cli", "schemaDir": "fluid_build/schemas", "previewSource": "fluid_build/schema_manager.py", - "ref": "v0.18.1", - "commit": "93e78d4386c70b9e040283f9c5d98a89f901c986" + "ref": "v0.21.0", + "commit": "91e071e2c2273e2ba06fd0deabfe42d78de82f2e" }, "latestStable": "0.7.5", "preview": ["0.7.6"], diff --git a/specs/0.7.3/fluid-spec.html b/specs/0.7.3/fluid-spec.html index 4e675f2..0aa14a6 100644 --- a/specs/0.7.3/fluid-spec.html +++ b/specs/0.7.3/fluid-spec.html @@ -13316,8 +13316,15 @@

/> tagsType: object
-

LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as awslakeformationresourcelftags.

- +
+

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

+ +
+
+ +
@@ -59980,8 +59987,15 @@

/> tagDefinitionsType: object
-

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.

- +
+

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlftag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and ab. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.

+ +
+
+ +
diff --git a/specs/0.7.4/fluid-spec.html b/specs/0.7.4/fluid-spec.html index 6160792..6106a4a 100644 --- a/specs/0.7.4/fluid-spec.html +++ b/specs/0.7.4/fluid-spec.html @@ -13295,8 +13295,15 @@

/> tagsType: object
-

LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as awslakeformationresourcelftags.

- +
+

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

+ +
+
+ +
@@ -64550,8 +64557,15 @@

/> tagDefinitionsType: object
-

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.

- +
+

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlftag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and ab. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.

+ +
+
+ +
diff --git a/specs/0.7.5/fluid-spec.html b/specs/0.7.5/fluid-spec.html index aa8ccf6..70e20e6 100644 --- a/specs/0.7.5/fluid-spec.html +++ b/specs/0.7.5/fluid-spec.html @@ -14990,8 +14990,15 @@

/> tagsType: object
-

LF-tag (TBAC) associations applied to this table. Keys must reference tag definitions declared in the contract's top-level governance.lakeFormation.tagDefinitions. Emitted as awslakeformationresourcelftags.

- +
+

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

+ +
+
+ +
@@ -67366,8 +67373,15 @@

/> tagDefinitionsType: object
-

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlf_tag per key; values are the tag's allowed-values list. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.

- +
+

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlftag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and ab. Field idiom borrowed from OpenMetadata's classification-label model โ€” see ODCS roles + OpenMetadata data contracts spec.

+ +
+
+ +
diff --git a/specs/0.7.6/fluid-spec.html b/specs/0.7.6/fluid-spec.html index 6dc7e21..5a43501 100644 --- a/specs/0.7.6/fluid-spec.html +++ b/specs/0.7.6/fluid-spec.html @@ -6656,7 +6656,7 @@

columnRestrictionsType: array of object
-

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS.

+

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.

@@ -6716,27 +6716,23 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - columnRestrictions items

Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
+ columnRestrictions items

+ No Additional Properties

+ +

+
+
root @@ -6787,32 +6783,42 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - principal

Type: string
-

Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud.

- - + allOf + + + + item 0
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
root @@ -6863,18 +6869,41 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - columns

Type: array of string
-

Columns of this expose's schema.

- + allOf + + + + item 0 + + + + if
Type: object
+ - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
root @@ -6925,40 +6954,46 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - columns + allOf - columns items

Type: string
- + item 0 + + + + if + + + + access
Type: const
+Specific value: "mask" -
-
-
-
-
-

- -

-
+
-
-
+ + +
+
root @@ -7009,35 +7044,41 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - access

Type: enum (of string)
-

deny: the principal may not read the columns. allow: only the principals an allow names may read them.

-
-

Must be one of:

-
  • "allow"
  • "deny"
-
+ allOf + + + + item 0 + + + + then
Type: object
+ +
+

The following properties are required:

+
  • mask
+
+ + +
-
-
-
-
-
-
-
-

- -

-
+ -
-
+ +
+
root @@ -7088,26 +7129,42 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - tags

Type: array of string
-

Access control rule tags.

-Same definition as tags -
-
-
-
-
-
-
-

- -

-
+ allOf + + + + item 1
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+
root @@ -7158,32 +7215,38 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - labels

Type: object
-

Access control rule labels.

-Same definition as labels -
-
-
-
- + allOf + + + + item 1 + + + + ifType: object
+ +
+

The following properties are required:

+
  • mask
- - + + + + + - -
-
-
-

- -

-
-
-
+ + +
+
root @@ -7220,85 +7283,54 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - tags

Type: array of string
-

Authorization policy tags.

-Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - exposes + columnRestrictions - exposes items + columnRestrictions items - policy + allOf - authz + item 1 - labels
Type: object
-

Authorization policy labels.

-Same definition as labels -
-
-
-
-
- - - -
+ then
Type: object
+ + + + + + + +
-
+

- +

-
+
root @@ -7329,82 +7361,87 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy

Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- -
root + authz - exposes + columnRestrictions - exposes items + columnRestrictions items - policy + allOf - privacy + item 1 - masking
Type: array of object
-
-

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

+ then + + + + access
Type: const
+Specific value: "mask" + + + + + +
-
- -
+
+
+
- No Additional Items

Each item of this array must be:

+ +
+ + + + + + +
-
- +
+

+ +

+
+ +
+
root @@ -7434,40 +7471,52 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + columnRestrictions - masking items

Type: object
- No Additional Properties + columnRestrictions items + + + + principal
Type: string
+

Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud.

+ -
+
+
+
+
+
-
+

- +

-
+
root @@ -7498,52 +7547,39 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + columnRestrictions - masking items + columnRestrictions items - column

Type: string
- + columns
Type: array of string
+

Columns of this expose's schema.

+ - -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+
root @@ -7573,62 +7609,60 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + columnRestrictions - masking items + columnRestrictions items - strategy

Type: enum (of string)
-
-

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

+ columns + + + + columns items
Type: string
-
-
- -
-

Must be one of:

-
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
-
+
+
-
+
-
+

- +

-
+
root @@ -7659,48 +7693,55 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + columnRestrictions - masking items + columnRestrictions items - params

Type: object
-

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

- + access
Type: enum (of string)
+

deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6).

+
+

Must be one of:

+
  • "allow"
  • "deny"
  • "mask"
+
-
+
+
+
+
+
-
+

- +

-
+
root @@ -7731,59 +7772,46 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy - - - - masking + authz - masking items + columnRestrictions - params + columnRestrictions items - saltEnv

Type: string Default: "FLUID_PII_HASH_SECRET"
-

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

-Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ - - - - - - + tags
Type: array of string
+

Access control rule tags.

+Same definition as tags
-
+
-
+

- +

-
+
root @@ -7814,59 +7842,46 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy - - - - masking + authz - masking items + columnRestrictions - params + columnRestrictions items - keyEnv

Type: string
-

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

-Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ - - - - - - + labels
Type: object
+

Access control rule labels.

+Same definition as labels
-
+
-
+

- +

-
+
root @@ -7897,59 +7912,68 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy - - - - masking + authz - masking items + columnRestrictions - params + columnRestrictions items - keepFirst

Type: integer Default: 0
-

NEW in v0.7.6: mask only. Leading characters left as they are.

- + mask
Type: enum (of string)
+
+

The masking rule an access: mask restriction applies, required with it and refused without it. lastfour: 'XXXXX' and the value's last four characters; firstfour: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATAMASKINGPOLICY, LASTFOURCHARACTERS, FIRSTFOURCHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6).

+ +
+
+ +
+

Must be one of:

+
  • "last_four"
  • "first_four"
  • "nullify"
+
-

Value must be greater or equal to 0 and lesser or equal to 64

+
-
+
+
+
+
+
+
+
-
+

- +

-
+
root @@ -7980,60 +8004,89 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + tags

Type: array of string
+

Authorization policy tags.

+Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root - masking items + exposes - params + exposes items - keepLast
Type: integer Default: 4
-

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

- - - - -

Value must be greater or equal to 0 and lesser or equal to 64

- - + policy + + + + authz + + + + labels
Type: object
+

Authorization policy labels.

+Same definition as labels
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
root @@ -8063,62 +8116,95 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + rowFilters

Type: array of object
+
+

Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filterpredicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (rowfilter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
root - masking items + exposes - params + exposes items - additionalProperties
Type: object
- + policy + + + + authz + + + + rowFilters + + + + rowFilters items
Type: object
+ No Additional Properties -
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
root @@ -8149,46 +8235,52 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + rowFilters - masking items + rowFilters items - tags

Type: array of string
-

Privacy masking rule tags.

-Same definition as tags + principal
Type: string
+

Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud.

+ + + + + + +
-
+
-
+

- +

-
+
root @@ -8219,52 +8311,52 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - masking + rowFilters - masking items + rowFilters items - labels

Type: object
-

Privacy masking rule labels.

-Same definition as labels -
-
-
-
-
-
+ name
Type: string
+

The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted.

+Must match regular expression: ^[a-z][a-z0-9_]{0,62}$ + + + + + +
-
+
-
+

- +

-
+
root @@ -8295,33 +8387,52 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - rowLevelPolicy

Type: object
- No Additional Properties + rowFilters + + + + rowFilters items + + + + where
Type: string
+

One SQL boolean expression over the expose's columns, e.g. consentdataanalytics = true AND is_suppressed = false.

+ - +

Must be at least 1 characters long

-
+
+
+
+
+
-
+

- +

-
+
root @@ -8352,23 +8463,29 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy + authz - rowLevelPolicy + rowFilters - expression

Type: string
-

Provider-specific predicate

- + rowFilters items + + + + description
Type: string
+ @@ -8379,18 +8496,28 @@

-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
root @@ -8421,39 +8548,26 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - privacy - - - - rowLevelPolicy - - - - tags

Type: array of string
-

Row-level policy tags.

-Same definition as tags -
-
-
-
-Type: object
+ No Additional Properties + + + + + + +
-
+

- +

-
+
root @@ -8491,37 +8605,25 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - rowLevelPolicy - - - - labels

Type: object
-

Row-level policy labels.

-Same definition as labels -
-
-
-
-
+ masking
Type: array of object
+
+

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

+
-
-
-
-
-
-

- -

+
+
+ -
-
+ + + + No Additional Items

Each item of this array must be:

+
+
+
root @@ -8558,25 +8660,33 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - tags

Type: array of string
-

Privacy policy tags.

-Same definition as tags -
-
-
-
-Type: object
+ No Additional Properties + + + + + + +
-
+

- +

-
+
root @@ -8614,64 +8724,22 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - labels

Type: object
-

Privacy policy labels.

-Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - exposes - - - - exposes items + masking - policy + masking items - classification
Type: enum (of string)
-
-

Must be one of:

-
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
-
+ column
Type: string
+ @@ -8682,18 +8750,18 @@

Must be one of:

-
+
-
+

- +

-
+
root @@ -8724,29 +8792,62 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy

Type: object
-

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

- + privacy + + + + masking + + + + masking items + + + + strategy
Type: enum (of string)
+
+

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

- No Additional Properties +
+
+ +
+

Must be one of:

+
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
+
-
+
+
+
+
+
-
+

- +

-
+
root @@ -8777,25 +8878,49 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - allowedModels

Type: array of string
-

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

+ masking + + + + masking items + + + + params
Type: object
+

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

 -

All items must be unique

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
root @@ -8825,62 +8950,59 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - allowedModels + masking - allowedModels items

Type: string
-Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ + masking items + + + + params + + + + saltEnv
Type: string Default: "FLUID_PII_HASH_SECRET"
+

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

+Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ -
-
Examples:
-
"gpt-4"
-
-
"claude-3-opus"
-
-
"claude-3-sonnet"
-
-
"gemini-pro"
-
-
"gemini-ultra"
-
-
"llama-3"
-
-
"mixtral-8x7b"
-
-
-
-
+
-
+
-
+

- +

-
+
root @@ -8911,94 +9033,59 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy - - - - deniedModels

Type: array of string
-

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

- - - - - - -

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- - -
root - - - - exposes - - - - exposes items + privacy - policy + masking - agentPolicy + masking items - deniedModels + params - deniedModels items
Type: string
- + keyEnv
Type: string
+

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

+Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ -
-
-
+
-
+

- +

-
+
root @@ -9029,108 +9116,59 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy - - - - maxTokensPerRequest

Type: integer
-

Maximum tokens per AI request. Prevents excessive data exposure in single queries.

- - - - -

Value must be greater or equal to 1

- -
-
Examples:
-
4096
-
-
8192
-
-
16384
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - exposes + privacy - exposes items + masking - policy + masking items - agentPolicy + params - maxTokensPerDay
Type: integer
-

Daily token limit for AI consumption. Enforces usage quotas.

+ keepFirst
Type: integer Default: 0
+

NEW in v0.7.6: mask only. Leading characters left as they are.

 -

Value must be greater or equal to 1

+

Value must be greater or equal to 0 and lesser or equal to 64

-
+
-
+

- +

-
+
root @@ -9161,112 +9199,60 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy - - - - allowedUseCases

Type: array of enum (of string)
-

Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

- - - - - - -

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- - -
root - - - - exposes - - - - exposes items + privacy - policy + masking - agentPolicy + masking items - allowedUseCases + params - allowedUseCases items
Type: enum (of string)
-
-

Must be one of:

-
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
-
+ keepLast
Type: integer Default: 4
+

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

+ - +

Value must be greater or equal to 0 and lesser or equal to 64

-
-

-
Examples:
-
[
-    "inference",
-    "reasoning",
-    "summarization"
-]
-
-
[
-    "search",
-    "qa",
-    "rag"
-]
-
-
-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

root @@ -9296,104 +9282,62 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy - - - - deniedUseCases

Type: array of enum (of string)
-

Prohibited AI use cases. Example: Allow reporting but deny model training.

- - - - - - -

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- - -
root - - - - exposes - - - - exposes items + privacy - policy + masking - agentPolicy + masking items - deniedUseCases + params - deniedUseCases items
Type: enum (of string)
-
-

Must be one of:

-
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
-
+ additionalProperties
Type: object
+ +
-

-
Example:
-
[
-    "training",
-    "fine_tuning"
-]
-
+
-
+
-
+

- +

-
+
root @@ -9424,38 +9368,46 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - canReason

Type: boolean Default: false
-

Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

- - - - - - - + masking + + + + masking items + + + + tags
Type: array of string
+

Privacy masking rule tags.

+Same definition as tags
-
+
-
+

- +

-
+
root @@ -9486,38 +9438,52 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - canStore

Type: boolean Default: false
-

Whether AI systems can cache/store this data. False = ephemeral processing only.

- - - - - - - + masking + + + + masking items + + + + labels
Type: object
+

Privacy masking rule labels.

+Same definition as labels
-
+
+
+
+
+
+
+
-
+

- +

-
+
root @@ -9548,34 +9514,33 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - retentionPolicy

Type: object
-

Data retention requirements for AI systems consuming this data.

- + rowLevelPolicy
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
root @@ -9606,45 +9571,45 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - retentionPolicy + rowLevelPolicy - maxRetentionDays

Type: integer
-

Maximum days AI system can retain this data. 0 = no retention allowed.

+ expression
Type: string
+

Provider-specific predicate

 -

Value must be greater or equal to 0

+
-
+
-
+

- +

-
+
root @@ -9675,49 +9640,39 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - retentionPolicy + rowLevelPolicy - requireDeletion

Type: boolean Default: true
-

Whether AI system must delete data after use.

- - - - - - - -
-
-
-
+ tags
Type: array of string
+

Row-level policy tags.

+Same definition as tags
-
+
-
+

- +

-
+
root @@ -9748,38 +9703,43 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - auditRequired

Type: boolean Default: true
-

Whether AI consumption must be logged/audited.

- - - - - - - + rowLevelPolicy + + + + labels
Type: object
+

Row-level policy labels.

+Same definition as labels
-
+
+
+
+
+
-
+

- +

-
+
root @@ -9810,38 +9770,32 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - purposeLimitation

Type: string
-

Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

- - - - - - - + tags
Type: array of string
+

Privacy policy tags.

+Same definition as tags
-
+
-
+

- +

-
+
root @@ -9872,32 +9826,36 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy + privacy - tags

Type: array of string
-

Agent policy tags for categorization.

-Same definition as tags + labels
Type: object
+

Privacy policy labels.

+Same definition as labels
-
+
+
+
+
+
-
+

- +

-
+
root @@ -9928,36 +9886,33 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - agentPolicy - - - - labels

Type: object
-

Agent policy labels for automation.

-Same definition as labels + classification
Type: enum (of string)
+
+

Must be one of:

+
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
-
-
-
+ + + + + +
-
+
-
+

- +

-
+
root @@ -9988,25 +9943,29 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - tags

Type: array of string
-

Policy-level tags for governance.

-Same definition as tags -
-
-
-
-Type: object
+

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

+ + + No Additional Properties + + + + + + +
-
+

- +

-
+
root @@ -10037,31 +9996,26 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - labels

Type: object
-

Policy-level labels for automation.

-Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- + agentPolicy + + + + allowedModels
Type: array of string
+

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

+ + + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ +
root d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding
Type: object
+ policy + + + + agentPolicy + + + + allowedModels + + + + allowedModels items
Type: string
+Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ + + + - No Additional Properties

- -

-
- +
+
Examples:
+
"gpt-4"
+
+
"claude-3-opus"
+
+
"claude-3-sonnet"
+
+
"gemini-pro"
+
+
"gemini-ultra"
+
+
"llama-3"
+
+
"mixtral-8x7b"
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
root @@ -10124,49 +10123,32 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0

Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: array of string
+

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

+ -
  • - Then -
    • -
    - -
    + + +

    All items must be unique

    No Additional Items

    Each item of this array must be:

    +
    +
    +
    root @@ -10189,28 +10171,28 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0 + deniedModels - if

    Type: object
    + deniedModels items
    Type: string
    @@ -10218,18 +10200,24 @@

    -
    +
    +

    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -10253,53 +10241,54 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding - - - - allOf - - - - item 0 + policy - if + agentPolicy - platform

    Type: const
    -Specific value: "gcp" + maxTokensPerRequest
    Type: integer
    +

    Maximum tokens per AI request. Prevents excessive data exposure in single queries.

    +     - +

    Value must be greater or equal to 1

    - +
    +
    Examples:
    +
    4096
+
    +
    8192
+
    +
    16384
+
    +
    -
    +
    +
    +
    +

    + +

    +
    - - -
    - +
    +
    root @@ -10322,47 +10311,45 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding - - - - allOf + policy - item 0 + agentPolicy - then

    Type: object
    - + maxTokensPerDay
    Type: integer
    +

    Daily token limit for AI consumption. Enforces usage quotas.

    +     - +

    Value must be greater or equal to 1

    -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -10386,55 +10373,32 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding - - - - allOf - - - - item 0 + policy - then + agentPolicy - encryption

    Type: object
    - + allowedUseCases
    Type: array of enum (of string)
    +

    Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

    +     - -
    +

    All items must be unique

    No Additional Items

    Each item of this array must be:

    -
    -

    - -

    -
    - -
    -
    +
    +
    root @@ -10457,148 +10421,198 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0 + allowedUseCases - then + allowedUseCases items

    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "inference"
    • "reasoning"
    • "analysis"
    • "summarization"
    • "classification"
    • "embedding"
    • "search"
    • "qa"
    • "code_generation"
    • "fine_tuning"
    • "training"
    • "rag"
    +
    + + + + + + +
    +

    +
    Examples:
    +
    [
+    "inference",
+    "reasoning",
+    "summarization"
+]
+
    +
    [
+    "search",
+    "qa",
+    "rag"
+]
+
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - encryption + exposes - kms

    -
    -

    Must not be:

    -
    -
    - - -
    root + exposes items - exposes + policy - exposes items + agentPolicy - binding + deniedUseCases
    Type: array of enum (of string)
    +

    Prohibited AI use cases. Example: Allow reporting but deny model training.

    +     + + + + + +

    All items must be unique

    No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root - allOf + exposes - item 0 + exposes items - then + policy - encryption + agentPolicy - kms + deniedUseCases - not
    Type: object
    -Must match regular expression: ^(alias/|arn:) - - - - - - -
    -
    -
    + deniedUseCases items
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "inference"
    • "reasoning"
    • "analysis"
    • "summarization"
    • "classification"
    • "embedding"
    • "search"
    • "qa"
    • "code_generation"
    • "fine_tuning"
    • "training"
    • "rag"
    +
    -
    -
    +

    +
    Example:
    +
    [
+    "training",
+    "fine_tuning"
+]
+
    -
    +
    -
    +

    - +

    -
    +
    root @@ -10622,55 +10636,46 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding - - - - allOf - - - - item 0 + policy - then + agentPolicy - principals

    Type: object
    - + canReason
    Type: boolean Default: false
    +

    Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

    +     -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    -

    Each additional property must conform to the following schema

    +
    +
    root @@ -10693,58 +10698,104 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0 + canStore

    Type: boolean Default: false
    +

    Whether AI systems can cache/store this data. False = ephemeral processing only.

    +     + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - then + exposes - principals + exposes items - additionalProperties

    -

    - -

    -
    - + policy + + + + agentPolicy + + + + retentionPolicy
    Type: object
    +

    Data retention requirements for AI systems consuming this data.

    +     + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    root @@ -10767,66 +10818,126 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0 + retentionPolicy - then + maxRetentionDays

    Type: integer
    +

    Maximum days AI system can retain this data. 0 = no retention allowed.

    +     + + + +

    Value must be greater or equal to 0

    + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - principals + exposes - additionalProperties + exposes items - anyOf + policy - item 0
    Type: string
    -Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + agentPolicy + + + + retentionPolicy + + + + requireDeletion
    Type: boolean Default: true
    +

    Whether AI system must delete data after use.

    +     -
    - +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    root @@ -10849,66 +10960,108 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0 + auditRequired

    Type: boolean Default: true
    +

    Whether AI consumption must be logged/audited.

    +     + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - then + exposes - principals + exposes items - additionalProperties + policy - anyOf + agentPolicy - item 1
    Type: array of string
    - + purposeLimitation
    Type: string
    +

    Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

    +     - No Additional Items

    Each item of this array must be:

    + +
    +
    +
    +
    +
    -
    - +
    +

    + +

    +
    + +
    +
    root @@ -10931,79 +11084,181 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding + policy - allOf + agentPolicy - item 0 + tags

    Type: array of string
    +

    Agent policy tags for categorization.

    +    Same definition as tags +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - then + exposes - principals + exposes items - additionalProperties + policy - anyOf + agentPolicy - item 1 + labels
    Type: object
    +

    Agent policy labels for automation.

    +    Same definition as labels +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - item 1 items
    Type: string
    -Must match regular expression: ^(user|group|serviceAccount|domain):.+$ - - - - - - + exposes + + + + exposes items + + + + policy + + + + tags
    Type: array of string
    +

    Policy-level tags for governance.

    +    Same definition as tags +
    -
    - +
    +
    +
    +
    +

    + +

    +
    - - +
    +
    - +
    root + + + + exposes + + + + exposes items + + + + policy + + + + labels
    Type: object
    +

    Policy-level labels for automation.

    +    Same definition as labels
    @@ -11012,15 +11267,59 @@

    -
    - +
    +
    +
    +

    + +

    +
    - - +
    +
    - -
    +
    root + + + + exposes + + + + exposes items + + + + binding
    Type: object
    + + + No Additional Properties

    + +

    +
    root @@ -11058,18 +11357,18 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1

    Type: object
    + item 0
    Type: object

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

    - + -
    - -
    +
    +
    +
    +

    + +

    +
    + +
    +
    root @@ -28610,54 +29050,76 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - builds + exposes - builds items + exposes items - allOf + mcp - item 0 + classification - if

    Type: object
    - + dataClass
    Type: enum (of string)
    +

    Surfaced on the describe tool so consumer agents can declare downstream handling. Advisory; the real gate is policy.agentPolicy.

    +
    +

    Must be one of:

    +
    • "public"
    • "internal"
    • "confidential"
    • "restricted"
    +
    -
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -28667,60 +29129,97 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - builds + consumes

    Type: array
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root - builds items + consumes - allOf + consumeRef
    Type: object
    + + + No Additional Properties + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - item 0 + consumes - if + consumes items - pattern
    Type: const
    -Specific value: "hybrid-reference" - - - - - - + productId
    Type: string
    +

    Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity.

    +    Same definition as id
    -
    +
    +
    +
    +

    + +

    +
    - - -
    - +
    +
    root @@ -28729,54 +29228,89 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - builds + consumes - builds items + consumes items - allOf + exposeId

    Type: string
    +

    The upstream exposeId being consumed.

    +    Same definition as id +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - item 0 + consumes - then
    Type: object
    + consumes items + + + + versionConstraint
    Type: string
    +

    Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces.

    +     + -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -28786,64 +29320,4037 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - builds + consumes - builds items + consumes items - allOf + qosExpectations

    Type: object
    + No Additional Properties + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - item 0 + consumes - then + consumes items - properties
    Type: object
    -

    Hybrid-reference pattern from v0.4.0 (dbt-style).

    -     - - No Additional Properties - + qosExpectations + + + + freshnessMax
    Type: string
    +

    ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

    +    Same definition as window +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    - - +
    +
    - -
    +
    root + + + + consumes + + + + consumes items + + + + qosExpectations + + + + maxStaleness
    Type: string
    +

    Maximum acceptable staleness for consumed data.

    +    Same definition as window +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + qosExpectations + + + + minCompleteness
    Type: number
    +

    Minimum completeness expectation.

    +     + + + +

    Value must be greater or equal to 0 and lesser or equal to 1

    + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + qosExpectations + + + + tags
    Type: array of string
    +

    QoS expectation tags.

    +    Same definition as tags +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + qosExpectations + + + + labels
    Type: object
    +

    QoS expectation labels.

    +    Same definition as labels +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + requiredPolicies
    Type: array of string
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + consumes + + + + consumes items + + + + requiredPolicies + + + + requiredPolicies items
    Type: string
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + purpose
    Type: string
    +

    Business purpose for consuming this data.

    +     + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + tags
    Type: array of string
    +

    Consumption relationship tags.

    +    Same definition as tags +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + labels
    Type: object
    +

    Consumption relationship labels.

    +    Same definition as labels +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + upstreamWorkspace
    Type: string
    +

    Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace.

    +     + + + +

    Must be at least 1 characters long

    + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + consumes + + + + consumes items + + + + upstreamDigest
    Type: string
    +

    Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. fluid apply re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set.

    +    Must match regular expression: ^sha256:[0-9a-f]{64}$ + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds
    Type: array
    +

    NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).

    +     + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + builds + + + + build
    Type: object
    +

    Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.

    +     + + No Additional Properties

    + +

    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0
    Type: object
    +

    + +

    +

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

    + + +
    + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + if
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + if + + + + pattern
    Type: const
    +Specific value: "hybrid-reference" + + + + + + +
    +
    +
    +
    +
    + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties
    Type: object
    +

    Hybrid-reference pattern from v0.4.0 (dbt-style).

    +     + + No Additional Properties + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + model
    Type: string
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + target
    Type: string
    +

    dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

    +     + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + select

    +

    dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

    +

    + +

    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + select + + + + oneOf + + + + item 0
    Type: string
    + + + + + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + select + + + + oneOf + + + + item 1
    Type: array of string
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + select + + + + oneOf + + + + item 1 + + + + item 1 items
    Type: string
    + + + + + + + +
    +
    +
    + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + models

    +

    Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

    +

    + +

    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + models + + + + oneOf + + + + item 0
    Type: string
    + + + + + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + models + + + + oneOf + + + + item 1
    Type: array of string
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + models + + + + oneOf + + + + item 1 + + + + item 1 items
    Type: string
    + + + + + + + +
    +
    +
    + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + vars
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +

    Additional Properties of any type are allowed.

    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + vars + + + + additionalProperties
    Type: object
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + materializations
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +

    Each additional property must conform to the following schema

    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + materializations + + + + additionalProperties
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "table"
    • "view"
    • "incremental"
    • "ephemeral"
    +
    + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + tags
    Type: array of string
    +

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as tags +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 0 + + + + then + + + + properties + + + + labels
    Type: object
    +

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as labels +
    +
    +
    +
    +
    +
    +
    +
    +
    + + + + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1
    Type: object
    +

    + +

    +

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

    + + +
    + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + if
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + if + + + + pattern
    Type: const
    +Specific value: "embedded-logic" + + + + + + +
    +
    +
    +
    +
    + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties
    Type: object
    +

    Embedded logic pattern from v0.4.0.

    +     + + No Additional Properties + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + sql
    Type: string
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + language
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "sql"
    • "flink_sql"
    • "pyspark"
    • "scala"
    • "python"
    • "r"
    +
    + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + parameters
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +

    Additional Properties of any type are allowed.

    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + parameters + + + + additionalProperties
    Type: object
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + tags
    Type: array of string
    +

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as tags +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + labels
    Type: object
    +

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as labels +
    +
    +
    +
    +
    +
    +
    +
    +
    + + + + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2
    Type: object
    +

    + +

    +

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

    + + +
    + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + if
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + if + + + + pattern
    Type: const
    +Specific value: "multi-stage" + + + + + + +
    +
    +
    +
    +
    + + + +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties
    Type: object
    +

    NEW in v0.5.5: Multi-stage orchestration pattern.

    +     + + No Additional Properties + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages
    Type: array of object
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items
    Type: object
    + No Additional Properties + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + name
    Type: string
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + pattern
    Type: enum (of string) Default: "hybrid-reference"
    +
    +

    Must be one of:

    +
    • "hybrid-reference"
    • "embedded-logic"
    +
    + + + + + + +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + properties
    Type: object
    +

    Base properties for all build patterns.

    +    Same definition as properties +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + dependsOn
    Type: array of string
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + dependsOn + + + + dependsOn items
    Type: string
    + + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + outputs
    Type: array
    + + + + + + + No Additional Items

    Each item of this array must be:

    +
    +
    + + +
    root + + + + builds + + + + builds items + + + + allOf + + + + item 2 + + + + then + + + + properties + + + + stages + + + + stages items + + + + outputs + + + + identifier
    Type: string
    +

    Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

    +    Same definition as id +
    +
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -28874,51 +33381,60 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 2 - then + then - properties + properties - model

    Type: string
    - - - - - - - + stages + + + + stages items + + + + tags
    Type: array of string
    +

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as tags
    -
    +
    -
    +

    - +

    -
    +
    root @@ -28949,52 +33465,66 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 2 - then + then - properties + properties - target

    Type: string
    -

    dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

    -     - - - - - - + stages + + + + stages items + + + + labels
    Type: object
    +

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as labels
    -
    +
    +
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -29025,45 +33555,48 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 2 - then + then - properties + properties - select


    -

    dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

    -

    - -

    -
    - + orchestration
    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    root @@ -29093,52 +33626,59 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 - - - - then + item 2 - properties + then - select + properties - oneOf + orchestration - item 0

    Type: string
    + parallelism
    Type: integer
    - +

    Value must be greater or equal to 1

    -
    - +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    + +
    +
    root @@ -29168,52 +33708,57 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 - - - - then + item 2 - properties + then - select + properties - oneOf + orchestration - item 1

    Type: array of string
    - - - - - - - No Additional Items

    Each item of this array must be:

    + retryPolicy
    Type: object
    +Same definition as retries +
    +
    +
    +
    +
    +
    +
    +
    +
    -
    - +
    +

    + +

    +
    + +
    +
    root @@ -29243,81 +33788,46 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 - - - - then - - - - properties - - - - select + item 2 - oneOf + then - item 1 + properties - item 1 items

    Type: string
    - - - - - - - -
    -
    -
    - - - - - - + tags
    Type: array of string
    +

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as tags
    -
    +
    -
    +

    - +

    -
    +
    root @@ -29348,44 +33858,47 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 2 - then + then - properties + properties - models


    -

    Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

    -

    - -

    -
    + labels
    Type: object
    +

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    +    Same definition as labels +
    +
    +
    +
    +
    +
    +
    +
    +
    + + + + + + +
    root @@ -29416,42 +33929,72 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3

    Type: object
    +

    + +

    +

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

    + + +
    + +
    + + +
    root - then + builds - properties + builds items - models + allOf - oneOf + item 3 - item 0
    Type: string
    + if
    Type: object
    @@ -29459,9 +34002,19 @@

    -

    - +
    +
    +
    +

    + +

    +
    + +
    +
    root @@ -29491,51 +34044,38 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 - - - - then - - - - properties - - - - models + item 3 - oneOf + if - item 1

    Type: array of string
    - + pattern
    Type: const
    +Specific value: "acquisition" - No Additional Items

    Each item of this array must be:

    -
    -
    + +
    +
    +
    +
    +
    + + + +
    root @@ -29566,81 +34106,100 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then

    Type: object
    + + + + + + + +
    +
    +
    +

    + +

    +
    + +
    +
    + +
    root - properties + builds - models + builds items - oneOf + allOf - item 1 + item 3 - item 1 items
    Type: string
    - - - - - + then + + + + properties
    Type: object
    +

    NEW in v0.7.3: Source-aligned acquisition pattern. The build ingests data from an external system (no transformation). Engine โˆˆ {duckdb, airbyte, meltano, dlt, kafka-connect, debezium}.

    +     - -
    -
    -
    + No Additional Properties -
    -
    -
    -
    -
    +
    -
    +

    - +

    -
    +
    root @@ -29671,48 +34230,51 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then - properties + properties - vars

    Type: object
    + source
    Type: object
    +

    External source system specification.

    +     + No Additional Properties -
    +
    -
    +

    - +

    -
    -

    Additional Properties of any type are allowed.

    +
    +
    root @@ -29742,36 +34304,37 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then - properties + properties - vars + source - additionalProperties

    Type: object
    - + kind
    Type: string
    +

    Source system kind: filesystem, postgres, mysql, sqlite, http, salesforce, stripe, github, kafka, etc. Engine-specific catalog of supported kinds.

    +     @@ -29782,22 +34345,18 @@

    -
    -
    -
    -
    -
    +
    -
    +

    - +

    -
    +
    root @@ -29828,48 +34387,56 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then - properties + properties - materializations

    Type: object
    - + source + + + + connection
    Type: object
    +

    Connection details. Use ${VAR} placeholders for env values; use secretRef for credentials. Inline secrets are forbidden by validator.

    +     -
    +
    -
    +

    - +

    -
    -

    Each additional property must conform to the following schema

    +
    +
    root @@ -29899,39 +34466,43 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then - properties + properties - materializations + source - additionalProperties

    Type: enum (of string)
    -
    -

    Must be one of:

    -
    • "table"
    • "view"
    • "incremental"
    • "ephemeral"
    -
    + connection + + + + uri
    Type: string
    + @@ -29942,22 +34513,18 @@

    Must be one of:

    -
    -
    -
    -
    -
    +
    -
    +

    - +

    -
    +
    root @@ -29988,46 +34555,65 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then - properties + properties - tags

    Type: array of string
    -

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as tags + source + + + + connection + + + + host
    Type: string
    + + + + + + +
    -
    +
    -
    +

    - +

    -
    +
    root @@ -30058,47 +34644,57 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 3 - then + then - properties + properties - labels

    Type: object
    -

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as labels -
    -
    -
    -
    -
    -
    -
    -
    -
    - - - - - - -
    + source + + + + connection + + + + port

    +

    + +

    +
    root @@ -30129,72 +34725,56 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1

    Type: object
    -

    - -

    -

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

    - - -
    - -
    - - -
    root + item 3 - builds + then - builds items + properties - allOf + source - item 1 + connection - if
    Type: object
    + port + + + + oneOf + + + + item 0
    Type: integer
    @@ -30202,19 +34782,9 @@

    -
    -
    -
    -

    - -

    -
    - -
    -
    +
    +
    root @@ -30244,95 +34814,87 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - if + then - pattern

    Type: const
    -Specific value: "embedded-logic" - - - - - - -
    -
    -
    -
    -

    - - - -
    - - -
    root + properties - builds + source - builds items + connection - allOf + port - item 1 + oneOf - then
    Type: object
    + item 1
    Type: string
    +

    Allows ${ENV_VAR} placeholders that resolve to an integer at runtime.

    +     + + + + +
    + -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -30363,43 +34925,65 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - then + then - properties

    Type: object
    -

    Embedded logic pattern from v0.4.0.

    -     + properties + + + + source + + + + connection + + + + database
    Type: string
    - No Additional Properties -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -30430,28 +35014,42 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - then + then - properties + properties - sql

    Type: string
    + source + + + + connection + + + + instance_url
    Type: string
    @@ -30463,18 +35061,18 @@

    -
    +
    -
    +

    - +

    -
    +
    root @@ -30505,32 +35103,44 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - then + then - properties + properties - language

    Type: enum (of string)
    -
    -

    Must be one of:

    -
    • "sql"
    • "flink_sql"
    • "pyspark"
    • "scala"
    • "python"
    • "r"
    -
    + source + + + + connection + + + + secretRef
    Type: string
    +

    Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://).

    +    Must match regular expression: ^[a-z][a-z0-9_+.-]*://.+ @@ -30541,18 +35151,18 @@

    Must be one of:

    -
    +
    -
    +

    - +

    -
    +
    root @@ -30569,62 +35179,81 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - builds items + builds items + + + + allOf + + + + item 3 - allOf + then - item 1 + properties - then + source - properties + connection - parameters

    Type: object
    - + user
    Type: string
    +

    NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) โ€” without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside secretRef (or an inline password) for the credential pair.

    +     -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    -

    Additional Properties of any type are allowed.

    +
    +
    root @@ -30654,36 +35283,44 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - then + then - properties + properties - parameters + source - additionalProperties

    Type: object
    - + connection + + + + schema
    Type: string
    +

    NEW in v0.7.3: Default schema/namespace within the source database to read from. Maps to engine-specific config: dlt sqldatabase schema=, Meltano tap-postgres filterschemas=[], Airbyte source-postgres schemas=[], Debezium schema.include.list, Kafka Connect JDBC schema.pattern. Use schemas (array) to scope to multiple schemas at once.

    +     @@ -30694,22 +35331,18 @@

    -
    -
    -
    -
    -
    +
    -
    +

    - +

    -
    +
    root @@ -30740,47 +35373,53 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - then + then - properties + properties - tags

    Type: array of string
    -

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as tags -
    -
    -
    -
    -
    -
    -
    -

    - -

    -
    + source + + + + connection + + + + schemas
    Type: array of string
    +

    NEW in v0.7.3: Multi-schema variant of schema. When both are set, schemas wins and schema is treated as a deprecated alias. Engines that don't natively support multi-schema reads will iterate the list.

    +     + -
    -
    + + + +

    All items must be unique

    No Additional Items

    Each item of this array must be:

    +
    +
    +
    root @@ -30810,48 +35449,75 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 1 + item 3 - then + then - properties + properties - labels

    Type: object
    -

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as labels -
    + source + + + + connection + + + + schemas + + + + schemas items
    Type: string
    + + + + + + +
    -
    -
    - - - - +
    +
    +
    +

    + +

    +
    - -
    - +
    +

    Additional Properties of any type are allowed.

    root @@ -30881,72 +35547,42 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2

    Type: object
    -

    - -

    -

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

    - - -
    - -
    - - -
    root + item 3 - builds + then - builds items + properties - allOf + source - item 2 + connection - if
    Type: object
    + additionalProperties
    Type: object
    @@ -30954,18 +35590,26 @@

    -
    +
    +

    +
    +
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -30996,22 +35640,39 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - if + then - pattern

    Type: const
    -Specific value: "multi-stage" + properties + + + + source + + + + mode
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "full_refresh"
    • "incremental_append"
    • "incremental_dedup"
    • "incremental_merge"
    • "cdc"
    • "streaming"
    +
    @@ -31022,13 +35683,19 @@

    -
    +
    +
    +
    +

    + +

    +
    - - -
    - +
    +
    root @@ -31058,33 +35725,59 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then

    Type: object
    - + then + + + + properties + + + + source + + + + cursor_field
    Type: string
    +

    Column used for incremental cursor (e.g., updated_at, SystemModstamp).

    +     -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -31115,43 +35808,54 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties

    Type: object
    -

    NEW in v0.5.5: Multi-stage orchestration pattern.

    -     - - No Additional Properties + properties + + + + source + + + + watermark
    Type: object
    + No Additional Properties -
    +
    -
    +

    - +

    -
    +
    root @@ -31182,38 +35886,69 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - stages

    Type: array of object
    - + source + + + + watermark + + + + strategy
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "high_water_mark"
    • "log_position"
    • "lsn"
    +
    - No Additional Items

    Each item of this array must be:

    + +
    +
    +
    +
    +
    -
    - +
    +

    + +

    +
    + +
    +
    root @@ -31243,54 +35978,64 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - stages + source - stages items

    Type: object
    - No Additional Properties - - - - - - -Type: string
    +

    ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

    +    Same definition as window +
    +
    +
    +
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -31321,66 +36066,46 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 - - - - then + item 3 - properties + then - stages + properties - stages items + source - name

    Type: string
    - + streams
    Type: array of string
    +

    List of streams/tables/objects to ingest.

    +     - -
    -
    -
    -
    -
    + No Additional Items

    Each item of this array must be:

    -
    -

    - -

    -
    - -
    -
    +
    +
    root @@ -31410,68 +36135,67 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - stages + source - stages items + streams - pattern

    Type: enum (of string) Default: "hybrid-reference"
    -
    -

    Must be one of:

    -
    • "hybrid-reference"
    • "embedded-logic"
    -
    + streams items
    Type: string
    + +
    +
    -
    +
    -
    +

    - +

    -
    +
    root @@ -31502,60 +36226,54 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 - - - - then + item 3 - properties + then - stages + properties - stages items + source - properties

    Type: object
    -

    Base properties for all build patterns.

    -    Same definition as properties -
    -
    -
    -
    -Type: object
    + + + + + + + +
    -
    +

    - +

    -
    +
    root @@ -31586,52 +36304,69 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - stages + source - stages items + reader - dependsOn

    Type: array of string
    - + format
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "csv"
    • "parquet"
    • "json"
    • "ndjson"
    • "avro"
    • "orc"
    +
    - No Additional Items

    Each item of this array must be:

    + +
    +
    +
    +
    +
    -
    - +
    +

    + +

    +
    + +
    +
    root @@ -31661,49 +36396,42 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 - - - - then + item 3 - properties + then - stages + properties - stages items + source - dependsOn + reader - dependsOn items

    Type: string
    + options
    Type: object
    @@ -31711,25 +36439,19 @@

    -

    -
    -
    -
    -
    -
    -
    +
    -
    +

    - +

    -
    -
    +
    +

    Additional Properties of any type are allowed.

    root @@ -31759,52 +36481,77 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - stages + source - stages items + reader - outputs

    Type: array
    + options + + + + additionalProperties
    Type: object
    - No Additional Items

    Each item of this array must be:

    + +
    +
    +
    +
    +
    +
    +
    +
    +
    -
    - +
    +

    + +

    +
    + +
    +

    Additional Properties of any type are allowed.

    root @@ -31834,69 +36581,73 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 - - - - then + item 3 - properties + then - stages + properties - stages items + source - outputs + reader - identifier

    Type: string
    -

    Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

    -    Same definition as id + additionalProperties
    Type: object
    + + + + + + + +
    +
    -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -31927,60 +36678,50 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 - - - - then - - - - properties + item 3 - stages + then - stages items + properties - tags

    Type: array of string
    -

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as tags -
    -
    -
    -
    -
    + sink
    Type: object
    +

    Destination format for ingested data. binding.platform on the expose remains the source of truth for where; sink.format describes how.

    +     + + No Additional Properties + + + + + + +
    -
    +

    - +

    -
    +
    root @@ -32011,66 +36752,61 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 - - - - then + item 3 - properties + then - stages + properties - stages items + sink - labels

    Type: object
    -

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as labels + format
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "iceberg"
    • "delta"
    • "parquet"
    • "csv"
    • "json"
    • "snowflake_table"
    • "bigquery_table"
    • "redshift_table"
    • "duckdb_table"
    -
    -
    -
    -
    -
    + + + + + +
    -
    +
    -
    +

    - +

    -
    +
    root @@ -32101,47 +36837,61 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - orchestration

    Type: object
    - + sink + + + + catalog
    Type: enum (of string)
    +
    +

    Must be one of:

    +
    • "rest"
    • "glue"
    • "nessie"
    • "unity"
    • "snowflake-managed"
    • "hive"
    • "lakekeeper"
    • "polaris"
    +
    -
    +
    +
    +
    +
    +
    -
    +

    - +

    -
    +
    root @@ -32172,59 +36922,45 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - orchestration + sink - parallelism

    Type: integer
    + partitionBy
    Type: array of string
    -

    Value must be greater or equal to 1

    - -
    -
    -
    -
    -
    -
    -
    -

    - -

    -
    -
    -
    + No Additional Items

    Each item of this array must be:

    +
    +
    +
    root @@ -32254,56 +36990,67 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - orchestration + sink - retryPolicy

    Type: object
    -Same definition as retries -
    + partitionBy + + + + partitionBy items
    Type: string
    + + + + + + +
    -
    -
    +
    -
    +

    - +

    -
    +
    root @@ -32334,159 +37081,142 @@

    d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 2 + item 3 - then + then - properties + properties - tags

    Type: array of string
    -

    Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as tags -
    -
    -
    -
    -
    -
    -
    -

    - -

    -
    - -
    -
    - -
    root + sink - builds + catalogAuth

    +
    +

    NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:<NAME>, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding clientid:clientsecret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is <polaris>/api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.

    + +
    +
    + +
    No Additional Properties

    + +

    +
    + + +
    root - builds items + builds - allOf + builds items - item 2 + allOf - then + item 3 - properties + then - labels
    Type: object
    -

    Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

    -    Same definition as labels -
    -
    -
    -
    -
    -
    -
    -
    -
    - - - - - - -
    - - -
    root + properties - builds + sink - builds items + catalogAuth - allOf + allOf - item 3
    Type: object
    + item 0
    Type: object

    If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

    -

    _masked`, that returns them masked by the same rule and applies the principal's row filter. On BigQuery it attaches a data policy to the columns' policy tag. Like the other descriptions of what `fluid apply` emits, this is non-normative. + +## `engine: flink` + + +```yaml +build: + pattern: embedded-logic + engine: flink + properties: + language: flink_sql + sql: | + INSERT INTO payments_reconciled + SELECT p.payment_id, p.amount, s.settled_at + FROM payments AS p JOIN settlements AS s ON p.payment_id = s.payment_id + parameters: + parallelism: 4 + upgradeMode: savepoint # the operator's savepoint | last-state | stateless + flinkConfiguration: + execution.checkpointing.interval: 60s +``` + +`flink` names Apache Flink as the build's engine, usually for a long-running Flink SQL job (`properties.language: flink_sql`). The embedded-logic properties are a closed set, so deployment settings go under `properties.parameters`; the schema's description recommends the names the Flink Kubernetes Operator's job spec uses (`parallelism`, `upgradeMode`) and `flinkConfiguration`. A 0.7.5 contract cannot declare `engine: flink`. + +The reference implementation validates and plans a `flink` build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. It never runs such a build's SQL on DuckDB or turns it into a warehouse statement, and it treats a build with `properties.language: flink_sql` the same way on any `fluidVersion`. + +## REST catalog credentials: `sink.catalogAuth` + + +```yaml +exposes: + - exposeId: events + kind: table + contract: + schema: + - { name: event_id, type: STRING, required: true } + binding: + platform: local + format: iceberg + location: { catalog: lakekeeper, uri: "http://lakekeeper:8181/catalog", warehouse: analytics, database: default, table: events } +builds: + - id: ingest + pattern: acquisition + engine: kafka-connect + outputs: [events] + properties: + source: { kind: postgres, mode: incremental_append } + sink: + format: iceberg + catalog: lakekeeper + catalogAuth: + type: oauth2 + credentialEnv: LAKEKEEPER_CLIENT # a variable holding client_id:client_secret + oauth2ServerUri: https://keycloak.example.com/realms/lake/protocol/openid-connect/token + scope: lakekeeper +``` + +`catalogAuth` says how the sink authenticates to an Iceberg REST catalog. It names environment variables and never holds a secret. + +| `type` | Requires | Refuses | +|---|---|---| +| `oauth2` (client credentials) | `credentialEnv`, `oauth2ServerUri`, `scope`; `audience` and `resource` are optional | `tokenEnv` | +| `bearer` (a static token) | `tokenEnv` | `credentialEnv`, `oauth2ServerUri`, `scope`, `audience`, `resource` | + +- `credentialEnv` and `tokenEnv` must be environment variable names. The schema refuses one shaped like a token: a run of 20 or more letters and digits, with a digit, between underscores. +- `oauth2ServerUri` must be an `http://` or `https://` URL with no `user:password@` part. It, `scope`, `audience` and `resource` may not hold a `${...}` expression or a control character, or start with `env:`. + +The reference implementation writes each named variable into the sink config it derives as `env:`, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink. It refuses `catalogAuth` on a catalog it does not reach over REST, and next to a hand-written override of the same settings. It also refuses an Iceberg sink unless an expose the build writes is bound with `format: iceberg`, as `events` is above; the catalog's address is on that binding. + ## Column restrictions: semantics written down `exposes[].policy.authz.columnRestrictions` exists in 0.7.5 with the one-line description "Column-level access control". The 0.7.6 description defines it: @@ -172,13 +280,14 @@ policy: - A column named in any restriction is restricted. - `access: deny` โ€” the principal may not read the columns. `access: allow` โ€” the columns are readable only by the principals an `allow` names. - A deny beats an allow, and a restriction never grants access: the readers are still the expose's readers. +- `access: mask` (added in 0.7.6) โ€” the principal reads the columns masked by the rule in `mask` ([above](#row-filters-and-masked-columns)). A deny beats a mask, and a mask beats an allow for the principal it names. - Principals are logical, resolved per platform through `binding.principals`. --- ## Two complete contracts -Both validate against the 0.7.6 JSON Schema (Draft 2020-12) and with the reference implementation, `data-product-forge` 0.18.1 (`fluid validate` prints `โœ… Valid FLUID contract (schema v0.7.6)`). +Both validate against the 0.7.6 JSON Schema (Draft 2020-12) and with the reference implementation, `data-product-forge` 0.21.0 (`fluid validate` prints `โœ… Valid FLUID contract (schema v0.7.6)`). **BigQuery:** column restriction, mapped principals, a product-owned key, expiring partitions, a percentile measure, packaging and a consumer. diff --git a/docs/releases/README.md b/docs/releases/README.md index 0908f15..beb937c 100644 --- a/docs/releases/README.md +++ b/docs/releases/README.md @@ -6,7 +6,7 @@ Release notes for the FLUID specification, newest first. Each entry links to its | Version | Status | Codename | Compatibility | Summary | |---|---|---|---|---| -| **[0.7.6](/fluid/releases/0.7.6)** | **Preview** | Declarative Packaging Modes | โœ… Additive over 0.7.5 | Adds top-level `packaging` and `consumers`, cross-mesh pins on `consumes[]` (`upstreamWorkspace` + `upstreamDigest`), `aggParams` on semantic measures, and on bindings `encryption.kms`, `principals` and Lake Formation `bucketPolicy`; an expose's `lifecycle` gains `expire`, and masking rules gain typed `params`. | +| **[0.7.6](/fluid/releases/0.7.6)** | **Preview** | Declarative Packaging Modes | โœ… Additive over 0.7.5 | Adds top-level `packaging` and `consumers`, cross-mesh pins on `consumes[]` (`upstreamWorkspace` + `upstreamDigest`), `aggParams` on semantic measures, and on bindings `encryption.kms`, `principals` and Lake Formation `bucketPolicy`; an expose's `lifecycle` gains `expire`, masking rules gain typed `params`, and access policies gain `rowFilters` and `access: mask`. Builds can declare `engine: flink`, and an Iceberg sink can name a REST catalog's credentials by environment variable (`sink.catalogAuth`). | | **[0.7.5](/fluid/releases/0.7.5)** | **Stable (latest)** | Streaming Kafka โ†’ Iceberg Sink & Confluent Tableflow | โœ… Additive over 0.7.4 | Adds an opt-in Iceberg streaming sink on the `kafka-connect` acquisition engine, the `confluent` (Tableflow) and `pgvector` binding platforms, the `pgvector_table` format and `binding.vectorConfig`, and location fields for Tableflow, Iceberg catalogs, Redshift Serverless and Kinesis. | | **[0.7.4](/fluid/releases/0.7.4)** | Stable | Runtime agentPolicy Enforcement at the MCP Gateway | โœ… Additive over 0.7.3 | Adds the optional `exposes[].mcp` block, the `postgres` binding platform, and the `postgres_table` / `athena_table` / `glue_table` formats. | | **[0.7.3](/fluid/releases/0.7.3)** | Stable | Source-Aligned Acquisition | โœ… Additive over 0.7.2 | The `acquisition` build pattern and six ingestion engines, top-level `retention`, `governance` (AWS Lake Formation) and `extensions`, `binding.governance`, `metadata.productType`, and `exposes[].contract.schemaPolicy`. | diff --git a/docs/schema/changelog.md b/docs/schema/changelog.md index 75ee722..cf3cb98 100644 --- a/docs/schema/changelog.md +++ b/docs/schema/changelog.md @@ -21,6 +21,11 @@ No narrowing change. **Preview:** the 0.7.6 schema can still change before it is - `exposes[].binding.governance.lakeFormation.bucketPolicy` (`cross-account` | `none` | `all-grantees`; `cross-account` is the default). Its description says that an emitted `aws_s3_bucket_policy` is authoritative and replaces every other statement on the bucket; see the [0.7.6 notes](/fluid/releases/0.7.6#binding-fields-encryption-principals-lake-formation-bucket-policy). - `exposes[].lifecycle` now refers to a new `exposeLifecycle` definition: the root `lifecycle` fields plus `expire` (default `false`). - Typed `exposes[].policy.privacy.masking[].params`: `keepFirst`, `keepLast`, `saltEnv`, `keyEnv`. +- `exposes[].policy.authz.rowFilters[]` (`principal` and `where` required; `name` a lowercase identifier; `description`). +- `exposes[].policy.authz.columnRestrictions[].access` value `mask`, with `columnRestrictions[].mask` (`last_four` | `first_four` | `nullify`), required with `access: mask` and refused without it. +- `exposes[].binding.governance.lakeFormation.maskedViews` (`definer` and `validationConnection`, both required). +- `build(s).engine` value `flink`; see the [0.7.6 notes](/fluid/releases/0.7.6#engine-flink). +- `build(s).properties.sink.catalog` values `lakekeeper` and `polaris`, and `build(s).properties.sink.catalogAuth` (`type`: `oauth2` | `bearer`), which names the environment variables holding a REST catalog's credentials. - `fluidVersion` accepts `"0.7.6"`. **Described:** `exposes[].policy.authz.columnRestrictions` gains a description of its semantics (deny beats allow; a restriction never grants access; principals are logical). @@ -147,3 +152,4 @@ No narrowing change. Note that 0.1.0 and 0.1.1 declare the same `$id` (see [Vers - **Lake Formation `admins` (0.7.3, 0.7.4, 0.7.5).** These schemas were first published with a description of `governance.lakeFormation.admins` saying that principals not listed are not removed. That is the opposite of what applying it does: the list is **authoritative**, and an admin not listed โ€” including the identity running the apply โ€” loses Lake Formation admin. The description was corrected in the reference implementation and the published files carry the corrected text. Only the description changed; the same documents validate. - **0.7.5** was re-published with the stable content (above). +- **Lake Formation tags (0.7.3, 0.7.4, 0.7.5, 0.7.6).** These schemas were first published with a description of `exposes[].binding.governance.lakeFormation.tags` saying that its keys must reference tag definitions in the contract's top-level `governance.lakeFormation.tagDefinitions`. The schema never required that, and a key the contract does not define names a tag owned outside it, such as a platform's central LF-tag ontology, which must already exist in the account. The descriptions of `tags` and `tagDefinitions` were corrected in the reference implementation, and the published files carry the corrected text. Only the descriptions changed; the same documents validate.