From 20cb1a87c15331e606f805f85970577752c828f2 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:34:09 +0100 Subject: [PATCH] chore: re-vendor the 0.7.7 preview from forge-cli v0.31.0 The vendored 0.7.7 schema was the v0.27.0 copy. Replace it with the reference implementation's current file, byte for byte, and regenerate what derives from it: specs/0.7.7/ and schema-diffs/diff-0.7.6-to-0.7.7.md. 0.7.7 gained rules about valid data (invariants, identifier formats, placeholders, shared attributes), relationships and validity windows, a schema registry definition, stream trigger/watermark/checkpoint/offsets, maskAtIngest, parameters.spark, the lakeflow engine, six packaging container kinds, Protobuf field numbers, and a per-scheme secretRef grammar. 0.7.6 is unchanged. The secretRef grammar is the one narrowing against 0.7.6: a vault:// reference needs a #field. The 0.7.7 acquisition cases use the new form, the narrowing is recorded in scripts/compat-waivers.txt, and meta_test keeps the waiver file sorted when it adds its own entry. The release note and changelog say so. upstream.ref is v0.31.0 and upstream.commit is the tag's peeled commit, 0b72fb1c327ae33acddfdf8bbf9f5d1e7849d19c. data-product-forge 0.31.0 validates the release note's complete contract and agrees with the corpus. --- docs/releases/0.7.7.md | 52 +- docs/releases/README.md | 2 +- docs/schema/changelog.md | 18 +- schema-diffs/diff-0.7.6-to-0.7.7.md | 621 +- schema/fluid-schema-0.7.7.json | 1493 +- scripts/compat-waivers.txt | 24 + scripts/schema-versions.json | 4 +- specs/0.7.7/fluid-spec.html | 36501 +++++++++++++++++++------- tests/0.7.7/acquisition.json | 6 +- tests/meta_test.py | 5 +- 10 files changed, 28951 insertions(+), 9775 deletions(-) diff --git a/docs/releases/0.7.7.md b/docs/releases/0.7.7.md index 49a7d49..91540bf 100644 --- a/docs/releases/0.7.7.md +++ b/docs/releases/0.7.7.md @@ -4,26 +4,48 @@ 0.7.7 is a **preview**. A contract uses it only by declaring `fluidVersion: "0.7.7"`, and its fields and their meaning can still change before it is promoted to stable. The latest stable version is [0.7.6](/fluid/releases/0.7.6). See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). ::: -0.7.7 lets a column declare nested data (structs, arrays and maps, to any depth) and lets an embedded-logic build describe the Kafka records it reads and how they become rows, in place of a SQL statement. +0.7.7 lets a column declare nested data (structs, arrays and maps, to any depth) and lets an embedded-logic build describe the Kafka records it reads and how they become rows, in place of a SQL statement. Since its first publication here it has also gained rules about valid data (invariants written in FLUID SQL/1), identifier formats, placeholder values and shared attributes, named relationships and the validity window of a slowly changing dimension, a schema registry definition, Spark builds, the `lakeflow` engine, and a stricter grammar for secret references. - **Schema:** [`fluid-schema-0.7.7.json`](/fluid/schema/fluid-schema-0.7.7.json) · **Reference:** [`specs/0.7.7/fluid-spec.html`](/fluid/specs/0.7.7/fluid-spec.html) · **Diff:** [`diff-0.7.6-to-0.7.7.md`](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.6-to-0.7.7.md) -- **Compatibility:** additive over 0.7.6. `scripts/check-compat.py --from 0.7.6 --to 0.7.7` reports no narrowing change, so a valid 0.7.6 contract stays valid when it declares `"0.7.7"`. +- **Compatibility:** additive over 0.7.6 except for one narrowing, in secret references (see [Secret references](#secret-references)). A valid 0.7.6 contract stays valid when it declares `"0.7.7"` unless it names a `vault://` secret without a `#field`. `scripts/check-compat.py --from 0.7.6 --to 0.7.7` reports that one case, and it is recorded as a waiver in `scripts/compat-waivers.txt`. --- ## What is new -Every addition below is optional. These are the property paths present in 0.7.7 and absent in 0.7.6. +Every addition below is optional. These are the property paths present in 0.7.7 and absent in 0.7.6. The text of each comes from the schema's own descriptions, which are the place to read the rules in full. | Path | What it declares | |---|---| | `schema[].fields`, `schema[].element`, `schema[].key`, `schema[].value` | The parts of a nested column: the child columns of a struct, the element type of an array, the key and value types of a map. They are available wherever a column is: `exposes[].contract.schema` and `stream.schema`. | | `build(s).properties.stream` | The Kafka records a build reads: topic, value format, value fields, an optional key and Kafka metadata. | | `build(s).properties.mapping` | How the stream's records become the output's rows: columns read from another path, renamed or cast, and an explode over an array. | +| `build(s).properties.stream.registry`, `binding.properties.registry` | A `schemaRegistry` definition: the registry that holds a topic's record schema, Glue or Confluent. | +| `build(s).properties.stream.trigger`, `.watermark`, `.checkpoint`, `.startingOffsets` | When the engine starts a micro-batch, the event-time watermark, where offsets and state are kept (an `s3://` URI), and where a job with no checkpoint starts reading. | +| `build(s).properties.mapping.columns[].maskAtIngest` | Opt-in `hash` of a mapped column before it lands. | +| `build(s).properties.parameters.spark` | Where and how a Spark build runs: `runtime` (`emr-serverless`, `emr-ec2`, `emr-eks` or `glue`), `mode`, the application or cluster, release, role, `sparkConf` and the Kafka `source`. | +| `build(s).engine` value `lakeflow` | A Databricks Lakeflow Declarative Pipeline: `properties.sql` is the `SELECT` of the dataset an expose binds. | +| `packaging.containers` keys `lfTagVocabulary`, `emrApplication`, `schemaRegistry`, `athenaWorkgroup`, `kmsKey`, `mwaaEnvironment` | Ownership of six AWS container kinds. `mwaaEnvironment` is shared only. | +| `exposes[].contract.invariants[]` | Rules about valid data: a `check`, a `derive` of a column (with `expr`, or with an `aggregate` over child rows) or a functional `dependency`, each with an optional `when` and a band of violating rows. | +| `schema[].identifierFormat`, `schema[].placeholders`, `schema[].sharedAttribute` | The identifier formats of a string column, its placeholder values, and the attribute it shares with other columns of the contract. | +| `schema[].protobuf` | The column's field number (`field`) when the topic's value is Protobuf. | +| `exposes[].semantics.relationships[]`, with `sharedColumns` | Named joins from one expose to another, composite keys included, and columns copied through a relationship. | +| `exposes[].semantics.dimensions[].typeParams.validityParams` | Marks a time dimension as one bound of the validity window of a type II slowly changing dimension. The entity type `natural` is described as that dimension's key. | +| `secretRef` (shared `$defs/secretRef`), `connection.password`, `auth.token` | A per-scheme grammar for secret references, and two write-only members that name what a contract may not hold. | -`fluidVersion` accepts `"0.7.7"` in addition to `"0.7.3"` to `"0.7.6"`. No enum gains a value. +`fluidVersion` accepts `"0.7.7"` in addition to `"0.7.3"` to `"0.7.6"`. `build.engine` gains `lakeflow`. -One existing rule changes shape. Through 0.7.6, an embedded-logic build's `properties` always required `sql`. In 0.7.7 it requires `sql` unless the build has a `stream` and its `parameters.flink` declares `management`; the build is then described by the stream and the mapping alone. A `mapping` without a `stream` is refused. Every document that satisfied the old rule satisfies the new one. +One existing rule changes shape. Through 0.7.6, an embedded-logic build's `properties` always required `sql`. In 0.7.7 it requires `sql` unless the build has a `stream` and either its `parameters.flink` declares `management`, or it has a `mapping` and a `parameters.spark`; the build is then described by the stream and the mapping alone. A `mapping` without a `stream` is refused. Every document that satisfied the old rule satisfies the new one. + +The schema carries a second change of meaning, in `secretRef`. See the next section. + +--- + +## Secret references + +`secretRef` is now one shared definition, `$defs/secretRef`, used by an acquisition source's `connection`, by the `auth` of an acquisition build's `deployment`, and by the `auth` of the `kafka-connect` `schema_registry`. It names a secret held in a store and never carries it. The grammar is `://[#field][?version=n]`. The schemes are `env`, `file`, `vault`, `aws-sm`, `gcp-sm` and `azure-kv`, with the aliases `openbao` (vault), `aws`, `gcp` and `azure`. Any other lower-case scheme, such as `op://`, is passed to the pack that resolves it. `#field` picks one key of a secret that holds several, and `?version=` pins a version where the store keeps them. Each scheme has its own pattern in the schema. + +Through 0.7.6 and in the first copy of 0.7.7 published here, a `secretRef` was any string of the form `://`. That is the one place 0.7.7 accepts less than 0.7.6: a `vault://` reference now needs a `#field`, so `vault://secret/data/crm-db` is refused and `vault://secret/data/crm-db#password` is valid. A reference with another scheme or with a `#field` is unaffected. `connection.password` and `auth.token` are declared `writeOnly`, with the description that a contract may not hold the value and names the secret with `secretRef` instead. --- @@ -142,9 +164,27 @@ build: --- +## Rules about valid data + +`exposes[].contract.invariants[]` declares what a valid row looks like, in FLUID SQL/1: a closed SQL expression subset with exact numbers and two-valued truth, at most 4,096 code points and 64 levels deep. An invariant has exactly one form: `check`, a predicate every covered row satisfies; `derive` with `expr`, a column's value from the row and the rows its links reach; `derive` with `aggregate`, a column's value as a summary over child rows (`function`, `column`, `from`, `via`, `filter`, `ifNone`); or `dependency`, where equal `from` values mean equal `to` values. `when` narrows the rows it covers, `severity` is `error`, `warning` or `info`, and one `mustBe*` operator with a `unit` of `rows` or `percent` bounds the violating rows. Without one, the invariant holds only with no violating row. SQL outside the subset belongs in `contract.quality[]`. + +A string column can also declare `identifierFormat`, a list of alternatives, each a mask (`9` a digit, `A` an upper-case letter, `X` either) with optional `prefixes`; `placeholders`, the values that stand in for a real value not known yet, which are exempt from the column's own value checks; and `sharedAttribute`, the attribute it carries with every other column that names the same one. Alternatives of one format must not overlap. A relationship's `sharedColumns` lists columns copied through it, each in both exposes' schemas under one name and of one type. + +The schema asks only for a text. `fluid validate` in the reference implementation parses it and type-checks it against the expose (non-normative). + +--- + +## Schema registries and Spark builds + +`$defs/schemaRegistry` is one definition of the registry that holds a topic's record schema: `kind` is `glue` (the AWS Glue Schema Registry, with IAM authentication and so no secret) or `confluent` (the Confluent Schema Registry, or one that serves its REST API), with `url`, `registryName`, `subject`, `version`, `compatibility`, `auth` and `rules`. It appears at `stream.registry` and at `binding.properties.registry`. Who owns the subject decides what happens to it: a stream a build reads belongs to its producer, so the subject is verified against the declared schema and never written, and a topic an expose produces has its subject registered with the declared compatibility. The older spellings, `kafka-connect`'s `schema_registry` and a Flink build's `parameters.flink.source.schemaRegistryUrl`, stay valid and read as this definition with `kind: confluent`. + +`parameters.spark` describes where a Spark build runs, on an embedded-logic build with `engine: spark`. `runtime` is required: `emr-serverless`, `emr-ec2`, `emr-eks` or `glue`. The build declares what it computes with `stream` and `mapping`, or with `sql` as Spark SQL; there is no entry point for user PySpark. `execution` keeps when the build runs and what happens when it fails, and `parameters.spark` keeps where Spark runs. `maskAtIngest: hash` on a mapped column needs `parameters.spark.maskKeySecretArn`, the Secrets Manager secret that holds the HMAC key; the schema declares that member and the check that links them lives in the Spark compiler. + +--- + ## A complete contract -The reference implementation can deploy a managed Flink job for a build like this one: a Kubernetes cluster that runs the Flink Kubernetes Operator, writing an Iceberg REST catalog table. That is behaviour of one implementation (non-normative), and it needs operator-side allow-lists for the cluster context, namespace, image, service account and secrets the contract names. The contract below validates against the 0.7.7 JSON Schema (Draft 2020-12) and with `data-product-forge` 0.27.0 (`fluid validate` prints `✅ Valid FLUID contract (schema v0.7.7)`). `fluid validate` refuses a managed Flink build that has no build `id` or no Iceberg output named in `outputs`, so the contract carries both. +The reference implementation can deploy a managed Flink job for a build like this one: a Kubernetes cluster that runs the Flink Kubernetes Operator, writing an Iceberg REST catalog table. That is behaviour of one implementation (non-normative), and it needs operator-side allow-lists for the cluster context, namespace, image, service account and secrets the contract names. The contract below validates against the 0.7.7 JSON Schema (Draft 2020-12) and with `data-product-forge` 0.31.0 (`fluid validate` prints `✅ Valid FLUID contract (schema v0.7.7)`). `fluid validate` refuses a managed Flink build that has no build `id` or no Iceberg output named in `outputs`, so the contract carries both. ```yaml fluidVersion: "0.7.7" diff --git a/docs/releases/README.md b/docs/releases/README.md index 8caaea6..3e93cf9 100644 --- a/docs/releases/README.md +++ b/docs/releases/README.md @@ -6,7 +6,7 @@ Release notes for the FLUID specification, newest first. Each entry links to its | Version | Status | Codename | Compatibility | Summary | |---|---|---|---|---| -| **[0.7.7](/fluid/releases/0.7.7)** | **Preview** | Recursive Streaming Types | ✅ Additive over 0.7.6 | Adds nested columns (`fields`, `element`, `key` and `value` on a column), and on an embedded-logic build the Kafka `stream` it reads and the `mapping` that turns its records into rows. `sql` is no longer required when a managed Flink build has a stream. | +| **[0.7.7](/fluid/releases/0.7.7)** | **Preview** | Recursive Streaming Types | ⚠️ Additive over 0.7.6, except that a `vault://` secret reference needs a `#field` | Adds nested columns (`fields`, `element`, `key` and `value` on a column), and on an embedded-logic build the Kafka `stream` it reads and the `mapping` that turns its records into rows, a schema registry definition, Spark builds and the `lakeflow` engine. Adds rules about valid data (`invariants`), identifier formats, placeholders and shared attributes, and named relationships. `sql` is no longer required when a managed Flink or Spark build has a stream. | | **[0.7.6](/fluid/releases/0.7.6)** | **Stable (latest)** | Declarative Packaging Modes | ✅ Additive over 0.7.5 | Adds top-level `packaging` and `consumers`, cross-mesh pins on `consumes[]` (`upstreamWorkspace` + `upstreamDigest`), `aggParams` on semantic measures, and on bindings `encryption.kms`, `principals` and Lake Formation `bucketPolicy`; an expose's `lifecycle` gains `expire`, masking rules gain typed `params`, and access policies gain `rowFilters` and `access: mask`. Builds can declare `engine: flink` and a `continuous` trigger, and an Iceberg sink can name a REST catalog's credentials by environment variable (`sink.catalogAuth`). Optional Bitol-shaped fields: `metadata.team`, an object root `description`, `consumes[].contractId` and `contractVersion`, `exposes[].contract.id`, column `quality`, `binding.location.host` and `binding.serialization`. | | **[0.7.5](/fluid/releases/0.7.5)** | Stable | Streaming Kafka → Iceberg Sink & Confluent Tableflow | ✅ Additive over 0.7.4 | Adds an opt-in Iceberg streaming sink on the `kafka-connect` acquisition engine, the `confluent` (Tableflow) and `pgvector` binding platforms, the `pgvector_table` format and `binding.vectorConfig`, and location fields for Tableflow, Iceberg catalogs, Redshift Serverless and Kinesis. | | **[0.7.4](/fluid/releases/0.7.4)** | Stable | Runtime agentPolicy Enforcement at the MCP Gateway | ✅ Additive over 0.7.3 | Adds the optional `exposes[].mcp` block, the `postgres` binding platform, and the `postgres_table` / `athena_table` / `glue_table` formats. | diff --git a/docs/schema/changelog.md b/docs/schema/changelog.md index f6ac007..7a10036 100644 --- a/docs/schema/changelog.md +++ b/docs/schema/changelog.md @@ -8,7 +8,7 @@ Human-readable summaries of what changed between consecutive versions of the FLU [Full diff →](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.6-to-0.7.7.md) · [Release note →](/fluid/releases/0.7.7) -No narrowing change. **Preview:** the 0.7.7 schema can still change before it is promoted. +One narrowing, in secret references (see **Changed**); everything else is additive. **Preview:** the 0.7.7 schema can still change before it is promoted. The file published here was re-vendored from forge-cli v0.31.0; it was first published from v0.27.0, and the items under "Added since the v0.27.0 copy" came after that. **Added** @@ -16,8 +16,20 @@ No narrowing change. **Preview:** the 0.7.7 schema can still change before it is - `build(s).properties.stream`, a new closed `streamSource` definition: `kind` (`kafka`), `topic`, `format` (`json` | `avro-confluent`) and `schema` required; `key` (`format` `raw` | `json` | `avro-confluent`, `schema`, `prefix`) and `metadata[]` (`name`, `from`: `topic` | `partition` | `offset` | `timestamp` | `timestamp-type` | `headers` | `leader-epoch`) optional. - `build(s).properties.mapping`, a new closed `recordMapping` definition: `explode` (`path` and `as`, both required) and `columns[]` (`name` and `from` required; `cast` optional). - `fluidVersion` accepts `"0.7.7"`. - -**Changed:** `embeddedLogicPattern` no longer lists `sql` as unconditionally required. It requires `sql` unless the build has a `stream` and `parameters.flink` declares `management`, and `mapping` requires `stream` (`dependentRequired`). Every document that carried `sql` is unaffected. +- Added since the v0.27.0 copy: + - Rules about valid data: `exposes[].contract.invariants[]` (the new `invariant`, `invariantAggregate` and `invariantDependency` definitions, with `fluidSqlText` and `violationBound`), and on a column `identifierFormat`, `placeholders` and `sharedAttribute`. + - `exposes[].semantics.relationships[]` (a new `semanticRelationship` definition, with `sharedColumns`) and `dimensions[].typeParams.validityParams`; the entity type `natural` gains a description. + - `$defs/schemaRegistry` (`kind` `glue` | `confluent`), used at `stream.registry` and `binding.properties.registry`; `stream.trigger`, `stream.watermark`, `stream.checkpoint` and `stream.startingOffsets`. + - `mapping.columns[].maskAtIngest` (`hash`), and `embeddedLogicPattern.properties.parameters.spark`, a new `sparkParameters` definition (`runtime` required). + - `column.protobuf.field`, for a topic whose value is Protobuf. + - `build.engine` value `lakeflow`. + - `packaging.containers` keys `lfTagVocabulary`, `emrApplication`, `schemaRegistry`, `athenaWorkgroup`, `kmsKey` and `mwaaEnvironment`. + - `connection.password` and `auth.token` as `writeOnly` members, on an acquisition source and on an acquisition build's `deployment.auth`. + +**Changed:** + +- `embeddedLogicPattern` no longer lists `sql` as unconditionally required. It requires `sql` unless the build has a `stream` and either `parameters.flink` declares `management`, or it has a `mapping` and `parameters.spark`; `mapping` requires `stream` (`dependentRequired`). Every document that carried `sql` is unaffected. +- `secretRef` (an acquisition source's `connection`, the `auth` of an acquisition build's `deployment`, and the `kafka-connect` `schema_registry.auth`) is now the shared `$defs/secretRef`: one pattern per scheme (`env`, `file`, `vault`, `aws-sm`, `gcp-sm`, `azure-kv`, and other lower-case schemes). It was any string of the form `://`. This is the narrowing: a `vault://` reference now needs a `#field`, so `vault://secret/data/crm-db` is refused and `vault://secret/data/crm-db#password` is valid. `scripts/check-compat.py` reports it and `scripts/compat-waivers.txt` records it. See the [0.7.7 notes](/fluid/releases/0.7.7#secret-references). ## 0.7.5 to 0.7.6: Declarative Packaging Modes diff --git a/schema-diffs/diff-0.7.6-to-0.7.7.md b/schema-diffs/diff-0.7.6-to-0.7.7.md index 908fa83..d424474 100644 --- a/schema-diffs/diff-0.7.6-to-0.7.7.md +++ b/schema-diffs/diff-0.7.6-to-0.7.7.md @@ -1,14 +1,57 @@ # Schema Changes: 0.7.6 → 0.7.7 -**Total changes:** 22 -- ✅ Added: 16 -- ❌ Removed: 1 -- 📝 Modified: 5 +**Total changes:** 61 +- ✅ Added: 45 +- ❌ Removed: 5 +- 📝 Modified: 11 --- ## ✅ Added Properties +### `$defs.acquisitionDeployment.properties.auth.properties.secretRef.$ref` +```json +"#/$defs/secretRef" +``` + +### `$defs.acquisitionDeployment.properties.auth.properties.token` +```json +{ + type: "string" + writeOnly: true + description: "Write-only: a contract may not hold this value. Name the secret with secretRef instead." +} +``` + +### `$defs.acquisitionPattern.properties.kafka-connect.properties.schema_registry.properties.auth.properties.secretRef.$ref` +```json +"#/$defs/secretRef" +``` + +### `$defs.acquisitionSource.properties.connection.properties.password` +```json +{ + type: "string" + writeOnly: true + description: "Write-only: a contract may not hold this value. Name the secret with secretRef instead." +} +``` + +### `$defs.acquisitionSource.properties.connection.properties.secretRef.$ref` +```json +"#/$defs/secretRef" +``` + +### `$defs.binding.properties.properties.properties` +```json +{ + registry: { + $ref: "#/$defs/schemaRegistry" + description: "NEW in v0.7.7 (preview): The registry a produced topic's subject is registered in, on a Kafka bindin..." + } +} +``` + ### `$defs.column.properties.element` ```json { @@ -28,6 +71,39 @@ } ``` +### `$defs.column.properties.identifierFormat` +```json +{ + type: "array" + minItems: 1 + description: "NEW in v0.7.7 (preview): The identifier formats of this string column (RFC-data-rules.md section 3),..." + items: { + type: "object" + additionalProperties: false + required: ["mask"] + properties: { + prefixes: { + type: "array" + minItems: 1 + uniqueItems: true + items: { + type: "string" + pattern: "^[ -~]*$" + } + description: "Literal prefixes, printable ASCII. One empty prefix when left out." + } + mask: { + type: "string" + minLength: 1 + maxLength: 64 + description: "COBOL PICTURE style: 9 is a digit, A an upper-case ASCII letter, X an upper-case ASCII letter or a d..." + } + } + } + examples: [[{"prefixes": ["MSC", "TWR"], "mask": "99999"}, {"prefixes": ["LEGACY-"], "mask": "A9-99AA"}]] +} +``` + ### `$defs.column.properties.key` ```json { @@ -35,6 +111,82 @@ } ``` +### `$defs.column.properties.placeholders` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): Placeholder values of this string column: stand-ins for a real value that i..." + properties: { + values: { + type: "array" + minItems: 1 + uniqueItems: true + items: { + type: "string" + minLength: 1 + } + description: "Whole values that are placeholders." + } + prefixes: { + type: "array" + minItems: 1 + uniqueItems: true + items: { + type: "string" + minLength: 1 + } + description: "A value that starts with one of these is a placeholder." + } + mustBe: { + $ref: "#/$defs/violationBound" + description: "Exactly this many placeholder rows." + } + mustBeLessThan: { + $ref: "#/$defs/violationBound" + description: "Fewer placeholder rows than this." + } + mustBeLessOrEqualTo: { + $ref: "#/$defs/violationBound" + description: "At most this many placeholder rows." + } + ... (4 more) + } + examples: [{"values": ["XXXXX"], "prefixes": ["TBD_"], "mustBeLessOrEqualTo": 1, "unit": "percent"}] +} +``` + +### `$defs.column.properties.protobuf` +```json +{ + type: "object" + additionalProperties: false + required: ["field"] + properties: { + field: { + type: "integer" + minimum: 1 + maximum: 536870911 + not: { + minimum: 19000 + maximum: 19999 + } + description: "The column's field number in the .proto forge generates when the topic's value is Protobuf (binding...." + } + } + description: "NEW in v0.7.7 (preview): How this column is written when the topic's value is Protobuf (binding.seri..." +} +``` + +### `$defs.column.properties.sharedAttribute` +```json +{ + type: "string" + pattern: "^[A-Za-z][A-Za-z0-9_.]*$" + description: "NEW in v0.7.7 (preview): The attribute this column carries with every other column of the contract t..." +} +``` + ### `$defs.column.properties.value` ```json { @@ -53,17 +205,7 @@ ```json { not: { - required: ["stream", "parameters"] - properties: { - parameters: { - required: ["flink"] - properties: { - flink: { - required: ["management"] - } - } - } - } + anyOf: [{"required": ["stream", "parameters"], "properties": {"parameters": {"required": ["flink"], "properties": {"flink": {"required": ["management"]}}}}}, {"required": ["stream", "mapping", "parameters"], "properties": {"parameters": {"required": ["spark"]}}}] } } ``` @@ -76,6 +218,16 @@ } ``` +### `$defs.embeddedLogicPattern.properties.parameters.properties` +```json +{ + spark: { + $ref: "#/$defs/sparkParameters" + description: "NEW in v0.7.7 (preview): Where and how a Spark build runs." + } +} +``` + ### `$defs.embeddedLogicPattern.properties.stream` ```json { @@ -91,6 +243,130 @@ } ``` +### `$defs.exposeContract.properties.invariants` +```json +{ + type: "array" + description: "NEW in v0.7.7 (preview): Rules about valid data (RFC-data-rules.md): checks, derived columns, summar..." + items: { + $ref: "#/$defs/invariant" + } +} +``` + +### `$defs.fluidSqlText` +```json +{ + type: "string" + minLength: 1 + description: "NEW in v0.7.7 (preview): A FLUID SQL/1 text (RFC-data-rules.md section 2): a closed SQL expression s..." +} +``` + +### `$defs.invariant` +```json +{ + type: "object" + description: "NEW in v0.7.7 (preview): A rule about valid data (RFC-data-rules.md section 1). Exactly one form: ch..." + additionalProperties: false + required: ["id"] + properties: { + id: { + type: "string" + pattern: "^[A-Za-z][A-Za-z0-9_]*$" + description: "Unique within the contract, across every expose and kind, and against dq.rules[].id and contract.qua..." + } + description: { + type: "string" + } + check: { + $ref: "#/$defs/fluidSqlText" + description: "A FLUID SQL/1 predicate over the row." + } + derive: { + type: "string" + minLength: 1 + description: "The column this invariant determines, with exactly one of expr and aggregate. Several derives of one..." + } + expr: { + $ref: "#/$defs/fluidSqlText" + description: "With derive: the column's value, a FLUID SQL/1 expression of the column's type whose static scale fi..." + } + ... (11 more) + } + ... (2 more) +} +``` + +### `$defs.invariantAggregate` +```json +{ + type: "object" + description: "NEW in v0.7.7 (preview): A summary over child rows (RFC-data-rules.md section 1.2): SQL's aggregate,..." + additionalProperties: false + required: ["function", "from", "via"] + properties: { + function: { + type: "string" + enum: [...4 items...] + } + column: { + type: "string" + minLength: 1 + description: "The child column summarized: none for count, a number of bounded scale for sum, any orderable type f..." + } + from: { + $ref: "#/$defs/identifier" + description: "The child expose, an exposeId of this contract." + } + via: { + type: "string" + minLength: 1 + description: "A relationship the child declares (its semantics.relationships[].name) whose to is this expose." + } + filter: { + $ref: "#/$defs/fluidSqlText" + description: "A FLUID SQL/1 predicate over the child row: only children for which it is TRUE count, as in SUM(x) F..." + } + ... (1 more) + } +} +``` + +### `$defs.invariantDependency` +```json +{ + type: "object" + description: "NEW in v0.7.7 (preview): A functional dependency (RFC-data-rules.md section 1.2): equal values in fr..." + additionalProperties: false + required: ["from", "to"] + properties: { + from: { + type: "array" + minItems: 1 + uniqueItems: true + items: { + type: "string" + minLength: 1 + } + } + to: { + type: "array" + minItems: 1 + uniqueItems: true + items: { + type: "string" + minLength: 1 + } + } + bothWays: { + type: "boolean" + default: false + } + } +} +``` + ### `$defs.nestedType` ```json { @@ -122,6 +398,54 @@ } ``` +### `$defs.packaging.properties.containers.properties.athenaWorkgroup` +```json +{ + $ref: "#/$defs/packagingContainerMode" + description: "NEW in v0.7.7 (preview): Ownership of the Athena workgroup (aws_athena_workgroup) the product's quer..." +} +``` + +### `$defs.packaging.properties.containers.properties.emrApplication` +```json +{ + $ref: "#/$defs/packagingContainerMode" + description: "NEW in v0.7.7 (preview): Ownership of the EMR Serverless application (aws_emrserverless_application)..." +} +``` + +### `$defs.packaging.properties.containers.properties.kmsKey` +```json +{ + $ref: "#/$defs/packagingContainerMode" + description: "NEW in v0.7.7 (preview): Ownership of the KMS key (aws_kms_key) that encrypts the product's data, ch..." +} +``` + +### `$defs.packaging.properties.containers.properties.lfTagVocabulary` +```json +{ + $ref: "#/$defs/packagingContainerMode" + description: "NEW in v0.7.7 (preview): Ownership of the Lake Formation tag vocabulary (aws_lakeformation_lf_tag). ..." +} +``` + +### `$defs.packaging.properties.containers.properties.mwaaEnvironment` +```json +{ + const: "shared" + description: "NEW in v0.7.7 (preview): The MWAA environment (aws_mwaa_environment) is the platform team's, so shar..." +} +``` + +### `$defs.packaging.properties.containers.properties.schemaRegistry` +```json +{ + $ref: "#/$defs/packagingContainerMode" + description: "NEW in v0.7.7 (preview): Ownership of the Glue schema registry (aws_glue_registry) that holds produc..." +} +``` + ### `$defs.recordExplode` ```json { @@ -191,7 +515,169 @@ description: { type: "string" } + maskAtIngest: { + enum: ["hash"] + description: "NEW in v0.7.7 (preview): Mask the value before it lands. hash = HMAC-SHA256 keyed by parameters.spar..." + } + } +} +``` + +### `$defs.schemaRegistry` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): The schema registry that holds a topic's record schema, one definition for ..." + required: ["kind"] + properties: { + kind: { + enum: ["glue", "confluent"] + description: "glue: AWS Glue Schema Registry. confluent: Confluent Schema Registry REST API." + } + url: { + type: "string" + format: "uri" + pattern: "^https?://[^/?#@\\s]+(?:/[^?#\\s]*)?(?![\\s\\S])" + description: "kind confluent only, and required there: the registry's REST endpoint, http or https, with no creden..." + } + registryName: { + type: "string" + pattern: "^[A-Za-z0-9_$#.-]{1,255}(?![\\s\\S])" + description: "kind glue only: the Glue registry's name. Absent: default-registry, the registry Glue uses when none..." + } + subject: { + type: "string" + pattern: "^[^\\u0000-\\u001f\\u007f]{1,255}(?![\\s\\S])" + description: "The subject (confluent) or schema name (glue) that holds the record value's schema. Absent: -..." + } + version: { + oneOf: [{"const": "latest"}, {"type": "integer", "minimum": 1}] + default: "latest" + description: "The schema version to verify against or read with: latest, or a version number from 1." + } + ... (3 more) + } + ... (1 more) +} +``` + +### `$defs.secretRef` +```json +{ + type: "string" + description: "NEW in v0.7.7: A reference to a secret held in a store, never the secret itself: a contract names a ..." + anyOf: [...7 items...] +} +``` + +### `$defs.semanticModel.properties.dimensions.items.properties.typeParams.properties.validityParams` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): Marks this time dimension as one bound of the validity window of a type II ..." + properties: { + isStart: { + type: "boolean" + description: "This dimension holds the time each version starts to apply." + } + isEnd: { + type: "boolean" + description: "This dimension holds the time each version stops applying." + } + } +} +``` + +### `$defs.semanticModel.properties.relationships` +```json +{ + type: "array" + description: "NEW in v0.7.7 (preview): Named joins from this expose to an expose of the same contract, declared on..." + items: { + $ref: "#/$defs/semanticRelationship" + } +} +``` + +### `$defs.semanticRelationship` +```json +{ + type: "object" + description: "NEW in v0.7.7 (preview): One foreign key, from the expose that declares it (the many side) to an exp..." + additionalProperties: false + required: [...5 items...] + properties: { + name: { + type: "string" + description: "Relationship name, unique within the expose (e.g., 'orders_to_customers')." + } + to: { + $ref: "#/$defs/identifier" + description: "exposeId of the target expose, in the same contract. It may be the declaring expose itself, for a se..." + } + fromColumns: { + type: "array" + minItems: 1 + items: { + type: "string" + } + description: "Foreign-key columns of the declaring expose, in key order." + } + toColumns: { + type: "array" + minItems: 1 + items: { + type: "string" + } + description: "Key columns of the target expose, in the order of fromColumns." + } + cardinality: { + type: "string" + enum: ["many_to_one", "one_to_one"] + description: "many_to_one: many rows of the declaring expose can match one target row (an ordinary foreign key). o..." + } + ... (2 more) } + ... (1 more) +} +``` + +### `$defs.sparkParameters` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): Where and how a Spark build runs, at builds[].properties.parameters.spark o..." + required: ["runtime"] + properties: { + runtime: { + enum: [...4 items...] + description: "The AWS service that runs the job: EMR Serverless, EMR on EC2, EMR on EKS or an AWS Glue Spark job. ..." + } + mode: { + enum: ["streaming", "batch"] + description: "streaming: a long-running Structured Streaming job. batch: a job that runs to completion. Absent: st..." + } + applicationId: { + type: "string" + pattern: "^[0-9a-z]{1,64}(?![\\s\\S])" + description: "emr-serverless only: an existing EMR Serverless application. Absent: the forge-managed application s..." + } + clusterId: { + type: "string" + pattern: "^j-[0-9A-Za-z]{1,254}(?![\\s\\S])" + description: "emr-ec2 only: a running EMR cluster (j-...). Absent: a transient cluster that ends with the job." + } + virtualClusterId: { + type: "string" + pattern: "^[0-9a-z]{1,64}(?![\\s\\S])" + description: "emr-eks only, and required there: the EMR on EKS virtual cluster." + } + ... (6 more) + } + ... (1 more) } ``` @@ -280,13 +766,42 @@ key: { $ref: "#/$defs/streamKey" } - ... (1 more) + ... (6 more) } } ``` +### `$defs.violationBound` +```json +{ + type: "number" + minimum: 0 + description: "NEW in v0.7.7 (preview): A number of rows, or a percentage (unit), read as an exact decimal from its..." +} +``` + ## ❌ Removed Properties +### `$defs.acquisitionDeployment.properties.auth.properties.secretRef.type` +```json +"string" +``` + +### `$defs.acquisitionPattern.properties.kafka-connect.properties.schema_registry.properties.auth.properties.secretRef.type` +```json +"string" +``` + +### `$defs.acquisitionSource.properties.connection.properties.secretRef.pattern` +```json +"^[a-z][a-z0-9_+.-]*://.+" +``` + +### `$defs.acquisitionSource.properties.connection.properties.secretRef.type` +```json +"string" +``` + ### `$defs.embeddedLogicPattern.required` ```json ["sql"] @@ -294,6 +809,78 @@ ## 📝 Modified Properties +### `$defs.acquisitionSource.properties.connection.properties.secretRef.description` + +**Before:** +```json +"Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://)." +``` + +**After:** +```json +"Reference to the connection's secret in the configured store, e.g. vault://kv/orgs/acme/pg#password ..." +``` + +### `$defs.acquisitionSource.properties.connection.properties.user.description` + +**Before:** +```json +"NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers ..." +``` + +**After:** +```json +"NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers ..." +``` + +### `$defs.build.properties.engine.anyOf` + +**Before:** +```json +[{"enum": ["dbt", "sql", "python", "spark", "glue", "custom", "duckdb", "airbyte", "meltano", "dlt", "kafka-connect", "debezium", "flink"]}, {"pattern": "^dbt-[a-z0-9]+([_-][a-z0-9]+)*$"}] +``` + +**After:** +```json +[{"enum": ["dbt", "sql", "python", "spark", "glue", "custom", "duckdb", "airbyte", "meltano", "dlt", "kafka-connect", "debezium", "flink", "lakeflow"]}, {"pattern": "^dbt-[a-z0-9]+([_-][a-z0-9]+)*$"}] +``` + +### `$defs.build.properties.engine.description` + +**Before:** +```json +"Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-quali..." +``` + +**After:** +```json +"Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-quali..." +``` + +### `$defs.packaging.properties.containers.description` + +**Before:** +```json +"NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over `mode` for that kind, yie..." +``` + +**After:** +```json +"NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over `mode` for that kind, yie..." +``` + +### `$defs.semanticModel.properties.entities.items.properties.type.description` + +**Before:** +```json +"Entity key type: primary (unique, complete), foreign (references another table), unique (unique, pos..." +``` + +**After:** +```json +"Entity key type: primary (unique, complete), foreign (references another table), unique (unique, pos..." +``` + ### `$id` **Before:** @@ -315,7 +902,7 @@ **After:** ```json -"Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 i..." +"Opt-in preview. Adds recursive struct fields, array elements and map key/value types, the lakeflow b..." ``` ### `properties.fluidVersion.description` diff --git a/schema/fluid-schema-0.7.7.json b/schema/fluid-schema-0.7.7.json index 393a2b9..c9f2e4a 100644 --- a/schema/fluid-schema-0.7.7.json +++ b/schema/fluid-schema-0.7.7.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", "title": "FLUID 0.7.7 — Preview: recursive streaming types", - "description": "Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 is unchanged.", + "description": "Opt-in preview. Adds recursive struct fields, array elements and map key/value types, the lakeflow build engine (Databricks Lakeflow Declarative Pipelines), and two semantics fields: named relationships between exposes, composite keys included (exposes[].semantics.relationships), and the validity window of a slowly changing dimension (dimensions[].typeParams.validityParams). For AWS streaming to Iceberg it adds one schemaRegistry definition (Glue or Confluent) at streamSource.registry, the stream's trigger, watermark, checkpoint and startingOffsets, an opt-in maskAtIngest hash on mapped columns, the Spark build's parameters.spark, and six AWS container kinds in packaging.containers. Also rules about valid data (RFC-data-rules.md): invariants written in FLUID SQL/1 (exposes[].contract.invariants), identifier formats, placeholder values and shared attributes (exposes[].contract.schema[].identifierFormat, .placeholders, .sharedAttribute) and columns copied through a relationship (relationships[].sharedColumns). Stable 0.7.6 is unchanged.", "type": "object", "additionalProperties": false, "required": [ @@ -377,6 +377,40 @@ } }, "$defs": { + "secretRef": { + "type": "string", + "description": "NEW in v0.7.7: A reference to a secret held in a store, never the secret itself: a contract names a secret and never carries one. Grammar: ://[#field][?version=n]. Schemes: env, file, vault, aws-sm, gcp-sm, azure-kv; aliases openbao (vault), aws (aws-sm), gcp (gcp-sm) and azure (azure-kv) are accepted; any other lower-case scheme (such as op://) is passed to the pack that resolves it. #field picks one key of a secret that holds several; ?version= pins a version where the store keeps them.", + "anyOf": [ + { + "title": "env", + "pattern": "^env://[A-Za-z_][A-Za-z0-9_]*$" + }, + { + "title": "file", + "pattern": "^file://(/(?!\\.{1,2}(/|$))[^/\\s#?\\x00-\\x1f\\x7f-\\x9f]+)+$" + }, + { + "title": "vault", + "pattern": "^(vault|openbao)://[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_@=+-][A-Za-z0-9_.@=+-]*)+#[A-Za-z0-9_.@-]+(\\?version=[1-9][0-9]*)?$" + }, + { + "title": "aws-sm", + "pattern": "^(aws-sm|aws)://(arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:[A-Za-z0-9/_+=.@-]+|[A-Za-z0-9/_+=.@-]{1,512})(#[A-Za-z0-9_.@-]+)?(\\?version=[A-Za-z0-9-]{32,64})?$" + }, + { + "title": "gcp-sm", + "pattern": "^(gcp-sm|gcp)://projects/[a-z0-9-]+/secrets/[A-Za-z0-9_-]{1,255}(/versions/([1-9][0-9]*|latest))?$" + }, + { + "title": "azure-kv", + "pattern": "^(azure-kv|azure)://[A-Za-z0-9-]{3,24}/[A-Za-z0-9-]{1,127}(\\?version=[0-9a-f]{32})?$" + }, + { + "title": "pack", + "pattern": "^(?!(env|file|vault|openbao|aws-sm|aws|gcp-sm|gcp|azure-kv|azure)://)[a-z][a-z0-9+.-]*://(?![^/#?]*:[^/#?]*@)[^\\s#?\\x00-\\x1f\\x7f-\\x9f]+(#[A-Za-z0-9_.@-]+)?(\\?version=[A-Za-z0-9._-]+)?$" + } + ] + }, "descriptionObject": { "type": "object", "additionalProperties": false, @@ -448,7 +482,7 @@ "containers": { "type": "object", "additionalProperties": false, - "description": "NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over `mode` for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket → aws_s3_bucket / google_storage_bucket; database → snowflake_database AND aws_glue_catalog_database; dataset → google_bigquery_dataset; schema → snowflake_schema; warehouse → snowflake_warehouse; cluster → Confluent environment/cluster (v1: shared only).", + "description": "NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over `mode` for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket → aws_s3_bucket / google_storage_bucket; database → snowflake_database AND aws_glue_catalog_database; dataset → google_bigquery_dataset; schema → snowflake_schema; warehouse → snowflake_warehouse; cluster → Confluent environment/cluster (v1: shared only). NEW in v0.7.7 (preview), AWS streaming kinds: lfTagVocabulary → aws_lakeformation_lf_tag (the domain's LF-tag keys and values); emrApplication → aws_emrserverless_application; schemaRegistry → aws_glue_registry; athenaWorkgroup → aws_athena_workgroup; kmsKey → aws_kms_key (the domain key); mwaaEnvironment → aws_mwaa_environment (shared only: the platform team owns it, and a product ships DAG files into it). These kinds are not part of the blanket mode's plan summary; a product reads each one's decision explicitly.", "properties": { "bucket": { "$ref": "#/$defs/packagingContainerMode", @@ -473,6 +507,30 @@ "cluster": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only — dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated')." + }, + "lfTagVocabulary": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.7 (preview): Ownership of the Lake Formation tag vocabulary (aws_lakeformation_lf_tag). Shared: the domain defines the tag keys and values and the product only associates them." + }, + "emrApplication": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.7 (preview): Ownership of the EMR Serverless application (aws_emrserverless_application) a Spark build runs on." + }, + "schemaRegistry": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.7 (preview): Ownership of the Glue schema registry (aws_glue_registry) that holds produced subjects." + }, + "athenaWorkgroup": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.7 (preview): Ownership of the Athena workgroup (aws_athena_workgroup) the product's queries and dbt builds run in." + }, + "kmsKey": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.7 (preview): Ownership of the KMS key (aws_kms_key) that encrypts the product's data, checkpoints and logs." + }, + "mwaaEnvironment": { + "const": "shared", + "description": "NEW in v0.7.7 (preview): The MWAA environment (aws_mwaa_environment) is the platform team's, so shared is the only value: a product ships its DAG files into the environment's bucket and never creates or changes the environment." } } } @@ -902,6 +960,13 @@ } } }, + "invariants": { + "type": "array", + "description": "NEW in v0.7.7 (preview): Rules about valid data (RFC-data-rules.md): checks, derived columns, summaries over child rows and functional dependencies, written in FLUID SQL/1, each with a band of violating rows. dbt, ODCS and FLUX consumers read them alike. contract.quality[] keeps free SQL in the warehouse's dialect, which FLUX never reads.", + "items": { + "$ref": "#/$defs/invariant" + } + }, "tags": { "$ref": "#/$defs/tags", "description": "Contract-level tags for data quality and governance." @@ -1160,6 +1225,155 @@ } } }, + "identifierFormat": { + "type": "array", + "minItems": 1, + "description": "NEW in v0.7.7 (preview): The identifier formats of this string column (RFC-data-rules.md section 3), a list of alternatives. A value has the format when it is one of an alternative's prefixes followed by a string its mask matches. Alternatives must not overlap, a foreign-key column repeats its target column's format or declares none, and placeholders are exempt. Consumers match one anchored regex compiled from it; uniqueness comes from the column's key, not from the format.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "mask" + ], + "properties": { + "prefixes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[ -~]*$" + }, + "description": "Literal prefixes, printable ASCII. One empty prefix when left out." + }, + "mask": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "description": "COBOL PICTURE style: 9 is a digit, A an upper-case ASCII letter, X an upper-case ASCII letter or a digit; a backslash makes the next character literal (\\A), and every other printable ASCII character stands for itself. At least one symbol; no repeat counts (write 99999, not 9(5))." + } + } + }, + "examples": [ + [ + { + "prefixes": [ + "MSC", + "TWR" + ], + "mask": "99999" + }, + { + "prefixes": [ + "LEGACY-" + ], + "mask": "A9-99AA" + } + ] + ] + }, + "placeholders": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): Placeholder values of this string column: stand-ins for a real value that is not known yet, such as TBD_0042 (RFC-data-rules.md section 4). A value is a placeholder when it equals one of values or starts with one of prefixes, case-sensitively; NULL never is. A placeholder is exempt from the column's own value checks (identifierFormat, validationRules, its enum) and an ordinary value everywhere else. One mustBe* operator with unit bounds the placeholder rows; without one they are declared and reported, never failed. A sentinel that stands for no value (UNKNOWN for NULL) is a quality[] missingValues rule instead.", + "properties": { + "values": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "Whole values that are placeholders." + }, + "prefixes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "A value that starts with one of these is a placeholder." + }, + "mustBe": { + "$ref": "#/$defs/violationBound", + "description": "Exactly this many placeholder rows." + }, + "mustBeLessThan": { + "$ref": "#/$defs/violationBound", + "description": "Fewer placeholder rows than this." + }, + "mustBeLessOrEqualTo": { + "$ref": "#/$defs/violationBound", + "description": "At most this many placeholder rows." + }, + "mustBeGreaterThan": { + "$ref": "#/$defs/violationBound", + "description": "More placeholder rows than this." + }, + "mustBeGreaterOrEqualTo": { + "$ref": "#/$defs/violationBound", + "description": "At least this many placeholder rows." + }, + "mustBeBetween": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/violationBound" + }, + "description": "From lo to hi placeholder rows, both included, with lo below hi." + }, + "unit": { + "type": "string", + "enum": [ + "rows", + "percent" + ], + "default": "rows", + "description": "rows counts placeholder rows; percent counts them per 100 rows of the expose." + } + }, + "examples": [ + { + "values": [ + "XXXXX" + ], + "prefixes": [ + "TBD_" + ], + "mustBeLessOrEqualTo": 1, + "unit": "percent" + } + ] + }, + "sharedAttribute": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_.]*$", + "description": "NEW in v0.7.7 (preview): The attribute this column carries with every other column of the contract that names the same one (RFC-data-rules.md section 5). The owner is the member that every other member reaches through relationships, following to, on exactly one path; on every row whose path reaches an owner row, a member IS NOT DISTINCT FROM the owner. Never inferred from column names, and not semanticType, which stays a kind of value such as email." + }, + "protobuf": { + "type": "object", + "additionalProperties": false, + "required": [ + "field" + ], + "properties": { + "field": { + "type": "integer", + "minimum": 1, + "maximum": 536870911, + "not": { + "minimum": 19000, + "maximum": 19999 + }, + "description": "The column's field number in the .proto forge generates when the topic's value is Protobuf (binding.serialization.value: protobuf). Unique within its message (the top level, or the struct it sits in); 1 to 536870911 except 19000 to 19999, which Protobuf reserves. A field number names the field on the wire, so it never changes once data is written." + } + }, + "description": "NEW in v0.7.7 (preview): How this column is written when the topic's value is Protobuf (binding.serialization.value: protobuf). Every column of such an expose, nested struct fields included, needs one." + }, "tags": { "$ref": "#/$defs/tags", "description": "Field-level tags for categorization and data governance." @@ -1190,7 +1404,7 @@ "streamSource": { "type": "object", "additionalProperties": false, - "description": "NEW in v0.7.7 (preview): The Kafka records a streaming build reads, described apart from the engine that reads them: one topic, how its values are encoded, the value's fields, an optional key, and Kafka record metadata read as columns. The value and key schemas are lists of columns, the same shape as an expose's contract.schema, so one fragment can describe a topic and a table that lands it. Brokers, consumer groups and registries are the engine's parameters, not part of the stream.", + "description": "NEW in v0.7.7 (preview): The Kafka records a streaming build reads, described apart from the engine that reads them: one topic, how its values are encoded, the value's fields, an optional key, and Kafka record metadata read as columns. The value and key schemas are lists of columns, the same shape as an expose's contract.schema, so one fragment can describe a topic and a table that lands it. Brokers and consumer groups are the engine's parameters, not part of the stream. NEW in v0.7.7 (preview): the registry that holds the value schema, and how the engine reads the topic (trigger, watermark, checkpoint, startingOffsets), are engine-neutral and live here.", "required": ["kind", "topic", "format", "schema"], "properties": { "kind": { @@ -1218,6 +1432,71 @@ "minItems": 1, "items": {"$ref": "#/$defs/streamMetadataColumn"}, "description": "Kafka record metadata, each read as a column." + }, + "registry": { + "$ref": "#/$defs/schemaRegistry", + "description": "NEW in v0.7.7 (preview): The registry that holds this topic's value schema. The topic belongs to its producer, so forge verifies the subject against schema and never registers it." + }, + "trigger": { + "description": "NEW in v0.7.7 (preview): When the engine starts a micro-batch. processingTime: one every ISO-8601 duration. availableNow: true: read what the topic holds now, in as many batches as it takes, then stop (a scheduled catch-up run). Absent: derived from the expose's qos.freshnessSLO.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "processingTime" + ], + "properties": { + "processingTime": { + "$ref": "#/$defs/isoDuration" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "availableNow" + ], + "properties": { + "availableNow": { + "const": true + } + } + } + ] + }, + "watermark": { + "type": "object", + "additionalProperties": false, + "required": [ + "column", + "delay" + ], + "description": "NEW in v0.7.7 (preview): The event-time watermark: the timestamp column it follows and how late a record may arrive before stateful operators drop it.", + "properties": { + "column": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*(?![\\s\\S])", + "description": "A timestamp column of the stream, or a nested field of one as a dotted path. On a Spark build it must be timestamp_ltz: Spark refuses a watermark on TIMESTAMP_NTZ (EVENT_TIME_IS_NOT_ON_TIMESTAMP_TYPE), and the Spark compiler refuses a timestamp (no time zone) watermark column." + }, + "delay": { + "$ref": "#/$defs/isoDuration", + "description": "The allowed lateness, as an ISO-8601 duration (PT10M)." + } + } + }, + "checkpoint": { + "type": "string", + "pattern": "^s3://[a-z0-9][a-z0-9.-]{1,61}[a-z0-9](?:/[^\\u0000-\\u001f\\u007f]*)?(?![\\s\\S])", + "description": "NEW in v0.7.7 (preview): Where the engine keeps its offsets and state, as an s3:// URI. Absent: derived from the product's Iceberg warehouse, one location per product and build. The checkpoint is the job's position in the topic: moving it starts the job again from startingOffsets." + }, + "startingOffsets": { + "enum": [ + "earliest", + "latest" + ], + "description": "NEW in v0.7.7 (preview): Where a job with no checkpoint yet starts reading: earliest (all the topic retains) or latest (new records only). Absent: the engine's default, latest for a Spark stream. A job that has a checkpoint resumes from it." } } }, @@ -1262,98 +1541,277 @@ "description": {"type": "string"} } }, - "recordMapping": { - "type": "object", - "additionalProperties": false, - "description": "NEW in v0.7.7 (preview): How the stream's records become the output's rows. An output column the mapping does not list, nested fields included, is read from the stream column or field at the same path, and keeps its type: so a stream landed as it is, flat or nested, needs no mapping. forge-cli writes the query from the mapping and checks every output column's type and nullability against it before anything deploys; no contract text reaches the job's SQL.", - "properties": { - "explode": {"$ref": "#/$defs/recordExplode"}, - "columns": { - "type": "array", - "minItems": 1, - "items": {"$ref": "#/$defs/recordMappingColumn"}, - "description": "Output columns, or nested fields of them, that are read from another path, renamed or converted." - } - } - }, - "recordExplode": { + "schemaRegistry": { "type": "object", "additionalProperties": false, - "description": "NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or empty produces no row.", - "required": ["path", "as"], + "description": "NEW in v0.7.7 (preview): The schema registry that holds a topic's record schema, one definition for every engine. kind glue is the AWS Glue Schema Registry (IAM authentication, so no secret); kind confluent is the Confluent Schema Registry or a registry that serves its REST API. Who owns the subject decides what forge does with it: a stream a build reads (streamSource.registry) is the producer's, so its subject is verified against the declared schema and never written; a topic an expose produces has its subject registered with this compatibility. The two older spellings stay valid and read as this definition: an acquisition build's kafka-connect schema_registry {url, auth} is {kind: confluent, url, auth}, and a managed Flink build's parameters.flink.source.schemaRegistryUrl is {kind: confluent, url}.", + "required": [ + "kind" + ], "properties": { - "path": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", - "description": "The array: a stream column or a nested field, such as items or order.lines." + "kind": { + "enum": [ + "glue", + "confluent" + ], + "description": "glue: AWS Glue Schema Registry. confluent: Confluent Schema Registry REST API." }, - "as": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", - "description": "The element's name in mapping paths: item.sku for a field of a struct element, item for a scalar element." - } - } - }, - "recordMappingColumn": { - "type": "object", - "additionalProperties": false, - "description": "NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.", - "required": ["name", "from"], - "properties": { - "name": { + "url": { "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*(?![\\s\\S])", - "description": "The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record." + "format": "uri", + "pattern": "^https?://[^/?#@\\s]+(?:/[^?#\\s]*)?(?![\\s\\S])", + "description": "kind confluent only, and required there: the registry's REST endpoint, http or https, with no credentials, query or fragment in it (credentials go in auth). Sending anything to the host is subject to the operator's egress policy." }, - "from": { + "registryName": { "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", - "description": "The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku)." - }, - "cast": { - "$ref": "#/$defs/column/properties/type", - "description": "Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision)." + "pattern": "^[A-Za-z0-9_$#.-]{1,255}(?![\\s\\S])", + "description": "kind glue only: the Glue registry's name. Absent: default-registry, the registry Glue uses when none is named." }, - "description": {"type": "string"} - } - }, - "dqSpec": { - "type": "object", - "additionalProperties": false, - "properties": { - "rules": { - "type": "array", - "items": { - "$ref": "#/$defs/dqRule" - } + "subject": { + "type": "string", + "pattern": "^[^\\u0000-\\u001f\\u007f]{1,255}(?![\\s\\S])", + "description": "The subject (confluent) or schema name (glue) that holds the record value's schema. Absent: -value, the Confluent TopicNameStrategy name, on both kinds." }, - "monitoring": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean", - "default": true - }, - "window": { - "$ref": "#/$defs/isoDuration" + "version": { + "oneOf": [ + { + "const": "latest" }, - "owner": { - "type": "string" + { + "type": "integer", + "minimum": 1 } - } + ], + "default": "latest", + "description": "The schema version to verify against or read with: latest, or a version number from 1." }, - "tags": { - "$ref": "#/$defs/tags", - "description": "Data quality specification tags." + "compatibility": { + "enum": [ + "NONE", + "DISABLED", + "BACKWARD", + "BACKWARD_ALL", + "FORWARD", + "FORWARD_ALL", + "FULL", + "FULL_ALL", + "BACKWARD_TRANSITIVE", + "FORWARD_TRANSITIVE", + "FULL_TRANSITIVE" + ], + "description": "The compatibility mode a produced subject is registered with. Glue accepts NONE, DISABLED, BACKWARD, BACKWARD_ALL, FORWARD, FORWARD_ALL, FULL and FULL_ALL; Confluent accepts NONE, BACKWARD, BACKWARD_TRANSITIVE, FORWARD, FORWARD_TRANSITIVE, FULL and FULL_TRANSITIVE. Absent: backward and transitive, BACKWARD_ALL on Glue and BACKWARD_TRANSITIVE on Confluent. A consumed subject's mode is the producer's, so this is not applied to it." }, - "labels": { - "$ref": "#/$defs/labels", - "description": "Data quality specification labels." + "auth": { + "$ref": "#/$defs/acquisitionPattern/properties/kafka-connect/properties/schema_registry/properties/auth", + "description": "kind confluent only: the registry credential, as a secret reference, the same shape as kafka-connect schema_registry.auth. Glue authenticates with the job's IAM role." + }, + "rules": { + "type": "boolean", + "default": false, + "description": "kind confluent only, on a subject an expose produces, and Avro only: true registers the expose's error-severity row checks (contract.invariants[] with a check and no mustBe* bound) as Confluent data-contract rules (CEL CONDITION rules in mode WRITE, onFailure ERROR) that Confluent's serializers enforce on every record. Other invariants (derive, aggregate, dependency, a check with a bound, a warning or info check) are reported as not enforced by the registry; a check with no exact CEL form is refused. Needs Confluent Platform Enterprise 7.4+ with the rule set extension, or Confluent Cloud's Stream Governance Advanced package." } - } - }, - "dqRule": { + }, + "allOf": [ + { + "if": { + "properties": { + "kind": { + "const": "glue" + } + } + }, + "then": { + "allOf": [ + { + "not": { + "required": [ + "url" + ] + }, + "description": "url is for kind: confluent; a Glue registry is named by registryName" + }, + { + "not": { + "required": [ + "auth" + ] + }, + "description": "auth is for kind: confluent; a Glue registry authenticates with the job's IAM role" + }, + { + "not": { + "required": [ + "rules" + ] + }, + "description": "rules is for kind: confluent; Glue has no data-contract rules" + }, + { + "properties": { + "subject": { + "pattern": "^[A-Za-z0-9_$#.-]{1,255}(?![\\s\\S])", + "description": "A Glue schema name: letters, digits and _ $ # . - (Glue CreateSchema SchemaName)." + } + } + }, + { + "properties": { + "compatibility": { + "enum": [ + "NONE", + "DISABLED", + "BACKWARD", + "BACKWARD_ALL", + "FORWARD", + "FORWARD_ALL", + "FULL", + "FULL_ALL" + ] + } + } + } + ] + } + }, + { + "if": { + "properties": { + "kind": { + "const": "confluent" + } + } + }, + "then": { + "required": [ + "url" + ], + "allOf": [ + { + "not": { + "required": [ + "registryName" + ] + }, + "description": "registryName is for kind: glue; a Confluent registry is addressed by url" + }, + { + "properties": { + "compatibility": { + "enum": [ + "NONE", + "BACKWARD", + "BACKWARD_TRANSITIVE", + "FORWARD", + "FORWARD_TRANSITIVE", + "FULL", + "FULL_TRANSITIVE" + ] + } + } + } + ] + } + } + ] + }, + "recordMapping": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): How the stream's records become the output's rows. An output column the mapping does not list, nested fields included, is read from the stream column or field at the same path, and keeps its type: so a stream landed as it is, flat or nested, needs no mapping. forge-cli writes the query from the mapping and checks every output column's type and nullability against it before anything deploys; no contract text reaches the job's SQL.", + "properties": { + "explode": {"$ref": "#/$defs/recordExplode"}, + "columns": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/recordMappingColumn"}, + "description": "Output columns, or nested fields of them, that are read from another path, renamed or converted." + } + } + }, + "recordExplode": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or empty produces no row.", + "required": ["path", "as"], + "properties": { + "path": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", + "description": "The array: a stream column or a nested field, such as items or order.lines." + }, + "as": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "description": "The element's name in mapping paths: item.sku for a field of a struct element, item for a scalar element." + } + } + }, + "recordMappingColumn": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.", + "required": ["name", "from"], + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*(?![\\s\\S])", + "description": "The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record." + }, + "from": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", + "description": "The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku)." + }, + "cast": { + "$ref": "#/$defs/column/properties/type", + "description": "Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision)." + }, + "description": {"type": "string"}, + "maskAtIngest": { + "enum": [ + "hash" + ], + "description": "NEW in v0.7.7 (preview): Mask the value before it lands. hash = HMAC-SHA256 keyed by parameters.spark.maskKeySecretArn, lowercase hex; null stays null. The raw value never reaches the table, and a keyed digest cannot be reversed by hashing a dictionary of likely values. Opt-in: the default masking is a protected view derived from the table." + } + } + }, + "dqSpec": { + "type": "object", + "additionalProperties": false, + "properties": { + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/dqRule" + } + }, + "monitoring": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "window": { + "$ref": "#/$defs/isoDuration" + }, + "owner": { + "type": "string" + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Data quality specification tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Data quality specification labels." + } + } + }, + "dqRule": { "type": "object", "additionalProperties": false, "required": [ @@ -2002,7 +2460,13 @@ "properties": { "type": "object", "description": "Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).", - "additionalProperties": true + "additionalProperties": true, + "properties": { + "registry": { + "$ref": "#/$defs/schemaRegistry", + "description": "NEW in v0.7.7 (preview): The registry a produced topic's subject is registered in, on a Kafka binding (platform: kafka, or platform: confluent with format: kafka_topic). The topic is this product's, so forge registers its value schema, derived from contract.schema, through OpenTofu with this compatibility: aws_glue_schema for kind glue, confluent_schema and confluent_subject_config for kind confluent. On any other platform nothing is registered and fluid validate warns; the value is still checked against this definition." + } + } }, "governance": { "$ref": "#/$defs/bindingGovernance", @@ -3234,7 +3698,8 @@ "dlt", "kafka-connect", "debezium", - "flink" + "flink", + "lakeflow" ] }, { @@ -3242,7 +3707,7 @@ } ], "default": "dbt", - "description": "Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration." + "description": "Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration. NEW in v0.7.7: lakeflow (a Databricks Lakeflow Declarative Pipeline: pattern embedded-logic, properties.sql is the SELECT of the dataset an expose binds; forge-cli writes the materialized view around it, with the expose's columns and its dq.rules as pipeline expectations, and fluid apply provisions one serverless pipeline per build)." }, "capabilities": { "type": "array", @@ -3478,11 +3943,17 @@ }, "parameters": { "type": "object", - "additionalProperties": true + "additionalProperties": true, + "properties": { + "spark": { + "$ref": "#/$defs/sparkParameters", + "description": "NEW in v0.7.7 (preview): Where and how a Spark build runs." + } + } }, "stream": { "$ref": "#/$defs/streamSource", - "description": "NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads." + "description": "NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional, and on a Spark build (parameters.spark) with a mapping it is too: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads." }, "mapping": { "$ref": "#/$defs/recordMapping", @@ -3497,24 +3968,42 @@ }, "if": { "not": { - "required": [ - "stream", - "parameters" - ], - "properties": { - "parameters": { + "anyOf": [ + { + "required": [ + "stream", + "parameters" + ], + "properties": { + "parameters": { + "required": [ + "flink" + ], + "properties": { + "flink": { + "required": [ + "management" + ] + } + } + } + } + }, + { "required": [ - "flink" + "stream", + "mapping", + "parameters" ], "properties": { - "flink": { + "parameters": { "required": [ - "management" + "spark" ] } } } - } + ] } }, "then": { @@ -3528,63 +4017,302 @@ ] } }, - "multiStagePattern": { + "sparkParameters": { "type": "object", - "description": "NEW in v0.5.5: Multi-stage orchestration pattern.", "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): Where and how a Spark build runs, at builds[].properties.parameters.spark of an embedded-logic build with engine: spark. forge runs one versioned PySpark runner with a generated job config; the build declares what it computes, either properties.stream with properties.mapping, or properties.sql as Spark SQL. There is no user PySpark entry point. The split with builds[].execution: this block says where Spark runs (runtime, application or cluster, release, role, Spark configuration); execution keeps when the build runs and what happens when it fails (trigger, retries, notifications). execution.runtime is the orchestrator's generic runtime (platform, image, resources) and does not choose a Spark runtime.", + "required": [ + "runtime" + ], "properties": { - "stages": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "type": "string" - }, - "pattern": { - "type": "string", - "enum": [ - "hybrid-reference", - "embedded-logic" - ], - "default": "hybrid-reference" - }, - "properties": { - "$ref": "#/$defs/buildProperties" - }, - "dependsOn": { - "type": "array", - "items": { - "type": "string" - } - }, - "outputs": { - "type": "array", - "items": { - "$ref": "#/$defs/identifier" - } - }, - "tags": { - "$ref": "#/$defs/tags" - }, - "labels": { - "$ref": "#/$defs/labels" - } - } - } - }, - "orchestration": { - "type": "object", - "properties": { - "parallelism": { - "type": "integer", - "minimum": 1 - }, - "retryPolicy": { - "$ref": "#/$defs/retryPolicy" - } - } + "runtime": { + "enum": [ + "emr-serverless", + "emr-ec2", + "emr-eks", + "glue" + ], + "description": "The AWS service that runs the job: EMR Serverless, EMR on EC2, EMR on EKS or an AWS Glue Spark job. One runner serves all four." + }, + "mode": { + "enum": [ + "streaming", + "batch" + ], + "description": "streaming: a long-running Structured Streaming job. batch: a job that runs to completion. Absent: streaming when properties.stream is set, else batch." + }, + "applicationId": { + "type": "string", + "pattern": "^[0-9a-z]{1,64}(?![\\s\\S])", + "description": "emr-serverless only: an existing EMR Serverless application. Absent: the forge-managed application shared by the product's domain." + }, + "clusterId": { + "type": "string", + "pattern": "^j-[0-9A-Za-z]{1,254}(?![\\s\\S])", + "description": "emr-ec2 only: a running EMR cluster (j-...). Absent: a transient cluster that ends with the job." + }, + "virtualClusterId": { + "type": "string", + "pattern": "^[0-9a-z]{1,64}(?![\\s\\S])", + "description": "emr-eks only, and required there: the EMR on EKS virtual cluster." + }, + "glueVersion": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?![\\s\\S])", + "default": "5.0", + "description": "glue only: the AWS Glue version of the job." + }, + "releaseLabel": { + "type": "string", + "pattern": "^emr-[0-9]+\\.[0-9]+\\.[0-9]+(?![\\s\\S])", + "default": "emr-7.12.0", + "description": "EMR runtimes only (emr-serverless, emr-ec2, emr-eks): the EMR release. 7.12 or later writes Iceberg with Lake Formation credentials." + }, + "executionRoleArn": { + "type": "string", + "pattern": "^arn:aws[a-zA-Z0-9-]*:iam::[0-9]{12}:role/[A-Za-z0-9_+=,.@/-]{1,512}(?![\\s\\S])", + "description": "The IAM role the job runs as. Absent: a least-privilege role forge creates for the build." + }, + "maskKeySecretArn": { + "type": "string", + "pattern": "^arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+(?![\\s\\S])", + "description": "The AWS Secrets Manager secret holding the HMAC key for maskAtIngest: hash. Required when any mapping column sets maskAtIngest (the cross-field check lives in the Spark compiler)." + }, + "sparkConf": { + "type": "object", + "propertyNames": { + "pattern": "^[A-Za-z0-9_.-]{1,256}(?![\\s\\S])" + }, + "additionalProperties": { + "type": "string" + }, + "description": "Spark configuration properties for the job, name to string value (spark.sql.shuffle.partitions: \"64\")." + }, + "source": { + "type": "object", + "additionalProperties": false, + "description": "The Kafka connection a streaming Spark build reads, as parameters.flink.source is for a Flink build. Exactly one of bootstrapServers or mskClusterArn. The schema registry is not here: it is properties.stream.registry ($defs/schemaRegistry).", + "properties": { + "bootstrapServers": { + "type": "string", + "description": "Comma-separated host:port list of Kafka brokers. Same name and shape as parameters.flink.source.bootstrapServers.", + "pattern": "^\\S(?:[^\\u0000-\\u001f\\u007f]*\\S)?(?![\\s\\S])" + }, + "mskClusterArn": { + "type": "string", + "pattern": "^arn:aws[a-z-]*:kafka:[a-z0-9-]+:[0-9]{12}:cluster/.+(?![\\s\\S])", + "description": "An Amazon MSK cluster. OpenTofu resolves its brokers through an MSK data source at plan time." + }, + "auth": { + "enum": [ + "msk-iam", + "none" + ], + "description": "How the job authenticates to the brokers. Absent: msk-iam with mskClusterArn, none with bootstrapServers. SASL/SCRAM is not supported yet." + }, + "groupIdPrefix": { + "type": "string", + "description": "Prefix for the consumer group ids Spark generates (the Kafka source's groupIdPrefix option). Absent: Spark's default.", + "pattern": "^\\S(?:[^\\u0000-\\u001f\\u007f]*\\S)?(?![\\s\\S])", + "maxLength": 255 + } + }, + "allOf": [ + { + "anyOf": [ + { + "required": [ + "bootstrapServers" + ] + }, + { + "required": [ + "mskClusterArn" + ] + } + ], + "description": "set bootstrapServers (a host:port list) or mskClusterArn (an Amazon MSK cluster)" + }, + { + "if": { + "required": [ + "bootstrapServers" + ] + }, + "then": { + "not": { + "required": [ + "mskClusterArn" + ] + }, + "description": "set one of bootstrapServers or mskClusterArn, not both" + } + } + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "runtime": { + "const": "emr-eks" + } + }, + "required": [ + "runtime" + ] + }, + "then": { + "required": [ + "virtualClusterId" + ] + } + }, + { + "if": { + "not": { + "properties": { + "runtime": { + "const": "emr-serverless" + } + } + } + }, + "then": { + "not": { + "required": [ + "applicationId" + ] + }, + "description": "applicationId is only valid with runtime: emr-serverless" + } + }, + { + "if": { + "not": { + "properties": { + "runtime": { + "const": "emr-ec2" + } + } + } + }, + "then": { + "not": { + "required": [ + "clusterId" + ] + }, + "description": "clusterId is only valid with runtime: emr-ec2" + } + }, + { + "if": { + "not": { + "properties": { + "runtime": { + "const": "emr-eks" + } + } + } + }, + "then": { + "not": { + "required": [ + "virtualClusterId" + ] + }, + "description": "virtualClusterId is only valid with runtime: emr-eks" + } + }, + { + "if": { + "properties": { + "runtime": { + "const": "glue" + } + }, + "required": [ + "runtime" + ] + }, + "then": { + "not": { + "required": [ + "releaseLabel" + ] + }, + "description": "releaseLabel is an EMR release; a glue runtime takes glueVersion" + }, + "else": { + "not": { + "required": [ + "glueVersion" + ] + }, + "description": "glueVersion is only valid with runtime: glue; EMR runtimes take releaseLabel" + } + } + ] + }, + "multiStagePattern": { + "type": "object", + "description": "NEW in v0.5.5: Multi-stage orchestration pattern.", + "additionalProperties": false, + "properties": { + "stages": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "pattern": { + "type": "string", + "enum": [ + "hybrid-reference", + "embedded-logic" + ], + "default": "hybrid-reference" + }, + "properties": { + "$ref": "#/$defs/buildProperties" + }, + "dependsOn": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/$defs/identifier" + } + }, + "tags": { + "$ref": "#/$defs/tags" + }, + "labels": { + "$ref": "#/$defs/labels" + } + } + } + }, + "orchestration": { + "type": "object", + "properties": { + "parallelism": { + "type": "integer", + "minimum": 1 + }, + "retryPolicy": { + "$ref": "#/$defs/retryPolicy" + } + } }, "tags": { "$ref": "#/$defs/tags" @@ -3739,7 +4467,7 @@ "additionalProperties": false, "properties": { "secretRef": { - "type": "string" + "$ref": "#/$defs/secretRef" } } } @@ -3873,13 +4601,17 @@ "type": "string" }, "secretRef": { + "$ref": "#/$defs/secretRef", + "description": "Reference to the connection's secret in the configured store, e.g. vault://kv/orgs/acme/pg#password or env://PG_PASSWORD." + }, + "password": { "type": "string", - "pattern": "^[a-z][a-z0-9_+.-]*://.+", - "description": "Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://)." + "writeOnly": true, + "description": "Write-only: a contract may not hold this value. Name the secret with secretRef instead." }, "user": { "type": "string", - "description": "NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) — without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside ``secretRef`` (or an inline ``password``) for the credential pair." + "description": "NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) — without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside ``secretRef`` for the credential pair." }, "schema": { "type": "string", @@ -4554,7 +5286,12 @@ "additionalProperties": true, "properties": { "secretRef": { - "type": "string" + "$ref": "#/$defs/secretRef" + }, + "token": { + "type": "string", + "writeOnly": true, + "description": "Write-only: a contract may not hold this value. Name the secret with secretRef instead." } } }, @@ -6160,7 +6897,7 @@ "unique", "natural" ], - "description": "Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier)." + "description": "Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier; the key of a slowly changing dimension, whose versions a validity window separates: see dimensions[].typeParams.validityParams)." }, "expr": { "type": "string", @@ -6172,6 +6909,13 @@ } } }, + "relationships": { + "type": "array", + "description": "NEW in v0.7.7 (preview): Named joins from this expose to an expose of the same contract, declared on the many side. Entities join by entity name only; a relationship names its target expose and the ordered key columns on both sides, so a composite key is one relationship. Names are unique within the expose.", + "items": { + "$ref": "#/$defs/semanticRelationship" + } + }, "measures": { "type": "array", "description": "Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.", @@ -6302,6 +7046,21 @@ "minute" ], "description": "Default time granularity for this dimension." + }, + "validityParams": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): Marks this time dimension as one bound of the validity window of a type II slowly changing dimension, field for field dbt's validity_params. A model with a window has exactly two such dimensions, one with isStart and one with isEnd, and one natural entity: the key whose versions the window separates.", + "properties": { + "isStart": { + "type": "boolean", + "description": "This dimension holds the time each version starts to apply." + }, + "isEnd": { + "type": "boolean", + "description": "This dimension holds the time each version stops applying." + } + } } } } @@ -6447,6 +7206,428 @@ } ] }, + "semanticRelationship": { + "type": "object", + "description": "NEW in v0.7.7 (preview): One foreign key, from the expose that declares it (the many side) to an expose of the same contract (the one side). fromColumns and toColumns are ordered and pair up by position, so they have the same length, and each names a column of its own expose's contract.schema. The shape is Apache Ossie's Relationship (from_columns, to_columns) plus the cardinality and join type Ossie leaves out.", + "additionalProperties": false, + "required": [ + "name", + "to", + "fromColumns", + "toColumns", + "cardinality" + ], + "properties": { + "name": { + "type": "string", + "description": "Relationship name, unique within the expose (e.g., 'orders_to_customers')." + }, + "to": { + "$ref": "#/$defs/identifier", + "description": "exposeId of the target expose, in the same contract. It may be the declaring expose itself, for a self-join such as employees to their managers." + }, + "fromColumns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + }, + "description": "Foreign-key columns of the declaring expose, in key order." + }, + "toColumns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + }, + "description": "Key columns of the target expose, in the order of fromColumns." + }, + "cardinality": { + "type": "string", + "enum": [ + "many_to_one", + "one_to_one" + ], + "description": "many_to_one: many rows of the declaring expose can match one target row (an ordinary foreign key). one_to_one: each target row matches at most one row of the declaring expose. Against a target with a validity window (a slowly changing dimension), the one target row is the version valid at the declaring row's point in time." + }, + "joinType": { + "type": "string", + "enum": [ + "left", + "inner" + ], + "default": "left", + "description": "How a consumer joins the target: left keeps the declaring expose's rows that have no match, inner drops them. Defaults to left." + }, + "sharedColumns": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "NEW in v0.7.7 (preview): Columns copied through this relationship (RFC-data-rules.md section 5): each is in both exposes' schemas under one name and of one type, and on every row whose link reaches a target row it IS NOT DISTINCT FROM the target's. Short for a derive of the column with expr . on the rows the link reaches. The relationship's own key columns are never listed: they agree by definition." + } + }, + "examples": [ + { + "name": "orders_to_customers", + "to": "customer_profiles", + "fromColumns": [ + "customer_id" + ], + "toColumns": [ + "id" + ], + "cardinality": "many_to_one", + "joinType": "left" + }, + { + "name": "order_lines_to_orders", + "to": "orders", + "fromColumns": [ + "order_id", + "region" + ], + "toColumns": [ + "order_id", + "region" + ], + "cardinality": "many_to_one" + } + ] + }, + "invariant": { + "type": "object", + "description": "NEW in v0.7.7 (preview): A rule about valid data (RFC-data-rules.md section 1). Exactly one form: check, a predicate every covered row satisfies; derive with expr, the column's value from the row and the rows its links reach (a generated column in PostgreSQL's sense); derive with aggregate, the column's value as a summary over child rows; or dependency, equal from values meaning equal to values. when narrows the rows it covers. One mustBe* operator with unit bounds the violating rows, as ODCS writes it; without one the invariant holds only with no violating row (mustBe: 0). A covered row violates a check when the predicate is not TRUE, and a derive when the column IS DISTINCT FROM the value.", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "description": "Unique within the contract, across every expose and kind, and against dq.rules[].id and contract.quality[].rule: reports, tests, ODCS rules and FLUX's seed streams are keyed by it." + }, + "description": { + "type": "string" + }, + "check": { + "$ref": "#/$defs/fluidSqlText", + "description": "A FLUID SQL/1 predicate over the row." + }, + "derive": { + "type": "string", + "minLength": 1, + "description": "The column this invariant determines, with exactly one of expr and aggregate. Several derives of one column, each with its own when, cover a row in declaration order, as SQL's CASE WHEN does." + }, + "expr": { + "$ref": "#/$defs/fluidSqlText", + "description": "With derive: the column's value, a FLUID SQL/1 expression of the column's type whose static scale fits the column's (wrap a product or a quotient in ROUND(x, s))." + }, + "aggregate": { + "$ref": "#/$defs/invariantAggregate" + }, + "dependency": { + "$ref": "#/$defs/invariantDependency" + }, + "when": { + "$ref": "#/$defs/fluidSqlText", + "description": "A FLUID SQL/1 predicate: the rows the invariant covers (every row without it). A row on which a compared value is NULL is covered and usually violates the invariant; write when to leave such rows out. Not on a dependency in v1." + }, + "severity": { + "type": "string", + "enum": [ + "error", + "warning", + "info" + ], + "default": "error", + "description": "How a consumer reports a violation, in contract.quality[]'s words." + }, + "mustBe": { + "$ref": "#/$defs/violationBound", + "description": "Exactly this many violating rows." + }, + "mustBeLessThan": { + "$ref": "#/$defs/violationBound", + "description": "Fewer violating rows than this." + }, + "mustBeLessOrEqualTo": { + "$ref": "#/$defs/violationBound", + "description": "At most this many violating rows." + }, + "mustBeGreaterThan": { + "$ref": "#/$defs/violationBound", + "description": "More violating rows than this." + }, + "mustBeGreaterOrEqualTo": { + "$ref": "#/$defs/violationBound", + "description": "At least this many violating rows." + }, + "mustBeBetween": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/violationBound" + }, + "description": "From lo to hi violating rows, both included, with lo below hi." + }, + "unit": { + "type": "string", + "enum": [ + "rows", + "percent" + ], + "default": "rows", + "description": "rows counts violating rows; percent is violating rows times 100 divided by covered rows, and 0 when no row is covered." + } + }, + "oneOf": [ + { + "required": [ + "check" + ], + "not": { + "anyOf": [ + { + "required": [ + "derive" + ] + }, + { + "required": [ + "expr" + ] + }, + { + "required": [ + "aggregate" + ] + }, + { + "required": [ + "dependency" + ] + } + ] + } + }, + { + "required": [ + "derive", + "expr" + ], + "not": { + "anyOf": [ + { + "required": [ + "check" + ] + }, + { + "required": [ + "aggregate" + ] + }, + { + "required": [ + "dependency" + ] + } + ] + } + }, + { + "required": [ + "derive", + "aggregate" + ], + "not": { + "anyOf": [ + { + "required": [ + "check" + ] + }, + { + "required": [ + "expr" + ] + }, + { + "required": [ + "dependency" + ] + } + ] + } + }, + { + "required": [ + "dependency" + ], + "not": { + "anyOf": [ + { + "required": [ + "check" + ] + }, + { + "required": [ + "derive" + ] + }, + { + "required": [ + "expr" + ] + }, + { + "required": [ + "aggregate" + ] + }, + { + "required": [ + "when" + ] + } + ] + } + } + ], + "examples": [ + { + "id": "gross_from_net", + "derive": "gross", + "expr": "ROUND(net * (1 + tax_rate), 2)" + }, + { + "id": "due_after_issue", + "check": "issue_date <= due_date", + "mustBeLessOrEqualTo": 0.5, + "unit": "percent" + }, + { + "id": "billed_base_fee_from_lines", + "derive": "billed_base_fee", + "aggregate": { + "function": "sum", + "column": "amount", + "from": "invoice_lines", + "via": "invoice", + "filter": "kind = 'base_fee'", + "ifNone": 0 + } + }, + { + "id": "group_name_by_id", + "dependency": { + "from": [ + "group_id" + ], + "to": [ + "group_name" + ], + "bothWays": true + } + } + ] + }, + "invariantAggregate": { + "type": "object", + "description": "NEW in v0.7.7 (preview): A summary over child rows (RFC-data-rules.md section 1.2): SQL's aggregate, with SQL's null rules and its FILTER clause, declared on the parent expose. count has scale 0, sum its column's scale, min and max their column's type.", + "additionalProperties": false, + "required": [ + "function", + "from", + "via" + ], + "properties": { + "function": { + "type": "string", + "enum": [ + "count", + "sum", + "min", + "max" + ] + }, + "column": { + "type": "string", + "minLength": 1, + "description": "The child column summarized: none for count, a number of bounded scale for sum, any orderable type for min and max." + }, + "from": { + "$ref": "#/$defs/identifier", + "description": "The child expose, an exposeId of this contract." + }, + "via": { + "type": "string", + "minLength": 1, + "description": "A relationship the child declares (its semantics.relationships[].name) whose to is this expose." + }, + "filter": { + "$ref": "#/$defs/fluidSqlText", + "description": "A FLUID SQL/1 predicate over the child row: only children for which it is TRUE count, as in SUM(x) FILTER (WHERE ...)." + }, + "ifNone": { + "type": [ + "string", + "number" + ], + "description": "A literal of the summary's type that replaces a NULL summary, as COALESCE would: when no child counts, or every counted child's column is NULL. A date is written 'YYYY-MM-DD', quoted. Without it, sum, min and max of nothing are NULL and count is 0." + } + } + }, + "invariantDependency": { + "type": "object", + "description": "NEW in v0.7.7 (preview): A functional dependency (RFC-data-rules.md section 1.2): equal values in from mean equal values in to. A row with a NULL from column is not covered; a NULL to value is a value. A key's violating rows are its covered rows minus those that carry its most common to values (the g3 measure). bothWays adds the reverse dependency, counted on its own, and takes one column a side.", + "additionalProperties": false, + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "to": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "bothWays": { + "type": "boolean", + "default": false + } + } + }, + "fluidSqlText": { + "type": "string", + "minLength": 1, + "description": "NEW in v0.7.7 (preview): A FLUID SQL/1 text (RFC-data-rules.md section 2): a closed SQL expression subset, with exact numbers and two-valued truth, at most 4,096 code points and 64 levels deep. fluid validate parses and type-checks it against the expose; SQL outside the subset belongs in contract.quality[]." + }, + "violationBound": { + "type": "number", + "minimum": 0, + "description": "NEW in v0.7.7 (preview): A number of rows, or a percentage (unit), read as an exact decimal from its text: at most 15 significant digits, so the text survives a round trip through a double." + }, "exposeMcp": { "type": "object", "additionalProperties": false, diff --git a/scripts/compat-waivers.txt b/scripts/compat-waivers.txt index bc550e8..c2d7947 100644 --- a/scripts/compat-waivers.txt +++ b/scripts/compat-waivers.txt @@ -65,3 +65,27 @@ # admits -- that would strand real documents rather than just old version # strings. 0.7.3->0.7.4 /properties/fluidVersion + +# --------------------------------------------------------------------------- +# 0.7.6 -> 0.7.7 -- DELIBERATE, in a PREVIEW. The only narrowing in the pair. +# +# secretRef (acquisitionSource.connection, acquisitionDeployment.auth and the +# kafka-connect schema_registry.auth) was a string matching +# ^[a-z][a-z0-9_+.-]*://.+ . It is now the shared $defs/secretRef: an anyOf of +# one grammar per scheme (env, file, vault, aws-sm, gcp-sm, azure-kv, and a +# catch-all for other lower-case schemes). The vault grammar requires a #field. +# +# Verified with a document from the 0.7.6 acquisition corpus: +# build.properties.source.connection.secretRef = "vault://secret/data/crm-db" +# 0.7.6: VALID 0.7.7: REJECTED (anyOf) +# "vault://secret/data/crm-db#password": VALID under 0.7.7 +# A secretRef that already carried #field, or used another scheme, stays valid. +# +# Why it stands: 0.7.7 is a preview, a contract reaches it only by declaring +# "0.7.7", and the change comes from the reference implementation's +# "secret references end to end" work. The 0.7.7 corpus (tests/0.7.7/ +# acquisition.json) was updated to the new grammar. docs/schema/changelog.md and +# docs/releases/0.7.7.md say the pair is not purely additive. +# +# Re-examine when 0.7.7 is promoted: a stable version may not narrow. +0.7.6->0.7.7 acquisition/streaming CDC acquisition reaching every sub-block: source, sink, delivery, schema evolution, quality gates and anomalies, cost budget, catalog, concurrency lock, lineage, kafka-connect (schema registry + streaming sink + managed deployment), and debezium diff --git a/scripts/schema-versions.json b/scripts/schema-versions.json index 54c50b7..23335a3 100644 --- a/scripts/schema-versions.json +++ b/scripts/schema-versions.json @@ -11,8 +11,8 @@ "repository": "Agenticstiger/forge-cli", "schemaDir": "fluid_build/schemas", "previewSource": "fluid_build/schema_manager.py", - "ref": "v0.27.0", - "commit": "c0e97c076f735205f4767f67f0bd05bcd78a47cd" + "ref": "v0.31.0", + "commit": "0b72fb1c327ae33acddfdf8bbf9f5d1e7849d19c" }, "latestStable": "0.7.6", "preview": ["0.7.7"], diff --git a/specs/0.7.7/fluid-spec.html b/specs/0.7.7/fluid-spec.html index 26f4b0e..fca95ad 100644 --- a/specs/0.7.7/fluid-spec.html +++ b/specs/0.7.7/fluid-spec.html @@ -18,8 +18,15 @@

FLUID 0.7.7 — Preview: recursive streaming types

Type: object
-

Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 is unchanged.

-
No Additional Properties +
+

Opt-in preview. Adds recursive struct fields, array elements and map key/value types, the lakeflow build engine (Databricks Lakeflow Declarative Pipelines), and two semantics fields: named relationships between exposes, composite keys included (exposes[].semantics.relationships), and the validity window of a slowly changing dimension (dimensions[].typeParams.validityParams). For AWS streaming to Iceberg it adds one schemaRegistry definition (Glue or Confluent) at streamSource.registry, the stream's trigger, watermark, checkpoint and startingOffsets, an opt-in maskAtIngest hash on mapped columns, the Spark build's parameters.spark, and six AWS container kinds in packaging.containers. Also rules about valid data (RFC-data-rules.md): invariants written in FLUID SQL/1 (exposes[].contract.invariants), identifier formats, placeholder values and shared attributes (exposes[].contract.schema[].identifierFormat, .placeholders, .sharedAttribute) and columns copied through a relationship (relationships[].sharedColumns). Stable 0.7.6 is unchanged.

+ +
+
+ +
No Additional Properties @@ -6384,18 +6391,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Field-level tags for categorization and data governance.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
+ identifierFormat
Type: array of object
+

NEW in v0.7.7 (preview): The identifier formats of this string column (RFC-data-rules.md section 3), a list of alternatives. A value has the format when it is one of an alternative's prefixes followed by a string its mask matches. Alternatives must not overlap, a foreign-key column repeats its target column's format or declares none, and placeholders are exempt. Consumers match one anchored regex compiled from it; uniqueness comes from the column's key, not from the format.

+
+ -
-
+ + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ Type: object
-

Field-level labels for metadata and automation.

-
Same definition as labels -
-
-
-
-
-
-
-
- - -Type: object
+ No Additional Properties + + + + + + +
-
+

- +

-
+
Type: stringFormat: uri
-

OpenAPI document URL (for APIs).

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
- No Additional Properties + prefixes
Type: array of string
+

Literal prefixes, printable ASCII. One empty prefix when left out.

+
- -
+

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

Compatibility promise for non-breaking evolution.

-
-

Must be one of:

-
  • "none"
  • "patch"
  • "minor"
  • "backward"
  • "forward"
  • "strict"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
- Type: string
+Must match regular expression: ^[ -~]*$ - No Additional Items

Each item of this array must be:

+ +
+
+
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: enum (of string)
-
-

Must be one of:

-
  • "additive_columns"
  • "additive_fields"
  • "additive_endpoints"
  • "nullable_to_required_disallowed"
  • "field_removal_with_notice"
  • "type_widening_allowed"
-
+ identifierFormat + + + + identifierFormat items + + + + mask
Type: string
+

COBOL PICTURE style: 9 is a digit, A an upper-case ASCII letter, X an upper-case ASCII letter or a digit; a backslash makes the next character literal (\A), and every other printable ASCII character stands for itself. At least one symbol; no repeat counts (write 99999, not 9(5)).

+
- +

Must be at least 1 characters long

Must be at most 64 characters long

-
-
+
+
+
+
Example:
+
[
+    {
+        "prefixes": [
+            "MSC",
+            "TWR"
+        ],
+        "mask": "99999"
+    },
+    {
+        "prefixes": [
+            "LEGACY-"
+        ],
+        "mask": "A9-99AA"
+    }
+]
+
-
+
-
+

- +

-
+
Type: object
-

Enhanced data quality rules including anomaly detection.

-
+ schema + + + + schema items + + + + placeholders
Type: object
+
+

NEW in v0.7.7 (preview): Placeholder values of this string column: stand-ins for a real value that is not known yet, such as TBD_0042 (RFC-data-rules.md section 4). A value is a placeholder when it equals one of values or starts with one of prefixes, case-sensitively; NULL never is. A placeholder is exempt from the column's own value checks (identifierFormat, validationRules, its enum) and an ordinary value everywhere else. One mustBe* operator with unit bounds the placeholder rows; without one they are declared and reported, never failed. A sentinel that stands for no value (UNKNOWN for NULL) is a quality[] missingValues rule instead.

- No Additional Properties +
+
+ +
No Additional Properties - -
+
+
Example:
+
{
+    "values": [
+        "XXXXX"
+    ],
+    "prefixes": [
+        "TBD_"
+    ],
+    "mustBeLessOrEqualTo": 1,
+    "unit": "percent"
+}
+
+
+
-
+

- +

-
+
Type: array
- + schema items + + + + placeholders + + + + values
Type: array of string
+

Whole values that are placeholders.

+
- No Additional Items

Each item of this array must be:

+

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

-
+
Type: object
- + placeholders + + + + values + + + + values items
Type: string
- No Additional Properties - +

Must be at least 1 characters long

-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
- + prefixes
Type: array of string
+

A value that starts with one of these is a placeholder.

+
- -
-
-
-
-
+

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "freshness"
  • "completeness"
  • "uniqueness"
  • "valid_values"
  • "accuracy"
  • "schema"
  • "anomaly_detection"
  • "drift_detection"
-
- + prefixes + + + + prefixes items
Type: string
+ +

Must be at least 1 characters long

+
+
-
+
-
+

- +

-
+
Type: string
+ mustBe
Type: number
+

Exactly this many placeholder rows.

+
+ - +

Value must be greater or equal to 0

-
+
-
+

- +

-
+
Type: number
- - - - - - - + mustBeLessThan
Type: number
+

Fewer placeholder rows than this.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • ">="
  • ">"
  • "<="
  • "<"
  • "=="
  • "!="
-
- - - - - - + mustBeLessOrEqualTo
Type: number
+

At most this many placeholder rows.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
- - Must match regular expression: ^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+S)?)?$ - - - - - - + mustBeGreaterThan
Type: number
+

More placeholder rows than this.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "info"
  • "warn"
  • "error"
  • "critical"
-
- - - - - - + mustBeGreaterOrEqualTo
Type: number
+

At least this many placeholder rows.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: string
- + mustBeBetween
Type: array
+

From lo to hi placeholder rows, both included, with lo below hi.

+
- -
-
-
-
-
+

Must contain a minimum of 2 items

Must contain a maximum of 2 items

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: array of string
-

DQ rule tags for categorization and automation.

-
Same definition as tags + mustBeBetween + + + + violationBound
Type: number
+

NEW in v0.7.7 (preview): A number of rows, or a percentage (unit), read as an exact decimal from its text: at most 15 significant digits, so the text survives a round trip through a double.

+
Same definition as mustBe +
+
-
+
-
+

- +

-
+
Type: object
-

DQ rule labels for metadata and routing.

-
Same definition as labels + unit
Type: enum (of string) Default: "rows"
+

rows counts placeholder rows; percent counts them per 100 rows of the expose.

+
+

Must be one of:

+
  • "rows"
  • "percent"
+
+ + + + + +
-
-
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + schema items + + + + sharedAttribute
Type: string
+

NEW in v0.7.7 (preview): The attribute this column carries with every other column of the contract that names the same one (RFC-data-rules.md section 5). The owner is the member that every other member reaches through relationships, following to, on exactly one path; on every row whose path reaches an owner row, a member IS NOT DISTINCT FROM the owner. Never inferred from column names, and not semanticType, which stays a kind of value such as email.

+
Must match regular expression: ^[A-Za-z][A-Za-z0-9_.]*$ -
+
+
+
+
+
-
+

- +

-
+
Type: boolean Default: true
- + protobuf
Type: object
+

NEW in v0.7.7 (preview): How this column is written when the topic's value is Protobuf (binding.serialization.value: protobuf). Every column of such an expose, nested struct fields included, needs one.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
+ protobuf + + + + field

+

The column's field number in the .proto forge generates when the topic's value is Protobuf (binding.serialization.value: protobuf). Unique within its message (the top level, or the struct it sits in); 1 to 536870911 except 19000 to 19999, which Protobuf reserves. A field number names the field on the wire, so it never changes once data is written.

+
+

Must not be:

+
+
+ Type: string
+ protobuf + + + + field + + + + not
Type: object
@@ -7986,6 +8012,15 @@

+

+
+
+ + + +

Value must be greater or equal to 1 and lesser or equal to 536870911

+ +
@@ -7994,18 +8029,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Data quality specification tags.

+ schema items + + + + tags
Type: array of string
+

Field-level tags for categorization and data governance.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Data quality specification labels.

+ schema items + + + + labels
Type: object
+

Field-level labels for metadata and automation.

Same definition as labels
+
+
-
+
-
+

- +

-
+
Type: array of object
-

Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).

+ openapiRef
Type: stringFormat: uri
+

OpenAPI document URL (for APIs).

- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- + guarantees
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: string
-

Rule identifier.

-
+ compatibility
Type: enum (of string)
+

Compatibility promise for non-breaking evolution.

+
+

Must be one of:

+
  • "none"
  • "patch"
  • "minor"
  • "backward"
  • "forward"
  • "strict"
+
@@ -8277,18 +8331,18 @@

-
+
-
+

- +

-
+
Type: string
-

SQL or boolean expression that must evaluate to true.

-
+ evolution
Type: array of enum (of string)
+ - -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

Severity when the rule fails.

-
Type: enum (of string)
+

Must be one of:

-
  • "error"
  • "warning"
  • "info"
+
  • "additive_columns"
  • "additive_fields"
  • "additive_endpoints"
  • "nullable_to_required_disallowed"
  • "field_removal_with_notice"
  • "type_widening_allowed"
@@ -8414,28 +8445,28 @@

Must be one of:

-
-
+
+
-
+
-
+

- +

-
+
Type: array of string
-

Contract-level tags for data quality and governance.

-
Same definition as tags -
-
-
-
-
+ dq
Type: object
+

Enhanced data quality rules including anomaly detection.

+
+ + No Additional Properties + + + + + + +
-
+

- +

-
+
Type: object
-

Contract-level labels for automation and metadata.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
+ dq + + + + rules
Type: array
-
-
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ Type: object
+ contract + + + + dq + + + + rules + + + + dqRule
Type: object
No Additional Properties @@ -8571,18 +8621,18 @@

-
+
-
+

- +

-
+
Type: string
-

Availability percentage (e.g., 99.9%).

-
+ dq + + + + rules + + + + rules items + + + + id
Type: string
- Must match regular expression: ^(100(\.0+)?|\d{2}(\.\d+)?|\d{1}\d(\.\d+)?)%$ @@ -8628,18 +8696,18 @@

-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

e.g., '0 rows'

-
+ type
Type: enum (of string)
+
+

Must be one of:

+
  • "freshness"
  • "completeness"
  • "uniqueness"
  • "valid_values"
  • "accuracy"
  • "schema"
  • "anomaly_detection"
  • "drift_detection"
+
@@ -8732,18 +8774,18 @@

-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: number
-

Target completeness ratio (0.0 to 1.0).

-
+ selector
Type: string
+ -

Value must be greater or equal to 0 and lesser or equal to 1

+
-
+
-
+

- +

-
+
Type: number
-

Acceptable error rate (0.0 to 1.0).

-
+ dq + + + + rules + + + + rules items + + + + threshold
Type: number
+ -

Value must be greater or equal to 0 and lesser or equal to 1

+
-
+
-
+

- +

-
+
Type: array of string
-

QoS-level tags for SLA management.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

QoS-level labels for automation and reporting.

-
Same definition as labels + operator
Type: enum (of string)
+
+

Must be one of:

+
  • ">="
  • ">"
  • "<="
  • "<"
  • "=="
  • "!="
-
-
-
+ + + + + +
-
+
-
+

- +

-
+
Type: object
- - - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-
-

Must be one of:

-
  • "oidc"
  • "oauth2"
  • "api_key"
  • "none"
  • "custom"
  • "iam"
  • "jwt"
-
+ window
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
+ + Must match regular expression: ^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+S)?)?$ @@ -9095,18 +9080,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: array of string
- + severity
Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warn"
  • "error"
  • "critical"
+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+ rules items + + + + description
Type: string
@@ -9256,24 +9229,22 @@

-

-
-
+
-
+

- +

-
+
Type: array of string
- - - - - - - No Additional Items

Each item of this array must be:

+ rules + + + + rules items + + + + tags
Type: array of string
+

DQ rule tags for categorization and automation.

+
Same definition as tags +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- - - - - - - + rules items + + + + labels
Type: object
+

DQ rule labels for metadata and routing.

+
Same definition as labels +
+
+
+
-
+
-
+

- +

-
+
Type: array of object
-
-

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.

- -
-
- -
+ monitoring
Type: object
+ No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+

- No Additional Properties

- -

-
- + enabled
Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- + window
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
-
- -
- +
+
Type: object
+ owner
Type: string
@@ -9665,18 +9631,26 @@

-
+
+

+
+
+
+
+
+
+
-
+

- +

-
+
Type: array of string
+

Data quality specification tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: const
-Specific value: "mask" - - - - - - + labels
Type: object
+

Data quality specification labels.

+
Same definition as labels
+
+
+
+
+
+
+

+ +

+
- - -
- +
+
Type: array of object
+

Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
- -
-

The following properties are required:

-
  • mask
-
+ quality items
Type: object
- -
- +
+
+
+

+ +

+
- -
- +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: string
+

Rule identifier.

+
+ - - + -
- -
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
- -
-

The following properties are required:

-
  • mask
-
+ expression
Type: string
+

SQL or boolean expression that must evaluate to true.

+
+ +
+
+
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- + severity
Type: enum (of string)
+

Severity when the rule fails.

+
+

Must be one of:

+
  • "error"
  • "warning"
  • "info"
+
-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: array
+

NEW in v0.7.7 (preview): Rules about valid data (RFC-data-rules.md): checks, derived columns, summaries over child rows and functional dependencies, written in FLUID SQL/1, each with a band of violating rows. dbt, ODCS and FLUX consumers read them alike. contract.quality[] keeps free SQL in the warehouse's dialect, which FLUX never reads.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: const
-Specific value: "mask" - + invariant
Type: object
+
+

NEW in v0.7.7 (preview): A rule about valid data (RFC-data-rules.md section 1). Exactly one form: check, a predicate every covered row satisfies; derive with expr, the column's value from the row and the rows its links reach (a generated column in PostgreSQL's sense); derive with aggregate, the column's value as a summary over child rows; or dependency, equal from values meaning equal to values. when narrows the rows it covers. One mustBe* operator with unit bounds the violating rows, as ODCS writes it; without one the invariant holds only with no violating row (mustBe: 0). A covered row violates a check when the predicate is not TRUE, and a derive when the column IS DISTINCT FROM the value.

- - - - -
-
-
-
- - - - - - -
- - - - - - -
-
-
-

- -

+
+
-
-
+ No Additional Properties

+ +

+
+ Type: string
-

Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud.

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+
+

Must not be:

+
+
+ Type: array of string
-

Columns of this expose's schema.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
+ item 0 + + + + not

+

+ +

+
Type: string
+ not + + + + anyOf + + + + item 0
Type: object
- +
+

The following properties are required:

+
  • derive
+
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6).

-
-

Must be one of:

-
  • "allow"
  • "deny"
  • "mask"
-
- + item 0 + + + + not + + + + anyOf + + + + item 1
Type: object
+ +
+

The following properties are required:

+
  • expr
+
-
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: array of string
-

Access control rule tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
+ +
+

The following properties are required:

+
  • aggregate
+
+ + + + + +
+ + + Type: object
-

Access control rule labels.

-
Same definition as labels -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-
-

The masking rule an access: mask restriction applies, required with it and refused without it. lastfour: 'XXXXX' and the value's last four characters; firstfour: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATAMASKINGPOLICY, LASTFOURCHARACTERS, FIRSTFOURCHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6).

+ item 3
Type: object
-
-
- -
-

Must be one of:

-
  • "last_four"
  • "first_four"
  • "nullify"
-
+
+

The following properties are required:

+
  • dependency
+
+ + + + + +
-
-
-
+
+
+

The following properties are required:

+
  • check
-
-
-
-
-
-
-
-

- -

-
-
-
+ + + + +
+ Type: array of string
-

Authorization policy tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- -
+
+

Must not be:

+
+
+ + + Type: object
-

Authorization policy labels.

-
Same definition as labels -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of object
-
-

Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filterpredicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (rowfilter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.

- -
-
- -
- - - - - - No Additional Items

Each item of this array must be:

-
-
+ not

+

+ +

+
Type: object
- No Additional Properties - + oneOf + + + + item 1 + + + + not + + + + anyOf + + + + item 0
Type: object
+ +
+

The following properties are required:

+
  • check
+
-
-
-
-

- -

-
- -
-
+
+ Type: string
-

Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud.

-
- + item 1 + + + + not + + + + anyOf + + + + item 1
Type: object
+ +
+

The following properties are required:

+
  • aggregate
+
-
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: string
-

The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted.

-
Must match regular expression: ^[a-z][a-z0-9_]{0,62}$ + item 1 + + + + not + + + + anyOf + + + + item 2
Type: object
+ +
+

The following properties are required:

+
  • dependency
+
+ + + + + +
-
-
-
-
-
-

- -

-
+
+
+

The following properties are required:

+
  • derive
  • expr
+
-
-
+ + + + +
+ Type: string
-

One SQL boolean expression over the expose's columns, e.g. consentdataanalytics = true AND is_suppressed = false.

-
- - - -

Must be at least 1 characters long

- - -
-
-
-
-
-
-
-

- -

-
- -
-
+
+

Must not be:

+
+
+ Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
+ item 2 + + + + not

+

+ +

+
+ Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: array of object
-
-

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

+ anyOf + + + + item 0
Type: object
-
-
- -
- +
+

The following properties are required:

+
  • check
+
- No Additional Items

Each item of this array must be:

-
-
+ +
Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: object
+ +
+

The following properties are required:

+
  • expr
+
+ + + + + +
+ + + Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-
-

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

+ item 2
Type: object
-
-
- -
-

Must be one of:

-
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
-
+
+

The following properties are required:

+
  • dependency
+
+ + + + + +
-
-
-
-
-
-

- -

-
+
+
+

The following properties are required:

+
  • derive
  • aggregate
+
-
-
+ + + + +
+ Type: object
-

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

-
- - - - - - -
-
-
-

- -

-
- -
-
+
+

Must not be:

+
+
+ Type: string Default: "FLUID_PII_HASH_SECRET"
-

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

-
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+ not

+

+ +

+
+ Type: string
-

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

-
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ - + not + + + + anyOf + + + + item 0
Type: object
+ +
+

The following properties are required:

+
  • check
+
-
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: integer Default: 0
-

NEW in v0.7.6: mask only. Leading characters left as they are.

-
- + not + + + + anyOf + + + + item 1
Type: object
- -

Value must be greater or equal to 0 and lesser or equal to 64

+
+

The following properties are required:

+
  • derive
+
-
-
-
-
-
-
-
-

- -

-
+ -
-
+ +
+ Type: integer Default: 4
-

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

-
- - - -

Value must be greater or equal to 0 and lesser or equal to 64

- - -
-
-
-
-
-
-
-

- -

-
- -
-

Additional Properties of any type are allowed.

- - Type: object
+ +
+

The following properties are required:

+
  • expr
+
+ + + + + +
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
+ +
+

The following properties are required:

+
  • aggregate
+
+ + + + + +
+ + + Type: array of string
-

Privacy masking rule tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Privacy masking rule labels.

-
Same definition as labels -
-
+ item 4
Type: object
+ +
+

The following properties are required:

+
  • when
-
-
+ + + + + +
+ + + + + +
-
-
+
+
+
+

The following properties are required:

+
  • dependency
+ + + + + +
+ + + + + +
+
Examples:
+
{
+    "id": "gross_from_net",
+    "derive": "gross",
+    "expr": "ROUND(net * (1 + tax_rate), 2)"
+}
+
+
{
+    "id": "due_after_issue",
+    "check": "issue_date <= due_date",
+    "mustBeLessOrEqualTo": 0.5,
+    "unit": "percent"
+}
+
+
{
+    "id": "billed_base_fee_from_lines",
+    "derive": "billed_base_fee",
+    "aggregate": {
+        "function": "sum",
+        "column": "amount",
+        "from": "invoice_lines",
+        "via": "invoice",
+        "filter": "kind = 'base_fee'",
+        "ifNone": 0
+    }
+}
+
+
{
+    "id": "group_name_by_id",
+    "dependency": {
+        "from": [
+            "group_id"
+        ],
+        "to": [
+            "group_name"
+        ],
+        "bothWays": true
+    }
+}
+
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + invariants items + + + + id
Type: string
+

Unique within the contract, across every expose and kind, and against dq.rules[].id and contract.quality[].rule: reports, tests, ODCS rules and FLUX's seed streams are keyed by it.

+
Must match regular expression: ^[A-Za-z][A-Za-z0-9_]*$ -
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Provider-specific predicate

-
+ description
Type: string
+ @@ -12372,18 +12284,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Row-level policy tags.

-
Same definition as tags + check
Type: string
+

A FLUID SQL/1 predicate over the row.

+
+ + + + + +

Must be at least 1 characters long

+ +
-
+
-
+

- +

-
+
Type: object
-

Row-level policy labels.

-
Same definition as labels -
-
-
-
+ derive
Type: string
+

The column this invariant determines, with exactly one of expr and aggregate. Several derives of one column, each with its own when, cover a row in declaration order, as SQL's CASE WHEN does.

+
+ + + +

Must be at least 1 characters long

+ +
-
+
-
+

- +

-
+
Type: array of string
-

Privacy policy tags.

-
Same definition as tags + invariants items + + + + expr
Type: string
+

With derive: the column's value, a FLUID SQL/1 expression of the column's type whose static scale fits the column's (wrap a product or a quotient in ROUND(x, s)).

+
Same definition as check
-
+
-
+

- +

-
+
Type: object
-

Privacy policy labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-Type: object
+

NEW in v0.7.7 (preview): A summary over child rows (RFC-data-rules.md section 1.2): SQL's aggregate, with SQL's null rules and its FILTER clause, declared on the parent expose. count has scale 0, sum its column's scale, min and max their column's type.

+
+ + No Additional Properties + + + + + + +
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

-
- - No Additional Properties + function
Type: enum (of string)
+
+

Must be one of:

+
  • "count"
  • "sum"
  • "min"
  • "max"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: array of string
-

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

+ invariants items + + + + aggregate + + + + column
Type: string
+

The child column summarized: none for count, a number of bounded scale for sum, any orderable type for min and max.

- +

Must be at least 1 characters long

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
-Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ - - - - - -
-
Examples:
-
"gpt-4"
-
-
"claude-3-opus"
-
-
"claude-3-sonnet"
-
-
"gemini-pro"
-
-
"gemini-ultra"
-
-
"llama-3"
-
-
"mixtral-8x7b"
-
-
-
-
+ aggregate + + + + from
Type: string
+

The child expose, an exposeId of this contract.

+
Same definition as id
-
+
-
+

- +

-
+
Type: array of string
-

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

+ invariants items + + + + aggregate + + + + via
Type: string
+

A relationship the child declares (its semantics.relationships[].name) whose to is this expose.

- +

Must be at least 1 characters long

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- - - - - - - -
-
+ aggregate + + + + filter
Type: string
+

A FLUID SQL/1 predicate over the child row: only children for which it is TRUE count, as in SUM(x) FILTER (WHERE ...).

+
Same definition as check
-
+
-
+

- +

-
+
Type: integer
-

Maximum tokens per AI request. Prevents excessive data exposure in single queries.

+ invariants items + + + + aggregate + + + + ifNone
Type: string or number
+

A literal of the summary's type that replaces a NULL summary, as COALESCE would: when no child counts, or every counted child's column is NULL. A date is written 'YYYY-MM-DD', quoted. Without it, sum, min and max of nothing are NULL and count is 0.

-

Value must be greater or equal to 1

+ -
-
Examples:
-
4096
-
-
8192
-
-
16384
-
+ +
+
+
-
+
-
+

- +

-
+
Type: integer
-

Daily token limit for AI consumption. Enforces usage quotas.

+ invariants items + + + + dependency
Type: object
+

NEW in v0.7.7 (preview): A functional dependency (RFC-data-rules.md section 1.2): equal values in from mean equal values in to. A row with a NULL from column is not covered; a NULL to value is a value. A key's violating rows are its covered rows minus those that carry its most common to values (the g3 measure). bothWays adds the reverse dependency, counted on its own, and takes one column a side.

+ + No Additional Properties -

Value must be greater or equal to 1

+ -
-
-
-
-
+
-
+

- +

-
+
Type: array of enum (of string)
-

Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

-
+ invariants items + + + + dependency + + + + from
Type: array of string
+ -

All items must be unique

No Additional Items

Each item of this array must be:

+

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
-
- + dependency + + + + from + + + + from items
Type: string
+ +

Must be at least 1 characters long

-

-
Examples:
-
[
-    "inference",
-    "reasoning",
-    "summarization"
-]
-
-
[
-    "search",
-    "qa",
-    "rag"
-]
-
-
+
-
+
-
+

- +

-
+
Type: array of enum (of string)
-

Prohibited AI use cases. Example: Allow reporting but deny model training.

-
+ invariants items + + + + dependency + + + + to
Type: array of string
+ -

All items must be unique

No Additional Items

Each item of this array must be:

+

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
-
- + dependency + + + + to + + + + to items
Type: string
+ +

Must be at least 1 characters long

-

-
Example:
-
[
-    "training",
-    "fine_tuning"
-]
-
-
+
-
+
-
+

- +

-
+
Type: boolean Default: false
-

Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

-
+ invariants items + + + + dependency + + + + bothWays
Type: boolean Default: false
+ @@ -13437,18 +13436,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: boolean Default: false
-

Whether AI systems can cache/store this data. False = ephemeral processing only.

-
- - - - - - + invariants items + + + + when
Type: string
+

A FLUID SQL/1 predicate: the rows the invariant covers (every row without it). A row on which a compared value is NULL is covered and usually violates the invariant; write when to leave such rows out. Not on a dependency in v1.

+
Same definition as check
-
+
-
+

- +

-
+
Type: object
-

Data retention requirements for AI systems consuming this data.

-
+ invariants items + + + + severity
Type: enum (of string) Default: "error"
+

How a consumer reports a violation, in contract.quality[]'s words.

+
+

Must be one of:

+
  • "error"
  • "warning"
  • "info"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: integer
-

Maximum days AI system can retain this data. 0 = no retention allowed.

-
- - - -

Value must be greater or equal to 0

- - + mustBe
Type: number
+

Exactly this many violating rows.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: boolean Default: true
-

Whether AI system must delete data after use.

-
- - - - - - -
-
-
-
+ mustBeLessThan
Type: number
+

Fewer violating rows than this.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: boolean Default: true
-

Whether AI consumption must be logged/audited.

-
- - - - - - + invariants items + + + + mustBeLessOrEqualTo
Type: number
+

At most this many violating rows.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: string
-

Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

-
- - - - - - + invariants items + + + + mustBeGreaterThan
Type: number
+

More violating rows than this.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: array of string
-

Agent policy tags for categorization.

-
Same definition as tags + invariants items + + + + mustBeGreaterOrEqualTo
Type: number
+

At least this many violating rows.

+
Same definition as mustBe
-
+
-
+

- +

-
+
Type: object
-

Agent policy labels for automation.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array
+

From lo to hi violating rows, both included, with lo below hi.

+
+ + + + + +

Must contain a minimum of 2 items

Must contain a maximum of 2 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: array of string
-

Policy-level tags for governance.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Policy-level labels for automation.

-
Same definition as labels -
+ violationBound
Type: number
+

NEW in v0.7.7 (preview): A number of rows, or a percentage (unit), read as an exact decimal from its text: at most 15 significant digits, so the text survives a round trip through a double.

+
Same definition as mustBe
-
-
+
-
+

- +

-
+
Type: object
- - - No Additional Properties

- -

-
- - - Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: enum (of string) Default: "rows"
+

rows counts violating rows; percent is violating rows times 100 divided by covered rows, and 0 when no row is covered.

+
+

Must be one of:

+
  • "rows"
  • "percent"
+
+ - - + -
- -
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
- - - - - - - -
+ tags
Type: array of string
+

Contract-level tags for data quality and governance.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
+
Type: object
+

Contract-level labels for automation and metadata.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: const
-Specific value: "gcp" + qos
Type: object
+ + + No Additional Properties -
+
+
+
+

+ +

-
-
-
- - -
- +
+
Type: object
+ availability
Type: string
+

Availability percentage (e.g., 99.9%).

+
+ Must match regular expression: ^(100(\.0+)?|\d{2}(\.\d+)?|\d{1}\d(\.\d+)?)%$ -
+
+
+
+
+
-
+

- +

-
+
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
- + qos + + + + dataLossSLO
Type: string
+

e.g., '0 rows'

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
-
-

Must not be:

-
-
- - - Type: number
+

Target completeness ratio (0.0 to 1.0).

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - binding - - - - allOf + qos - item 0 + errorBudget
Type: number
+

Acceptable error rate (0.0 to 1.0).

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
-Must match regular expression: ^(alias/|arn:) - - - - - - -
-
-
- - - - - - -
-
-
-
+ tags
Type: array of string
+

QoS-level tags for SLA management.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: object
+

QoS-level labels for automation and reporting.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ policy
Type: object
+ + No Additional Properties -
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+
Type: enum (of string)
+
+

Must be one of:

+
  • "oidc"
  • "oauth2"
  • "api_key"
  • "none"
  • "custom"
  • "iam"
  • "jwt"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
-

- -

-
- + authz
Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
-Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + readers + + + + readers items
Type: string
+ -
- +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: array of string
+ writers + + + + writers items
Type: string
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: array of object
+
+

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
-Must match regular expression: ^(user|group|serviceAccount|domain):.+$ - - - - - - -
-
-
- - - - - - -
-
-
-
-
-
-
-
-
- - - - - - -

+ No Additional Properties

+ +

+
Type: object
+ columnRestrictions + + + + columnRestrictions items + + + + allOf + + + + item 0
Type: object

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. Otherwise, the conditions in the "Else" tab should be respected.

-
- - - - - - -
+
-
+

- +

-
+
Type: enum (of string)
-

Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target — emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafka→Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws').

-
-

Must be one of:

-
  • "gcp"
  • "aws"
  • "azure"
  • "snowflake"
  • "databricks"
  • "kafka"
  • "confluent"
  • "local"
  • "kubernetes"
  • "postgres"
  • "pgvector"
  • "other"
-
- - - - - - -
-
-
+ authz + + + + columnRestrictions + + + + columnRestrictions items + + + + access
Type: enum (of string)
+

deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6).

+
+

Must be one of:

+
  • "allow"
  • "deny"
  • "mask"
+
+ + + + + + +
+
+
-
+
-
+

- +

-
+
Type: enum (of string)
-

Binding format. NEW in v0.7.5: 'pgvectortable' — a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgrestable' (PostgreSQL driver), 'athenatable' / 'gluetable' (AWS Athena driver) for the MCP output port.

-
-

Must be one of:

-
  • "bigquery_table"
  • "snowflake_table"
  • "snowflake_view"
  • "gcs_file"
  • "s3_file"
  • "http_api"
  • "grpc_api"
  • "pubsub_topic"
  • "kafka_topic"
  • "delta_table"
  • "iceberg"
  • "parquet"
  • "csv"
  • "json"
  • "redshift_table"
  • "redshift_serverless"
  • "redshift_external_schema"
  • "postgres_table"
  • "athena_table"
  • "glue_table"
  • "pgvector_table"
  • "other"
-
- - - - - - + authz + + + + columnRestrictions + + + + columnRestrictions items + + + + tags
Type: array of string
+

Access control rule tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Apache Iceberg table format configuration

-
No Additional Properties - - - - - - -Type: object
+

Access control rule labels.

+
Same definition as labels +
+
+
+
+
-
+

- +

-
+
Type: enum (of integer)
-

Iceberg table format version

-
Type: enum (of string)
+
+

The masking rule an access: mask restriction applies, required with it and refused without it. lastfour: 'XXXXX' and the value's last four characters; firstfour: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATAMASKINGPOLICY, LASTFOURCHARACTERS, FIRSTFOURCHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6).

+ +
+
+ +

Must be one of:

-
  • 1
  • 2
+
  • "last_four"
  • "first_four"
  • "nullify"
@@ -16171,18 +16401,24 @@

Must be one of:

-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Underlying file format for data files

-
-

Must be one of:

-
  • "parquet"
  • "orc"
  • "avro"
-
- - - - - - + tags
Type: array of string
+

Authorization policy tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: array of object
-

Partition transformation specifications

-
- - - - - - No Additional Items

Each item of this array must be:

+ labels
Type: object
+

Authorization policy labels.

+
Same definition as labels +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
+ rowFilters
Type: array of object
+
+

Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filterpredicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (rowfilter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.

+
+
+ +
- -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Partition field name

-
+ rowFilters items
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Source column for partitioning

+ principal
Type: string
+

Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud.

@@ -16500,18 +16714,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-

Partition transform function

-
-

Must be one of:

-
  • "identity"
  • "year"
  • "month"
  • "day"
  • "hour"
  • "bucket"
  • "truncate"
-
+ name
Type: string
+

The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted.

+
Must match regular expression: ^[a-z][a-z0-9_]{0,62}$ @@ -16579,18 +16790,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: integer
-

Number of buckets (for bucket transform)

+ where
Type: string
+

One SQL boolean expression over the expose's columns, e.g. consentdataanalytics = true AND is_suppressed = false.

- +

Must be at least 1 characters long

-
+
-
+

- +

-
+
Type: integer
-

Truncate width (for truncate transform)

-
+ description
Type: string
+ @@ -16737,18 +16947,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: array of object
-

Sort order specifications for data files

-
+ privacy
Type: object
+ No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: object
- - + masking
Type: array of object
+
+

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

- +
+
+ +
-
-
-
-

- -

-
+ -
-
+ No Additional Items

Each item of this array must be:

+
+
+ Type: string
-

Column to sort by

-
+ masking items
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-

Sort direction

-
-

Must be one of:

-
  • "asc"
  • "desc"
-
+ column
Type: string
+ @@ -17004,18 +17195,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string)
-

NULL value ordering

-
Type: enum (of string)
+
+

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

+ +
+
+ +

Must be one of:

-
  • "nulls-first"
  • "nulls-last"
+
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
@@ -17083,24 +17281,18 @@

Must be one of:

-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Iceberg table properties

+ masking + + + + masking items + + + + params
Type: object
+

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

@@ -17147,19 +17353,19 @@

-
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+
Type: string
- + masking items + + + + params + + + + saltEnv
Type: string Default: "FLUID_PII_HASH_SECRET"
+

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

+
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ @@ -17215,26 +17436,18 @@

-
-
-
-
-

-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvectortable'). Consumed by fluidbuild.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.

-
No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: integer
-

Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector(<dimensions>).

-
+ keyEnv
Type: string
+

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

+
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ -

Value must be greater or equal to 1 and lesser or equal to 16000

+
-
+
-
+

- +

-
+
Type: string
-

Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity.

+ masking + + + + masking items + + + + params + + + + keepFirst
Type: integer Default: 0
+

NEW in v0.7.6: mask only. Leading characters left as they are.

- +

Value must be greater or equal to 0 and lesser or equal to 64

-
+
-
+

- +

-
+
Type: enum (of string) Default: "vector"
-

pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims).

-
-

Must be one of:

-
  • "vector"
  • "halfvec"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string) Default: "hnsw"
-

ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index — exact search only).

-
-

Must be one of:

-
  • "hnsw"
  • "ivfflat"
  • "none"
-
+ keepLast
Type: integer Default: 4
+

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

+
- +

Value must be greater or equal to 0 and lesser or equal to 64

-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: enum (of string) Default: "cosine"
-

Distance function the index optimises for → pgvector operator class: cosine→vectorcosineops, l2→vectorl2ops, innerproduct→vectoripops, l1→vectorl1_ops.

-
-

Must be one of:

-
  • "cosine"
  • "l2"
  • "inner_product"
  • "l1"
-
+ masking + + + + masking items + + + + params + + + + additionalProperties
Type: object
+ @@ -17593,18 +17767,22 @@

Must be one of:

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Target embeddings table name. Defaults to '<exposeId>_embeddings' when omitted.

-
- - - - - - + masking + + + + masking items + + + + tags
Type: array of string
+

Privacy masking rule tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: string
-

Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store.

-
- - - - - - + masking + + + + masking items + + + + labels
Type: object
+

Privacy masking rule labels.

+
Same definition as labels
-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
-

HNSW index tuning (used when indexType is 'hnsw').

-
No Additional Properties + rowLevelPolicy
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: integer
-

Max bi-directional links per node (pgvector default 16).

+ expression
Type: string
+

Provider-specific predicate

-

Value must be greater or equal to 1

+
-
+
-
+

- +

-
+
Type: integer
-

Build-time dynamic candidate-list size (pgvector default 64).

-
- - - -

Value must be greater or equal to 1

- - -
-
-
-
+ tags
Type: array of string
+

Row-level policy tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

IVFFlat index tuning (used when indexType is 'ivfflat').

-
No Additional Properties - - - - - - -Type: object
+

Row-level policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: integer
-

Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows).

-
- - - -

Value must be greater or equal to 1

- - -
-
-
-
-
-
-
-
+ tags
Type: array of string
+

Privacy policy tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Provider-native addressing with tightened structure for better interop.

-
- - No Additional Properties - - - - - - -
+ privacy + + + + labels
Type: object
+

Privacy policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Cloud account ID

-
+ classification
Type: enum (of string)
+
+

Must be one of:

+
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
+
@@ -18167,18 +18346,18 @@

-
+
-
+

- +

-
+
Type: string
-

GCP project ID

+ agentPolicy
Type: object
+

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

+ + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

BigQuery dataset or database name

+ allowedModels
Type: array of string
+

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Database name

-
+ allowedModels + + + + allowedModels items
Type: string
+Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ - +
+
Examples:
+
"gpt-4"
+
+
"claude-3-opus"
+
+
"claude-3-sonnet"
+
+
"gemini-pro"
+
+
"gemini-ultra"
+
+
"llama-3"
+
+
"mixtral-8x7b"
+
+
+
+
-
+
-
+

- +

-
+
Type: string
-

Schema name

+ deniedModels
Type: array of string
+

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Table name

-
+ deniedModels + + + + deniedModels items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: string
-

Storage bucket name

+ maxTokensPerRequest
Type: integer
+

Maximum tokens per AI request. Prevents excessive data exposure in single queries.

- +

Value must be greater or equal to 1

- +
+
Examples:
+
4096
+
+
8192
+
+
16384
+
+
-
+
-
+

- +

-
+
Type: string
-

File or object path

+ maxTokensPerDay
Type: integer
+

Daily token limit for AI consumption. Enforces usage quotas.

- +

Value must be greater or equal to 1

-
+
-
+

- +

-
+
Type: string
-

API gateway endpoint

+ allowedUseCases
Type: array of enum (of string)
+

Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads host and falls back to account, so until now the only valid way to set the bootstrap servers was a field named for something else — and with neither set, the exported ODCS server is the placeholder ${KAFKA_BOOTSTRAP_SERVERS}. Usually set per environment in an overlay rather than in the base contract.

-
+ allowedUseCases + + + + allowedUseCases items
Type: enum (of string)
+
+

Must be one of:

+
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
+
+
+

+
Examples:
+
[
+    "inference",
+    "reasoning",
+    "summarization"
+]
+
+
[
+    "search",
+    "qa",
+    "rag"
+]
+
+
-
+
-
+

- +

-
+
Type: stringFormat: uri
-

Base URL for API endpoints

+ deniedUseCases
Type: array of enum (of string)
+

Prohibited AI use cases. Example: Allow reporting but deny model training.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Pub/Sub or Kafka topic name

-
+ deniedUseCases + + + + deniedUseCases items
Type: enum (of string)
+
+

Must be one of:

+
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
+
+
+

+
Example:
+
[
+    "training",
+    "fine_tuning"
+]
+
+
-
+
-
+

- +

-
+
Type: string
-

Pub/Sub subscription name

+ canReason
Type: boolean Default: false
+

Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

@@ -18911,18 +19108,18 @@

-
+
-
+

- +

-
+
Type: string
-

Cloud region

+ canStore
Type: boolean Default: false
+

Whether AI systems can cache/store this data. False = ephemeral processing only.

@@ -18973,18 +19170,18 @@

-
+
-
+

- +

-
+
Type: string
-

Cloud zone

+ retentionPolicy
Type: object
+

Data retention requirements for AI systems consuming this data.

@@ -19031,22 +19228,18 @@

-

-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter.

+ retentionPolicy + + + + maxRetentionDays
Type: integer
+

Maximum days AI system can retain this data. 0 = no retention allowed.

- +

Value must be greater or equal to 0

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to.

+ retentionPolicy + + + + requireDeletion
Type: boolean Default: true
+

Whether AI system must delete data after use.

@@ -19159,18 +19366,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: AWS Kinesis Data Stream name — maps to awskinesisstream in the streaming IaC emitter.

+ auditRequired
Type: boolean Default: true
+

Whether AI consumption must be logged/audited.

@@ -19221,18 +19432,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Amazon Redshift Serverless namespace name — maps to awsredshiftserverlessnamespace.

+ purposeLimitation
Type: string
+

Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

@@ -19283,18 +19494,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Amazon Redshift Serverless workgroup name — maps to awsredshiftserverlessworkgroup.

-
- - - - - - + tags
Type: array of string
+

Agent policy tags for categorization.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge.

-
- - - - - - + labels
Type: object
+

Agent policy labels for automation.

+
Same definition as labels
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge.

-
- - - - - - + tags
Type: array of string
+

Policy-level tags for governance.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema.

-
- - - - - - + labels
Type: object
+

Policy-level labels for automation.

+
Same definition as labels
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byobaws). Its trust policy is updated post-apply with the provider integration's externalid (two-phase IAM).

-
- - - - + binding
Type: object
- -
-
-
-
-
-
-
-

- -

-
-
-
+ No Additional Properties

+ +

+
+ Type: string
-
-

NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'', so icebergrest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under <host>/catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/icebergcatalog.py.

- -
-
- -
- + item 0
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: string
-

NEW in v0.7.5: Iceberg warehouse — the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog.

-
+ item 0 + + + + if
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: REST Iceberg catalog endpoint URL — the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri).

-
+ item 0 + + + + if + + + + platform
Type: const
+Specific value: "gcp" @@ -19786,19 +19957,13 @@

-
-
-
-

- -

-
+
-
-
+ + +
+ Type: array of string
-
-

NEW in v0.7.5: Iceberg partition columns — a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition retention after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written).

+ item 0 + + + + then
Type: object
-
-
- -
- No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: string
+ then + + + + encryption
Type: object
@@ -19905,28 +20086,18 @@

-

-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Binding configuration tags for infrastructure.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Binding configuration labels for automation.

-
Same definition as labels -
-
-
-
-
-
-
-

- -

-
- -
-
+ kms

+
+

Must not be:

+
+
+ Type: object
-

Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).

-
- - - - - - -
-
-
-

- -

-
- -
-

Additional Properties of any type are allowed.

- - Type: object
+ not
Type: object
+Must match regular expression: ^(alias/|arn:) + + + + +
+
+
+ @@ -20129,18 +20251,18 @@

-
+
-
+

- +

-
+
Type: object
-

Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block.

-
+ allOf + + + + item 0 + + + + then + + + + principals
Type: object
- No Additional Properties -
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: object
-

Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).

-
No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- -
+

+ +

+
+ + + Type: boolean Default: false
-

When true, emit awslakeformationresource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path.

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string) Default: "cross-account"
-
-

NEW in v0.7.6: Which grants[] principals get a statement in the awss3bucketpolicy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted awss3bucketpolicy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted.

- -
-
- -
-

Must be one of:

-
  • "cross-account"
  • "none"
  • "all-grantees"
-
+ anyOf + + + + item 0
Type: string
+Must match regular expression: ^(user|group|serviceAccount|domain):.+$ -
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: array of object
-

Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as awslakeformationpermissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: string
-

IAM principal ARN (role or user) receiving the grant.

-
Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + item 1 items
Type: string
+Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + + + + + + +
+
+
@@ -20597,19 +20672,20 @@

-
-
-
-

- -

+
+
+
+
+ -
-
+ + + + +
+ Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: array of enum (of string)
-

LF permissions granted. Valid values per the hashicorp/aws awslakeformationpermissions resource.

-
- - - - - -

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- - - Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
-
-

Must be one of:

-
  • "ALL"
  • "SELECT"
  • "ALTER"
  • "DROP"
  • "DELETE"
  • "INSERT"
  • "DESCRIBE"
  • "CREATE_TABLE"
  • "CREATE_DATABASE"
  • "DATA_LOCATION_ACCESS"
-
+ if + + + + platform
Type: const
+Specific value: "aws" -
-
-
-
-
-

- -

-
+
-
-
+ + +
+ Type: array of string
-

Subset of permissions the principal can re-grant. Maps to awslakeformationpermissions.permissionswithgrant_option.

-
+ then
Type: object
+ -

All items must be unique

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: string
+ encryption
Type: object
@@ -20914,24 +21006,18 @@

-

-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Optional column-level restriction. When set, the grant is emitted against awslakeformationpermissions.tablewithcolumns instead of .table, allowing the principal to read only the listed columns.

-
- - - - - -

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- + kms

+
+

Must not be:

+
+
+ Type: string
+ not
Type: object
+Must match regular expression: ^projects/ + + + + +
+
+
+ +
+
-
+
-
+

- +

-
+
Type: array of string
-

Opposite of columns: every column EXCEPT these is granted. Maps to tablewithcolumns.excludedcolumnnames. Mutually exclusive with columns.

-
+ principals
Type: object
+ -

All items must be unique

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+

Each additional property must conform to the following schema


+

+ +

+
+ + + Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-
-

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

- -
-
- -
- - - - - - -
-
-
-

- -

-
- -
-

Each additional property must conform to the following schema

- - Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+ + + Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as awslakeformationdatacellsfilter. The filter must then be granted to principals via grants[].permissions including SELECT — LF enforces both layers.

-
No Additional Properties + item 1
Type: array of string
+ - -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Filter name (must be unique per table).

-
+ principals + + + + additionalProperties + + + + anyOf + + + + item 1 + + + + item 1 items
Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+
+
@@ -21538,18 +21592,36 @@

-
+
+
+
+
+
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: string
-

PartiQL-flavoured row predicate. ALL lets every row through (use when only column filtering is intended).

-
+ platform
Type: enum (of string)
+

Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target — emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafka→Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws').

+
+

Must be one of:

+
  • "gcp"
  • "aws"
  • "azure"
  • "snowflake"
  • "databricks"
  • "kafka"
  • "confluent"
  • "local"
  • "kubernetes"
  • "postgres"
  • "pgvector"
  • "other"
+
@@ -21614,18 +21668,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
+

Binding format. NEW in v0.7.5: 'pgvectortable' — a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgrestable' (PostgreSQL driver), 'athenatable' / 'gluetable' (AWS Athena driver) for the MCP output port.

+
+

Must be one of:

+
  • "bigquery_table"
  • "snowflake_table"
  • "snowflake_view"
  • "gcs_file"
  • "s3_file"
  • "http_api"
  • "grpc_api"
  • "pubsub_topic"
  • "kafka_topic"
  • "delta_table"
  • "iceberg"
  • "parquet"
  • "csv"
  • "json"
  • "redshift_table"
  • "redshift_serverless"
  • "redshift_external_schema"
  • "postgres_table"
  • "athena_table"
  • "glue_table"
  • "pgvector_table"
  • "other"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
-

Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.

-
+ binding + + + + icebergConfig
Type: object
+

Apache Iceberg table format configuration

+
No Additional Properties -

All items must be unique

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: enum (of integer)
+

Iceberg table format version

+
+

Must be one of:

+
  • 1
  • 2
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
- + binding + + + + icebergConfig + + + + fileFormat
Type: enum (of string)
+

Underlying file format for data files

+
+

Must be one of:

+
  • "parquet"
  • "orc"
  • "avro"
+
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Columns hidden through the filter (everything else is visible).

+ partitionSpec
Type: array of object
+

Partition transformation specifications

-

All items must be unique

No Additional Items

Each item of this array must be:

+ No Additional Items

Each item of this array must be:

-
+
Type: string
+ partitionSpec items
Type: object
@@ -21900,24 +22019,18 @@

-

-
-
-
-
-
-
+
-
+

- +

-
+
Type: boolean Default: false
-

Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted.

+ name
Type: string
+

Partition field name

@@ -21982,22 +22095,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, <table>_masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.

-
No Additional Properties + partitionSpec items + + + + sourceColumn
Type: string
+

Source column for partitioning

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole).

-
+ transform
Type: enum (of string)
+

Partition transform function

+
+

Must be one of:

+
  • "identity"
  • "year"
  • "month"
  • "day"
  • "hour"
  • "bucket"
  • "truncate"
+
@@ -22127,18 +22250,18 @@

-
+
-
+

- +

-
+
Type: string
-

The Glue connection of type VIEWVALIDATIONATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it.

+ numBuckets
Type: integer
+

Number of buckets (for bucket transform)

@@ -22203,30 +22326,18 @@

-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: Per-exposure packaging override — key-wise precedence over the contract-wide top-level packaging block (binding.packaging > packaging > absent-legacy).

+ icebergConfig + + + + partitionSpec + + + + partitionSpec items + + + + width
Type: integer
+

Truncate width (for truncate transform)

- - No Additional Properties -
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in containers.

-
-

Must be one of:

-
  • "isolated"
  • "shared"
-
+ sortOrder
Type: array of object
+

Sort order specifications for data files

+
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' — a pool must be addressable. Propagated as the fluid_pool label/tag for cost attribution.

-
+ sortOrder + + + + sortOrder items
Type: object
+ -

Must be at least 1 characters long

+ -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans).

+ sortOrder + + + + sortOrder items + + + + column
Type: string
+

Column to sort by

-

Must be at least 1 characters long

+
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over mode for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket → awss3bucket / googlestoragebucket; database → snowflakedatabase AND awsgluecatalogdatabase; dataset → googlebigquerydataset; schema → snowflakeschema; warehouse → snowflakewarehouse; cluster → Confluent environment/cluster (v1: shared only).

-
No Additional Properties + sortOrder + + + + sortOrder items + + + + direction
Type: enum (of string)
+

Sort direction

+
+

Must be one of:

+
  • "asc"
  • "desc"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the object-store bucket (awss3bucket / googlestoragebucket).

-
- - Type: enum (of string)
+

NULL value ordering

+

Must be one of:

-
  • "isolated"
  • "shared"
+
  • "nulls-first"
  • "nulls-last"
@@ -22589,18 +22754,24 @@

Must be one of:

-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the database container — covers Snowflake database AND AWS glue_database.

-
Same definition as bucket -
-
-
-
-Type: object
+

Iceberg table properties

+
+ + + + + + +
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: enum (of string)
-

NEW in v0.7.6: Ownership of the BigQuery dataset (googlebigquerydataset).

-
Same definition as bucket + additionalProperties
Type: string
+ + + + + + +
-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
+

NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvectortable'). Consumed by fluidbuild.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
-

NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema).

-
Same definition as bucket + exposes items + + + + binding + + + + vectorConfig + + + + dimensions
Type: integer
+

Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector(<dimensions>).

+
+ + + +

Value must be greater or equal to 1 and lesser or equal to 16000

+ +
-
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) — isolated gives per-product cost attribution.

-
Same definition as bucket + embeddingModel
Type: string
+

Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity.

+
+ + + + + +
-
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only — dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated').

-
Same definition as bucket -
-
-
-
+ vectorType
Type: enum (of string) Default: "vector"
+

pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims).

+
+

Must be one of:

+
  • "vector"
  • "halfvec"
-
-
-
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.

-
No Additional Properties + vectorConfig + + + + indexType
Type: enum (of string) Default: "hnsw"
+

ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index — exact search only).

+
+

Must be one of:

+
  • "hnsw"
  • "ivfflat"
  • "none"
+
-
+
+
+
+
+
-
+

- +

-
+
Default: "product"
-
-

NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): fluid apply creates a customer managed key for each bucket this product owns (awskmskey, rotation on, alias alias/fluid/<contract id>/<bucket>) and makes it the bucket's default encryption (awss3bucketserversideencryptionconfiguration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. tofu destroy deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; fluid apply refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/<name>' or a key or alias ARN: an existing key, looked up at plan time (data awskmskey, so the identity running fluid apply needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRICDEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. fluid verify checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (googlekmskeyring fluid-<contract id>-<dataset> and googlekmscryptokey 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's defaultencryptionconfiguration and the table's encryptionconfiguration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: tofu destroy removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/<p>/locations/<l>/keyRings/<r>/cryptoKeys/<k>': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so fluid apply refuses that plan without --allow-data-loss; the next build lands the data again. fluid verify checks kmsKeyName on the dataset and the table.

+ distanceMetric
Type: enum (of string) Default: "cosine"
+

Distance function the index optimises for → pgvector operator class: cosine→vectorcosineops, l2→vectorl2ops, innerproduct→vectoripops, l1→vectorl1_ops.

+
+

Must be one of:

+
  • "cosine"
  • "l2"
  • "inner_product"
  • "l1"
+
+ + + + + +
-
- -

- -

-
- +
+
+
+
+
+

+ +

+
+ +
+
Type: string
+

Target embeddings table name. Defaults to '<exposeId>_embeddings' when omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
-
-

Must be one of:

-
  • "product"
  • "none"
-
- - - - - - -
- - - Type: string
+

Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
-Must match regular expression: ^alias/[A-Za-z0-9/_-]{1,250}$ + vectorConfig + + + + hnsw
Type: object
+

HNSW index tuning (used when indexType is 'hnsw').

+
No Additional Properties -
- +
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: ^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$ + m
Type: integer
+

Max bi-directional links per node (pgvector default 16).

+
- +

Value must be greater or equal to 1

-
- +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: ^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$ - - - - - - -
+ efConstruction
Type: integer
+

Build-time dynamic candidate-list size (pgvector default 64).

+
- +

Value must be greater or equal to 1

@@ -23306,18 +23588,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: object
-
-

NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.

- -
-
- -
+ vectorConfig + + + + ivfflat
Type: object
+

IVFFlat index tuning (used when indexType is 'ivfflat').

+
No Additional Properties -
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+

-

- -

-
- - - Type: integer
+

Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows).

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+ Type: object
+

Provider-native addressing with tightened structure for better interop.

+
+ No Additional Properties -

Must be at least 1 characters long

+ -
- +
+
+
+

+ +

+
+ +
+
Type: array of string
- + account
Type: string
+

Cloud account ID

+
-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- - - - -

Must be at least 1 characters long

- - -
-
-
+ project
Type: string
+

GCP project ID

+
@@ -23630,22 +23900,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only format: kafka_topic, which tells a consumer nothing about whether the values are Avro or JSON — the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server format field; the two -confluent values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.

-
No Additional Properties + location + + + + dataset
Type: string
+

BigQuery dataset or database name

+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

How message values are serialized.

-
-

Must be one of:

-
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
-
+ database
Type: string
+

Database name

+
@@ -23750,18 +24024,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string)
-

How message keys are serialized, for a topic whose keys are not opaque bytes.

-
-

Must be one of:

-
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
-
+ schema
Type: string
+

Schema name

+
@@ -23815,26 +24086,18 @@

Must be one of:

-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus expire, which is only meaningful where there is storage to expire.

+ binding + + + + location + + + + table
Type: string
+

Table name

- - No Additional Properties -
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Unified lifecycle state vocabulary.

+ location + + + + bucket
Type: string
+

Storage bucket name

- -
-

Must be one of:

-
  • "preview"
  • "active"
  • "deprecated"
  • "retired"
-
@@ -23929,18 +24210,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless expire is true. With expire: true on an AWS S3 binding, fluid apply expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one.

-
Same definition as window + location + + + + path
Type: string
+

File or object path

+
+ + + + + +
-
+
-
+

- +

-
+
Type: boolean Default: false
-
-

NEW in v0.7.6: Delete stored data once it is older than retention. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: fluid apply writes one rule per expose into the bucket's awss3bucketlifecycleconfiguration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version retention after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that fluid apply did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and fluid verify checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing expire later deletes the lifecycle configuration, which fluid apply refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); tofu destroy deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (timepartitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expirationms = retention in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than retention after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than retention lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding expire to a live table replaces it: fluid apply refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of retention is an in-place update. Removing expire from a live table removes the partitioning's replacement trigger, which fluid apply also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer retention instead. fluid verify checks the live table's partition expiration.

- -
-
- -
+ location + + + + gateway
Type: string
+

API gateway endpoint

+
@@ -24040,18 +24334,18 @@

-
+
-
+

- +

-
+
Type: object
- No Additional Properties + location + + + + host
Type: string
+

NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads host and falls back to account, so until now the only valid way to set the bootstrap servers was a field named for something else — and with neither set, the exported ODCS server is the placeholder ${KAFKA_BOOTSTRAP_SERVERS}. Usually set per environment in an overlay rather than in the base contract.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window + baseUrl
Type: stringFormat: uri
+

Base URL for API endpoints

+
+ + + + + +
-
+
-
+

- +

-
+
Type: string
- + topic
Type: string
+

Pub/Sub or Kafka topic name

+
@@ -24207,18 +24520,18 @@

-
+
-
+

- +

-
+
Type: string
-

Reference to replacement data product or expose.

+ subscription
Type: string
+

Pub/Sub subscription name

@@ -24269,18 +24582,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Deprecation policy tags.

-
Same definition as tags + region
Type: string
+

Cloud region

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Deprecation policy labels.

-
Same definition as labels -
-
-
-
+ zone
Type: string
+

Cloud zone

+
+ + + + + +
-
+
-
+

- +

-
+
Type: array of string
-

Lifecycle tags for automation.

-
Same definition as tags + location + + + + environment_id
Type: string
+

NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter.

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Lifecycle labels for governance.

-
Same definition as labels -
-
-
-
+ location + + + + kafka_cluster_id
Type: string
+

NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to.

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
- - - No Additional Properties + binding + + + + location + + + + stream
Type: string
+

NEW in v0.7.5: AWS Kinesis Data Stream name — maps to awskinesisstream in the streaming IaC emitter.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: array of object
- + location + + + + namespace
Type: string
+

NEW in v0.7.5: Amazon Redshift Serverless namespace name — maps to awsredshiftserverlessnamespace.

+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + workgroup
Type: string
+

NEW in v0.7.5: Amazon Redshift Serverless workgroup name — maps to awsredshiftserverlessworkgroup.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
- + iam_role_arn
Type: string
+

NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge.

+
@@ -24697,18 +25078,18 @@

-
+
-
+

- +

-
+
Type: string
- + external_schema
Type: string
+

NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge.

+
@@ -24765,18 +25140,18 @@

-
+
-
+

- +

-
+
Type: string
-

Human/machine-readable SLI check description

+ glue_database
Type: string
+

NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema.

@@ -24834,24 +25202,18 @@

-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of object
- + location + + + + confluent_role_arn
Type: string
+

NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byobaws). Its trust policy is updated post-apply with the provider integration's externalid (two-phase IAM).

+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + catalog
Type: string
+
+

NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'', so icebergrest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under <host>/catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/icebergcatalog.py.

+ +
+
+ +
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "slack"
  • "email"
  • "webhook"
  • "pagerduty"
  • "custom"
-
+ warehouse
Type: string
+

NEW in v0.7.5: Iceberg warehouse — the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog.

+
@@ -25008,18 +25395,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
- + uri
Type: string
+

NEW in v0.7.5: REST Iceberg catalog endpoint URL — the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri).

+
@@ -25076,18 +25457,18 @@

-
+
-
+

- +

-
+
Type: string
+ partitionBy
Type: array of string
+
+

NEW in v0.7.5: Iceberg partition columns — a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition retention after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written).

+
+
+ +
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: stringFormat: uri
+ partitionBy items
Type: string
@@ -25208,22 +25576,28 @@

+

+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
-
- - - - - - + tags
Type: array of string
+

Binding configuration tags for infrastructure.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: array of string
-

Alert configuration tags.

-
Same definition as tags + labels
Type: object
+

Binding configuration labels for automation.

+
Same definition as labels
-
+
-
+

- +

-
+
Type: object
-

Alert configuration labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-Type: object
+

Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).

+
+ + + + + + +
-
+

- +

-
+
Type: object
-

Default SLI trio (freshness, completeness, latency) by kind.

+ properties + + + + registry
Type: object
+

NEW in v0.7.7 (preview): The registry a produced topic's subject is registered in, on a Kafka binding (platform: kafka, or platform: confluent with format: kafkatopic). The topic is this product's, so forge registers its value schema, derived from contract.schema, through OpenTofu with this compatibility: awsglueschema for kind glue, confluentschema and confluentsubjectconfig for kind confluent. On any other platform nothing is registered and fluid validate warns; the value is still checked against this definition.

- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
+ No Additional Properties

+ +

+
+ Type: boolean Default: true
-

Enable auto-generation of default SLIs.

-
- + registry + + + + allOf + + + + item 0
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
-

Default freshness SLI configuration.

-
No Additional Properties + registry + + + + allOf + + + + item 0 + + + + if
Type: object
+ -
+
-
+

- +

-
+
Type: boolean Default: true
- + allOf + + + + item 0 + + + + if + + + + kind
Type: const
+Specific value: "glue" @@ -25656,19 +26037,13 @@

-
-
-
-

- -

-
+
-
-
+ + +
+ Type: string
-

Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- -
+

+ +

+
+ + + Type: enum (of string) Default: "warning"
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
-

Freshness SLI tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
+

url is for kind: confluent; a Glue registry is named by registryName

+
+

Must not be:

+
+
+ Type: object
-

Freshness SLI labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Default completeness SLI configuration.

-
No Additional Properties + not
Type: object
+ +
+

The following properties are required:

+
  • url
+
+ + +
+
+
-
-
-
-

- -

-
+ -
-
+ +
+ Type: boolean Default: true
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -
+

auth is for kind: confluent; a Glue registry authenticates with the job's IAM role

+
+

Must not be:

+
+
+ + + Type: number Default: 0.95
-

Completeness threshold (auto-set by kind: table=0.95, stream=0.99)

-
- - - -

Value must be greater or equal to 0 and lesser or equal to 1

- - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string) Default: "warning"
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
+ +
+

The following properties are required:

+
  • auth
+
+ + + + + +
+
+
+ + + + + + +
+ + + Type: array of string
-

Completeness SLI tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Completeness SLI labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
+ allOf + + + + item 2

+

rules is for kind: confluent; Glue has no data-contract rules

+
+

Must not be:

+
+
+ Type: object
-

Default latency SLI configuration.

-
No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: boolean Default: true
+ not
Type: object
- +
+

The following properties are required:

+
  • rules
+
-
-
-
-
-
-

- -

-
+
+ -
-
+ + + + +
+ Type: string
-

P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: number Default: 95
-

Latency percentile to measure.

-
+ item 3
Type: object
+ -

Value must be greater or equal to 0 and lesser or equal to 100

+ -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string) Default: "warning"
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
-

Latency SLI tags.

-
Same definition as tags + subject
Type: object
+

A Glue schema name: letters, digits and _ $ # . - (Glue CreateSchema SchemaName).

+
Must match regular expression: ^[A-Za-z0-9_$#.-]{1,255}(?![\s\S]) + + + + + +
-
-
-
-

- -

-
- -
-
+
+ Type: object
-

Latency SLI labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
-

Default SLI configuration tags.

-
Same definition as tags -
-
-
-
-Type: object
+ + + + + + + +
-
+

- +

-
+
Type: object
-

Default SLI configuration labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).

-
No Additional Properties + allOf + + + + item 4 + + + + compatibility
Type: enum (of string)
+
+

Must be one of:

+
  • "NONE"
  • "DISABLED"
  • "BACKWARD"
  • "BACKWARD_ALL"
  • "FORWARD"
  • "FORWARD_ALL"
  • "FULL"
  • "FULL_ALL"
+
-
-
-
-

- -

+
+
+
+
+ -
-
+ + + + +
+ + + + + + +
+ Type: array of object
+ registry + + + + allOf + + + + item 1
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
- - - No Additional Items

Each item of this array must be:

-
-
- Type: object
- No Additional Properties + allOf + + + + item 1 + + + + if
Type: object
+ -
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "log"
  • "file"
  • "webhook"
-
+ item 1 + + + + if + + + + kind
Type: const
+Specific value: "confluent" @@ -27131,19 +27291,13 @@

Must be one of:

-
-
-
-

- -

-
+
-
-
+ + +
+ Type: string
-

For kind=file: NDJSON sink path (env-vars expanded).

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+ item 1 + + + + then

+

+ +

+
+ Type: stringFormat: uri
-

For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time.

-
- - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
+ item 1 + + + + then + + + + allOf + + + + item 0

+

registryName is for kind: glue; a Confluent registry is addressed by url

+
+

Must not be:

+
+
+ Type: array of string
-

Observability configuration tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Observability configuration labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
+ not
Type: object
+
+

The following properties are required:

+
  • registryName
+
- No Additional Properties + +
+
+
-
-
-
-

- -

-
+ -
-
+ +
+ Type: stringFormat: uri
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: stringFormat: uri
+ then + + + + allOf + + + + item 1
Type: object
@@ -27544,22 +27630,18 @@

-

-
-
-
-
+
-
+

- +

-
+
Type: stringFormat: uri
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: stringFormat: uri
- + then + + + + allOf + + + + item 1 + + + + compatibility
Type: enum (of string)
+
+

Must be one of:

+
  • "NONE"
  • "BACKWARD"
  • "BACKWARD_TRANSITIVE"
  • "FORWARD"
  • "FORWARD_TRANSITIVE"
  • "FULL"
  • "FULL_TRANSITIVE"
+
@@ -27656,18 +27736,42 @@

-
+
+
+

The following properties are required:

+
  • url
+
+ + + + + +
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: array of string
-

Documentation tags for organization.

-
Same definition as tags + properties + + + + registry + + + + kind
Type: enum (of string)
+

glue: AWS Glue Schema Registry. confluent: Confluent Schema Registry REST API.

+
+

Must be one of:

+
  • "glue"
  • "confluent"
+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Documentation labels for automation.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.

-
- - No Additional Properties + url
Type: stringFormat: uri
+

kind confluent only, and required there: the registry's REST endpoint, http or https, with no credentials, query or fragment in it (credentials go in auth). Sending anything to the host is subject to the operator's egress policy.

+
Must match regular expression: ^https?://[^/?#@\s]+(?:/[^?#\s]*)?(?![\s\S]) -
-
Example:
-
{
-    "name": "orders_revenue_model",
-    "description": "Canonical semantic model for revenue analytics",
-    "defaultAggTimeDimension": "order_date",
-    "entities": [
-        {
-            "name": "order",
-            "type": "primary",
-            "expr": "order_id"
-        },
-        {
-            "name": "customer",
-            "type": "foreign",
-            "expr": "customer_id"
-        }
-    ],
-    "measures": [
-        {
-            "name": "order_amount",
-            "agg": "sum",
-            "expr": "amount",
-            "description": "Sum of all order amounts"
-        },
-        {
-            "name": "order_count",
-            "agg": "count_distinct",
-            "expr": "order_id",
-            "createMetric": true
-        }
-    ],
-    "dimensions": [
-        {
-            "name": "order_date",
-            "type": "time",
-            "expr": "completed_at",
-            "typeParams": {
-                "timeGranularity": "day"
-            }
-        },
-        {
-            "name": "region",
-            "type": "categorical"
-        }
-    ],
-    "metrics": [
-        {
-            "name": "net_revenue",
-            "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds",
-            "type": "derived",
-            "inputMetrics": [
-                "gross_revenue",
-                "total_discounts"
-            ],
-            "expr": "gross_revenue - total_discounts",
-            "owner": "finance-data@company.com"
-        },
-        {
-            "name": "gross_revenue",
-            "description": "Total value of completed orders",
-            "type": "simple",
-            "measure": "order_amount",
-            "filter": "status = 'completed'"
-        }
-    ]
-}
-
+ +
+
+
-
+
-
+

- +

-
+
Type: string
-

Human-readable name for this semantic model.

-
+ properties + + + + registry + + + + registryName
Type: string
+

kind glue only: the Glue registry's name. Absent: default-registry, the registry Glue uses when none is named.

+
Must match regular expression: ^[A-Za-z0-9_$#.-]{1,255}(?![\s\S]) @@ -27927,18 +27970,18 @@

-
+
-
+

- +

-
+
Type: string
-

Business context explaining what this semantic model represents.

-
+ properties + + + + registry + + + + subject
Type: string
+

The subject (confluent) or schema name (glue) that holds the record value's schema. Absent: <topic>-value, the Confluent TopicNameStrategy name, on both kinds.

+
Must match regular expression: ^[^\u0000-\u001f\u007f]{1,255}(?![\s\S]) @@ -27982,18 +28039,18 @@

-
+
-
+

- +

-
+
Type: string
-

Default time dimension used for measure aggregation. Can be overridden per-measure.

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Default: "latest"
+

The schema version to verify against or read with: latest, or a version number from 1.

+

+ +

+
+ + + Type: array of object
-

Join keys with type annotations. Entities connect this table to others in the semantic graph.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- No Additional Properties + oneOf + + + + item 0
Type: const
+Specific value: "latest" -
-
-
-

- -

-
- -
-
+
+ Type: string
-

Entity name (e.g., 'order', 'customer').

-
+ version + + + + oneOf + + + + item 1
Type: integer
+ + + + +

Value must be greater or equal to 1

+ + +
@@ -28204,18 +28257,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-

Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier).

-
Type: enum (of string)
+

The compatibility mode a produced subject is registered with. Glue accepts NONE, DISABLED, BACKWARD, BACKWARDALL, FORWARD, FORWARDALL, FULL and FULLALL; Confluent accepts NONE, BACKWARD, BACKWARDTRANSITIVE, FORWARD, FORWARDTRANSITIVE, FULL and FULLTRANSITIVE. Absent: backward and transitive, BACKWARDALL on Glue and BACKWARDTRANSITIVE on Confluent. A consumed subject's mode is the producer's, so this is not applied to it.

+

Must be one of:

-
  • "primary"
  • "foreign"
  • "unique"
  • "natural"
+
  • "NONE"
  • "DISABLED"
  • "BACKWARD"
  • "BACKWARD_ALL"
  • "FORWARD"
  • "FORWARD_ALL"
  • "FULL"
  • "FULL_ALL"
  • "BACKWARD_TRANSITIVE"
  • "FORWARD_TRANSITIVE"
  • "FULL_TRANSITIVE"
@@ -28276,18 +28329,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

Column expression. Defaults to entity name if omitted.

+ auth
Type: object
+

kind confluent only: the registry credential, as a secret reference, the same shape as kafka-connect schema_registry.auth. Glue authenticates with the job's IAM role.

+ + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
+ auth + + + + secretRef
Type: string
+

NEW in v0.7.7: A reference to a secret held in a store, never the secret itself: a contract names a secret and never carries one. Grammar: <scheme>://<locator>[#field][?version=n]. Schemes: env, file, vault, aws-sm, gcp-sm, azure-kv; aliases openbao (vault), aws (aws-sm), gcp (gcp-sm) and azure (azure-kv) are accepted; any other lower-case scheme (such as op://) is passed to the pack that resolves it. #field picks one key of a secret that holds several; ?version= pins a version where the store keeps them.

+
-
-
+

+ +

+
+ Type: array of object
-

Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- No Additional Properties + env

env

Type: object
+Must match regular expression: ^env://[A-Za-z_][A-Za-z0-9_]*$ -
-
-
-

- -

-
- -
-
+
+ Type: string
-

Measure name (e.g., 'totalamount', 'ordercount').

-
+ auth + + + + secretRef + + + + anyOf + + + + file

file

Type: object
+Must match regular expression: ^file://(/(?!\.{1,2}(/|$))[^/\s#?\x00-\x1f\x7f-\x9f]+)+$ -
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -

vault

Type: object
+Must match regular expression: ^(vault|openbao)://[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_@=+-][A-Za-z0-9_.@=+-]*)+#[A-Za-z0-9_.@-]+(\?version=[1-9][0-9]*)?$ + + + + + + +
+ + + Type: enum (of string)
-

Aggregation function applied to the expression.

-
-

Must be one of:

-
  • "sum"
  • "avg"
  • "count"
  • "count_distinct"
  • "min"
  • "max"
  • "median"
  • "percentile"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

SQL expression to aggregate. Defaults to measure name if omitted.

-
+ aws-sm

aws-sm

Type: object
+Must match regular expression: ^(aws-sm|aws)://(arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:[A-Za-z0-9/_+=.@-]+|[A-Za-z0-9/_+=.@-]{1,512})(#[A-Za-z0-9_.@-]+)?(\?version=[A-Za-z0-9-]{32,64})?$ -
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: object
-

Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.

-
No Additional Properties + auth + + + + secretRef + + + + anyOf + + + + gcp-sm

gcp-sm

Type: object
+Must match regular expression: ^(gcp-sm|gcp)://projects/[a-z0-9-]+/secrets/[A-Za-z0-9_-]{1,255}(/versions/([1-9][0-9]*|latest))?$ -
-
-
-

- -

-
- -
-
+
+ Type: number
-

Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted.

-
+ secretRef + + + + anyOf + + + + azure-kv

azure-kv

Type: object
+Must match regular expression: ^(azure-kv|azure)://[A-Za-z0-9-]{3,24}/[A-Za-z0-9-]{1,127}(\?version=[0-9a-f]{32})?$ -

Value must be greater or equal to 0 and lesser or equal to 1

- -
-
-
-
-
-
-
-

- -

-
-
-
+ +
+ Type: boolean
-

Use the discrete percentile (PERCENTILEDISC / usediscrete_percentile) instead of continuous interpolation.

-
+ secretRef + + + + anyOf + + + + pack

pack

Type: object
+Must match regular expression: ^(?!(env|file|vault|openbao|aws-sm|aws|gcp-sm|gcp|azure-kv|azure)://)[a-z][a-z0-9+.-]*://(?![^/#?]*:[^/#?]*@)[^\s#?\x00-\x1f\x7f-\x9f]+(#[A-Za-z0-9_.@-]+)?(\?version=[A-Za-z0-9._-]+)?$ + + + + + + +
@@ -29016,18 +29041,18 @@

-
+
-
+

- +

-
+
Type: string
-

Override the model-level default time dimension for this measure.

-
+ rules
Type: boolean Default: false
+
+

kind confluent only, on a subject an expose produces, and Avro only: true registers the expose's error-severity row checks (contract.invariants[] with a check and no mustBe* bound) as Confluent data-contract rules (CEL CONDITION rules in mode WRITE, onFailure ERROR) that Confluent's serializers enforce on every record. Other invariants (derive, aggregate, dependency, a check with a bound, a warning or info check) are reported as not enforced by the registry; a check with no exact CEL form is refused. Needs Confluent Platform Enterprise 7.4+ with the rule set extension, or Confluent Cloud's Stream Governance Advanced package.

+ +
+
+ +
@@ -29085,19 +29117,23 @@

-
+
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: object
-

Dimension over which this measure cannot be naively aggregated (e.g., account balances).

-
No Additional Properties + additionalProperties
Type: object
+ -
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
+

Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
- + binding + + + + governance + + + + lakeFormation
Type: object
+

Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-

Which value to pick when the non-additive dimension is present.

-
-

Must be one of:

-
  • "min"
  • "max"
-
+ registerLocation
Type: boolean Default: false
+

When true, emit awslakeformationresource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path.

+
@@ -29304,22 +29366,18 @@

Must be one of:

-
-
-
-
-
+
-
+

- +

-
+
Type: boolean Default: false
-

If true, automatically create a simple metric from this measure.

-
+ bucketPolicy
Type: enum (of string) Default: "cross-account"
+
+

NEW in v0.7.6: Which grants[] principals get a statement in the awss3bucketpolicy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted awss3bucketpolicy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted.

+ +
+
+ +
+

Must be one of:

+
  • "cross-account"
  • "none"
  • "all-grantees"
+
@@ -29377,24 +29445,18 @@

-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of object
-

Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.

+ governance + + + + lakeFormation + + + + grants
Type: array of object
+

Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as awslakeformationpermissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.

@@ -29435,7 +29511,7 @@

No Additional Items

Each item of this array must be:

-
+
Type: object
+ lakeFormation + + + + grants + + + + grants items
Type: object
No Additional Properties @@ -29481,18 +29571,18 @@

-
+
-
+

- +

-
+
Type: string
-

Dimension name (e.g., 'orderdate', 'region', 'productcategory').

-
- - - - - - + grants + + + + grants items + + + + principal
Type: string
+

IAM principal ARN (role or user) receiving the grant.

+
Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + +
-
+
-
+

- +

-
+
Type: enum (of string)
-

Dimension type: categorical (text/enum grouping) or time (temporal axis).

-
-

Must be one of:

-
  • "categorical"
  • "time"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of enum (of string)
+

LF permissions granted. Valid values per the hashicorp/aws awslakeformationpermissions resource.

+
+ + + + + +

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
-

SQL expression. Defaults to dimension name if omitted.

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
- Type: enum (of string)
+
+

Must be one of:

+
  • "ALL"
  • "SELECT"
  • "ALTER"
  • "DROP"
  • "DELETE"
  • "INSERT"
  • "DESCRIBE"
  • "CREATE_TABLE"
  • "CREATE_DATABASE"
  • "DATA_LOCATION_ACCESS"
+
+
+
-
+
-
+

- +

-
+
Type: object
-

Type-specific parameters.

-
No Additional Properties + grants + + + + grants items + + + + permissionsWithGrantOption
Type: array of string
+

Subset of permissions the principal can re-grant. Maps to awslakeformationpermissions.permissionswithgrant_option.

+
- -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

Default time granularity for this dimension.

-
-

Must be one of:

-
  • "day"
  • "week"
  • "month"
  • "quarter"
  • "year"
  • "hour"
  • "minute"
-
+ grants items + + + + permissionsWithGrantOption + + + + permissionsWithGrantOption items
Type: string
+ -
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of object
-

Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- No Additional Properties + columns
Type: array of string
+

Optional column-level restriction. When set, the grant is emitted against awslakeformationpermissions.tablewithcolumns instead of .table, allowing the principal to read only the listed columns.

+
- -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Metric name (e.g., 'netrevenue', 'customerchurn_rate').

-
+ grants + + + + grants items + + + + columns + + + + columns items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: string
-

Business definition of this metric.

+ grants + + + + grants items + + + + excludedColumns
Type: array of string
+

Opposite of columns: every column EXCEPT these is granted. Maps to tablewithcolumns.excludedcolumnnames. Mutually exclusive with columns.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator).

-
-

Must be one of:

-
  • "simple"
  • "derived"
  • "ratio"
-
+ grants + + + + grants items + + + + excludedColumns + + + + excludedColumns items
Type: string
+ +
+
-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Reference to a measure name (for simple metrics).

-
+ tags
Type: object
+
+

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

+ +
+
+ +
-
-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: string
-

SQL WHERE clause filter (e.g., "status = 'completed' AND is_refunded = FALSE").

-
- - - - - + tags + + + + additionalProperties
Type: string
+ + + + + +
-
+
+
+
+
+
-
+

- +

-
+
Type: array of string
-

References to other metric names (for derived/ratio metrics).

-
+ rowFilter
Type: object
+

Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as awslakeformationdatacellsfilter. The filter must then be granted to principals via grants[].permissions including SELECT — LF enforces both layers.

+
No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: string
- + name
Type: string
+

Filter name (must be unique per table).

+
-
-
-
+
-
+

- +

-
+
Type: string
-

Mathematical expression combining input metrics (for derived metrics).

+ rowFilter + + + + rowExpression
Type: string
+

PartiQL-flavoured row predicate. ALL lets every row through (use when only column filtering is intended).

@@ -30560,18 +30671,18 @@

-
+
-
+

- +

-
+
Type: string
-

Numerator metric name (for ratio metrics).

+ rowFilter + + + + columnNames
Type: array of string
+

Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Denominator metric name (for ratio metrics).

-
+ rowFilter + + + + columnNames + + + + columnNames items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: string
-

Business owner of this metric definition.

+ rowFilter + + + + excludedColumnNames
Type: array of string
+

Columns hidden through the filter (everything else is visible).

- -
-
-
-
-
-
-
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: array of string
-

Semantic model tags for discovery and categorization.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Semantic model labels for automation and metadata.

-
Same definition as labels -
+ excludedColumnNames items
Type: string
+ + + + + + +
-
-
+
-
+

- +

-
+
Type: object
-

AWS Glue crawler configuration for auto-discovery of schema from S3 paths.

-
- - - - - - -
-
-
-

- -

-
- -
-
- - Type: string
-

Glue crawler name.

+ allColumns
Type: boolean Default: false
+

Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted.

@@ -30974,18 +31039,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

IAM role ARN for the crawler.

-
+ governance + + + + lakeFormation + + + + maskedViews
Type: object
+

Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, <table>_masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Cron schedule expression (e.g., 'cron(0 6 * * ? *)').

+ governance + + + + lakeFormation + + + + maskedViews + + + + definer
Type: string
+

ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole).

@@ -31084,18 +31184,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Custom classifier names to apply.

-
- - - - - - No Additional Items

Each item of this array must be:

-
- Type: string
+

The Glue connection of type VIEWVALIDATIONATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it.

+
+
+
-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Policy for schema changes detected by the crawler.

+ packaging
Type: object
+

NEW in v0.7.6: Per-exposure packaging override — key-wise precedence over the contract-wide top-level packaging block (binding.packaging > packaging > absent-legacy).

+ + No Additional Properties -
+
-
+

- -

+ +

-
+
Type: enum (of string)
-
+ mode
Type: enum (of string)
+

NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in containers.

+

Must be one of:

-
  • "UPDATE_IN_DATABASE"
  • "LOG"
+
  • "isolated"
  • "shared"
@@ -31303,18 +31390,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "LOG"
  • "DELETE_FROM_DATABASE"
  • "DEPRECATE_IN_DATABASE"
-
+ pool
Type: string
+

NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' — a pool must be addressable. Propagated as the fluid_pool label/tag for cost attribution.

+
- +

Must be at least 1 characters long

-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

AWS Glue Iceberg table management configuration (snapshots, compaction).

-
- - - - - - -
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-

Underlying file format for Iceberg data files.

-
-

Must be one of:

-
  • "parquet"
  • "orc"
  • "avro"
-
+ poolManifest
Type: string
+

NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans).

+
- +

Must be at least 1 characters long

-
+
-
+

- +

-
+
Type: object
-

Snapshot retention policy.

-
+ packaging + + + + containers
Type: object
+
+

NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over mode for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket → awss3bucket / googlestoragebucket; database → snowflakedatabase AND awsgluecatalogdatabase; dataset → googlebigquerydataset; schema → snowflakeschema; warehouse → snowflakewarehouse; cluster → Confluent environment/cluster (v1: shared only). NEW in v0.7.7 (preview), AWS streaming kinds: lfTagVocabulary → awslakeformationlftag (the domain's LF-tag keys and values); emrApplication → awsemrserverlessapplication; schemaRegistry → awsglueregistry; athenaWorkgroup → awsathenaworkgroup; kmsKey → awskmskey (the domain key); mwaaEnvironment → awsmwaa_environment (shared only: the platform team owns it, and a product ships DAG files into it). These kinds are not part of the blanket mode's plan summary; a product reads each one's decision explicitly.

+ +
+
+ +
No Additional Properties -
+
-
+

- +

-
+
Type: integer
-

Maximum age of snapshots in milliseconds.

+ containers + + + + bucket
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the object-store bucket (awss3bucket / googlestoragebucket).

+ +
+

Must be one of:

+
  • "isolated"
  • "shared"
+
@@ -31590,18 +31653,18 @@

-
+
-
+

- +

-
+
Type: integer
-

Minimum number of snapshots to retain.

-
- - - - - - -
-
-
-
+ containers + + + + database
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the database container — covers Snowflake database AND AWS glue_database.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: object
-

Iceberg compaction settings.

-
- - - - - - -
+ packaging + + + + containers + + + + dataset
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the BigQuery dataset (googlebigquerydataset).

+
Same definition as bucket +
+
+
+
+
-
+

- +

-
+
Type: boolean
- - - - - - - + containers + + + + schema
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema).

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: integer
-

Target file size in MB after compaction.

-
- - - - - - -
-
-
-
-
-
-
-
+ containers + + + + warehouse
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) — isolated gives per-product cost attribution.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (fluid mcp output-port serve). Its presence opts the expose into the Fluid MCP gateway, where policy.agentPolicy.allowedModels / allowedUseCases are enforced at every read.

-
- - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: object
- No Additional Properties - - - - - - -Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only — dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated').

+
Same definition as bucket +
+
+
+
+
-
+

- +

-
+
Type: integer
-

Hard cap for the sample MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100.

-
- - - -

Value must be greater or equal to 1

- - -
-
-
-
+ containers + + + + lfTagVocabulary
Type: enum (of string)
+

NEW in v0.7.7 (preview): Ownership of the Lake Formation tag vocabulary (awslakeformationlf_tag). Shared: the domain defines the tag keys and values and the product only associates them.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: object
- No Additional Properties - - - - - - -Type: enum (of string)
+

NEW in v0.7.7 (preview): Ownership of the EMR Serverless application (awsemrserverlessapplication) a Spark build runs on.

+
Same definition as bucket +
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Surfaced on the describe tool so consumer agents can declare downstream handling. Advisory; the real gate is policy.agentPolicy.

-
-

Must be one of:

-
  • "public"
  • "internal"
  • "confidential"
  • "restricted"
-
- - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
+ containers + + + + schemaRegistry
Type: enum (of string)
+

NEW in v0.7.7 (preview): Ownership of the Glue schema registry (awsglueregistry) that holds produced subjects.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: array
- - - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- - - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: string
-

Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity.

-
Same definition as id + athenaWorkgroup
Type: enum (of string)
+

NEW in v0.7.7 (preview): Ownership of the Athena workgroup (awsathenaworkgroup) the product's queries and dbt builds run in.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: string
-

The upstream exposeId being consumed.

-
Same definition as id -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own — a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived <productId>.<exposeId>; until now consumes[] was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it.

-
- - - - - - + kmsKey
Type: enum (of string)
+

NEW in v0.7.7 (preview): Ownership of the KMS key (awskmskey) that encrypts the product's data, checkpoints and logs.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: string
-

Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces.

-
- - - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-
-

NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within versionConstraint. Exported as the ODPS inputPorts[].version, and as the version of the matching outputPorts[].inputContracts[] entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. versionConstraint stays the range this product accepts; this is the single version its exported lineage claims. fluid validate does not check that it falls inside versionConstraint.

- -
-
- -
- - Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ + mwaaEnvironment
Type: const
+

NEW in v0.7.7 (preview): The MWAA environment (awsmwaaenvironment) is the platform team's, so shared is the only value: a product ships its DAG files into the environment's bucket and never creates or changes the environment.

+
Specific value: "shared" -
-
Example:
-
"2.1.0"
-
+ +
+
+
-
+
+
+
+
+
-
+

- +

-
+
Type: object
- No Additional Properties + binding + + + + encryption
Type: object
+

NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.

+
No Additional Properties -
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Default: "product"
+
+

NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): fluid apply creates a customer managed key for each bucket this product owns (awskmskey, rotation on, alias alias/fluid/<contract id>/<bucket>) and makes it the bucket's default encryption (awss3bucketserversideencryptionconfiguration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. tofu destroy deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; fluid apply refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/<name>' or a key or alias ARN: an existing key, looked up at plan time (data awskmskey, so the identity running fluid apply needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRICDEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. fluid verify checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (googlekmskeyring fluid-<contract id>-<dataset> and googlekmscryptokey 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's defaultencryptionconfiguration and the table's encryptionconfiguration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: tofu destroy removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/<p>/locations/<l>/keyRings/<r>/cryptoKeys/<k>': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so fluid apply refuses that plan without --allow-data-loss; the next build lands the data again. fluid verify checks kmsKeyName on the dataset and the table.

+ +
+
+ +

+ +

+
+ + + Type: string
-

Maximum acceptable staleness for consumed data.

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: number
-

Minimum completeness expectation.

-
+ item 0
Type: enum (of string)
+
+

Must be one of:

+
  • "product"
  • "none"
+
-

Value must be greater or equal to 0 and lesser or equal to 1

- -
-
-
-
-
-
-
-

- -

-
-
-
+ +
+ Type: array of string
-

QoS expectation tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
+ encryption + + + + kms + + + + anyOf + + + + item 1
Type: object
+Must match regular expression: ^alias/[A-Za-z0-9/_-]{1,250}$ + -
-
+ + + + +
+ Type: object
-

QoS expectation labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
- + item 2
Type: object
+Must match regular expression: ^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$ - No Additional Items

Each item of this array must be:

-
-
+ +
Type: object
+Must match regular expression: ^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$ + + + + + + +
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+
+

NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.

+ +
+
+ +
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ +
+

+ +

+
+ + + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+ + + Type: array of string
+ + + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only format: kafka_topic, which tells a consumer nothing about whether the values are Avro or JSON — the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server format field; the two -confluent values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

How message values are serialized.

+
+

Must be one of:

+
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

How message keys are serialized, for a topic whose keys are not opaque bytes.

+
+

Must be one of:

+
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus expire, which is only meaningful where there is storage to expire.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Unified lifecycle state vocabulary.

+
+ +
+

Must be one of:

+
  • "preview"
  • "active"
  • "deprecated"
  • "retired"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless expire is true. With expire: true on an AWS S3 binding, fluid apply expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one.

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+
+

NEW in v0.7.6: Delete stored data once it is older than retention. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: fluid apply writes one rule per expose into the bucket's awss3bucketlifecycleconfiguration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version retention after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that fluid apply did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and fluid verify checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing expire later deletes the lifecycle configuration, which fluid apply refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); tofu destroy deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (timepartitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expirationms = retention in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than retention after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than retention lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding expire to a live table replaces it: fluid apply refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of retention is an in-place update. Removing expire from a live table removes the partitioning's replacement trigger, which fluid apply also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer retention instead. fluid verify checks the live table's partition expiration.

+ +
+
+ +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Reference to replacement data product or expose.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Deprecation policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Deprecation policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Lifecycle tags for automation.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Lifecycle labels for governance.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human/machine-readable SLI check description

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "slack"
  • "email"
  • "webhook"
  • "pagerduty"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Alert configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Alert configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default SLI trio (freshness, completeness, latency) by kind.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+

Enable auto-generation of default SLIs.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default freshness SLI configuration.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Freshness SLI tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Freshness SLI labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default completeness SLI configuration.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number Default: 0.95
+

Completeness threshold (auto-set by kind: table=0.95, stream=0.99)

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Completeness SLI tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Completeness SLI labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default latency SLI configuration.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number Default: 95
+

Latency percentile to measure.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 100

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Latency SLI tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Latency SLI labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Default SLI configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default SLI configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "log"
  • "file"
  • "webhook"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

For kind=file: NDJSON sink path (env-vars expanded).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+

For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Observability configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Observability configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Documentation tags for organization.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Documentation labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.

+
+ + No Additional Properties + + + + + +
+
Example:
+
{
+    "name": "orders_revenue_model",
+    "description": "Canonical semantic model for revenue analytics",
+    "defaultAggTimeDimension": "order_date",
+    "entities": [
+        {
+            "name": "order",
+            "type": "primary",
+            "expr": "order_id"
+        },
+        {
+            "name": "customer",
+            "type": "foreign",
+            "expr": "customer_id"
+        }
+    ],
+    "measures": [
+        {
+            "name": "order_amount",
+            "agg": "sum",
+            "expr": "amount",
+            "description": "Sum of all order amounts"
+        },
+        {
+            "name": "order_count",
+            "agg": "count_distinct",
+            "expr": "order_id",
+            "createMetric": true
+        }
+    ],
+    "dimensions": [
+        {
+            "name": "order_date",
+            "type": "time",
+            "expr": "completed_at",
+            "typeParams": {
+                "timeGranularity": "day"
+            }
+        },
+        {
+            "name": "region",
+            "type": "categorical"
+        }
+    ],
+    "metrics": [
+        {
+            "name": "net_revenue",
+            "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds",
+            "type": "derived",
+            "inputMetrics": [
+                "gross_revenue",
+                "total_discounts"
+            ],
+            "expr": "gross_revenue - total_discounts",
+            "owner": "finance-data@company.com"
+        },
+        {
+            "name": "gross_revenue",
+            "description": "Total value of completed orders",
+            "type": "simple",
+            "measure": "order_amount",
+            "filter": "status = 'completed'"
+        }
+    ]
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human-readable name for this semantic model.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business context explaining what this semantic model represents.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Default time dimension used for measure aggregation. Can be overridden per-measure.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Join keys with type annotations. Entities connect this table to others in the semantic graph.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Entity name (e.g., 'order', 'customer').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier; the key of a slowly changing dimension, whose versions a validity window separates: see dimensions[].typeParams.validityParams).

+
+

Must be one of:

+
  • "primary"
  • "foreign"
  • "unique"
  • "natural"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Column expression. Defaults to entity name if omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

NEW in v0.7.7 (preview): Named joins from this expose to an expose of the same contract, declared on the many side. Entities join by entity name only; a relationship names its target expose and the ordered key columns on both sides, so a composite key is one relationship. Names are unique within the expose.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

NEW in v0.7.7 (preview): One foreign key, from the expose that declares it (the many side) to an expose of the same contract (the one side). fromColumns and toColumns are ordered and pair up by position, so they have the same length, and each names a column of its own expose's contract.schema. The shape is Apache Ossie's Relationship (fromcolumns, tocolumns) plus the cardinality and join type Ossie leaves out.

+
+ + No Additional Properties + + + + + +
+
Examples:
+
{
+    "name": "orders_to_customers",
+    "to": "customer_profiles",
+    "fromColumns": [
+        "customer_id"
+    ],
+    "toColumns": [
+        "id"
+    ],
+    "cardinality": "many_to_one",
+    "joinType": "left"
+}
+
+
{
+    "name": "order_lines_to_orders",
+    "to": "orders",
+    "fromColumns": [
+        "order_id",
+        "region"
+    ],
+    "toColumns": [
+        "order_id",
+        "region"
+    ],
+    "cardinality": "many_to_one"
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Relationship name, unique within the expose (e.g., 'orderstocustomers').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

exposeId of the target expose, in the same contract. It may be the declaring expose itself, for a self-join such as employees to their managers.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Foreign-key columns of the declaring expose, in key order.

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Key columns of the target expose, in the order of fromColumns.

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

manytoone: many rows of the declaring expose can match one target row (an ordinary foreign key). onetoone: each target row matches at most one row of the declaring expose. Against a target with a validity window (a slowly changing dimension), the one target row is the version valid at the declaring row's point in time.

+
+

Must be one of:

+
  • "many_to_one"
  • "one_to_one"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "left"
+

How a consumer joins the target: left keeps the declaring expose's rows that have no match, inner drops them. Defaults to left.

+
+

Must be one of:

+
  • "left"
  • "inner"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

NEW in v0.7.7 (preview): Columns copied through this relationship (RFC-data-rules.md section 5): each is in both exposes' schemas under one name and of one type, and on every row whose link reaches a target row it IS NOT DISTINCT FROM the target's. Short for a derive of the column with expr <name>.<column> on the rows the link reaches. The relationship's own key columns are never listed: they agree by definition.

+
+ + + + + +

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Measure name (e.g., 'totalamount', 'ordercount').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Aggregation function applied to the expression.

+
+

Must be one of:

+
  • "sum"
  • "avg"
  • "count"
  • "count_distinct"
  • "min"
  • "max"
  • "median"
  • "percentile"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL expression to aggregate. Defaults to measure name if omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+

Use the discrete percentile (PERCENTILEDISC / usediscrete_percentile) instead of continuous interpolation.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Override the model-level default time dimension for this measure.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Dimension over which this measure cannot be naively aggregated (e.g., account balances).

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Which value to pick when the non-additive dimension is present.

+
+

Must be one of:

+
  • "min"
  • "max"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

If true, automatically create a simple metric from this measure.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Dimension name (e.g., 'orderdate', 'region', 'productcategory').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Dimension type: categorical (text/enum grouping) or time (temporal axis).

+
+

Must be one of:

+
  • "categorical"
  • "time"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL expression. Defaults to dimension name if omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Type-specific parameters.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Default time granularity for this dimension.

+
+

Must be one of:

+
  • "day"
  • "week"
  • "month"
  • "quarter"
  • "year"
  • "hour"
  • "minute"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.7 (preview): Marks this time dimension as one bound of the validity window of a type II slowly changing dimension, field for field dbt's validity_params. A model with a window has exactly two such dimensions, one with isStart and one with isEnd, and one natural entity: the key whose versions the window separates.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+

This dimension holds the time each version starts to apply.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+

This dimension holds the time each version stops applying.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Metric name (e.g., 'netrevenue', 'customerchurn_rate').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business definition of this metric.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator).

+
+

Must be one of:

+
  • "simple"
  • "derived"
  • "ratio"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Reference to a measure name (for simple metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL WHERE clause filter (e.g., "status = 'completed' AND is_refunded = FALSE").

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

References to other metric names (for derived/ratio metrics).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Mathematical expression combining input metrics (for derived metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Numerator metric name (for ratio metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Denominator metric name (for ratio metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business owner of this metric definition.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Semantic model tags for discovery and categorization.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Semantic model labels for automation and metadata.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

AWS Glue crawler configuration for auto-discovery of schema from S3 paths.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Glue crawler name.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

IAM role ARN for the crawler.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cron schedule expression (e.g., 'cron(0 6 * * ? *)').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Custom classifier names to apply.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Policy for schema changes detected by the crawler.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "UPDATE_IN_DATABASE"
  • "LOG"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "LOG"
  • "DELETE_FROM_DATABASE"
  • "DEPRECATE_IN_DATABASE"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

AWS Glue Iceberg table management configuration (snapshots, compaction).

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Underlying file format for Iceberg data files.

+
+

Must be one of:

+
  • "parquet"
  • "orc"
  • "avro"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Snapshot retention policy.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Maximum age of snapshots in milliseconds.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Minimum number of snapshots to retain.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Iceberg compaction settings.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Target file size in MB after compaction.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (fluid mcp output-port serve). Its presence opts the expose into the Fluid MCP gateway, where policy.agentPolicy.allowedModels / allowedUseCases are enforced at every read.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Hard cap for the sample MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100.

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Surfaced on the describe tool so consumer agents can declare downstream handling. Advisory; the real gate is policy.agentPolicy.

+
+

Must be one of:

+
  • "public"
  • "internal"
  • "confidential"
  • "restricted"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The upstream exposeId being consumed.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own — a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived <productId>.<exposeId>; until now consumes[] was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces.

+
+ + + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+
+

NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within versionConstraint. Exported as the ODPS inputPorts[].version, and as the version of the matching outputPorts[].inputContracts[] entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. versionConstraint stays the range this product accepts; this is the single version its exported lineage claims. fluid validate does not check that it falls inside versionConstraint.

+ +
+
+ +
+ + Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ + + + + + +
+
Example:
+
"2.1.0"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Maximum acceptable staleness for consumed data.

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Minimum completeness expectation.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

QoS expectation tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

QoS expectation labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business purpose for consuming this data.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Consumption relationship tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Consumption relationship labels.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. fluid apply re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set.

+
Must match regular expression: ^sha256:[0-9a-f]{64}$ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.

+
+ + No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "hybrid-reference" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Hybrid-reference pattern from v0.4.0 (dbt-style).

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

+

+ +

+
+ + + Type: string
+ + + + + + + +
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

+

+ +

+
+ + + Type: string
+ + + + + + + +
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: enum (of string)
+
+

Must be one of:

+
  • "table"
  • "view"
  • "incremental"
  • "ephemeral"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "embedded-logic" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Embedded logic pattern from v0.4.0.

+
+ + No Additional Properties

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+
+

Must not be:

+
+
+ + +
+

+ +

+
+ + + Type: object
+ +
+

The following properties are required:

+
  • stream
+
+ + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + + +
+
+
+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • stream
  • mapping
+
+ + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ +
+

The following properties are required:

+
  • spark
+
+ + + + + +
+
+
+
+
+ + + + + + +
+
+
+ + + + + + +
+ + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • sql
+
+ + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "sql"
  • "flink_sql"
  • "pyspark"
  • "scala"
  • "python"
  • "r"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.7 (preview): Where and how a Spark build runs.

+
+ + No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "emr-eks" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • virtualClusterId
+
+ + + + + +
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "emr-serverless" + + + + + + +
+
+
+
+
+
+
+ + + + + + +
+ + + +
+ + +
+

applicationId is only valid with runtime: emr-serverless

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • applicationId
+
+ + + + + +
+
+
+ + + + + + +
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "emr-ec2" + + + + + + +
+
+
+
+
+
+
+ + + + + + +
+ + + +
+ + +
+

clusterId is only valid with runtime: emr-ec2

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • clusterId
+
+ + + + + +
+
+
+ + + + + + +
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "emr-eks" + + + + + + +
+
+
+
+
+
+
+ + + + + + +
+ + + +
+ + +
+

virtualClusterId is only valid with runtime: emr-eks

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • virtualClusterId
+
+ + + + + +
+
+
+ + + + + + +
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "glue" + + + + + + +
+
+
+
+
+ + + +
+ + +
+

releaseLabel is an EMR release; a glue runtime takes glueVersion

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • releaseLabel
+
+ + + + + +
+
+
+ + + + + + +
+
+ + +
+

glueVersion is only valid with runtime: glue; EMR runtimes take releaseLabel

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • glueVersion
+
+ + + + + +
+
+
+ + + + + + +
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

Business purpose for consuming this data.

-
+ runtime
Type: enum (of string)
+

The AWS service that runs the job: EMR Serverless, EMR on EC2, EMR on EKS or an AWS Glue Spark job. One runner serves all four.

+
+

Must be one of:

+
  • "emr-serverless"
  • "emr-ec2"
  • "emr-eks"
  • "glue"
+
@@ -32864,18 +49561,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Consumption relationship tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Consumption relationship labels.

-
Same definition as labels -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace.

-
+ mode
Type: enum (of string)
+

streaming: a long-running Structured Streaming job. batch: a job that runs to completion. Absent: streaming when properties.stream is set, else batch.

+
+

Must be one of:

+
  • "streaming"
  • "batch"
+
-

Must be at least 1 characters long

+
-
+
-
+

- +

-
+
Type: string
-

Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. fluid apply re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set.

-
Must match regular expression: ^sha256:[0-9a-f]{64}$ - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array
-

NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
-

Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.

-
+ properties + + + + parameters + + + + spark + + + + applicationId
Type: string
+

emr-serverless only: an existing EMR Serverless application. Absent: the forge-managed application shared by the product's domain.

+
Must match regular expression: ^[0-9a-z]{1,64}(?![\s\S]) + - No Additional Properties

- -

-
- + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
- -
- - - Type: object
- + clusterId
Type: string
+

emr-ec2 only: a running EMR cluster (j-...). Absent: a transient cluster that ends with the job.

+
Must match regular expression: ^j-[0-9A-Za-z]{1,254}(?![\s\S]) -
+
+
+
+
+
-
+

- +

-
+
Type: const
-Specific value: "hybrid-reference" + properties + + + + parameters + + + + spark + + + + virtualClusterId
Type: string
+

emr-eks only, and required there: the EMR on EKS virtual cluster.

+
Must match regular expression: ^[0-9a-z]{1,64}(?![\s\S]) @@ -33298,13 +49924,19 @@

-
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- + then + + + + properties + + + + parameters + + + + spark + + + + glueVersion
Type: string Default: "5.0"
+

glue only: the AWS Glue version of the job.

+
Must match regular expression: ^[0-9]+\.[0-9]+(?![\s\S]) -
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Hybrid-reference pattern from v0.4.0 (dbt-style).

-
- - No Additional Properties + properties + + + + parameters + + + + spark + + + + releaseLabel
Type: string Default: "emr-7.12.0"
+

EMR runtimes only (emr-serverless, emr-ec2, emr-eks): the EMR release. 7.12 or later writes Iceberg with Lake Formation credentials.

+
Must match regular expression: ^emr-[0-9]+\.[0-9]+\.[0-9]+(?![\s\S]) -
+
+
+
+
+
-
+

- +

-
+
Type: string
- + parameters + + + + spark + + + + executionRoleArn
Type: string
+

The IAM role the job runs as. Absent: a least-privilege role forge creates for the build.

+
Must match regular expression: ^arn:aws[a-zA-Z0-9-]*:iam::[0-9]{12}:role/[A-Za-z0-9_+=,.@/-]{1,512}(?![\s\S]) @@ -33491,18 +50194,18 @@

-
+
-
+

- +

-
+
Type: string
-

dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

-
+ parameters + + + + spark + + + + maskKeySecretArn
Type: string
+

The AWS Secrets Manager secret holding the HMAC key for maskAtIngest: hash. Required when any mapping column sets maskAtIngest (the cross-field check lives in the Spark compiler).

+
Must match regular expression: ^arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+(?![\s\S]) @@ -33567,18 +50284,18 @@

-
+
-
+

- +

-
+

-

dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

-

- -

-
- + parameters + + + + spark + + + + sparkConf
Type: object
+

Spark configuration properties for the job, name to string value (spark.sql.shuffle.partitions: "64").

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

Type: string
+ sparkConf + + + + additionalProperties
Type: string
@@ -33720,9 +50462,27 @@

-

- +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: array of string
- - - - - - - No Additional Items

Each item of this array must be:

-
-
+

The Kafka connection a streaming Spark build reads, as parameters.flink.source is for a Flink build. Exactly one of bootstrapServers or mskClusterArn. The schema registry is not here: it is properties.stream.registry ($defs/schemaRegistry).

+
No Additional Properties

+ +

+
Type: string
- - - - - - - -
-
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+ allOf + + + + item 0

+

set bootstrapServers (a host:port list) or mskClusterArn (an Amazon MSK cluster)

+

+ +

+
+
-

Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

-

- -

-
- - - Type: object
+ +
+

The following properties are required:

+
  • bootstrapServers
+
+ + + + + +
+ + + Type: string
- - - - - - - -
- - - Type: array of string
+ anyOf + + + + item 1
Type: object
+ +
+

The following properties are required:

+
  • mskClusterArn
+
+ +
- No Additional Items

Each item of this array must be:

-
-
+ + + + +
Type: string
- - - - - - - -
-
-
- + allOf + + + + item 1
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
+ parameters + + + + spark + + + + source + + + + allOf + + + + item 1 + + + + if
Type: object
- +
+

The following properties are required:

+
  • bootstrapServers
+
-
-
-
-

- -

-
+
-
-

Additional Properties of any type are allowed.

+ + +
+ Type: object
- - - - - - - -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- -
+

set one of bootstrapServers or mskClusterArn, not both

+
+

Must not be:

+
+
+ + + Type: object
- - - - - - - -
-
-
-

- -

-
- -
-

Each additional property must conform to the following schema

- - Type: enum (of string)
-
-

Must be one of:

-
  • "table"
  • "view"
  • "incremental"
  • "ephemeral"
-
- + item 1 + + + + then + + + + not
Type: object
+ +
+

The following properties are required:

+
  • mskClusterArn
+
-
-
-
-
-
-
+ + + + + + +
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: array of string
-

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as tags + bootstrapServers
Type: string
+

Comma-separated host:port list of Kafka brokers. Same name and shape as parameters.flink.source.bootstrapServers.

+
Must match regular expression: ^\S(?:[^\u0000-\u001f\u007f]*\S)?(?![\s\S]) + + + + + +
-
+
-
+

- +

-
+
Type: object
-

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as labels -
-
-
-
-
-
-
-
-
+ parameters + + + + spark + + + + source + + + + mskClusterArn
Type: string
+

An Amazon MSK cluster. OpenTofu resolves its brokers through an MSK data source at plan time.

+
Must match regular expression: ^arn:aws[a-z-]*:kafka:[a-z0-9-]+:[0-9]{12}:cluster/.+(?![\s\S]) -
- +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
- -
- - - Type: object
- + source + + + + auth
Type: enum (of string)
+

How the job authenticates to the brokers. Absent: msk-iam with mskClusterArn, none with bootstrapServers. SASL/SCRAM is not supported yet.

+
+

Must be one of:

+
  • "msk-iam"
  • "none"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: const
-Specific value: "embedded-logic" - - - - - - -
-
-
-
-
- - - -
- - - Type: object
- + groupIdPrefix
Type: string
+

Prefix for the consumer group ids Spark generates (the Kafka source's groupIdPrefix option). Absent: Spark's default.

+
Must match regular expression: ^\S(?:[^\u0000-\u001f\u007f]*\S)?(?![\s\S]) - +

Must be at most 255 characters long

-
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: object
-

Embedded logic pattern from v0.4.0.

-
+ properties + + + + parameters + + + + additionalProperties
Type: object
- No Additional Properties

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- + -
- -
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

-
-

Must not be:

-
-
- + stream
Type: object
+

NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional, and on a Spark build (parameters.spark) with a mapping it is too: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
Type: object
- -
-

The following properties are required:

-
  • stream
-
+ kind
Type: const
+

The streaming system. Only kafka in this preview.

+
Specific value: "kafka" + -
+
+
+
+
+
-
+

- +

-
+
Type: object
- + topic
Type: string
+

One Kafka topic, by name: Kafka's legal characters (not . or ..), no pattern and no list.

+
Must match regular expression: ^(?!\.\.?(?![\s\S]))[A-Za-z0-9._-]{1,249}(?![\s\S]) - +
+
+
+
- -
+
+
+
+

+ +

-
-
-
-
-
- - - - - -
- - - -
- +
+
Type: object
- -
-

The following properties are required:

-
  • sql
-
- - - - - -
+ stream + + + + schema
Type: array
+

The record value's fields, as columns. Nested data uses fields (struct), element (array) and key/value (map).

+
- -
+

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
- - - - - - - + stream + + + + schema + + + + column
Type: object
+Same definition as exposes_items_contract_schema_items +
+
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "sql"
  • "flink_sql"
  • "pyspark"
  • "scala"
  • "python"
  • "r"
-
+ stream + + + + key
Type: object
+

NEW in v0.7.7 (preview): The record key, read as columns next to the value's. A record without a key reads its key columns as null.

+
+ + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: object
- + stream + + + + key + + + + format
Type: enum (of string)
+

raw: the key's bytes as one column, of type string (UTF-8) or binary. json and avro-confluent: a key record whose fields are the schema's columns.

+
+

Must be one of:

+
  • "raw"
  • "json"
  • "avro-confluent"
+
-
+
+
+
+
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- + key + + + + schema
Type: array
+

The key's fields, as columns: exactly one for raw.

+
- -
-
-
-
-
-
-
-
-
+

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: object
-

NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads.

-
- - No Additional Properties - - - - - - -
+ stream + + + + key + + + + schema + + + + column
Type: object
+Same definition as exposes_items_contract_schema_items +
+
+
+
+
+
+
-
+

- +

-
+
Type: const
-

The streaming system. Only kafka in this preview.

-
Specific value: "kafka" + key + + + + prefix
Type: string
+

Put in front of every key column's name, so a key field can share its name with a value field (a Debezium key repeats the value's id). A mapping names the prefixed column.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) @@ -35856,18 +52705,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

One Kafka topic, by name: Kafka's legal characters (not . or ..), no pattern and no list.

-
Must match regular expression: ^(?!\.\.?(?![\s\S]))[A-Za-z0-9._-]{1,249}(?![\s\S]) + metadata
Type: array
+

Kafka record metadata, each read as a column.

+
- -
-
-
-
-
+

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

How record values are encoded. json: one JSON object per record; a field the schema does not declare is ignored, and a declared field that is missing reads as null. avro-confluent: Avro in the Confluent Schema Registry wire format. The declared schema is the reader schema, so the producers' subject must stay readable by it: FORWARD or FULL compatibility, transitive.

-
-

Must be one of:

-
  • "json"
  • "avro-confluent"
-
+ metadata + + + + streamMetadataColumn
Type: object
+

NEW in v0.7.7 (preview): One piece of Kafka record metadata, read as a column. Its type is fixed by what it reads: topic string, partition int, offset long, timestamp timestamp_ltz(3), timestamp-type string, headers map<string, binary> and leader-epoch int. leader-epoch is null when the broker does not report it; the others are never null.

+
+ + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: array
-

The record value's fields, as columns. Nested data uses fields (struct), element (array) and key/value (map).

+ metadata + + + + metadata items + + + + name
Type: string
+

The column's name, unique among the stream's value, key and metadata columns.

- +

Must be at least 1 characters long

-

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
-Same definition as exposes_items_contract_schema_items -
-
+ metadata items + + + + from
Type: enum (of string)
+

The metadata the column reads.

+
+

Must be one of:

+
  • "topic"
  • "partition"
  • "offset"
  • "timestamp"
  • "timestamp-type"
  • "headers"
  • "leader-epoch"
+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.7 (preview): The record key, read as columns next to the value's. A record without a key reads its key columns as null.

-
+ metadata + + + + metadata items + + + + description
Type: string
- No Additional Properties -
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

raw: the key's bytes as one column, of type string (UTF-8) or binary. json and avro-confluent: a key record whose fields are the schema's columns.

-
-

Must be one of:

-
  • "raw"
  • "json"
  • "avro-confluent"
-
- - - - - - + registry
Type: object
+

NEW in v0.7.7 (preview): The registry that holds this topic's value schema. The topic belongs to its producer, so forge verifies the subject against schema and never registers it.

+
Same definition as registry
-
+
-
+

- +

-
+
Type: array
-

The key's fields, as columns: exactly one for raw.

-
- - - - - -

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

-
-
+

NEW in v0.7.7 (preview): When the engine starts a micro-batch. processingTime: one every ISO-8601 duration. availableNow: true: read what the topic holds now, in as many batches as it takes, then stop (a scheduled catch-up run). Absent: derived from the expose's qos.freshnessSLO.

+

+ +

+
Type: object
-Same definition as exposes_items_contract_schema_items -
-
-
-
-
-
-Type: object
+ No Additional Properties + + + + + + +
-
+

- +

-
+
Type: string
-

Put in front of every key column's name, so a key field can share its name with a value field (a Debezium key repeats the value's id). A mapping names the prefixed column.

-
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) - - - - - - -
-
-
-
+ oneOf + + + + item 0 + + + + processingTime
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window
-
-
-
-

- -

-
- -
-
+
+ Type: array
-

Kafka record metadata, each read as a column.

-
+ trigger + + + + oneOf + + + + item 1
Type: object
+ No Additional Properties -

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: object
-

NEW in v0.7.7 (preview): One piece of Kafka record metadata, read as a column. Its type is fixed by what it reads: topic string, partition int, offset long, timestamp timestamp_ltz(3), timestamp-type string, headers map<string, binary> and leader-epoch int. leader-epoch is null when the broker does not report it; the others are never null.

-
+ oneOf + + + + item 1 + + + + availableNow
Type: const
+Specific value: true + - No Additional Properties + + + + +
+
+
+
+
-
+
+
+
+
+
-
+

- +

-
+
Type: object
+

NEW in v0.7.7 (preview): The event-time watermark: the timestamp column it follows and how late a record may arrive before stateful operators drop it.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
-

The column's name, unique among the stream's value, key and metadata columns.

-
+ builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + stream + + + + watermark + + + + column
Type: string
+

A timestamp column of the stream, or a nested field of one as a dotted path. On a Spark build it must be timestampltz: Spark refuses a watermark on TIMESTAMPNTZ (EVENTTIMEISNOTONTIMESTAMPTYPE), and the Spark compiler refuses a timestamp (no time zone) watermark column.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![\s\S]) -

Must be at least 1 characters long

+
-
+
-
+

- +

-
+
Type: enum (of string)
-

The metadata the column reads.

-
-

Must be one of:

-
  • "topic"
  • "partition"
  • "offset"
  • "timestamp"
  • "timestamp-type"
  • "headers"
  • "leader-epoch"
+ delay
Type: string
+

The allowed lateness, as an ISO-8601 duration (PT10M).

+
Same definition as window
- - - - - - +
+
+
-
+
-
+

- +

-
+
Type: string
+

NEW in v0.7.7 (preview): Where the engine keeps its offsets and state, as an s3:// URI. Absent: derived from the product's Iceberg warehouse, one location per product and build. The checkpoint is the job's position in the topic: moving it starts the job again from startingOffsets.

+
Must match regular expression: ^s3://[a-z0-9][a-z0-9.-]{1,61}[a-z0-9](?:/[^\u0000-\u001f\u007f]*)?(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
- + builds items + + + + allOf + + + + item 1 + + + + then + + + + properties + + + + stream + + + + startingOffsets
Type: enum (of string)
+

NEW in v0.7.7 (preview): Where a job with no checkpoint yet starts reading: earliest (all the topic retains) or latest (new records only). Absent: the engine's default, latest for a Spark stream. A job that has a checkpoint resumes from it.

+
+

Must be one of:

+
  • "earliest"
  • "latest"
+
-
-
-
-
-
-
@@ -37560,29 +54618,126 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - recordMappingColumn

Type: object
-

NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.

-
- - No Additional Properties + recordMappingColumn
Type: object
+

NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![\s\S]) -
+
+
+
+
+
-
+

- +

-
+
Type: string
-

The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record.

-
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![\s\S]) + from
Type: string
+

The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku).

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S]) @@ -37668,18 +54823,18 @@

-
+
-
+

- +

-
+
Type: string
-

The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku).

-
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S]) - - - - - - + cast
Type: object
+

Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision).

+
Same definition as type
-
+
-
+

- +

-
+
Type: object
-

Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision).

-
Same definition as type + description
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: string
- + maskAtIngest
Type: enum (of string)
+

NEW in v0.7.7 (preview): Mask the value before it lands. hash = HMAC-SHA256 keyed by parameters.spark.maskKeySecretArn, lowercase hex; null stays null. The raw value never reaches the table, and a keyed digest cannot be reversed by hashing a dictionary of likely values. Opt-in: the default masking is a protected view derived from the table.

+
+

Must be one of:

+
  • "hash"
+
@@ -40949,8 +58107,92 @@

/> secretRef

Type: string
-

Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://).

-
Must match regular expression: ^[a-z][a-z0-9_+.-]*://.+ +

Reference to the connection's secret in the configured store, e.g. vault://kv/orgs/acme/pg#password or env://PG_PASSWORD.

+
Same definition as secretRef +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Write-only: a contract may not hold this value. Name the secret with secretRef instead.

+
@@ -41039,7 +58281,7 @@

/> user

Type: string
-

NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) — without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside secretRef (or an inline password) for the credential pair.

+

NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) — without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside secretRef for the credential pair.

@@ -53094,7 +70336,99 @@

/> secretRef

Type: string
+

NEW in v0.7.7: A reference to a secret held in a store, never the secret itself: a contract names a secret and never carries one. Grammar: <scheme>://<locator>[#field][?version=n]. Schemes: env, file, vault, aws-sm, gcp-sm, azure-kv; aliases openbao (vault), aws (aws-sm), gcp (gcp-sm) and azure (azure-kv) are accepted; any other lower-case scheme (such as op://) is passed to the pack that resolves it. #field picks one key of a secret that holds several; ?version= pins a version where the store keeps them.

+
Same definition as secretRef +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ Type: string
+

Write-only: a contract may not hold this value. Name the secret with secretRef instead.

+
@@ -56368,13 +73702,8 @@

/> secretRef

Type: string
- - - - - - - +

NEW in v0.7.7: A reference to a secret held in a store, never the secret itself: a contract names a secret and never carries one. Grammar: <scheme>://<locator>[#field][?version=n]. Schemes: env, file, vault, aws-sm, gcp-sm, azure-kv; aliases openbao (vault), aws (aws-sm), gcp (gcp-sm) and azure (azure-kv) are accepted; any other lower-case scheme (such as op://) is passed to the pack that resolves it. #field picks one key of a secret that holds several; ?version= pins a version where the store keeps them.

+
Same definition as secretRef
@@ -58598,7 +75927,7 @@

engine

Default: "dbt"
-

Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration.

+

Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration. NEW in v0.7.7: lakeflow (a Databricks Lakeflow Declarative Pipeline: pattern embedded-logic, properties.sql is the SELECT of the dataset an expose binds; forge-cli writes the materialized view around it, with the expose's columns and its dq.rules as pipeline expectations, and fluid apply provisions one serverless pipeline per build).

@@ -58660,7 +75989,7 @@

item 0

Type: enum (of string)

Must be one of:

-
  • "dbt"
  • "sql"
  • "python"
  • "spark"
  • "glue"
  • "custom"
  • "duckdb"
  • "airbyte"
  • "meltano"
  • "dlt"
  • "kafka-connect"
  • "debezium"
  • "flink"
+
  • "dbt"
  • "sql"
  • "python"
  • "spark"
  • "glue"
  • "custom"
  • "duckdb"
  • "airbyte"
  • "meltano"
  • "dlt"
  • "kafka-connect"
  • "debezium"
  • "flink"
  • "lakeflow"
diff --git a/tests/0.7.7/acquisition.json b/tests/0.7.7/acquisition.json index 6ffef97..75ad44b 100644 --- a/tests/0.7.7/acquisition.json +++ b/tests/0.7.7/acquisition.json @@ -50,7 +50,7 @@ "host": "db.internal", "port": 5432, "database": "crm", - "secretRef": "vault://secret/data/crm-db", + "secretRef": "vault://secret/data/crm-db#password", "schema": "public", "schemas": [ "public", @@ -165,7 +165,7 @@ "schema_registry": { "url": "http://schema-registry:8081", "auth": { - "secretRef": "vault://secret/data/sr-auth" + "secretRef": "vault://secret/data/sr-auth#password" } }, "streamingSink": { @@ -941,7 +941,7 @@ "errors": [ { "pointer": "/build/properties/source/connection/secretRef", - "keyword": "pattern" + "keyword": "anyOf" } ] }, diff --git a/tests/meta_test.py b/tests/meta_test.py index 61a8396..85bf112 100644 --- a/tests/meta_test.py +++ b/tests/meta_test.py @@ -283,7 +283,10 @@ def compat_gate_is_load_bearing() -> None: json.dumps(_set_in(json.loads(target.read_text()), ["properties", "exposes", "minItems"], 2), indent=2) ) waivers = root / "scripts" / "compat-waivers.txt" - waivers.write_text(waivers.read_text() + "0.7.4->0.7.5 /properties/exposes\n") + # the file must stay sorted, so the new line goes before the last pair's entry + text = waivers.read_text() + at = text.rindex("\n0.7.6->0.7.7 ") + 1 + waivers.write_text(text[:at] + "0.7.4->0.7.5 /properties/exposes\n" + text[at:]) r = run(root, "scripts/check-compat.py", "--from", "0.7.4", "--to", "0.7.5") check("a waiver silences its own pointer", r.returncode == 0, f"exit={r.returncode}")