diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.yml index ffd45735..d5b17eab 100644 --- a/.github/workflows/release-java.yml +++ b/.github/workflows/release-java.yml @@ -188,7 +188,8 @@ jobs: test -s "$artifact" done - python3 tools/verify_binary_artifact.py --jar "$jar_file" + python3 tools/verify_binary_artifact.py --jar "$jar_file" \ + --sources-jar "$sources_jar" --javadoc-jar "$javadoc_jar" for entry in \ META-INF/DEPENDENCIES \ diff --git a/.github/workflows/release-vote-gate.yml b/.github/workflows/release-vote-gate.yml index 87c0dadb..e7739c17 100644 --- a/.github/workflows/release-vote-gate.yml +++ b/.github/workflows/release-vote-gate.yml @@ -29,6 +29,10 @@ on: - "java/src/test/java/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.java" - "tools/create_source_release.sh" - "tools/deploy_java_staging.sh" + - "tools/release_checks.py" + - "tools/verify_binary_artifact.py" + - "tools/tests/test_release_checks.py" + - "tools/tests/test_verify_binary_artifact.py" - "tools/update_branch_version.sh" - "tools/verify_release_versions.py" - "tools/verify_source_archive.py" @@ -49,6 +53,10 @@ on: - "java/src/test/java/org/apache/paimon/mosaic/MosaicNativeLoaderSmokeTest.java" - "tools/create_source_release.sh" - "tools/deploy_java_staging.sh" + - "tools/release_checks.py" + - "tools/verify_binary_artifact.py" + - "tools/tests/test_release_checks.py" + - "tools/tests/test_verify_binary_artifact.py" - "tools/update_branch_version.sh" - "tools/verify_release_versions.py" - "tools/verify_source_archive.py" @@ -77,12 +85,19 @@ jobs: python-version: "3.12" - name: Install test dependencies - run: python -m pip install pytest PyYAML + run: | + sudo apt-get update + sudo apt-get install -y subversion + python -m pip install pytest PyYAML + svn --version --quiet + svnadmin --version --quiet - name: Run release vote tests run: | python -m pytest -q \ tools/tests/test_create_source_release.py \ + tools/tests/test_release_checks.py \ + tools/tests/test_verify_binary_artifact.py \ tools/tests/test_release_vote_workflow.py \ tools/tests/test_update_branch_version.py \ tools/tests/test_verify_release_versions.py \ @@ -117,6 +132,8 @@ jobs: set -euo pipefail python -m compileall -q \ tools/verify_release_versions.py \ + tools/release_checks.py \ + tools/verify_binary_artifact.py \ tools/verify_source_archive.py \ tools/tests/test_create_source_release.py \ tools/tests/test_release_vote_workflow.py \ diff --git a/docs/creating-a-release.html b/docs/creating-a-release.html index eb687671..cad80fca 100644 --- a/docs/creating-a-release.html +++ b/docs/creating-a-release.html @@ -95,6 +95,8 @@
Source release checks require python3 with tomllib (Python 3.11 or newer). This is a release-tool requirement; it does not change the Python binding's supported runtime versions. Select the required interpreter before starting source packaging.
If only the Python publish job fails due to a transient PyPI or TestPyPI issue, use Re-run failed jobs on the Release workflow. The successful Rust, Java, and Python wheel jobs do not need to be repeated.
Use a complete JDK 8 for Java release packaging. On Apple silicon use its ARM64 build. The script checks the JVM actually used by Maven, uses that same JDK for JNI checks, and builds with the UTC timezone. A JDK 11 release build is rejected because its generated Javadoc adds resources that are not covered by this release's JDK 8 license layout.
Check out the exact RC tag and copy the numeric run id from its successful push-triggered Release workflow. Run the local preflight first, then repeat without --dry-run:
RELEASE_RUN_ID="12345678901"
git checkout ${RC_TAG}
@@ -240,27 +243,25 @@ Deploy Java Artifacts to Nexus Staging
./tools/deploy_java_staging.sh --release-version ${RELEASE_VERSION} --rc ${RC_NUM} --run-id ${RELEASE_RUN_ID} --dry-run
./tools/deploy_java_staging.sh --release-version ${RELEASE_VERSION} --rc ${RC_NUM} --run-id ${RELEASE_RUN_ID}
The script downloads the four native artifacts and the CI java-package, checks and smoke-tests the CI JAR, then locally rebuilds and signs the Maven artifacts with the Release Manager's GPG and apache.releases.https settings. The CI main JAR is a validation reference and is not directly uploaded to Nexus; the local Maven rebuild is deployed. Record the resulting orgapachepaimon-XXXX repository id and leave it staged for the vote.
The release profile checks the main, sources and Javadoc JARs and verifies all four actual artifact signatures after GPG signing and before Nexus deployment. The signing key or subkey must be RSA with at least 2048 bits, as required by the ASF distribution policy; an RSA primary key does not make an Ed25519 signing subkey compliant. The same signature check runs before publishing the local source artifact directory. Unsigned -Dgpg.skip=true builds remain available for CI and local dry runs.
Create source release artifacts from the same commit as the RC tag:
+Run from the repository root. Create source release artifacts from the same commit as the RC tag and capture the verified output directory:
git checkout ${RC_TAG}
-cd tools
-RELEASE_VERSION=${RELEASE_VERSION} RC_TAG=${RC_TAG} ./create_source_release.sh
- This creates the following under tools/release/:
This creates the following under tools/release/${RC_TAG}/${RC_COMMIT_SHA}/, where RC_COMMIT_SHA is the full commit resolved from the verified signed tag:
apache-paimon-mosaic-${RELEASE_VERSION}-src.tgz — source archiveapache-paimon-mosaic-${RELEASE_VERSION}-src.tgz.asc — GPG signatureapache-paimon-mosaic-${RELEASE_VERSION}-src.tgz.sha512 — SHA-512 checksumBuild and verification messages go to stderr; stdout contains only the absolute output directory after all checks succeed. A failed run leaves SOURCE_RELEASE_DIR empty, even if it previously held an older candidate's directory. Existing candidate output is never overwritten.
svn checkout https://dist.apache.org/repos/dist/dev/paimon/ paimon-dist-dev --depth=immediates
-cd paimon-dist-dev
-mkdir paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}
-cp ../paimon-mosaic/tools/release/apache-paimon-mosaic-${RELEASE_VERSION}-src.* \
- paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}/
-svn add paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}
-svn commit -m "Add paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM}"
+ In the same shell, upload only the directory returned by source packaging. The required-variable check stops the command if packaging failed or has not run in this shell. Do not upload the parent tools/release/ directory; it may contain older candidates or legacy artifacts.
svn import "${SOURCE_RELEASE_DIR:?Source packaging must succeed first}" \
+ "https://dist.apache.org/repos/dist/dev/paimon/paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}" \
+ -m "Add paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM}"
Checklist:
If the vote reveals issues:
# From the paimon-mosaic checkout
-mv tools/release "../paimon-mosaic-${RC_TAG}-artifacts"
- cd paimon-dist-dev
-svn remove paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}
-svn commit -m "Remove paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM} (superseded)"
+svn remove "https://dist.apache.org/repos/dist/dev/paimon/paimon-mosaic-${RELEASE_VERSION}-rc${RC_NUM}" \
+ -m "Remove paimon-mosaic ${RELEASE_VERSION} RC${RC_NUM} (superseded)"
RC_NUM, then go back to Build a release candidate.(.*?)", documentation, re.DOTALL,
+ )]
+ preparation = next(line for block in blocks for line in block.splitlines()
+ if line.startswith("SOURCE_RELEASE_DIR="))
+ upload = next(block for block in blocks if block.startswith("svn import "))
+ upload = upload.replace("https://dist.apache.org/repos/dist/dev/paimon", svn_url)
+ # Intentionally no `set -e` or &&: execute the actual documented upload
+ # even after packaging fails, just as separately pasted RM commands do.
+ result = subprocess.run(
+ ["bash", "-c", preparation + "\n" + upload],
+ cwd=repo, env=env, capture_output=True,
+ )
+ if failure:
+ assert result.returncode != 0, output(result)
+ assert "Source packaging must succeed first" in output(result)
+ assert run(["svn", "list", svn_url], cwd=repo).stdout == b""
+ else:
+ assert result.returncode == 0, output(result)
+ exported = tmp_path / "exported"
+ run(["svn", "export", f"{svn_url}/paimon-mosaic-{VERSION}-rc2", str(exported)], cwd=repo)
+ artifacts = artifact_paths(repo, env["RC_TAG"])
+ assert {path.name for path in exported.iterdir()} == {path.name for path in artifacts}
+ for artifact in artifacts:
+ assert (exported / artifact.name).read_bytes() == artifact.read_bytes()
+ assert [path.read_bytes() for path in previous] == previous_bytes
+
+
def test_release_version_must_match_rc_tag(
tmp_path: Path,
signing_home: tuple[Path, str],
diff --git a/tools/tests/test_release_checks.py b/tools/tests/test_release_checks.py
new file mode 100644
index 00000000..7176d55d
--- /dev/null
+++ b/tools/tests/test_release_checks.py
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import release_checks as checks # noqa: E402
+
+
+@pytest.fixture(scope="module")
+def signing_keys(tmp_path_factory):
+ if shutil.which("gpg") is None:
+ pytest.skip("gpg is required")
+ home = tmp_path_factory.mktemp("signature-keys")
+ home.chmod(0o700)
+ command = ["gpg", "--homedir", str(home), "--batch", "--pinentry-mode", "loopback", "--passphrase", ""]
+
+ def run(*args):
+ return subprocess.check_output(command + list(args), stderr=subprocess.PIPE, text=True)
+
+ def fingerprints():
+ return [line.split(":")[9] for line in run("--with-colons", "--list-secret-keys").splitlines() if line.startswith("fpr:")]
+
+ run("--quick-gen-key", "Release Policy Test RC_NUM"
- )
+ assert "Keep the previous RC's local source artifacts" in retry_section
+ assert 'tools/release/${RC_TAG}/${RC_COMMIT_SHA}/' in source
+ assert 'mv tools/release' not in retry_section
def test_release_vote_gate_matches_blocking_contract() -> None:
diff --git a/tools/tests/test_verify_binary_artifact.py b/tools/tests/test_verify_binary_artifact.py
index deb54bc4..6501e495 100644
--- a/tools/tests/test_verify_binary_artifact.py
+++ b/tools/tests/test_verify_binary_artifact.py
@@ -35,6 +35,12 @@ def repository(tmp_path, monkeypatch):
"LICENSE-binary": b"JNI third-party licenses\n",
"LICENSE-binary-ffi": b"FFI third-party licenses\n",
"java/src/main/binary-resources/META-INF/NOTICE": b"Binary notices\n",
+ "java/pom.xml": (
+ b'