From dd2ae82cfa205b4b1447644264a3f0a2cb1910c9 Mon Sep 17 00:00:00 2001 From: Jeff Pignataro Date: Tue, 25 Aug 2026 10:18:42 -0600 Subject: [PATCH 01/15] fix: restore remote execution support for apko_image The apko build action was pinned to local execution in #332 to work around #331. The underlying incompatibility is that apko writes into its --cache-dir even with --offline (it creates directories on cache lookup and expands packages into the cache), while Bazel provides the prepopulated cache tree artifact as a read-only action input, which remote executors enforce strictly. Local sandboxes keep directories writable, so this only surfaced under remote execution. - Copy the cache input into a writable scratch directory inside the action and point --cache-dir at it, so the action works on read-only input trees. - Give HOME a writable fallback when the executor does not provide one. - Bind the apko_image and apko_show_config actions to the resolved apko toolchain so an executor never receives a binary built for a different platform than the action is scheduled on. - Remove the no-remote-exec execution requirement from #332. - Add an opt-in use_default_shell_env attribute (default True, the previous behavior) so users who want a hermetic action environment can drop host env inheritance per target. Verified locally (all tests, examples, and e2e/smoke) and against a Buildbarn-based remote executor: both e2e/smoke image targets build with the apko actions executing on remote linux-amd64 workers from a macOS host. --- apko/private/apko_image.bzl | 42 +++++++++++++++++++++++++++---- apko/private/apko_show_config.bzl | 4 +++ docs/rules.md | 3 ++- e2e/smoke/BUILD | 10 ++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index 8c40947..0eb25f8 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -12,6 +12,10 @@ _ATTRS = { "architecture": attr.string(doc = "the CPU architecture which this image should be built to run on. See https://github.com/chainguard-dev/apko/blob/main/docs/apko_file.md#archs-top-level-element"), "tag": attr.string(doc = "tag to apply to the resulting docker tarball. only applicable when `output` is `docker`", mandatory = True), "args": attr.string_list(doc = "additional arguments to provide when running the `apko build` command."), + "use_default_shell_env": attr.bool( + doc = "whether the apko build action inherits the default shell environment (the client environment as filtered by `--action_env` / `--incompatible_strict_action_env`). Set to `False` to run apko with a hermetic action environment.", + default = True, + ), } def _impl(ctx): @@ -81,7 +85,6 @@ def _impl(ctx): inputs.append(lockfile_copy) args.add("--lockfile={}".format(lockfile.short_path)) - args.add("--cache-dir={}".format(cache_name)) args.add("--offline") if ctx.attr.architecture: @@ -102,15 +105,39 @@ def _impl(ctx): copy_to_workdir(ctx, apko_info.binary, apko_binary) inputs.append(apko_binary) + # apko writes into its --cache-dir even with --offline (it expands + # packages and creates directories on cache lookup). Bazel provides the + # prepopulated cache tree artifact as a read-only input, which remote + # executors enforce strictly, so we copy the cache into a writable + # scratch directory before invoking apko. HOME falls back to a scratch + # directory on executors that don't provide a writable one. + command = "\n".join([ + "set -e", + 'SCRATCH="$(mktemp -d)"', + "trap 'rm -rf \"$SCRATCH\"' EXIT", + 'mkdir "$SCRATCH/cache" "$SCRATCH/home"', + 'cp -RL {cache_src}/. "$SCRATCH/cache/"', + 'chmod -R u+w "$SCRATCH/cache"', + 'export HOME="${{HOME:-$SCRATCH/home}}"', + "cd {workdir}", + '{apko} "$@" --cache-dir="$SCRATCH/cache"', + ]).format( + cache_src = cache_dir.path, + workdir = paths.join(ctx.bin_dir.path, ctx.label.workspace_root, ctx.label.package, workdir), + apko = apko_info.binary.short_path, + ) + ctx.actions.run_shell( - command = "cd {} && {} $@".format(paths.join(ctx.bin_dir.path, ctx.label.workspace_root, ctx.label.package, workdir), apko_info.binary.short_path), + command = command, arguments = [args], inputs = inputs, tools = [apko_info.binary], outputs = [output], - use_default_shell_env = True, - execution_requirements = {"no-remote-exec": "1"}, - toolchain = None, + use_default_shell_env = ctx.attr.use_default_shell_env, + # Bind the action to the platform the apko toolchain was resolved + # for, so remote executors never receive a binary built for a + # different platform. + toolchain = "@rules_apko//apko:toolchain_type", ) return DefaultInfo( @@ -137,6 +164,7 @@ def apko_image( output = "oci", architecture = None, args = [], + use_default_shell_env = True, **kwargs): """Build OCI images from APK packages directly without Dockerfile @@ -181,6 +209,9 @@ def apko_image( architecture: the CPU architecture which this image should be built to run on. See https://github.com/chainguard-dev/apko/blob/main/docs/apko_file.md#archs-top-level-element"), tag: tag to apply to the resulting docker tarball. only applicable when `output` is `docker` args: additional arguments to provide when running the `apko build` command. + use_default_shell_env: whether the apko build action inherits the default shell environment + (the client environment as filtered by `--action_env` / `--incompatible_strict_action_env`). + Set to `False` to run apko with a hermetic action environment. **kwargs: other common arguments like: tags, visibility. """ _apko_image( @@ -191,6 +222,7 @@ def apko_image( architecture = architecture, tag = tag, args = args, + use_default_shell_env = use_default_shell_env, **kwargs ) config_label = native.package_relative_label(config) diff --git a/apko/private/apko_show_config.bzl b/apko/private/apko_show_config.bzl index c0884be..1ed76d3 100644 --- a/apko/private/apko_show_config.bzl +++ b/apko/private/apko_show_config.bzl @@ -45,6 +45,10 @@ def _impl(ctx): inputs = inputs, tools = [apko_info.binary], outputs = [output], + # Bind the action to the platform the apko toolchain was resolved + # for, so remote executors never receive a binary built for a + # different platform. + toolchain = "@rules_apko//apko:toolchain_type", ) return DefaultInfo( diff --git a/docs/rules.md b/docs/rules.md index 22ede14..2dda526 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -75,7 +75,7 @@ When referencing other files in the config yaml file use paths relative to your 
 load("@rules_apko//apko:defs.bzl", "apko_image")
 
-apko_image(name, contents, config, tag, output, architecture, args, **kwargs)
+apko_image(name, contents, config, tag, output, architecture, args, use_default_shell_env, **kwargs)
Build OCI images from APK packages directly without Dockerfile @@ -125,6 +125,7 @@ For more examples checkout the [examples](/examples) directory. | output | "oci" of "docker", | `"oci"` | | architecture | the CPU architecture which this image should be built to run on. See https://github.com/chainguard-dev/apko/blob/main/docs/apko_file.md#archs-top-level-element"), | `None` | | args | additional arguments to provide when running the `apko build` command. | `[]` | +| use_default_shell_env | whether the apko build action inherits the default shell environment (the client environment as filtered by `--action_env` / `--incompatible_strict_action_env`). Set to `False` to run apko with a hermetic action environment. | `True` | | kwargs | other common arguments like: tags, visibility. | none | diff --git a/e2e/smoke/BUILD b/e2e/smoke/BUILD index 581aef2..deb771f 100644 --- a/e2e/smoke/BUILD +++ b/e2e/smoke/BUILD @@ -10,9 +10,19 @@ apko_image( tag = "lock:latest", ) +# Exercises the hermetic action environment opt-in. +apko_image( + name = "lock_hermetic_env", + config = ":apko.yaml", + contents = "@example_lock//:contents", + tag = "lock:latest", + use_default_shell_env = False, +) + build_test( name = "test", targets = [ ":lock", + ":lock_hermetic_env", ], ) From a57c67949b088056244d70691e63b7a64589ff5b Mon Sep 17 00:00:00 2001 From: Jeff Pignataro Date: Tue, 25 Aug 2026 13:35:47 -0600 Subject: [PATCH 02/15] chore: drop misleading comment on toolchain binding apko cross-builds images for any target arch, so the comment's framing about arch mismatches was wrong; the toolchain param needs no explanation beyond what it says. --- apko/private/apko_image.bzl | 3 --- apko/private/apko_show_config.bzl | 3 --- 2 files changed, 6 deletions(-) diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index 0eb25f8..349f41b 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -134,9 +134,6 @@ def _impl(ctx): tools = [apko_info.binary], outputs = [output], use_default_shell_env = ctx.attr.use_default_shell_env, - # Bind the action to the platform the apko toolchain was resolved - # for, so remote executors never receive a binary built for a - # different platform. toolchain = "@rules_apko//apko:toolchain_type", ) diff --git a/apko/private/apko_show_config.bzl b/apko/private/apko_show_config.bzl index 1ed76d3..a7909dc 100644 --- a/apko/private/apko_show_config.bzl +++ b/apko/private/apko_show_config.bzl @@ -45,9 +45,6 @@ def _impl(ctx): inputs = inputs, tools = [apko_info.binary], outputs = [output], - # Bind the action to the platform the apko toolchain was resolved - # for, so remote executors never receive a binary built for a - # different platform. toolchain = "@rules_apko//apko:toolchain_type", ) From 32c6f1092af2151222d63ba60caa8cc97ecd9a13 Mon Sep 17 00:00:00 2001 From: Jeff Pignataro Date: Tue, 25 Aug 2026 16:09:19 -0600 Subject: [PATCH 03/15] perf: symlink the cache into the scratch dir instead of copying Mirror the read-only cache tree artifact into the writable scratch directory as real directories containing per-file symlinks rather than cp -RL'ing the whole cache. apko only adds new entries (package expansions) to its --cache-dir and never rewrites existing files, so new writes land in the scratch directories while existing entries are read through the symlinks. This avoids duplicating the full APK cache on every image build. The chmod is no longer needed since apko never writes to the symlinked files. Verified locally (darwin-sandbox) and on a Buildbarn-based remote executor with linux-amd64 workers; layer digests are identical to the copy-based version. --- apko/private/apko_image.bzl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index 349f41b..c3ff84a 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -108,16 +108,19 @@ def _impl(ctx): # apko writes into its --cache-dir even with --offline (it expands # packages and creates directories on cache lookup). Bazel provides the # prepopulated cache tree artifact as a read-only input, which remote - # executors enforce strictly, so we copy the cache into a writable - # scratch directory before invoking apko. HOME falls back to a scratch - # directory on executors that don't provide a writable one. + # executors enforce strictly, so we mirror it into a writable scratch + # directory as real directories containing per-file symlinks. apko only + # adds new cache entries and never rewrites existing files, so the + # symlinks stay read-only while new writes land in the scratch + # directories, avoiding a full copy of the cache. HOME falls back to a + # scratch directory on executors that don't provide a writable one. command = "\n".join([ "set -e", 'SCRATCH="$(mktemp -d)"', "trap 'rm -rf \"$SCRATCH\"' EXIT", 'mkdir "$SCRATCH/cache" "$SCRATCH/home"', - 'cp -RL {cache_src}/. "$SCRATCH/cache/"', - 'chmod -R u+w "$SCRATCH/cache"', + 'CACHE_SRC="$(cd {cache_src} && pwd)"', + '(cd "$CACHE_SRC" && find . -type d -exec mkdir -p "$SCRATCH/cache/{{}}" \';\' -o -exec ln -s "$CACHE_SRC/{{}}" "$SCRATCH/cache/{{}}" \';\')', 'export HOME="${{HOME:-$SCRATCH/home}}"', "cd {workdir}", '{apko} "$@" --cache-dir="$SCRATCH/cache"', From a50d7493d077687daadcfa20fcd87b51e4353eed Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 12:24:03 +0000 Subject: [PATCH 04/15] ci: build examples/wolfi-base through actiond remote execution Adds an x86_64-only workflow that starts actiond, a local hermetic Remote Execution API worker that runs Bazel actions inside a Linux VM, and points Bazel at it to build //examples/wolfi-base. x86_64 only: GitHub's arm64 Linux runners have no /dev/kvm because the underlying Azure DPDsv6 SKU does not support nested virtualization, and the macOS arm64 runners cannot nest under Apple's Virtualization Framework. actiond's own CI has the same split. --- .github/workflows/actiond.yaml | 141 +++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 .github/workflows/actiond.yaml diff --git a/.github/workflows/actiond.yaml b/.github/workflows/actiond.yaml new file mode 100644 index 0000000..331e0a4 --- /dev/null +++ b/.github/workflows/actiond.yaml @@ -0,0 +1,141 @@ +# Copyright 2026 Chainguard, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Experiment: build examples/wolfi-base through actiond, a local hermetic +# Remote Execution API worker that runs Bazel actions inside a Linux VM. +# https://github.com/hermeticbuild/actiond +# +# x86_64 only. GitHub's arm64 Linux runners have no /dev/kvm (the Azure +# DPDsv6 SKU does not support nested virtualization), and the macOS arm64 +# runners cannot nest under Apple's Virtualization Framework. +name: actiond + +on: + push: + branches: ['main'] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +env: + ACTIOND_VERSION: v0.0.6 + ACTIOND_ENDPOINT: 127.0.0.1:8980 + +jobs: + remote-execution: + name: Build examples/wolfi-base via actiond + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read # Clone the repository + + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Enable KVM and vhost-vsock + run: | + set -euxo pipefail + test -c /dev/kvm + sudo chmod a+rw /dev/kvm + if [[ ! -e /dev/vhost-vsock ]]; then + sudo modprobe vhost_vsock + fi + test -c /dev/vhost-vsock + sudo chmod a+rw /dev/vhost-vsock + + - name: Download actiond + env: + GH_TOKEN: ${{ github.token }} + working-directory: ${{ runner.temp }} + run: | + set -euxo pipefail + gh release download "${ACTIOND_VERSION}" \ + --repo hermeticbuild/actiond \ + --pattern 'linux-actiond_linux_x86_64' \ + --pattern 'SHA256.txt' \ + --clobber + # SHA256.txt is sha256sum-format. This only proves the download is + # intact; actiond publishes no build provenance to verify against. + sha256sum -c --ignore-missing SHA256.txt + chmod +x linux-actiond_linux_x86_64 + + - name: Start actiond worker + working-directory: ${{ runner.temp }} + run: | + set -euxo pipefail + setsid ./linux-actiond_linux_x86_64 serve-vm \ + --listen="${ACTIOND_ENDPOINT}" \ + --root="${RUNNER_TEMP}/actiond-root" \ + --cas-image-size-mib=8192 \ + --memory-mib=4096 \ + --cpus=4 \ + --start-timeout-ms=180000 \ + >"${RUNNER_TEMP}/actiond.log" 2>&1 & + echo "$!" > "${RUNNER_TEMP}/actiond.pid" + + # The worker is ready when it logs the gRPC bridge endpoint. + for _ in $(seq 1 240); do + if grep -Fq "actiond VM raw gRPC bridge listening on ${ACTIOND_ENDPOINT}" "${RUNNER_TEMP}/actiond.log"; then + exit 0 + fi + if ! kill -0 "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null; then + echo "actiond exited before becoming ready" >&2 + cat "${RUNNER_TEMP}/actiond.log" >&2 + exit 1 + fi + sleep 1 + done + echo "timed out waiting for ${ACTIOND_ENDPOINT}" >&2 + cat "${RUNNER_TEMP}/actiond.log" >&2 + exit 1 + + - name: Set up Bazel + uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0 + with: + repository-cache: true + bazelrc: | + common --announce_rc --color=yes + + - name: Build examples/wolfi-base with remote execution + run: | + set -euxo pipefail + # --spawn_strategy is left at its default (remote,worker,sandboxed, + # local) on purpose: apko_image tags its build action no-remote-exec + # (#332), and forcing remote-only would leave it with no usable + # strategy. requires-bash is needed because rules_apko stages inputs + # with run_shell actions. + bazel build //examples/wolfi-base \ + --remote_executor="grpc://${ACTIOND_ENDPOINT}" \ + --remote_cache="grpc://${ACTIOND_ENDPOINT}" \ + --remote_default_exec_properties=requires-bash= \ + --remote_upload_local_results=false \ + --noremote_cache_compression \ + --remote_local_fallback=false \ + --verbose_failures + + - name: Stop actiond + if: always() + run: | + if [[ -f "${RUNNER_TEMP}/actiond.pid" ]]; then + kill "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null || true + fi + + - name: Upload actiond log + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: actiond-log + if-no-files-found: warn + path: ${{ runner.temp }}/actiond.log From bbfc39530e08aed0343cf1ac1e2620402e64ca03 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 12:30:06 +0000 Subject: [PATCH 05/15] ci: surface the apko action failure mode under actiond CopyToWorkdir fails at exit 127 before the apko action is ever scheduled, which hides what #379 changed. Add a non-blocking follow-up build that runs the staging actions locally, so the apko action reaches the VM and its own missing-coreutils failure shows up in the log. Also trigger on actiond** branches so experiments do not have to land on main. --- .github/workflows/actiond.yaml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/actiond.yaml b/.github/workflows/actiond.yaml index 331e0a4..7eb9a0c 100644 --- a/.github/workflows/actiond.yaml +++ b/.github/workflows/actiond.yaml @@ -12,7 +12,7 @@ name: actiond on: push: - branches: ['main'] + branches: ['main', 'actiond**'] workflow_dispatch: concurrency: @@ -108,14 +108,13 @@ jobs: bazelrc: | common --announce_rc --color=yes + # Expected to fail today. rules_apko stages action inputs with + # run_shell commands (mkdir/cp, see apko/private/apko_config.bzl), and + # actiond's guest chroot ships only bash plus an /usr/bin/env stub, so + # these die with "mkdir: command not found" (exit 127). - name: Build examples/wolfi-base with remote execution run: | set -euxo pipefail - # --spawn_strategy is left at its default (remote,worker,sandboxed, - # local) on purpose: apko_image tags its build action no-remote-exec - # (#332), and forcing remote-only would leave it with no usable - # strategy. requires-bash is needed because rules_apko stages inputs - # with run_shell actions. bazel build //examples/wolfi-base \ --remote_executor="grpc://${ACTIOND_ENDPOINT}" \ --remote_cache="grpc://${ACTIOND_ENDPOINT}" \ @@ -125,6 +124,25 @@ jobs: --remote_local_fallback=false \ --verbose_failures + # CopyToWorkdir fails before the apko action is ever scheduled, which + # hides what #379 changed. Run the staging actions locally so the apko + # action itself reaches the VM and its own failure mode is visible. + # Non-blocking: this step exists to produce a log, not a verdict. + - name: Diagnostic - isolate the apko action + if: always() + continue-on-error: true + run: | + set -uxo pipefail + bazel build //examples/wolfi-base \ + --remote_executor="grpc://${ACTIOND_ENDPOINT}" \ + --remote_cache="grpc://${ACTIOND_ENDPOINT}" \ + --remote_default_exec_properties=requires-bash= \ + --remote_upload_local_results=false \ + --noremote_cache_compression \ + --remote_local_fallback=false \ + --strategy=CopyToWorkdir=local \ + --verbose_failures + - name: Stop actiond if: always() run: | From 33581ef3f1d1115439812672ef49a473748b1ab6 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 12:33:31 +0000 Subject: [PATCH 06/15] ci: run the actiond workflow on fix-apko-image-remote-exec So the workflow triggers on the branch #379 is developed on, not just on main and actiond** experiment branches. --- .github/workflows/actiond.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actiond.yaml b/.github/workflows/actiond.yaml index 7eb9a0c..32b186c 100644 --- a/.github/workflows/actiond.yaml +++ b/.github/workflows/actiond.yaml @@ -12,7 +12,7 @@ name: actiond on: push: - branches: ['main', 'actiond**'] + branches: ['main', 'actiond**', 'fix-apko-image-remote-exec'] workflow_dispatch: concurrency: From 2c7fef72a3b03f123e8f163108e81076295bad0f Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 12:53:12 +0000 Subject: [PATCH 07/15] fix: carry mkdir/cp/mktemp as action inputs instead of inheriting them The run_shell actions in copy_to_workdir and apko_image call mkdir, cp, mktemp, find, ln, and rm, all resolved from the executor's PATH. That holds on a conventional RBE image and fails on an executor that ships no userspace: against actiond, a local RE worker that runs each action in an empty chroot, staging dies with "mkdir: command not found" (exit 127) and the apko action with "mktemp: command not found". Take the binary from aspect_bazel_lib's coreutils toolchain and declare it in tools, so it is uploaded as an action input and travels to whichever executor the action lands on. aspect_bazel_lib is already a bazel_dep and registers the toolchain itself, so this adds no dependency and needs no registration. The Linux assets are statically linked against musl, so they need nothing from the executor either. The find/ln mirror collapses into a single cp -Rs, which has the same effect - real directories, per-file symlinks into the read-only input tree - without needing find, which uutils does not ship. These actions still need a shell; making them fully shell-free would mean replacing copy_to_workdir with copy_to_directory and giving the apko action a wrapper binary. --- apko/private/apko_config.bzl | 27 +++++++++++++++++++++++++-- apko/private/apko_image.bzl | 25 ++++++++++++++++++------- apko/private/apko_show_config.bzl | 7 +++++-- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/apko/private/apko_config.bzl b/apko/private/apko_config.bzl index 7ab9c52..8fa9e7c 100644 --- a/apko/private/apko_config.bzl +++ b/apko/private/apko_config.bzl @@ -2,6 +2,23 @@ load("@bazel_skylib//lib:paths.bzl", "paths") +# Prebuilt, statically linked (musl) uutils coreutils, registered by +# aspect_bazel_lib. Declaring it as an action tool means mkdir/cp travel with +# the action instead of being inherited from the executor image, which is what +# lets these actions run on executors that ship no userspace of their own. +COREUTILS_TOOLCHAIN_TYPE = "@aspect_bazel_lib//lib:coreutils_toolchain_type" + +def coreutils_bin(ctx): + """The coreutils binary from the resolved toolchain. + + Args: + ctx: rule context + + Returns: + The coreutils File. + """ + return ctx.toolchains[COREUTILS_TOOLCHAIN_TYPE].coreutils_info.bin + _PATH_CONVENTION_DOC = """ When referencing other files in the config yaml file use paths relative to your Bazel workspace root. For example, if you want to reference source file foo/bar/baz use foo/bar/baz. If you want to reference output file of foo/bar:rule and rule's @@ -69,7 +86,8 @@ def copy_to_workdir(ctx, src, dst): Uses cp instead of ctx.actions.symlink because Bazel 9 sandbox changes cause symlink outputs to dangle when used as inputs to subsequent - sandboxed actions. + sandboxed actions. mkdir and cp come from the coreutils toolchain rather + than the executor's PATH, so the action carries its own tools. When src is a list, dst must be a declared directory (tree artifact); each source is staged at its workspace-relative path inside dst. This @@ -86,10 +104,13 @@ def copy_to_workdir(ctx, src, dst): dst: declared output File or directory """ + coreutils = coreutils_bin(ctx) + # When f is a directory, append "/." so its contents land directly in # target instead of nesting at target/ (Bazel pre-creates # declared output directories, so plain cp -r src tgt would nest). - copy_file_cmd = lambda f, target: 'mkdir -p "{dir}" && cp -rfL "{src}" "{tgt}"'.format( + copy_file_cmd = lambda f, target: '"{cu}" mkdir -p "{dir}" && "{cu}" cp -rfL "{src}" "{tgt}"'.format( + cu = coreutils.path, dir = paths.dirname(target), src = (f.path + "/.") if f.is_directory else f.path, tgt = target, @@ -110,9 +131,11 @@ def copy_to_workdir(ctx, src, dst): ctx.actions.run_shell( inputs = inputs, outputs = [dst], + tools = [coreutils], command = "\n".join(cmds), mnemonic = "CopyToWorkdir", progress_message = "Staging %s" % dst.short_path, + toolchain = COREUTILS_TOOLCHAIN_TYPE, ) def prepare_apko_config_in_workdir(workdir, ctx): diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index c3ff84a..9093afe 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -2,7 +2,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:versions.bzl", "versions") -load("//apko/private:apko_config.bzl", "copy_to_workdir", "prepare_apko_config_in_workdir") +load("//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "coreutils_bin", "copy_to_workdir", "prepare_apko_config_in_workdir") load("//apko/private:apko_run.bzl", "apko_run") _ATTRS = { @@ -114,17 +114,25 @@ def _impl(ctx): # symlinks stay read-only while new writes land in the scratch # directories, avoiding a full copy of the cache. HOME falls back to a # scratch directory on executors that don't provide a writable one. + # + # Every command below is either a bash builtin or a coreutils subcommand, + # so the action depends on nothing from the executor image beyond a shell. + # COREUTILS is resolved to an absolute path up front because the EXIT trap + # fires after the cd into the workdir. + coreutils = coreutils_bin(ctx) command = "\n".join([ "set -e", - 'SCRATCH="$(mktemp -d)"', - "trap 'rm -rf \"$SCRATCH\"' EXIT", - 'mkdir "$SCRATCH/cache" "$SCRATCH/home"', + 'COREUTILS="$PWD/{coreutils}"', + 'SCRATCH="$("$COREUTILS" mktemp -d)"', + "trap '\"$COREUTILS\" rm -rf \"$SCRATCH\"' EXIT", + '"$COREUTILS" mkdir "$SCRATCH/cache" "$SCRATCH/home"', 'CACHE_SRC="$(cd {cache_src} && pwd)"', - '(cd "$CACHE_SRC" && find . -type d -exec mkdir -p "$SCRATCH/cache/{{}}" \';\' -o -exec ln -s "$CACHE_SRC/{{}}" "$SCRATCH/cache/{{}}" \';\')', + '"$COREUTILS" cp -Rs "$CACHE_SRC/." "$SCRATCH/cache/"', 'export HOME="${{HOME:-$SCRATCH/home}}"', "cd {workdir}", '{apko} "$@" --cache-dir="$SCRATCH/cache"', ]).format( + coreutils = coreutils.path, cache_src = cache_dir.path, workdir = paths.join(ctx.bin_dir.path, ctx.label.workspace_root, ctx.label.package, workdir), apko = apko_info.binary.short_path, @@ -134,7 +142,7 @@ def _impl(ctx): command = command, arguments = [args], inputs = inputs, - tools = [apko_info.binary], + tools = [apko_info.binary, coreutils], outputs = [output], use_default_shell_env = ctx.attr.use_default_shell_env, toolchain = "@rules_apko//apko:toolchain_type", @@ -147,7 +155,10 @@ def _impl(ctx): apko_image_lib = struct( attrs = _ATTRS, implementation = _impl, - toolchains = ["@rules_apko//apko:toolchain_type"], + toolchains = [ + "@rules_apko//apko:toolchain_type", + COREUTILS_TOOLCHAIN_TYPE, + ], ) _apko_image = rule( diff --git a/apko/private/apko_show_config.bzl b/apko/private/apko_show_config.bzl index a7909dc..9b1f7ff 100644 --- a/apko/private/apko_show_config.bzl +++ b/apko/private/apko_show_config.bzl @@ -1,7 +1,7 @@ "A rule for expanding apko config" load("@bazel_skylib//lib:paths.bzl", "paths") -load("//apko/private:apko_config.bzl", "copy_to_workdir", "prepare_apko_config_in_workdir") +load("//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "copy_to_workdir", "prepare_apko_config_in_workdir") def _impl(ctx): output = ctx.actions.declare_file(ctx.attr.name) @@ -64,5 +64,8 @@ For more advanced use-cases (multi-file configuration), use target providing `Ap ), }, doc = "Wrapper around `apko show-config` command to generate expanded config as bazel build action.", - toolchains = ["@rules_apko//apko:toolchain_type"], + toolchains = [ + "@rules_apko//apko:toolchain_type", + COREUTILS_TOOLCHAIN_TYPE, + ], ) From 13e2805f77c7896b2c7ddd62569009a6cbb49fc2 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 12:56:15 +0000 Subject: [PATCH 08/15] chore: sort load symbols for buildifier --- apko/private/apko_image.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index 9093afe..acb7051 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -2,7 +2,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:versions.bzl", "versions") -load("//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "coreutils_bin", "copy_to_workdir", "prepare_apko_config_in_workdir") +load("//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "copy_to_workdir", "coreutils_bin", "prepare_apko_config_in_workdir") load("//apko/private:apko_run.bzl", "apko_run") _ATTRS = { From bbefbc4e94dcbcbc881eeb5c939f8f9e63e284ba Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 12:59:34 +0000 Subject: [PATCH 09/15] ci: run the actiond workflow on pull requests Without a pull_request trigger the job never runs on a PR: a fork PR generates no push event in the upstream repo, so the workflow this PR proposes as a check would not have fired on the PR proposing it. --- .github/workflows/actiond.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/actiond.yaml b/.github/workflows/actiond.yaml index 32b186c..7e475fd 100644 --- a/.github/workflows/actiond.yaml +++ b/.github/workflows/actiond.yaml @@ -11,6 +11,8 @@ name: actiond on: + pull_request: + branches: ['main'] push: branches: ['main', 'actiond**', 'fix-apko-image-remote-exec'] workflow_dispatch: From 941a1c635d3959432355d1d3ccd1a3c67decb2ee Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 13:01:56 +0000 Subject: [PATCH 10/15] ci: make the actiond job a real remote-execution gate At the default spawn strategy an action that opts out of remote execution just runs on the host and the build still passes, so a regression of the kind #332 introduced would not have been caught. --spawn_strategy=remote with no local fallback turns that into a hard failure. Drop the diagnostic step: it existed to surface the apko action's failure when staging failed first, which no longer happens, and a continue-on-error step reads as a passing check when it is not one. --- .github/workflows/actiond.yaml | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/workflows/actiond.yaml b/.github/workflows/actiond.yaml index 7e475fd..ae22cf5 100644 --- a/.github/workflows/actiond.yaml +++ b/.github/workflows/actiond.yaml @@ -110,10 +110,13 @@ jobs: bazelrc: | common --announce_rc --color=yes - # Expected to fail today. rules_apko stages action inputs with - # run_shell commands (mkdir/cp, see apko/private/apko_config.bzl), and - # actiond's guest chroot ships only bash plus an /usr/bin/env stub, so - # these die with "mkdir: command not found" (exit 127). + # Regression gate for remote execution. --spawn_strategy=remote with no + # local fallback means an action that opts out of remote execution (a + # no-remote-exec tag, as in #332) fails the build outright rather than + # quietly running on the host and passing. requires-bash is needed + # because rules_apko stages inputs with run_shell actions; everything + # they call is either a bash builtin or comes from the coreutils + # toolchain, so nothing else is inherited from the executor. - name: Build examples/wolfi-base with remote execution run: | set -euxo pipefail @@ -123,28 +126,10 @@ jobs: --remote_default_exec_properties=requires-bash= \ --remote_upload_local_results=false \ --noremote_cache_compression \ + --spawn_strategy=remote \ --remote_local_fallback=false \ --verbose_failures - # CopyToWorkdir fails before the apko action is ever scheduled, which - # hides what #379 changed. Run the staging actions locally so the apko - # action itself reaches the VM and its own failure mode is visible. - # Non-blocking: this step exists to produce a log, not a verdict. - - name: Diagnostic - isolate the apko action - if: always() - continue-on-error: true - run: | - set -uxo pipefail - bazel build //examples/wolfi-base \ - --remote_executor="grpc://${ACTIOND_ENDPOINT}" \ - --remote_cache="grpc://${ACTIOND_ENDPOINT}" \ - --remote_default_exec_properties=requires-bash= \ - --remote_upload_local_results=false \ - --noremote_cache_compression \ - --remote_local_fallback=false \ - --strategy=CopyToWorkdir=local \ - --verbose_failures - - name: Stop actiond if: always() run: | From 9a1cbebf611015470a298e348babb49d0effee3a Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 26 Aug 2026 13:03:33 +0000 Subject: [PATCH 11/15] ci: fold the actiond job into ci.yaml so test-roundup gates it needs: can only reference jobs in the same workflow, so a separate actiond.yaml could never be a dependency of the summary gate. Moving the job into ci.yaml and adding it to test-roundup's needs means branch protection keeps requiring exactly one context while remote execution becomes part of what that context covers. --- .github/workflows/actiond.yaml | 146 --------------------------------- .github/workflows/ci.yaml | 127 ++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 146 deletions(-) delete mode 100644 .github/workflows/actiond.yaml diff --git a/.github/workflows/actiond.yaml b/.github/workflows/actiond.yaml deleted file mode 100644 index ae22cf5..0000000 --- a/.github/workflows/actiond.yaml +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2026 Chainguard, Inc. -# SPDX-License-Identifier: Apache-2.0 - -# Experiment: build examples/wolfi-base through actiond, a local hermetic -# Remote Execution API worker that runs Bazel actions inside a Linux VM. -# https://github.com/hermeticbuild/actiond -# -# x86_64 only. GitHub's arm64 Linux runners have no /dev/kvm (the Azure -# DPDsv6 SKU does not support nested virtualization), and the macOS arm64 -# runners cannot nest under Apple's Virtualization Framework. -name: actiond - -on: - pull_request: - branches: ['main'] - push: - branches: ['main', 'actiond**', 'fix-apko-image-remote-exec'] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: {} - -env: - ACTIOND_VERSION: v0.0.6 - ACTIOND_ENDPOINT: 127.0.0.1:8980 - -jobs: - remote-execution: - name: Build examples/wolfi-base via actiond - runs-on: ubuntu-24.04 - timeout-minutes: 60 - permissions: - contents: read # Clone the repository - - steps: - - name: Harden Runner - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 - with: - egress-policy: audit - - - name: Check out code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Enable KVM and vhost-vsock - run: | - set -euxo pipefail - test -c /dev/kvm - sudo chmod a+rw /dev/kvm - if [[ ! -e /dev/vhost-vsock ]]; then - sudo modprobe vhost_vsock - fi - test -c /dev/vhost-vsock - sudo chmod a+rw /dev/vhost-vsock - - - name: Download actiond - env: - GH_TOKEN: ${{ github.token }} - working-directory: ${{ runner.temp }} - run: | - set -euxo pipefail - gh release download "${ACTIOND_VERSION}" \ - --repo hermeticbuild/actiond \ - --pattern 'linux-actiond_linux_x86_64' \ - --pattern 'SHA256.txt' \ - --clobber - # SHA256.txt is sha256sum-format. This only proves the download is - # intact; actiond publishes no build provenance to verify against. - sha256sum -c --ignore-missing SHA256.txt - chmod +x linux-actiond_linux_x86_64 - - - name: Start actiond worker - working-directory: ${{ runner.temp }} - run: | - set -euxo pipefail - setsid ./linux-actiond_linux_x86_64 serve-vm \ - --listen="${ACTIOND_ENDPOINT}" \ - --root="${RUNNER_TEMP}/actiond-root" \ - --cas-image-size-mib=8192 \ - --memory-mib=4096 \ - --cpus=4 \ - --start-timeout-ms=180000 \ - >"${RUNNER_TEMP}/actiond.log" 2>&1 & - echo "$!" > "${RUNNER_TEMP}/actiond.pid" - - # The worker is ready when it logs the gRPC bridge endpoint. - for _ in $(seq 1 240); do - if grep -Fq "actiond VM raw gRPC bridge listening on ${ACTIOND_ENDPOINT}" "${RUNNER_TEMP}/actiond.log"; then - exit 0 - fi - if ! kill -0 "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null; then - echo "actiond exited before becoming ready" >&2 - cat "${RUNNER_TEMP}/actiond.log" >&2 - exit 1 - fi - sleep 1 - done - echo "timed out waiting for ${ACTIOND_ENDPOINT}" >&2 - cat "${RUNNER_TEMP}/actiond.log" >&2 - exit 1 - - - name: Set up Bazel - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0 - with: - repository-cache: true - bazelrc: | - common --announce_rc --color=yes - - # Regression gate for remote execution. --spawn_strategy=remote with no - # local fallback means an action that opts out of remote execution (a - # no-remote-exec tag, as in #332) fails the build outright rather than - # quietly running on the host and passing. requires-bash is needed - # because rules_apko stages inputs with run_shell actions; everything - # they call is either a bash builtin or comes from the coreutils - # toolchain, so nothing else is inherited from the executor. - - name: Build examples/wolfi-base with remote execution - run: | - set -euxo pipefail - bazel build //examples/wolfi-base \ - --remote_executor="grpc://${ACTIOND_ENDPOINT}" \ - --remote_cache="grpc://${ACTIOND_ENDPOINT}" \ - --remote_default_exec_properties=requires-bash= \ - --remote_upload_local_results=false \ - --noremote_cache_compression \ - --spawn_strategy=remote \ - --remote_local_fallback=false \ - --verbose_failures - - - name: Stop actiond - if: always() - run: | - if [[ -f "${RUNNER_TEMP}/actiond.pid" ]]; then - kill "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null || true - fi - - - name: Upload actiond log - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: actiond-log - if-no-files-found: warn - path: ${{ runner.temp }}/actiond.log diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index de2db88..eb54ff4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -113,6 +113,132 @@ jobs: if: ${{ matrix.bazelversion <= '8.' && matrix.folder == '.' }} run: bazel query 'tests(//...) except docs/...' | xargs bazel test + # Runs the build through actiond, a local Remote Execution API worker that + # executes each action inside an empty chroot in a Linux VM on the runner. + # This is what keeps remote-execution support from regressing: an action that + # depends on the executor's userspace, or that opts out of remote execution, + # fails here and nowhere else. x86_64 only - GitHub's arm64 Linux runners + # have no /dev/kvm. + remote-execution: + name: Build examples/wolfi-base via actiond + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read # Clone the repository + env: + ACTIOND_VERSION: v0.0.6 + ACTIOND_ENDPOINT: 127.0.0.1:8980 + + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Enable KVM and vhost-vsock + run: | + set -euxo pipefail + test -c /dev/kvm + sudo chmod a+rw /dev/kvm + if [[ ! -e /dev/vhost-vsock ]]; then + sudo modprobe vhost_vsock + fi + test -c /dev/vhost-vsock + sudo chmod a+rw /dev/vhost-vsock + + - name: Download actiond + env: + GH_TOKEN: ${{ github.token }} + working-directory: ${{ runner.temp }} + run: | + set -euxo pipefail + gh release download "${ACTIOND_VERSION}" \ + --repo hermeticbuild/actiond \ + --pattern 'linux-actiond_linux_x86_64' \ + --pattern 'SHA256.txt' \ + --clobber + # SHA256.txt is sha256sum-format. This only proves the download is + # intact; actiond publishes no build provenance to verify against. + sha256sum -c --ignore-missing SHA256.txt + chmod +x linux-actiond_linux_x86_64 + + - name: Start actiond worker + working-directory: ${{ runner.temp }} + run: | + set -euxo pipefail + setsid ./linux-actiond_linux_x86_64 serve-vm \ + --listen="${ACTIOND_ENDPOINT}" \ + --root="${RUNNER_TEMP}/actiond-root" \ + --cas-image-size-mib=8192 \ + --memory-mib=4096 \ + --cpus=4 \ + --start-timeout-ms=180000 \ + >"${RUNNER_TEMP}/actiond.log" 2>&1 & + echo "$!" > "${RUNNER_TEMP}/actiond.pid" + + # The worker is ready when it logs the gRPC bridge endpoint. + for _ in $(seq 1 240); do + if grep -Fq "actiond VM raw gRPC bridge listening on ${ACTIOND_ENDPOINT}" "${RUNNER_TEMP}/actiond.log"; then + exit 0 + fi + if ! kill -0 "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null; then + echo "actiond exited before becoming ready" >&2 + cat "${RUNNER_TEMP}/actiond.log" >&2 + exit 1 + fi + sleep 1 + done + echo "timed out waiting for ${ACTIOND_ENDPOINT}" >&2 + cat "${RUNNER_TEMP}/actiond.log" >&2 + exit 1 + + - name: Set up Bazel + uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0 + with: + repository-cache: true + bazelrc: | + common --announce_rc --color=yes + + # Regression gate for remote execution. --spawn_strategy=remote with no + # local fallback means an action that opts out of remote execution (a + # no-remote-exec tag, as in #332) fails the build outright rather than + # quietly running on the host and passing. requires-bash is needed + # because rules_apko stages inputs with run_shell actions; everything + # they call is either a bash builtin or comes from the coreutils + # toolchain, so nothing else is inherited from the executor. + - name: Build examples/wolfi-base with remote execution + run: | + set -euxo pipefail + bazel build //examples/wolfi-base \ + --remote_executor="grpc://${ACTIOND_ENDPOINT}" \ + --remote_cache="grpc://${ACTIOND_ENDPOINT}" \ + --remote_default_exec_properties=requires-bash= \ + --remote_upload_local_results=false \ + --noremote_cache_compression \ + --spawn_strategy=remote \ + --remote_local_fallback=false \ + --verbose_failures + + - name: Stop actiond + if: always() + run: | + if [[ -f "${RUNNER_TEMP}/actiond.pid" ]]; then + kill "$(cat "${RUNNER_TEMP}/actiond.pid")" 2>/dev/null || true + fi + + - name: Upload actiond log + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: actiond-log + if-no-files-found: warn + path: ${{ runner.temp }}/actiond.log + # Single summary gate that aggregates the dynamic test matrix. Branch # protection only needs to require this one context, so the Bazel version # matrix above can change without an infra change to the required checks. @@ -121,6 +247,7 @@ jobs: needs: - matrix-prep-bazelversion - matrix-prep-os + - remote-execution - test if: always() steps: From fc8d96bbf4938137815e6af7b36ce56d545ef2bd Mon Sep 17 00:00:00 2001 From: alex weidner Date: Mon, 26 Oct 2026 17:54:57 -0400 Subject: [PATCH 12/15] fix: stage directories without "src/." so uutils 0.0.16 works aspect_bazel_lib 1.x, the version rules_apko_dependencies() fetches for WORKSPACE users, registers uutils coreutils 0.0.16. That release treats "cp -R src/. dst" as a copy of a directory named after src and nests it at dst/. With it, the apko action's cache mirror lands one level too deep and apko fails at --offline with "failed to read ... in offline cache", and copy_to_workdir nests directory inputs the same way. uutils 0.0.23 and later, and GNU cp, are not affected. Copy to a destination that does not exist yet instead, which every uutils release aspect_bazel_lib has shipped (0.0.16, 0.0.23, 0.0.26, 0.0.27, 0.1.0) and GNU cp handle the same way: the apko action no longer pre-creates the cache mirror directory, and copy_to_workdir removes the pre-created output directory before copying a directory into its place. Resolve the cache source with pwd -P so cp -Rs always starts from a real directory rather than copying a symlink to one. --- apko/private/apko_config.bzl | 21 +++++++++++++-------- apko/private/apko_image.bzl | 11 +++++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/apko/private/apko_config.bzl b/apko/private/apko_config.bzl index 8fa9e7c..24f8f37 100644 --- a/apko/private/apko_config.bzl +++ b/apko/private/apko_config.bzl @@ -2,9 +2,12 @@ load("@bazel_skylib//lib:paths.bzl", "paths") -# Prebuilt, statically linked (musl) uutils coreutils, registered by -# aspect_bazel_lib. Declaring it as an action tool means mkdir/cp travel with -# the action instead of being inherited from the executor image, which is what +# Prebuilt uutils coreutils, registered by aspect_bazel_lib under bzlmod. +# WORKSPACE users register it themselves (see e2e/smoke/WORKSPACE.bazel). The +# Linux builds are statically linked against musl in recent aspect_bazel_lib +# releases; older ones, including the 1.x rules_apko_dependencies() fetches, +# link glibc. Declaring it as an action tool means mkdir/cp travel with the +# action instead of being inherited from the executor image, which is what # lets these actions run on executors that ship no userspace of their own. COREUTILS_TOOLCHAIN_TYPE = "@aspect_bazel_lib//lib:coreutils_toolchain_type" @@ -106,13 +109,15 @@ def copy_to_workdir(ctx, src, dst): coreutils = coreutils_bin(ctx) - # When f is a directory, append "/." so its contents land directly in - # target instead of nesting at target/ (Bazel pre-creates - # declared output directories, so plain cp -r src tgt would nest). - copy_file_cmd = lambda f, target: '"{cu}" mkdir -p "{dir}" && "{cu}" cp -rfL "{src}" "{tgt}"'.format( + # When f is a directory, remove the pre-created output directory first so + # cp creates it as a copy of f rather than nesting at target/. + # Copying "f/." into it does not work with uutils 0.0.16 (aspect_bazel_lib + # 1.x), which nests that as well. + copy_file_cmd = lambda f, target: '"{cu}" mkdir -p "{dir}" && {rm}"{cu}" cp -rfL "{src}" "{tgt}"'.format( cu = coreutils.path, dir = paths.dirname(target), - src = (f.path + "/.") if f.is_directory else f.path, + rm = '"{}" rm -rf "{}" && '.format(coreutils.path, target) if f.is_directory else "", + src = f.path, tgt = target, ) diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index acb7051..64d86e6 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -118,16 +118,19 @@ def _impl(ctx): # Every command below is either a bash builtin or a coreutils subcommand, # so the action depends on nothing from the executor image beyond a shell. # COREUTILS is resolved to an absolute path up front because the EXIT trap - # fires after the cd into the workdir. + # fires after the cd into the workdir. The mirror is copied to a path that + # does not exist yet rather than as "src/." into an existing directory, + # because uutils 0.0.16 (registered by aspect_bazel_lib 1.x) nests the + # latter at dst/. coreutils = coreutils_bin(ctx) command = "\n".join([ "set -e", 'COREUTILS="$PWD/{coreutils}"', 'SCRATCH="$("$COREUTILS" mktemp -d)"', "trap '\"$COREUTILS\" rm -rf \"$SCRATCH\"' EXIT", - '"$COREUTILS" mkdir "$SCRATCH/cache" "$SCRATCH/home"', - 'CACHE_SRC="$(cd {cache_src} && pwd)"', - '"$COREUTILS" cp -Rs "$CACHE_SRC/." "$SCRATCH/cache/"', + '"$COREUTILS" mkdir "$SCRATCH/home"', + 'CACHE_SRC="$(cd {cache_src} && pwd -P)"', + '"$COREUTILS" cp -Rs "$CACHE_SRC" "$SCRATCH/cache"', 'export HOME="${{HOME:-$SCRATCH/home}}"', "cd {workdir}", '{apko} "$@" --cache-dir="$SCRATCH/cache"', From 4b84263b44b2ab3ab7317e8c0f9ddf8c8b3350b6 Mon Sep 17 00:00:00 2001 From: alex weidner Date: Mon, 26 Oct 2026 17:55:20 -0400 Subject: [PATCH 13/15] fix: register the coreutils toolchain in the WORKSPACE setup Taking mkdir/cp/mktemp from aspect_bazel_lib's coreutils toolchain makes analysis fail in WORKSPACE mode with "No matching toolchains found for @@aspect_bazel_lib//lib:coreutils_toolchain_type", because nothing in the documented WORKSPACE setup registers it. Bzlmod users are not affected: aspect_bazel_lib registers it from its MODULE.bazel. The aspect_bazel_lib 1.34.5 that rules_apko_dependencies() fetches already defines the toolchain and register_coreutils_toolchains(), so no version bump is needed. Registration cannot move into rules_apko_dependencies() itself, since repositories.bzl is loaded before @aspect_bazel_lib exists, so the setup gains one load and one call. Also load local_repository from @bazel_tools. @bazel does not exist in WORKSPACE mode, so this file could not be loaded at all. --- e2e/smoke/WORKSPACE.bazel | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/e2e/smoke/WORKSPACE.bazel b/e2e/smoke/WORKSPACE.bazel index cbdf567..b43facd 100644 --- a/e2e/smoke/WORKSPACE.bazel +++ b/e2e/smoke/WORKSPACE.bazel @@ -1,5 +1,5 @@ # Override http_archive for local testing -load("@bazel//tools/build_defs/repo:local.bzl", "local_repository") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository") local_repository( name = "rules_apko", @@ -20,6 +20,13 @@ load("@rules_apko//apko:repositories.bzl", "apko_register_toolchains", "rules_ap rules_apko_dependencies() +# apko_image and apko_show_config take mkdir/cp/mktemp from aspect_bazel_lib's +# coreutils toolchain. Skip this if your WORKSPACE already registers it, for +# example through aspect_bazel_lib_dependencies() on aspect_bazel_lib 1.x. +load("@aspect_bazel_lib//lib:repositories.bzl", "register_coreutils_toolchains") + +register_coreutils_toolchains() + apko_register_toolchains(name = "apko") load("@rules_apko//apko:translate_lock.bzl", "translate_apko_lock") From 2709e3d43c12f5849b3b149adbc6246a63ab812c Mon Sep 17 00:00:00 2001 From: alex weidner Date: Mon, 26 Oct 2026 17:55:28 -0400 Subject: [PATCH 14/15] ci: test e2e/smoke in WORKSPACE mode on Bazel 7 and 8 CI only ever ran with --enable_bzlmod, so neither the missing coreutils registration nor the unloadable e2e/smoke WORKSPACE file showed up. Run e2e/smoke a second time with --noenable_bzlmod --enable_workspace on the Bazel versions that still support WORKSPACE. Also fix the setup-bazel pin comments: the action's tags have no "v" prefix, so zizmor reported "v0.19.0" as an unknown ref. --- .github/workflows/ci.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eb54ff4..10b0784 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,7 +86,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0 + - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 with: repository-cache: true bazelrc: | @@ -113,6 +113,12 @@ jobs: if: ${{ matrix.bazelversion <= '8.' && matrix.folder == '.' }} run: bazel query 'tests(//...) except docs/...' | xargs bazel test + # Bazel 9 dropped WORKSPACE; on 7 and 8 the published snippet still has to work. + - name: Test WORKSPACE mode + working-directory: ${{ matrix.folder }} + if: ${{ matrix.bazelversion < '9.' && matrix.folder == 'e2e/smoke' }} + run: bazel test //... --noenable_bzlmod --enable_workspace + # Runs the build through actiond, a local Remote Execution API worker that # executes each action inside an empty chroot in a Linux VM on the runner. # This is what keeps remote-execution support from regressing: an action that @@ -198,7 +204,7 @@ jobs: exit 1 - name: Set up Bazel - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0 + uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 with: repository-cache: true bazelrc: | From 87b9ff9802e93ee3a4e0cc0c8ea82cf3cd4c4421 Mon Sep 17 00:00:00 2001 From: alex weidner Date: Mon, 26 Oct 2026 19:17:20 -0400 Subject: [PATCH 15/15] address review nits: pin actiond digest, test tree-artifact staging Pin the actiond binary by its SHA-256 in ci.yaml instead of reading the checksum from a file served beside the release asset. Add e2e/smoke tree_staging, which stages a directory through copy_to_workdir and checks its contents land at the destination; it fails with the previous "src/." copy form on uutils 0.0.16. Describe the condition under which apko_image's HOME fallback applies, and drop the issue reference from the remote-execution gate comment. --- .github/workflows/ci.yaml | 12 +++++------ apko/private/apko_image.bzl | 3 ++- e2e/smoke/tree_staging/BUILD | 14 +++++++++++++ e2e/smoke/tree_staging/tree.bzl | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 e2e/smoke/tree_staging/BUILD create mode 100644 e2e/smoke/tree_staging/tree.bzl diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10b0784..410ab62 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -133,6 +133,7 @@ jobs: contents: read # Clone the repository env: ACTIOND_VERSION: v0.0.6 + ACTIOND_SHA256: 006dc798d4363596fe8ab997606fc93766a0cc427c2d005cf4fc1765fa4c2052 ACTIOND_ENDPOINT: 127.0.0.1:8980 steps: @@ -166,11 +167,10 @@ jobs: gh release download "${ACTIOND_VERSION}" \ --repo hermeticbuild/actiond \ --pattern 'linux-actiond_linux_x86_64' \ - --pattern 'SHA256.txt' \ --clobber - # SHA256.txt is sha256sum-format. This only proves the download is - # intact; actiond publishes no build provenance to verify against. - sha256sum -c --ignore-missing SHA256.txt + # actiond publishes no build provenance, so the digest is pinned + # here instead of read from a checksum file next to the asset. + echo "${ACTIOND_SHA256} linux-actiond_linux_x86_64" | sha256sum -c - chmod +x linux-actiond_linux_x86_64 - name: Start actiond worker @@ -212,8 +212,8 @@ jobs: # Regression gate for remote execution. --spawn_strategy=remote with no # local fallback means an action that opts out of remote execution (a - # no-remote-exec tag, as in #332) fails the build outright rather than - # quietly running on the host and passing. requires-bash is needed + # no-remote-exec tag) fails the build outright rather than quietly + # running on the host and passing. requires-bash is needed # because rules_apko stages inputs with run_shell actions; everything # they call is either a bash builtin or comes from the coreutils # toolchain, so nothing else is inherited from the executor. diff --git a/apko/private/apko_image.bzl b/apko/private/apko_image.bzl index 64d86e6..e7a3ac0 100644 --- a/apko/private/apko_image.bzl +++ b/apko/private/apko_image.bzl @@ -113,7 +113,8 @@ def _impl(ctx): # adds new cache entries and never rewrites existing files, so the # symlinks stay read-only while new writes land in the scratch # directories, avoiding a full copy of the cache. HOME falls back to a - # scratch directory on executors that don't provide a writable one. + # scratch directory when the action environment leaves it unset or empty, + # as with use_default_shell_env = False or an executor that sets none. # # Every command below is either a bash builtin or a coreutils subcommand, # so the action depends on nothing from the executor image beyond a shell. diff --git a/e2e/smoke/tree_staging/BUILD b/e2e/smoke/tree_staging/BUILD new file mode 100644 index 0000000..704a713 --- /dev/null +++ b/e2e/smoke/tree_staging/BUILD @@ -0,0 +1,14 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load(":tree.bzl", "staged_tree_check", "tree") + +tree(name = "input_tree") + +staged_tree_check( + name = "staged_tree", + src = ":input_tree", +) + +build_test( + name = "test", + targets = [":staged_tree"], +) diff --git a/e2e/smoke/tree_staging/tree.bzl b/e2e/smoke/tree_staging/tree.bzl new file mode 100644 index 0000000..68f2455 --- /dev/null +++ b/e2e/smoke/tree_staging/tree.bzl @@ -0,0 +1,37 @@ +"Fixtures for staging a tree artifact through copy_to_workdir." + +# buildifier: disable=bzl-visibility +load("@rules_apko//apko/private:apko_config.bzl", "COREUTILS_TOOLCHAIN_TYPE", "copy_to_workdir") + +def _tree_impl(ctx): + out = ctx.actions.declare_directory(ctx.attr.name) + ctx.actions.run_shell( + outputs = [out], + command = 'mkdir -p "$1/sub" && echo extra > "$1/sub/file"', + arguments = [out.path], + ) + return DefaultInfo(files = depset([out])) + +tree = rule(implementation = _tree_impl) + +def _staged_tree_check_impl(ctx): + src = ctx.attr.src[DefaultInfo].files.to_list()[0] + staged = ctx.actions.declare_directory(ctx.attr.name + "_staged") + copy_to_workdir(ctx, src, staged) + + # Staging a directory must put its contents at the destination, not nest + # the directory one level down at /. + marker = ctx.actions.declare_file(ctx.attr.name + ".ok") + ctx.actions.run_shell( + inputs = [staged], + outputs = [marker], + command = 'test -f "$1/sub/file" && ! test -e "$1/$3" && touch "$2"', + arguments = [staged.path, marker.path, src.basename], + ) + return DefaultInfo(files = depset([marker])) + +staged_tree_check = rule( + implementation = _staged_tree_check_impl, + attrs = {"src": attr.label(mandatory = True)}, + toolchains = [COREUTILS_TOOLCHAIN_TYPE], +)