From 8892f659f66a9eb4f5a591101d9029e42763bd17 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 14 Oct 2026 10:08:35 +0000 Subject: [PATCH 1/5] Reduce repeated CLI builds and isolate CI publication --- .github/workflows/cli-deno-tests.yml | 103 ++++++++------ .github/workflows/cli-docker-validation.yml | 87 ++++++++++++ .github/workflows/cli-docker.yml | 78 +++++------ .github/workflows/cli-p2p-tests.yml | 88 ++++++++++++ .github/workflows/deploy-pages.yml | 20 ++- .github/workflows/unit-ci.yml | 24 ++++ src/apps/cli/testdeno/ci-tasks.json | 17 +++ src/apps/cli/testdeno/run-ci-suite.ts | 18 +-- src/apps/cli/testdeno/test_dev_deno.md | 6 +- utils/ci-pages.unit.spec.ts | 72 ++++++++++ utils/cli-ci.unit.spec.ts | 143 ++++++++++++++++++++ 11 files changed, 547 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/cli-docker-validation.yml create mode 100644 .github/workflows/cli-p2p-tests.yml create mode 100644 src/apps/cli/testdeno/ci-tasks.json create mode 100644 utils/ci-pages.unit.spec.ts create mode 100644 utils/cli-ci.unit.spec.ts diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-deno-tests.yml index 98423fcd..76abc996 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-deno-tests.yml @@ -8,14 +8,34 @@ on: paths: - '.github/workflows/cli-deno-tests.yml' - 'src/apps/cli/**' - - 'test/bench-network/**' + - '!src/apps/cli/**/*.md' + - 'src/LiveSyncBaseCore.ts' + - 'src/common/**' + - 'src/serviceFeatures/**' + - 'src/serviceModules/**' + - 'src/integrations/**' + - 'src/modules/AbstractModule.ts' + - 'src/modules/main/**' + - 'src/modules/coreFeatures/**' + - 'manifest.json' + - 'tsconfig.json' - 'package.json' - 'package-lock.json' pull_request: paths: - '.github/workflows/cli-deno-tests.yml' - 'src/apps/cli/**' - - 'test/bench-network/**' + - '!src/apps/cli/**/*.md' + - 'src/LiveSyncBaseCore.ts' + - 'src/common/**' + - 'src/serviceFeatures/**' + - 'src/serviceModules/**' + - 'src/integrations/**' + - 'src/modules/AbstractModule.ts' + - 'src/modules/main/**' + - 'src/modules/coreFeatures/**' + - 'manifest.json' + - 'tsconfig.json' - 'package.json' - 'package-lock.json' workflow_dispatch: @@ -40,23 +60,34 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: prepare: runs-on: ubuntu-latest + timeout-minutes: 15 + env: + NODE_OPTIONS: --max-old-space-size=3072 outputs: task_matrix: ${{ steps.select.outputs.task_matrix }} steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Select task matrix id: select shell: bash + env: + SELECTED_TASK: ${{ github.event_name == 'workflow_dispatch' && inputs.test_task || 'test:ci' }} run: | set -euo pipefail - SELECTED_TASK="${{ github.event_name == 'workflow_dispatch' && inputs.test_task || 'test:ci' }}" echo "[INFO] Selected task set: $SELECTED_TASK" case "$SELECTED_TASK" in test:ci) - TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon","test:daemon-startup","test:push-pull","test:decoupled-vault","test:sync-two-local","test:sync-locked-remote","test:remote-commands","test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' + TASK_MATRIX=$(jq -c . src/apps/cli/testdeno/ci-tasks.json) ;; test:local) TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon"]' @@ -72,6 +103,27 @@ jobs: echo "task_matrix=$TASK_MATRIX" >> "$GITHUB_OUTPUT" + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: '24.x' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build CLI + working-directory: src/apps/cli + run: npm run build + + - name: Upload CLI bundle + uses: actions/upload-artifact@v6 + with: + name: cli-bundle + path: src/apps/cli/dist/ + if-no-files-found: error + retention-days: 1 + test: needs: prepare runs-on: ubuntu-latest @@ -109,9 +161,11 @@ jobs: - name: Install dependencies run: npm ci - - name: Build CLI - working-directory: src/apps/cli - run: npm run build + - name: Download CLI bundle + uses: actions/download-artifact@v8 + with: + name: cli-bundle + path: src/apps/cli/dist/ - name: Create .test.env working-directory: src/apps/cli @@ -145,38 +199,3 @@ jobs: run: | docker stop couchdb-test minio-test relay-test coturn-test >/dev/null 2>&1 || true docker rm couchdb-test minio-test relay-test coturn-test >/dev/null 2>&1 || true - - compose-p2p-e2e: - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - name: Checkout - uses: actions/checkout@v5 - - - name: Show Docker versions - run: | - docker --version - docker compose version - - - name: Run Compose CLI P2P E2E - env: - CLI_E2E_TASK: test:p2p:ci - RELAY: ws://nostr-relay:7777/ - PEERS_TIMEOUT: '20' - SYNC_TIMEOUT: '60' - LIVESYNC_USE_COTURN: '0' - TURN_SERVERS: none - LIVESYNC_P2P_PEERS_RETRY: '1' - LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: '60000' - BENCH_LIVESYNC_TEST_TEE: '0' - run: docker compose -f test/bench-network/compose.yml run --build --rm bench-runner run-livesync-cli-e2e - - - name: Show Compose diagnostics - if: failure() - run: | - docker compose -f test/bench-network/compose.yml ps - docker compose -f test/bench-network/compose.yml logs --no-color couchdb nostr-relay || true - - - name: Stop Compose services - if: always() - run: docker compose -f test/bench-network/compose.yml down -v --remove-orphans diff --git a/.github/workflows/cli-docker-validation.yml b/.github/workflows/cli-docker-validation.yml new file mode 100644 index 00000000..fc58f972 --- /dev/null +++ b/.github/workflows/cli-docker-validation.yml @@ -0,0 +1,87 @@ +name: CLI Docker validation + +on: + pull_request: + paths: + - '.github/workflows/cli-docker.yml' + - '.github/workflows/cli-docker-validation.yml' + - 'src/apps/cli/**' + - '!src/apps/cli/**/*.md' + - 'src/LiveSyncBaseCore.ts' + - 'src/common/**' + - 'src/serviceFeatures/**' + - 'src/serviceModules/**' + - 'src/integrations/**' + - 'src/modules/AbstractModule.ts' + - 'src/modules/main/**' + - 'src/modules/coreFeatures/**' + - 'manifest.json' + - 'tsconfig.json' + - 'package.json' + - 'package-lock.json' + - '.dockerignore' + workflow_call: + inputs: + force: + type: boolean + default: false + outputs: + e2e_passed: + description: Whether the Docker CLI E2E suite passed + value: ${{ jobs.validate.outputs.e2e_passed }} + +permissions: + contents: read + +concurrency: + group: cli-docker-validation-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + e2e_passed: ${{ steps.e2e.outcome == 'success' }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build Docker test image + uses: docker/build-push-action@v7 + with: + context: . + file: src/apps/cli/Dockerfile + load: true + tags: livesync-cli + platforms: linux/amd64 + cache-from: type=gha,scope=cli-image + cache-to: type=gha,scope=cli-image,mode=max + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: '24.x' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run CLI E2E (docker) + id: e2e + continue-on-error: ${{ github.event_name == 'workflow_dispatch' && inputs.force }} + working-directory: src/apps/cli + env: + CI: true + run: npm --ignore-scripts run test:e2e:docker:all + + - name: Stop test containers + if: always() + working-directory: src/apps/cli + run: | + bash ./util/couchdb-stop.sh >/dev/null 2>&1 || true + bash ./util/minio-stop.sh >/dev/null 2>&1 || true + bash ./util/p2p-stop.sh >/dev/null 2>&1 || true diff --git a/.github/workflows/cli-docker.yml b/.github/workflows/cli-docker.yml index ed43e230..b963506f 100644 --- a/.github/workflows/cli-docker.yml +++ b/.github/workflows/cli-docker.yml @@ -1,6 +1,6 @@ -# Build and push the CLI Docker image to GitHub Container Registry (GHCR).# -# Image tag format: --cli -# Example: 0.25.56-1743500000-cli +# Build and push the CLI Docker image to GitHub Container Registry (GHCR). +# Image tag format: -cli +# Example: 1.0.36-cli # # Stable releases are also tagged with their major-minor version and 'latest'. # Pre-releases receive immutable version and SHA-qualified tags only. @@ -13,16 +13,24 @@ on: - main tags: - "*.*.*-cli" - paths-ignore: - - "docs/**" - - "*.md" - - "images/**" - - "assets/**" - - "instruction_images/**" - - "src/apps/webapp/**" - - "src/apps/webpeer/**" - - ".github/workflows/release.yml" - - ".github/workflows/unit-ci.yml" + paths: + - ".github/workflows/cli-docker.yml" + - ".github/workflows/cli-docker-validation.yml" + - "src/apps/cli/**" + - "!src/apps/cli/**/*.md" + - "src/LiveSyncBaseCore.ts" + - "src/common/**" + - "src/serviceFeatures/**" + - "src/serviceModules/**" + - "src/integrations/**" + - "src/modules/AbstractModule.ts" + - "src/modules/main/**" + - "src/modules/coreFeatures/**" + - "manifest.json" + - "tsconfig.json" + - "package.json" + - "package-lock.json" + - ".dockerignore" workflow_dispatch: inputs: dry_run: @@ -36,8 +44,21 @@ on: type: boolean default: false +concurrency: + group: cli-docker-publish-${{ github.ref }} + cancel-in-progress: false + jobs: + validate: + uses: ./.github/workflows/cli-docker-validation.yml + with: + force: ${{ github.event_name == 'workflow_dispatch' && inputs.force || false }} + permissions: + contents: read + build-and-push: + needs: validate + if: ${{ needs.validate.outputs.e2e_passed == 'true' || (github.event_name == 'workflow_dispatch' && inputs.force) }} runs-on: ubuntu-latest timeout-minutes: 90 permissions: @@ -100,34 +121,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Setup Node.js - uses: actions/setup-node@v5 - with: - node-version: "24.x" - cache: "npm" - - - name: Install dependencies - run: npm ci - - - name: Run CLI E2E (docker) - id: e2e - continue-on-error: ${{ github.event_name == 'workflow_dispatch' && inputs.force }} - working-directory: src/apps/cli - env: - CI: true - run: npm run test:e2e:docker:all - - - name: Stop test containers (safety net) - if: always() - working-directory: src/apps/cli - run: | - # Keep this as a safety net for future suites/steps that may leave containers running. - bash ./util/couchdb-stop.sh >/dev/null 2>&1 || true - bash ./util/minio-stop.sh >/dev/null 2>&1 || true - bash ./util/p2p-stop.sh >/dev/null 2>&1 || true - - name: Build and push - if: ${{ steps.e2e.outcome == 'success' || (github.event_name == 'workflow_dispatch' && inputs.force) }} uses: docker/build-push-action@v7 with: context: . @@ -135,5 +129,5 @@ jobs: push: ${{ steps.meta.outputs.push }} tags: ${{ steps.meta.outputs.tags }} platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=cli-image + cache-to: type=gha,scope=cli-image,mode=max diff --git a/.github/workflows/cli-p2p-tests.yml b/.github/workflows/cli-p2p-tests.yml new file mode 100644 index 00000000..17ffa039 --- /dev/null +++ b/.github/workflows/cli-p2p-tests.yml @@ -0,0 +1,88 @@ +name: CLI P2P Compose tests + +on: + push: + branches: + - main + - beta + paths: + - '.github/workflows/cli-p2p-tests.yml' + - 'src/apps/cli/**' + - '!src/apps/cli/**/*.md' + - 'src/LiveSyncBaseCore.ts' + - 'src/common/**' + - 'src/serviceFeatures/**' + - 'src/serviceModules/**' + - 'src/integrations/**' + - 'src/modules/AbstractModule.ts' + - 'src/modules/main/**' + - 'src/modules/coreFeatures/**' + - 'manifest.json' + - 'tsconfig.json' + - 'package.json' + - 'package-lock.json' + - '.dockerignore' + - 'test/bench-network/**' + pull_request: + paths: + - '.github/workflows/cli-p2p-tests.yml' + - 'src/apps/cli/**' + - '!src/apps/cli/**/*.md' + - 'src/LiveSyncBaseCore.ts' + - 'src/common/**' + - 'src/serviceFeatures/**' + - 'src/serviceModules/**' + - 'src/integrations/**' + - 'src/modules/AbstractModule.ts' + - 'src/modules/main/**' + - 'src/modules/coreFeatures/**' + - 'manifest.json' + - 'tsconfig.json' + - 'package.json' + - 'package-lock.json' + - '.dockerignore' + - 'test/bench-network/**' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + compose-p2p-e2e: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Show Docker versions + run: | + docker --version + docker compose version + + - name: Run Compose CLI P2P E2E + env: + CLI_E2E_TASK: test:p2p:ci + RELAY: ws://nostr-relay:7777/ + PEERS_TIMEOUT: '20' + SYNC_TIMEOUT: '60' + LIVESYNC_USE_COTURN: '0' + TURN_SERVERS: none + LIVESYNC_P2P_PEERS_RETRY: '1' + LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: '60000' + BENCH_LIVESYNC_TEST_TEE: '0' + run: docker compose -f test/bench-network/compose.yml run --build --rm bench-runner run-livesync-cli-e2e + + - name: Show Compose diagnostics + if: failure() + run: | + docker compose -f test/bench-network/compose.yml ps + docker compose -f test/bench-network/compose.yml logs --no-color couchdb nostr-relay || true + + - name: Stop Compose services + if: always() + run: docker compose -f test/bench-network/compose.yml down -v --remove-orphans diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index bd440243..7683c7da 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -9,6 +9,7 @@ on: - 'aggregator.html' - 'src/**' - 'test/browser-apps/**' + - 'manifest.json' - 'package.json' - 'package-lock.json' - '.github/workflows/deploy-pages.yml' @@ -17,18 +18,17 @@ on: - 'aggregator.html' - 'src/**' - 'test/browser-apps/**' + - 'manifest.json' - 'package.json' - 'package-lock.json' - '.github/workflows/deploy-pages.yml' permissions: contents: read - pages: write - id-token: write concurrency: - group: pages - cancel-in-progress: false + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: build: @@ -84,9 +84,6 @@ jobs: - name: Validate packaged Pages site run: npm run test:browser-apps:pages - - name: Configure GitHub Pages - uses: actions/configure-pages@v6 - - name: Upload GitHub Pages artifact uses: actions/upload-pages-artifact@v5 with: @@ -97,10 +94,19 @@ jobs: if: github.event_name != 'pull_request' needs: build runs-on: ubuntu-latest + concurrency: + group: pages + cancel-in-progress: false + permissions: + pages: write + id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: + - name: Configure GitHub Pages + uses: actions/configure-pages@v6 + - name: Deploy GitHub Pages id: deployment uses: actions/deploy-pages@v5 diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index f8b0f80c..326567b6 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -8,8 +8,11 @@ on: - main - beta paths: + - 'manifest.json' + - 'updates.md' - 'src/**' - 'test/**' + - '_tools/**' - 'package.json' - 'package-lock.json' - 'tsconfig.json' @@ -19,8 +22,13 @@ on: - 'eslint.config.mjs' - 'eslint.config.common.mjs' - 'eslint.community.config.mjs' + - 'terser.config.mjs' - 'update-workspaces.mjs' - 'version-bump.mjs' + - 'utils/check-compatibility.js' + - 'utils/livesync-commonlib-version.test.ts' + - 'utils/livesync-commonlib-version.ts' + - 'utils/*ci*.unit.spec.ts' - 'utils/release-*.mjs' - 'utils/release-*.unit.spec.ts' - 'utils/couchdb/**' @@ -29,11 +37,18 @@ on: - '.github/workflows/prepare-release.yml' - '.github/workflows/finalise-release.yml' - '.github/workflows/release.yml' + - '.github/workflows/cli-deno-tests.yml' + - '.github/workflows/cli-p2p-tests.yml' + - '.github/workflows/cli-docker-validation.yml' + - '.github/workflows/cli-docker.yml' - '.github/workflows/unit-ci.yml' pull_request: paths: + - 'manifest.json' + - 'updates.md' - 'src/**' - 'test/**' + - '_tools/**' - 'package.json' - 'package-lock.json' - 'tsconfig.json' @@ -43,8 +58,13 @@ on: - 'eslint.config.mjs' - 'eslint.config.common.mjs' - 'eslint.community.config.mjs' + - 'terser.config.mjs' - 'update-workspaces.mjs' - 'version-bump.mjs' + - 'utils/check-compatibility.js' + - 'utils/livesync-commonlib-version.test.ts' + - 'utils/livesync-commonlib-version.ts' + - 'utils/*ci*.unit.spec.ts' - 'utils/release-*.mjs' - 'utils/release-*.unit.spec.ts' - 'utils/couchdb/**' @@ -53,6 +73,10 @@ on: - '.github/workflows/prepare-release.yml' - '.github/workflows/finalise-release.yml' - '.github/workflows/release.yml' + - '.github/workflows/cli-deno-tests.yml' + - '.github/workflows/cli-p2p-tests.yml' + - '.github/workflows/cli-docker-validation.yml' + - '.github/workflows/cli-docker.yml' - '.github/workflows/unit-ci.yml' permissions: diff --git a/src/apps/cli/testdeno/ci-tasks.json b/src/apps/cli/testdeno/ci-tasks.json new file mode 100644 index 00000000..4748666a --- /dev/null +++ b/src/apps/cli/testdeno/ci-tasks.json @@ -0,0 +1,17 @@ +[ + "test:settings-writeback", + "test:remote-administration-exit-codes", + "test:setup-put-cat", + "test:mirror", + "test:daemon", + "test:daemon-startup", + "test:push-pull", + "test:decoupled-vault", + "test:sync-two-local", + "test:sync-locked-remote", + "test:remote-commands", + "test:e2e-matrix:couchdb-enc0", + "test:e2e-matrix:couchdb-enc1", + "test:e2e-matrix:minio-enc0", + "test:e2e-matrix:minio-enc1" +] diff --git a/src/apps/cli/testdeno/run-ci-suite.ts b/src/apps/cli/testdeno/run-ci-suite.ts index 7c391ce9..0eb1edfc 100644 --- a/src/apps/cli/testdeno/run-ci-suite.ts +++ b/src/apps/cli/testdeno/run-ci-suite.ts @@ -1,20 +1,4 @@ -const TASKS = [ - "test:settings-writeback", - "test:remote-administration-exit-codes", - "test:setup-put-cat", - "test:mirror", - "test:daemon", - "test:daemon-startup", - "test:push-pull", - "test:decoupled-vault", - "test:sync-two-local", - "test:sync-locked-remote", - "test:remote-commands", - "test:e2e-matrix:couchdb-enc0", - "test:e2e-matrix:couchdb-enc1", - "test:e2e-matrix:minio-enc0", - "test:e2e-matrix:minio-enc1", -] as const; +import TASKS from "./ci-tasks.json" with { type: "json" }; for (const [index, task] of TASKS.entries()) { console.log(`\n[CLI E2E ${index + 1}/${TASKS.length}] ${task}`); diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index 889a4fe2..bb00f8d2 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -323,7 +323,11 @@ deno task test:sync-two-local ## Continuous Integration -The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automatically on pushes and pull requests affecting the CLI, executing the non-P2P test suite (`test:ci`). P2P tests (`test:p2p`) are excluded from automatic execution and must be run via manual dispatch (`workflow_dispatch`). You can optionally check the "Enable verbose and debug logging" checkbox during a manual dispatch to produce detailed trace logs for troubleshooting. +The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automatically on pushes and pull requests affecting the CLI or its shared runtime inputs. It builds the CLI once, then distributes the bundle to the non-P2P test jobs. The local `test:ci` runner and the automatic matrix both read `ci-tasks.json`. During a manual dispatch, select a test subset and optionally enable 'Enable verbose and debug logging' for detailed troubleshooting traces. + +The separate `.github/workflows/cli-p2p-tests.yml` workflow runs the canonical Compose P2P suite automatically for changes to the CLI, its shared runtime inputs, or `test/bench-network/`. A change confined to the Compose fixtures does not start the non-P2P matrix. Host-direct `test:p2p` tasks remain available for explicit diagnostics. + +The `.github/workflows/cli-docker-validation.yml` workflow tests the production Docker image on relevant pull requests and is also called before container publication. Validation runs without registry credentials. The publishing workflow, `.github/workflows/cli-docker.yml`, reuses the Docker build cache and retains the existing main-branch, release-tag, and manual publication conditions. --- diff --git a/utils/ci-pages.unit.spec.ts b/utils/ci-pages.unit.spec.ts new file mode 100644 index 00000000..5130c2bb --- /dev/null +++ b/utils/ci-pages.unit.spec.ts @@ -0,0 +1,72 @@ +import { readFileSync } from "node:fs"; +import { minimatch } from "minimatch"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +type Trigger = "push" | "pull_request"; +type Concurrency = { + group?: string; + "cancel-in-progress"?: boolean | string; +}; +type Workflow = { + on?: Partial>; + concurrency?: Concurrency; + jobs?: Record; +}; + +function readWorkflow(filename: string): Workflow { + const source = readFileSync(new URL(`../.github/workflows/${filename}`, import.meta.url), "utf8"); + return parse(source) as Workflow; +} + +function matchesAnyPath(patterns: string[], filename: string): boolean { + return patterns.some((pattern) => minimatch(filename, pattern)); +} + +const pagesWorkflow = readWorkflow("deploy-pages.yml"); +const unitWorkflow = readWorkflow("unit-ci.yml"); + +describe("CI workflow path filters", () => { + it.each(["push", "pull_request"] as const)("runs Pages checks for the manifest on %s", (trigger) => { + const patterns = pagesWorkflow.on?.[trigger]?.paths ?? []; + expect(matchesAnyPath(patterns, "manifest.json")).toBe(true); + }); + + it.each(["push", "pull_request"] as const)("runs unit CI for required inputs on %s", (trigger) => { + const patterns = unitWorkflow.on?.[trigger]?.paths ?? []; + const requiredInputs = [ + "manifest.json", + "updates.md", + "_tools/bakei18n.ts", + "terser.config.mjs", + "utils/check-compatibility.js", + "utils/livesync-commonlib-version.test.ts", + "utils/livesync-commonlib-version.ts", + "utils/ci-pages.unit.spec.ts", + "utils/cli-ci.unit.spec.ts", + ".github/workflows/cli-docker.yml", + ".github/workflows/cli-deno-tests.yml", + ".github/workflows/cli-p2p-tests.yml", + ".github/workflows/cli-docker-validation.yml", + ]; + + expect(requiredInputs.filter((filename) => !matchesAnyPath(patterns, filename))).toEqual([]); + expect(matchesAnyPath(patterns, "utils/unrelated.unit.spec.ts")).toBe(false); + }); +}); + +describe("Pages workflow concurrency", () => { + it("isolates pull requests from publishing refs and cancels superseded pull request runs", () => { + expect(pagesWorkflow.concurrency).toEqual({ + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}", + "cancel-in-progress": "${{ github.event_name == 'pull_request' }}", + }); + }); + + it("serialises deployments without cancelling an active deployment", () => { + expect(pagesWorkflow.jobs?.deploy?.concurrency).toEqual({ + group: "pages", + "cancel-in-progress": false, + }); + }); +}); diff --git a/utils/cli-ci.unit.spec.ts b/utils/cli-ci.unit.spec.ts new file mode 100644 index 00000000..42934c62 --- /dev/null +++ b/utils/cli-ci.unit.spec.ts @@ -0,0 +1,143 @@ +import { readFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { minimatch } from "minimatch"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const root = new URL("../", import.meta.url); +type Step = { id?: string; run?: string; uses?: string; with?: Record }; +type Workflow = { + on: Record; + jobs: Record< + string, + { steps?: Step[]; needs?: string | string[]; uses?: string; if?: string; permissions?: Record } + >; +}; +function workflow(name: string): Workflow { + return parse(readFileSync(new URL(`.github/workflows/${name}`, root), "utf8")) as Workflow; +} +function matches(paths: string[], file: string): boolean { + let included = false; + for (const pattern of paths) { + const negative = pattern.startsWith("!"); + if (minimatch(file, negative ? pattern.slice(1) : pattern)) included = !negative; + } + return included; +} +function localTasks(): string[] { + const runner = readFileSync(new URL("src/apps/cli/testdeno/run-ci-suite.ts", root), "utf8"); + expect(runner).toContain('import TASKS from "./ci-tasks.json" with { type: "json" }'); + return JSON.parse(readFileSync(new URL("src/apps/cli/testdeno/ci-tasks.json", root), "utf8")) as string[]; +} +function selectedTasks(task: string): string[] { + const select = workflow("cli-deno-tests.yml").jobs.prepare.steps?.find((step) => step.id === "select"); + expect(select?.run).toBeDefined(); + const directory = mkdtempSync(join(tmpdir(), "livesync-ci-matrix-")); + const output = join(directory, "output"); + try { + const script = select!.run!.replace(/\$\{\{[^}]+\}\}/g, task); + const result = spawnSync("bash", ["-c", script], { + cwd: root, + env: { ...process.env, SELECTED_TASK: task, GITHUB_OUTPUT: output }, + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + const line = readFileSync(output, "utf8") + .split("\n") + .find((line) => line.startsWith("task_matrix=")); + return JSON.parse(line!.slice("task_matrix=".length)) as string[]; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +describe("CLI CI selection", () => { + it("runs every task in the local standard suite automatically", () => { + expect(selectedTasks("test:ci")).toEqual(localTasks()); + expect(selectedTasks("test:ci")).toEqual( + expect.arrayContaining(["test:settings-writeback", "test:remote-administration-exit-codes"]) + ); + const tasks = JSON.parse(readFileSync(new URL("src/apps/cli/testdeno/deno.json", root), "utf8")).tasks; + expect(localTasks().filter((task) => !(task in tasks))).toEqual([]); + expect(new Set(localTasks()).size).toBe(localTasks().length); + }); + it("keeps the selected manual subsets", () => { + expect(selectedTasks("test:local")).toEqual(["test:setup-put-cat", "test:mirror", "test:daemon"]); + expect(selectedTasks("test:e2e-matrix")).toEqual( + localTasks().filter((task) => task.startsWith("test:e2e-matrix:")) + ); + }); + it.each([ + ["cli-deno-tests.yml", "push"], + ["cli-deno-tests.yml", "pull_request"], + ["cli-p2p-tests.yml", "push"], + ["cli-p2p-tests.yml", "pull_request"], + ["cli-docker.yml", "push"], + ["cli-docker-validation.yml", "pull_request"], + ])("validates shared CLI runtime inputs in %s on %s", (name, event) => { + const paths = workflow(name).on[event].paths ?? []; + const inputs = [ + "src/apps/cli/main.ts", + "src/LiveSyncBaseCore.ts", + "src/serviceFeatures/replicationScheduling.ts", + "src/common/replicatorProviders.ts", + "src/common/messages/en.ts", + "src/modules/AbstractModule.ts", + "src/modules/main/ModuleLiveSyncMain.ts", + "src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts", + "src/serviceModules/FileHandler.ts", + "src/integrations/cloudflare/turnCredentials.ts", + "manifest.json", + "package.json", + "package-lock.json", + "tsconfig.json", + ]; + expect(inputs.filter((file) => !matches(paths, file))).toEqual([]); + expect(matches(paths, "src/modules/features/ModuleObsidianSettingTab.ts")).toBe(false); + expect(matches(paths, "docs/settings.md")).toBe(false); + expect(matches(paths, "src/apps/cli/testdeno/CONTRIBUTING_TESTS.md")).toBe(false); + }); + it("runs Compose alone for a change to its fixture", () => { + const file = "test/bench-network/compose.yml"; + expect(matches(workflow("cli-p2p-tests.yml").on.pull_request.paths ?? [], file)).toBe(true); + expect(matches(workflow("cli-deno-tests.yml").on.pull_request.paths ?? [], file)).toBe(false); + expect(matches(workflow("cli-docker-validation.yml").on.pull_request.paths ?? [], file)).toBe(false); + }); + it("passes a single CLI build to every standard test job", () => { + const { prepare, test } = workflow("cli-deno-tests.yml").jobs; + expect(test.needs).toBe("prepare"); + expect(prepare.steps?.filter((step) => step.run === "npm run build")).toHaveLength(1); + expect(test.steps?.some((step) => step.run?.includes("npm run build"))).toBe(false); + const upload = prepare.steps?.find((step) => step.uses?.startsWith("actions/upload-artifact@")); + const download = test.steps?.find((step) => step.uses?.startsWith("actions/download-artifact@")); + expect(upload?.with?.["if-no-files-found"]).toBe("error"); + expect(download?.with?.name).toBe(upload?.with?.name); + expect(download?.with?.path).toBe(upload?.with?.path); + }); + it("keeps registry credentials out of Docker validation and gates publication on it", () => { + const validation = workflow("cli-docker-validation.yml"); + const publication = workflow("cli-docker.yml"); + expect(validation.on.push).toBeUndefined(); + const validationJob = validation.jobs.validate; + expect(validationJob.steps?.some((step) => step.uses?.startsWith("docker/login-action@"))).toBe(false); + const build = validationJob.steps?.find((step) => step.uses?.startsWith("docker/build-push-action@")); + expect(build?.with?.load).toBe(true); + expect(build?.with?.push).toBeUndefined(); + expect(validationJob.steps?.find((step) => step.id === "e2e")?.run).toBe( + "npm --ignore-scripts run test:e2e:docker:all" + ); + expect(publication.jobs.validate.uses).toBe("./.github/workflows/cli-docker-validation.yml"); + expect(publication.jobs.validate.permissions).toEqual({ contents: "read" }); + expect(publication.jobs["build-and-push"].needs).toBe("validate"); + expect(publication.jobs["build-and-push"].if).toBe( + "${{ needs.validate.outputs.e2e_passed == 'true' || (github.event_name == 'workflow_dispatch' && inputs.force) }}" + ); + const publishBuild = publication.jobs["build-and-push"].steps?.find((step) => + step.uses?.startsWith("docker/build-push-action@") + ); + expect(publishBuild?.with?.["cache-from"]).toBe(build?.with?.["cache-from"]); + expect(publishBuild?.with?.["cache-to"]).toBe(build?.with?.["cache-to"]); + }); +}); From 4a51253299e83824178915a96f6ccd6f11e17de9 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 14 Oct 2026 10:17:59 +0000 Subject: [PATCH 2/5] Preserve the multi-platform CLI build cache during validation --- .github/workflows/cli-docker-validation.yml | 7 +++++-- .github/workflows/cli-docker.yml | 4 +++- utils/cli-ci.unit.spec.ts | 8 +++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-docker-validation.yml b/.github/workflows/cli-docker-validation.yml index fc58f972..c21f855f 100644 --- a/.github/workflows/cli-docker-validation.yml +++ b/.github/workflows/cli-docker-validation.yml @@ -58,8 +58,11 @@ jobs: load: true tags: livesync-cli platforms: linux/amd64 - cache-from: type=gha,scope=cli-image - cache-to: type=gha,scope=cli-image,mode=max + cache-from: | + type=gha,scope=cli-image-amd64 + type=gha,scope=cli-image + # Keep amd64 validation from replacing the cache for both platforms. + cache-to: type=gha,scope=cli-image-amd64,mode=max - name: Setup Node.js uses: actions/setup-node@v5 diff --git a/.github/workflows/cli-docker.yml b/.github/workflows/cli-docker.yml index b963506f..0016104d 100644 --- a/.github/workflows/cli-docker.yml +++ b/.github/workflows/cli-docker.yml @@ -129,5 +129,7 @@ jobs: push: ${{ steps.meta.outputs.push }} tags: ${{ steps.meta.outputs.tags }} platforms: linux/amd64,linux/arm64 - cache-from: type=gha,scope=cli-image + cache-from: | + type=gha,scope=cli-image-amd64 + type=gha,scope=cli-image cache-to: type=gha,scope=cli-image,mode=max diff --git a/utils/cli-ci.unit.spec.ts b/utils/cli-ci.unit.spec.ts index 42934c62..cb74f4c4 100644 --- a/utils/cli-ci.unit.spec.ts +++ b/utils/cli-ci.unit.spec.ts @@ -138,6 +138,12 @@ describe("CLI CI selection", () => { step.uses?.startsWith("docker/build-push-action@") ); expect(publishBuild?.with?.["cache-from"]).toBe(build?.with?.["cache-from"]); - expect(publishBuild?.with?.["cache-to"]).toBe(build?.with?.["cache-to"]); + const validationScope = String(build?.with?.["cache-to"]).match(/scope=([^,\s]+)/)?.[1]; + const publicationScope = String(publishBuild?.with?.["cache-to"]).match(/scope=([^,\s]+)/)?.[1]; + expect(validationScope).toBeDefined(); + expect(publicationScope).toBeDefined(); + expect(publicationScope).not.toBe(validationScope); + expect(publishBuild?.with?.["cache-from"]).toContain(`scope=${validationScope}`); + expect(publishBuild?.with?.["cache-from"]).toContain(`scope=${publicationScope}`); }); }); From 8854eb9a155363f9158feb5d32da0c3512a0ad5d Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 14 Oct 2026 11:22:49 +0000 Subject: [PATCH 3/5] Run CLI Docker validation on native architectures and publish tested images --- .github/workflows/cli-docker-validation.yml | 126 +++++++++++-- .github/workflows/cli-docker.yml | 174 ++++++++++++++---- .github/workflows/unit-ci.yml | 2 + .gitignore | 1 + src/apps/cli/Dockerfile | 17 +- src/apps/cli/Dockerfile.publish | 3 + src/apps/cli/testdeno/test_dev_deno.md | 2 +- utils/cli-ci.unit.spec.ts | 193 ++++++++++++++++++-- utils/cli-docker-artifacts-ci.unit.spec.ts | 157 ++++++++++++++++ utils/cli-docker-artifacts.mjs | 177 ++++++++++++++++++ utils/cli-dockerfile-ci.unit.spec.ts | 37 ++++ 11 files changed, 821 insertions(+), 68 deletions(-) create mode 100644 src/apps/cli/Dockerfile.publish create mode 100644 utils/cli-docker-artifacts-ci.unit.spec.ts create mode 100644 utils/cli-docker-artifacts.mjs create mode 100644 utils/cli-dockerfile-ci.unit.spec.ts diff --git a/.github/workflows/cli-docker-validation.yml b/.github/workflows/cli-docker-validation.yml index c21f855f..a993e9ed 100644 --- a/.github/workflows/cli-docker-validation.yml +++ b/.github/workflows/cli-docker-validation.yml @@ -20,6 +20,7 @@ on: - 'package.json' - 'package-lock.json' - '.dockerignore' + - 'utils/cli-docker-*.mjs' workflow_call: inputs: force: @@ -28,7 +29,7 @@ on: outputs: e2e_passed: description: Whether the Docker CLI E2E suite passed - value: ${{ jobs.validate.outputs.e2e_passed }} + value: ${{ jobs.summary.outputs.e2e_passed }} permissions: contents: read @@ -38,15 +39,61 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - validate: - runs-on: ubuntu-latest - timeout-minutes: 45 - outputs: - e2e_passed: ${{ steps.e2e.outcome == 'success' }} + bundle: + name: Build shared Docker CLI bundle + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + NODE_OPTIONS: --max-old-space-size=3072 steps: - name: Checkout uses: actions/checkout@v5 + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: '22.x' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build CLI + working-directory: src/apps/cli + run: npm run build + + - name: Upload Docker CLI bundle + uses: actions/upload-artifact@v6 + with: + name: cli-docker-bundle + path: src/apps/cli/dist/ + if-no-files-found: error + overwrite: true + retention-days: 1 + + validate: + name: Docker E2E (${{ matrix.arch }}) + needs: bundle + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download Docker CLI bundle + uses: actions/download-artifact@v8 + with: + name: cli-docker-bundle + path: _cli-bundle/ + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -55,14 +102,18 @@ jobs: with: context: . file: src/apps/cli/Dockerfile + build-args: CLI_BUNDLE_STAGE=prebuilt load: true + outputs: type=oci,dest=${{ runner.temp }}/cli-image.tar tags: livesync-cli - platforms: linux/amd64 - cache-from: | - type=gha,scope=cli-image-amd64 - type=gha,scope=cli-image - # Keep amd64 validation from replacing the cache for both platforms. - cache-to: type=gha,scope=cli-image-amd64,mode=max + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=cli-runtime-${{ matrix.arch }} + cache-to: type=gha,scope=cli-runtime-${{ matrix.arch }},mode=max + + - name: Verify image architecture + env: + ARCH: ${{ matrix.arch }} + run: test "$(docker image inspect livesync-cli --format '{{.Architecture}}')" = "$ARCH" - name: Setup Node.js uses: actions/setup-node@v5 @@ -81,6 +132,34 @@ jobs: CI: true run: npm --ignore-scripts run test:e2e:docker:all + - name: Upload validated image + if: ${{ success() || (github.event_name == 'workflow_dispatch' && inputs.force && steps.e2e.outcome == 'failure') }} + uses: actions/upload-artifact@v6 + with: + name: cli-docker-image-${{ matrix.arch }} + path: ${{ runner.temp }}/cli-image.tar + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 1 + + - name: Record E2E result + if: always() + env: + ARCH: ${{ matrix.arch }} + OUTCOME: ${{ steps.e2e.outcome }} + run: node utils/cli-docker-artifacts.mjs record-result "$ARCH" "$OUTCOME" "$RUNNER_TEMP/cli-results" + + - name: Upload E2E result + if: always() + uses: actions/upload-artifact@v6 + with: + name: cli-docker-result-${{ matrix.arch }} + path: ${{ runner.temp }}/cli-results/ + if-no-files-found: error + overwrite: true + retention-days: 1 + - name: Stop test containers if: always() working-directory: src/apps/cli @@ -88,3 +167,26 @@ jobs: bash ./util/couchdb-stop.sh >/dev/null 2>&1 || true bash ./util/minio-stop.sh >/dev/null 2>&1 || true bash ./util/p2p-stop.sh >/dev/null 2>&1 || true + + summary: + name: Collect both Docker E2E results + needs: validate + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + e2e_passed: ${{ steps.summary.outputs.e2e_passed }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download E2E results + uses: actions/download-artifact@v8 + with: + pattern: cli-docker-result-* + merge-multiple: true + path: ${{ runner.temp }}/cli-results/ + + - name: Require both architecture results + id: summary + run: node utils/cli-docker-artifacts.mjs summary "$RUNNER_TEMP/cli-results" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/cli-docker.yml b/.github/workflows/cli-docker.yml index 0016104d..47347626 100644 --- a/.github/workflows/cli-docker.yml +++ b/.github/workflows/cli-docker.yml @@ -31,6 +31,7 @@ on: - "package.json" - "package-lock.json" - ".dockerignore" + - "utils/cli-docker-*.mjs" workflow_dispatch: inputs: dry_run: @@ -49,28 +50,26 @@ concurrency: cancel-in-progress: false jobs: - validate: - uses: ./.github/workflows/cli-docker-validation.yml - with: - force: ${{ github.event_name == 'workflow_dispatch' && inputs.force || false }} + metadata: + name: Select image publication metadata + runs-on: ubuntu-24.04 + timeout-minutes: 5 permissions: contents: read - - build-and-push: - needs: validate - if: ${{ needs.validate.outputs.e2e_passed == 'true' || (github.event_name == 'workflow_dispatch' && inputs.force) }} - runs-on: ubuntu-latest - timeout-minutes: 90 - permissions: - contents: read - packages: write - + outputs: + image: ${{ steps.meta.outputs.image }} + tags: ${{ steps.meta.outputs.tags }} + push: ${{ steps.meta.outputs.push }} steps: - name: Checkout uses: actions/checkout@v5 - name: Derive image tag id: meta + env: + REF: ${{ github.ref }} + EVENT_NAME: ${{ github.event_name }} + DRY_RUN: ${{ inputs.dry_run }} run: | VERSION=$(jq -r '.version' manifest.json) MAJOR_MINOR=$(echo "${VERSION}" | cut -d. -f1,2) @@ -79,7 +78,7 @@ jobs: # Build tag list based on the event and git ref TAGS="" - if [[ "${{ github.ref }}" == refs/tags/* ]]; then + if [[ "${REF}" == refs/tags/* ]]; then if [[ "${VERSION}" == *-* ]]; then # Pre-release builds must not advance stable moving tags. TAGS="${IMAGE}:${VERSION}-cli,${IMAGE}:${VERSION}-sha-${SHORT_SHA}-cli" @@ -87,7 +86,7 @@ jobs: # Stable release builds TAGS="${IMAGE}:${VERSION}-cli,${IMAGE}:${MAJOR_MINOR}-cli,${IMAGE}:latest,${IMAGE}:${VERSION}-sha-${SHORT_SHA}-cli" fi - elif [[ "${{ github.ref }}" == refs/heads/main ]]; then + elif [[ "${REF}" == refs/heads/main ]]; then # Bleeding-edge / nightly builds TAGS="${IMAGE}:edge" else @@ -97,39 +96,144 @@ jobs: # Determine if the image should be pushed PUSH="true" - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - if [[ "${{ inputs.dry_run }}" == "true" ]]; then + if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then + if [[ "${DRY_RUN}" == "true" ]]; then PUSH="false" fi fi - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "push=${PUSH}" >> $GITHUB_OUTPUT + echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + echo "push=${PUSH}" >> "$GITHUB_OUTPUT" + + validate: + uses: ./.github/workflows/cli-docker-validation.yml + with: + force: ${{ github.event_name == 'workflow_dispatch' && inputs.force || false }} + permissions: + contents: read + + publish-images: + name: Prepare image publication (${{ matrix.arch }}) + needs: [metadata, validate] + if: ${{ needs.validate.outputs.e2e_passed == 'true' || (github.event_name == 'workflow_dispatch' && inputs.force) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download validated image + uses: actions/download-artifact@v8 + with: + name: cli-docker-image-${{ matrix.arch }} + path: ${{ runner.temp }}/cli-image-archive/ + + - name: Extract OCI image + id: image-context + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/cli-image" + tar -xf "$RUNNER_TEMP/cli-image-archive/cli-image.tar" -C "$RUNNER_TEMP/cli-image" + DIGEST=$(jq -er '.manifests | if length == 1 then .[0].digest else error("Expected one OCI image") end' "$RUNNER_TEMP/cli-image/index.json") + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" - name: Log in to GitHub Container Registry + if: needs.metadata.outputs.push == 'true' uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Prepare architecture image + id: image + uses: docker/build-push-action@v7 with: - platforms: arm64 + context: . + file: src/apps/cli/Dockerfile.publish + build-contexts: validated-image=oci-layout://${{ runner.temp }}/cli-image@${{ steps.image-context.outputs.digest }} + platforms: linux/${{ matrix.arch }} + outputs: type=image,name=${{ needs.metadata.outputs.image }},push-by-digest=true,name-canonical=true,push=${{ needs.metadata.outputs.push }} + + - name: Record image digest + env: + ARCH: ${{ matrix.arch }} + DIGEST: ${{ steps.image.outputs.digest }} + run: node utils/cli-docker-artifacts.mjs record-digest "$ARCH" "$DIGEST" "$RUNNER_TEMP/cli-digests" + + - name: Upload image digest + uses: actions/upload-artifact@v6 + with: + name: cli-docker-digest-${{ matrix.arch }} + path: ${{ runner.temp }}/cli-digests/ + if-no-files-found: error + overwrite: true + retention-days: 1 + + manifest: + name: Finalise multi-platform image + needs: [metadata, publish-images] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download image digests + uses: actions/download-artifact@v8 + with: + pattern: cli-docker-digest-* + merge-multiple: true + path: ${{ runner.temp }}/cli-digests/ + + - name: Log in to GitHub Container Registry + if: needs.metadata.outputs.push == 'true' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Build and push - uses: docker/build-push-action@v7 - with: - context: . - file: src/apps/cli/Dockerfile - push: ${{ steps.meta.outputs.push }} - tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64,linux/arm64 - cache-from: | - type=gha,scope=cli-image-amd64 - type=gha,scope=cli-image - cache-to: type=gha,scope=cli-image,mode=max + - name: Assemble image publication + env: + IMAGE: ${{ needs.metadata.outputs.image }} + IMAGE_TAGS: ${{ needs.metadata.outputs.tags }} + PUSH: ${{ needs.metadata.outputs.push }} + shell: bash + run: | + set -euo pipefail + node utils/cli-docker-artifacts.mjs references "$IMAGE" "$RUNNER_TEMP/cli-digests" > "$RUNNER_TEMP/image-references.txt" + mapfile -t REFERENCES < "$RUNNER_TEMP/image-references.txt" + IFS=',' read -ra TAGS <<< "$IMAGE_TAGS" + TAG_ARGS=() + for tag in "${TAGS[@]}"; do + TAG_ARGS+=(--tag "$tag") + done + if [[ "$PUSH" == "true" ]]; then + docker buildx imagetools create "${TAG_ARGS[@]}" "${REFERENCES[@]}" + else + printf 'Dry run: both architecture images built; publication skipped.\n' + printf 'Image: %s\n' "${REFERENCES[@]}" + printf 'Tag: %s\n' "${TAGS[@]}" + fi diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index 326567b6..a3b94cfe 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -29,6 +29,7 @@ on: - 'utils/livesync-commonlib-version.test.ts' - 'utils/livesync-commonlib-version.ts' - 'utils/*ci*.unit.spec.ts' + - 'utils/cli-docker-*.mjs' - 'utils/release-*.mjs' - 'utils/release-*.unit.spec.ts' - 'utils/couchdb/**' @@ -65,6 +66,7 @@ on: - 'utils/livesync-commonlib-version.test.ts' - 'utils/livesync-commonlib-version.ts' - 'utils/*ci*.unit.spec.ts' + - 'utils/cli-docker-*.mjs' - 'utils/release-*.mjs' - 'utils/release-*.unit.spec.ts' - 'utils/couchdb/**' diff --git a/.gitignore b/.gitignore index f883e1a3..5c9289b6 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ cov_profile/** coverage src/apps/cli/dist/* +_cli-bundle/ src/apps/webapp/playwright-report/ src/apps/webapp/test-results/ _testdata/** diff --git a/src/apps/cli/Dockerfile b/src/apps/cli/Dockerfile index 985034f8..9c07939b 100644 --- a/src/apps/cli/Dockerfile +++ b/src/apps/cli/Dockerfile @@ -50,7 +50,8 @@ # Stage 1 — builder # Full Node.js environment to compile native modules and bundle the CLI. # ───────────────────────────────────────────────────────────────────────────── -FROM node:22-slim AS builder +ARG CLI_BUNDLE_STAGE=builder +FROM --platform=$BUILDPLATFORM node:22-slim AS builder # Build tools required by native Node.js addons (mainly leveldown) RUN apt-get update \ @@ -67,8 +68,14 @@ RUN npm install COPY . . RUN cd src/apps/cli && npm run build +# CI may provide a prebuilt bundle in the build context. +FROM scratch AS prebuilt +COPY _cli-bundle/ /build/src/apps/cli/dist/ + +FROM ${CLI_BUNDLE_STAGE} AS cli-bundle + # ───────────────────────────────────────────────────────────────────────────── -# Stage 2 — runtime-deps +# Stage 4 — runtime-deps # Install only the external (unbundled) packages that the CLI requires at # runtime. Native addons are compiled here against the same base image that # the final runtime stage uses. @@ -89,7 +96,7 @@ RUN npm pkg delete devDependencies \ && npm install --omit=dev # ───────────────────────────────────────────────────────────────────────────── -# Stage 3 — runtime +# Stage 5 — runtime # Minimal image: pre-compiled native modules + CLI bundle only. # No build tools are included, keeping the image small. # ───────────────────────────────────────────────────────────────────────────── @@ -100,8 +107,8 @@ WORKDIR /app # Copy pre-compiled external node_modules from runtime-deps stage COPY --from=runtime-deps /deps/node_modules ./node_modules -# Copy the built CLI bundle from builder stage -COPY --from=builder /build/src/apps/cli/dist ./dist +# Copy the built CLI bundle from the selected stage +COPY --from=cli-bundle /build/src/apps/cli/dist ./dist # Install the entrypoint wrapper with a deterministic mode, regardless of # source checkout permissions. diff --git a/src/apps/cli/Dockerfile.publish b/src/apps/cli/Dockerfile.publish new file mode 100644 index 00000000..3cdd15ea --- /dev/null +++ b/src/apps/cli/Dockerfile.publish @@ -0,0 +1,3 @@ +# syntax=docker/dockerfile:1 +# Export the image which passed Docker E2E without rebuilding its dependencies. +FROM validated-image diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index bb00f8d2..0141c9a3 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -327,7 +327,7 @@ The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automati The separate `.github/workflows/cli-p2p-tests.yml` workflow runs the canonical Compose P2P suite automatically for changes to the CLI, its shared runtime inputs, or `test/bench-network/`. A change confined to the Compose fixtures does not start the non-P2P matrix. Host-direct `test:p2p` tasks remain available for explicit diagnostics. -The `.github/workflows/cli-docker-validation.yml` workflow tests the production Docker image on relevant pull requests and is also called before container publication. Validation runs without registry credentials. The publishing workflow, `.github/workflows/cli-docker.yml`, reuses the Docker build cache and retains the existing main-branch, release-tag, and manual publication conditions. +The `.github/workflows/cli-docker-validation.yml` workflow builds one production JavaScript bundle with Node.js 22, then tests Docker images in parallel on native amd64 and arm64 runners. Each image uses the shared bundle and prepares its own runtime dependencies. Validation runs without registry credentials, and both architecture results are collected before publication. The publishing workflow, `.github/workflows/cli-docker.yml`, imports the validated OCI image from each architecture, collects the image digests, and publishes the combined manifest. It retains the existing publication conditions for main, release tags, and manual dispatch. A dry run builds both images and checks the digest pair while skipping registry uploads and manifest publication. --- diff --git a/utils/cli-ci.unit.spec.ts b/utils/cli-ci.unit.spec.ts index cb74f4c4..c6841b57 100644 --- a/utils/cli-ci.unit.spec.ts +++ b/utils/cli-ci.unit.spec.ts @@ -1,4 +1,4 @@ -import { readFileSync, mkdtempSync, rmSync } from "node:fs"; +import { readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; @@ -7,12 +7,20 @@ import { describe, expect, it } from "vitest"; import { parse } from "yaml"; const root = new URL("../", import.meta.url); -type Step = { id?: string; run?: string; uses?: string; with?: Record }; +type Step = { id?: string; run?: string; uses?: string; if?: string; with?: Record }; type Workflow = { on: Record; jobs: Record< string, - { steps?: Step[]; needs?: string | string[]; uses?: string; if?: string; permissions?: Record } + { + steps?: Step[]; + needs?: string | string[]; + uses?: string; + if?: string; + permissions?: Record; + "runs-on"?: string; + strategy?: { matrix?: { include?: Array<{ arch: string; runner: string }> } }; + } >; }; function workflow(name: string): Workflow { @@ -130,20 +138,175 @@ describe("CLI CI selection", () => { ); expect(publication.jobs.validate.uses).toBe("./.github/workflows/cli-docker-validation.yml"); expect(publication.jobs.validate.permissions).toEqual({ contents: "read" }); - expect(publication.jobs["build-and-push"].needs).toBe("validate"); - expect(publication.jobs["build-and-push"].if).toBe( + const publishJob = publication.jobs["publish-images"]; + expect(publishJob.needs).toEqual(["metadata", "validate"]); + expect(publishJob.if).toBe( "${{ needs.validate.outputs.e2e_passed == 'true' || (github.event_name == 'workflow_dispatch' && inputs.force) }}" ); - const publishBuild = publication.jobs["build-and-push"].steps?.find((step) => - step.uses?.startsWith("docker/build-push-action@") + expect(publishJob.strategy?.matrix?.include).toEqual(validationJob.strategy?.matrix?.include); + expect(publishJob["runs-on"]).toBe("${{ matrix.runner }}"); + const publishBuild = publishJob.steps?.find((step) => step.uses?.startsWith("docker/build-push-action@")); + expect(publishBuild?.with?.file).toBe("src/apps/cli/Dockerfile.publish"); + expect(publishBuild?.with?.["build-contexts"]).toBe( + "validated-image=oci-layout://${{ runner.temp }}/cli-image@${{ steps.image-context.outputs.digest }}" ); - expect(publishBuild?.with?.["cache-from"]).toBe(build?.with?.["cache-from"]); - const validationScope = String(build?.with?.["cache-to"]).match(/scope=([^,\s]+)/)?.[1]; - const publicationScope = String(publishBuild?.with?.["cache-to"]).match(/scope=([^,\s]+)/)?.[1]; - expect(validationScope).toBeDefined(); - expect(publicationScope).toBeDefined(); - expect(publicationScope).not.toBe(validationScope); - expect(publishBuild?.with?.["cache-from"]).toContain(`scope=${validationScope}`); - expect(publishBuild?.with?.["cache-from"]).toContain(`scope=${publicationScope}`); + const imageUpload = validationJob.steps?.find( + (step) => step.with?.name === "cli-docker-image-${{ matrix.arch }}" + ); + const imageDownload = publishJob.steps?.find( + (step) => step.with?.name === "cli-docker-image-${{ matrix.arch }}" + ); + expect(imageUpload?.with?.path).toBe("${{ runner.temp }}/cli-image.tar"); + expect(build?.with?.outputs).toBe("type=oci,dest=${{ runner.temp }}/cli-image.tar"); + expect(imageDownload?.with?.name).toBe(imageUpload?.with?.name); + const publishingInstructions = readFileSync(new URL("src/apps/cli/Dockerfile.publish", root), "utf8") + .split("\n") + .filter((line) => line && !line.startsWith("#")); + expect(publishingInstructions).toEqual(["FROM validated-image"]); + expect(publishBuild?.with?.outputs).toBe( + "type=image,name=${{ needs.metadata.outputs.image }},push-by-digest=true,name-canonical=true,push=${{ needs.metadata.outputs.push }}" + ); + expect(publishJob.steps?.some((step) => step.uses?.startsWith("docker/setup-qemu-action@"))).toBe(false); + expect(publishJob.steps?.find((step) => step.uses?.startsWith("docker/login-action@"))?.if).toBe( + "needs.metadata.outputs.push == 'true'" + ); + const manifestJob = publication.jobs.manifest; + expect(manifestJob.needs).toEqual(["metadata", "publish-images"]); + expect(manifestJob.steps?.find((step) => step.run?.includes("imagetools create"))?.run).toContain( + 'if [[ "$PUSH" == "true" ]]; then' + ); + }); + it("builds and tests Docker images on both native architectures", () => { + const job = workflow("cli-docker-validation.yml").jobs.validate; + expect(job.strategy?.matrix?.include).toEqual([ + { arch: "amd64", runner: "ubuntu-24.04" }, + { arch: "arm64", runner: "ubuntu-24.04-arm" }, + ]); + expect(job["runs-on"]).toBe("${{ matrix.runner }}"); + expect(job.steps?.some((step) => step.uses?.startsWith("docker/setup-qemu-action@"))).toBe(false); + }); + it("builds the Docker JavaScript bundle once and passes it to the native image builds", () => { + const jobs = workflow("cli-docker-validation.yml").jobs; + expect(jobs.bundle?.steps?.filter((step) => step.run === "npm run build") ?? []).toHaveLength(1); + expect(jobs.validate.needs).toBe("bundle"); + const upload = jobs.bundle?.steps?.find((step) => step.uses?.startsWith("actions/upload-artifact@")); + const download = jobs.validate.steps?.find((step) => step.uses?.startsWith("actions/download-artifact@")); + expect(download?.with?.name).toBe(upload?.with?.name); + expect(download?.with?.path).toBe("_cli-bundle/"); + const build = jobs.validate.steps?.find((step) => step.uses?.startsWith("docker/build-push-action@")); + expect(build?.with?.["build-args"]).toBe("CLI_BUNDLE_STAGE=prebuilt"); + }); + it.each([false, true])("assembles both image digests with publication=%s", (push) => { + const step = workflow("cli-docker.yml").jobs.manifest.steps?.find((step) => + step.run?.includes("imagetools create") + ); + expect(step?.run).toBeDefined(); + const directory = mkdtempSync(join(tmpdir(), "livesync-cli-manifest-")); + try { + const digests = join(directory, "cli-digests"); + const bin = join(directory, "bin"); + mkdirSync(digests); + mkdirSync(bin); + for (const [arch, char] of [ + ["amd64", "a"], + ["arm64", "b"], + ]) { + writeFileSync( + join(digests, `${arch}.json`), + JSON.stringify({ arch, digest: `sha256:${char.repeat(64)}` }) + ); + } + const capture = join(directory, "docker-args"); + writeFileSync(join(bin, "docker"), '#!/bin/sh\nprintf \'%s\\n\' "$@" > "$CLI_TEST_CAPTURE"\n', { + mode: 0o755, + }); + const image = "ghcr.io/test-owner/livesync-cli"; + const tags = [`${image}:1.0.36-cli`, `${image}:latest`]; + const result = spawnSync("bash", ["-c", step!.run!], { + cwd: root, + env: { + ...process.env, + RUNNER_TEMP: directory, + IMAGE: image, + IMAGE_TAGS: tags.join(","), + PUSH: String(push), + PATH: `${bin}:${process.env.PATH}`, + CLI_TEST_CAPTURE: capture, + }, + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + if (push) { + expect(readFileSync(capture, "utf8").trim().split("\n")).toEqual([ + "buildx", + "imagetools", + "create", + "--tag", + tags[0], + "--tag", + tags[1], + `${image}@sha256:${"a".repeat(64)}`, + `${image}@sha256:${"b".repeat(64)}`, + ]); + } else { + expect(existsSync(capture)).toBe(false); + expect(result.stdout).toContain("publication skipped"); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + it.each([ + [ + "1.0.36", + "refs/tags/1.0.36-cli", + "push", + "false", + true, + ["1.0.36-cli", "1.0-cli", "latest", "1.0.36-sha-deadbee-cli"], + ], + [ + "1.0.36-rc.1", + "refs/tags/1.0.36-rc.1-cli", + "push", + "false", + true, + ["1.0.36-rc.1-cli", "1.0.36-rc.1-sha-deadbee-cli"], + ], + ["1.0.36", "refs/heads/main", "push", "false", true, ["edge"]], + ["1.0.36", "refs/heads/test", "workflow_dispatch", "true", false, ["1.0.36-dev-sha-deadbee-cli"]], + ] as const)("preserves tag selection for %s on %s", (version, ref, event, dryRun, push, tags) => { + const metadata = workflow("cli-docker.yml").jobs.metadata.steps?.find((step) => step.id === "meta"); + expect(metadata?.run).toBeDefined(); + const directory = mkdtempSync(join(tmpdir(), "livesync-cli-image-metadata-")); + try { + const bin = join(directory, "bin"); + const output = join(directory, "output"); + mkdirSync(bin); + writeFileSync(join(directory, "manifest.json"), JSON.stringify({ version })); + writeFileSync(join(bin, "git"), "#!/bin/sh\nprintf 'deadbee\\n'\n", { mode: 0o755 }); + const script = metadata!.run!.replace("${{ github.repository_owner }}", "test-owner"); + const result = spawnSync("bash", ["-e", "-c", script], { + cwd: directory, + env: { + ...process.env, + GITHUB_OUTPUT: output, + REF: ref, + EVENT_NAME: event, + DRY_RUN: dryRun, + PATH: `${bin}:${process.env.PATH}`, + }, + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + const image = "ghcr.io/test-owner/livesync-cli"; + expect(readFileSync(output, "utf8").trim().split("\n")).toEqual([ + `image=${image}`, + `tags=${tags.map((tag) => `${image}:${tag}`).join(",")}`, + `push=${push}`, + ]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } }); }); diff --git a/utils/cli-docker-artifacts-ci.unit.spec.ts b/utils/cli-docker-artifacts-ci.unit.spec.ts new file mode 100644 index 00000000..468e7c09 --- /dev/null +++ b/utils/cli-docker-artifacts-ci.unit.spec.ts @@ -0,0 +1,157 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { imageReferences, summariseResults } from "./cli-docker-artifacts.mjs"; + +const helper = fileURLToPath(new URL("./cli-docker-artifacts.mjs", import.meta.url)); +const temporaryDirectories: string[] = []; +const amd64Digest = `sha256:${"a".repeat(64)}`; +const arm64Digest = `sha256:${"b".repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function makeTemporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), "livesync-cli-docker-artifacts-")); + temporaryDirectories.push(directory); + return directory; +} + +function runCli(args: string[]) { + return spawnSync(process.execPath, [helper, ...args], { encoding: "utf8" }); +} + +function writeJson(directory: string, name: string, value: unknown): void { + writeFileSync(join(directory, name), `${JSON.stringify(value)}\n`, "utf8"); +} + +describe("CLI Docker artifact aggregation", () => { + it.each([ + [ + [ + { arch: "amd64", e2e_passed: false }, + { arch: "arm64", e2e_passed: true }, + ], + false, + ], + [ + [ + { arch: "arm64", e2e_passed: true }, + { arch: "amd64", e2e_passed: false }, + ], + false, + ], + ])("summarises mixed results independent of record order", (records, expected) => { + expect(summariseResults(records)).toBe(expected); + }); + + it("reports success only when both architecture results pass", () => { + expect( + summariseResults([ + { arch: "amd64", e2e_passed: true }, + { arch: "arm64", e2e_passed: true }, + ]) + ).toBe(true); + }); + + it("rejects malformed, missing, and duplicate result records", () => { + expect(() => + summariseResults([ + { arch: "amd64", e2e_passed: "true" }, + { arch: "arm64", e2e_passed: true }, + ]) + ).toThrow(/boolean/); + expect(() => summariseResults([{ arch: "amd64", e2e_passed: true }])).toThrow(/Exactly one/); + expect(() => + summariseResults([ + { arch: "amd64", e2e_passed: true }, + { arch: "amd64", e2e_passed: false }, + ]) + ).toThrow(/Duplicate/); + }); + + it("builds deterministic architecture-ordered references and rejects invalid or duplicate digests", () => { + const records = [ + { arch: "arm64", digest: arm64Digest }, + { arch: "amd64", digest: amd64Digest }, + ]; + expect(imageReferences("ghcr.io/example/cli", records)).toEqual([ + `ghcr.io/example/cli@${amd64Digest}`, + `ghcr.io/example/cli@${arm64Digest}`, + ]); + expect(() => + imageReferences("ghcr.io/example/cli", [ + { arch: "amd64", digest: "sha256:BAD" }, + { arch: "arm64", digest: arm64Digest }, + ]) + ).toThrow(/Digest/); + expect(() => + imageReferences("ghcr.io/example/cli", [ + { arch: "amd64", digest: amd64Digest }, + { arch: "arm64", digest: amd64Digest }, + ]) + ).toThrow(/distinct/); + expect(() => imageReferences("../../escape", records)).toThrow(/canonical image/); + }); + + it("rejects extra JSON files and filenames whose records name another architecture", () => { + const directory = makeTemporaryDirectory(); + writeJson(directory, "amd64.json", { arch: "amd64", e2e_passed: true }); + writeJson(directory, "arm64.json", { arch: "arm64", e2e_passed: true }); + writeJson(directory, "extra.json", { arch: "arm64", e2e_passed: false }); + expect(runCli(["summary", directory]).status).toBe(1); + + rmSync(join(directory, "extra.json")); + writeJson(directory, "amd64.json", { arch: "arm64", e2e_passed: true }); + expect(runCli(["summary", directory]).status).toBe(1); + }); + + it("prints a false summary successfully and fails when either result file is missing", () => { + const directory = makeTemporaryDirectory(); + const results = join(directory, "results"); + expect(runCli(["record-result", "amd64", "failure", results]).status).toBe(0); + expect(runCli(["record-result", "arm64", "success", results]).status).toBe(0); + + const summary = runCli(["summary", results]); + expect(summary.status, summary.stderr).toBe(0); + expect(summary.stdout).toBe("e2e_passed=false\n"); + + const missing = join(directory, "missing"); + mkdirSync(missing); + expect(runCli(["summary", missing]).status).toBe(1); + }); + + it("records results and digests without overwriting and prints references in canonical order", () => { + const directory = makeTemporaryDirectory(); + const results = join(directory, "results"); + const digests = join(directory, "digests"); + + expect(runCli(["record-result", "arm64", "success", results]).status).toBe(0); + expect(runCli(["record-result", "amd64", "skipped", results]).status).toBe(0); + expect(runCli(["record-result", "amd64", "success", results]).status).toBe(1); + expect(JSON.parse(readFileSync(join(results, "amd64.json"), "utf8"))).toEqual({ + arch: "amd64", + e2e_passed: false, + }); + + expect(runCli(["record-digest", "arm64", arm64Digest, digests]).status).toBe(0); + expect(runCli(["record-digest", "amd64", amd64Digest, digests]).status).toBe(0); + const invalid = join(directory, "invalid"); + expect(runCli(["record-digest", "amd64", "sha256:BAD", invalid]).status).toBe(1); + expect(existsSync(invalid)).toBe(false); + + const references = runCli(["references", "ghcr.io/example/cli", digests]); + expect(references.status, references.stderr).toBe(0); + expect(references.stdout).toBe(`ghcr.io/example/cli@${amd64Digest}\nghcr.io/example/cli@${arm64Digest}\n`); + + const traversal = runCli(["record-result", "../escape", "success", join(directory, "nested")]); + expect(traversal.status).toBe(1); + expect(readdirSync(directory)).not.toContain("escape.json"); + }); +}); diff --git a/utils/cli-docker-artifacts.mjs b/utils/cli-docker-artifacts.mjs new file mode 100644 index 00000000..71584bf2 --- /dev/null +++ b/utils/cli-docker-artifacts.mjs @@ -0,0 +1,177 @@ +import { readdir, mkdir, readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const ARCHITECTURES = ["amd64", "arm64"]; +const OUTCOMES = ["success", "failure", "cancelled", "skipped"]; +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; +const IMAGE_COMPONENT = "[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*"; +const IMAGE_PATTERN = new RegExp( + `^(?:[a-z0-9]+(?:[.-][a-z0-9]+)*(?::[0-9]+)?/)?${IMAGE_COMPONENT}(?:/${IMAGE_COMPONENT})*(?::[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?$` +); + +function assertArchitecture(arch) { + if (!ARCHITECTURES.includes(arch)) { + throw new Error(`Unsupported architecture: ${String(arch)}`); + } +} + +function assertExactKeys(record, expectedKeys) { + if (record === null || typeof record !== "object" || Array.isArray(record)) { + throw new Error("Artifact record must be a JSON object"); + } + const keys = Object.keys(record).sort(); + const expected = [...expectedKeys].sort(); + if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) { + throw new Error(`Artifact record must contain exactly: ${expected.join(", ")}`); + } +} + +function orderedPair(records, expectedKeys) { + if (!Array.isArray(records) || records.length !== ARCHITECTURES.length) { + throw new Error("Exactly one artifact record is required for each architecture"); + } + + const recordsByArchitecture = new Map(); + for (const record of records) { + assertExactKeys(record, expectedKeys); + assertArchitecture(record.arch); + if (recordsByArchitecture.has(record.arch)) { + throw new Error(`Duplicate artifact record for ${record.arch}`); + } + recordsByArchitecture.set(record.arch, record); + } + + return ARCHITECTURES.map((arch) => { + const record = recordsByArchitecture.get(arch); + if (!record) { + throw new Error(`Missing artifact record for ${arch}`); + } + return record; + }); +} + +function validateImage(image) { + if (typeof image !== "string" || !image || /\s|@/.test(image) || !IMAGE_PATTERN.test(image)) { + throw new Error("Image must be a canonical image name without whitespace or a digest"); + } +} + +function validateDigest(digest) { + if (typeof digest !== "string" || !DIGEST_PATTERN.test(digest)) { + throw new Error("Digest must match sha256:<64 lowercase hexadecimal characters>"); + } +} + +export function summariseResults(records) { + const orderedRecords = orderedPair(records, ["arch", "e2e_passed"]); + for (const { e2e_passed } of orderedRecords) { + if (typeof e2e_passed !== "boolean") { + throw new Error("e2e_passed must be a boolean"); + } + } + return orderedRecords.every(({ e2e_passed }) => e2e_passed); +} + +export function imageReferences(image, records) { + validateImage(image); + const orderedRecords = orderedPair(records, ["arch", "digest"]); + const digests = orderedRecords.map(({ digest }) => { + validateDigest(digest); + return digest; + }); + if (new Set(digests).size !== digests.length) { + throw new Error("Each architecture must have a distinct image digest"); + } + return digests.map((digest) => `${image}@${digest}`); +} + +async function writeRecord(arch, directory, record) { + assertArchitecture(arch); + await mkdir(directory, { recursive: true }); + await writeFile(resolve(directory, `${arch}.json`), `${JSON.stringify(record)}\n`, { flag: "wx" }); +} + +async function readRecords(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + const jsonEntries = entries.filter((entry) => entry.name.toLowerCase().endsWith(".json")); + const expectedNames = ARCHITECTURES.map((arch) => `${arch}.json`); + const actualNames = jsonEntries.map((entry) => entry.name).sort(); + if (actualNames.length !== expectedNames.length || expectedNames.some((name) => !actualNames.includes(name))) { + throw new Error(`Directory must contain exactly ${expectedNames.join(" and ")}`); + } + + return Promise.all( + ARCHITECTURES.map(async (arch) => { + const name = `${arch}.json`; + const entry = jsonEntries.find((candidate) => candidate.name === name); + if (!entry?.isFile()) { + throw new Error(`${name} must be a regular file`); + } + let record; + try { + record = JSON.parse(await readFile(resolve(directory, name), "utf8")); + } catch (error) { + throw new Error(`Could not read ${name}: ${error instanceof Error ? error.message : String(error)}`); + } + if (record?.arch !== arch) { + throw new Error(`${name} must contain an ${arch} record`); + } + return record; + }) + ); +} + +async function main(args) { + const [command, ...values] = args; + switch (command) { + case "record-result": { + if (values.length !== 3) { + throw new Error("Usage: record-result ARCH OUTCOME DIRECTORY"); + } + const [arch, outcome, directory] = values; + assertArchitecture(arch); + if (!OUTCOMES.includes(outcome)) { + throw new Error(`Unsupported outcome: ${outcome}`); + } + await writeRecord(arch, directory, { arch, e2e_passed: outcome === "success" }); + return; + } + case "summary": { + if (values.length !== 1) { + throw new Error("Usage: summary DIRECTORY"); + } + const records = await readRecords(values[0]); + process.stdout.write(`e2e_passed=${summariseResults(records)}\n`); + return; + } + case "record-digest": { + if (values.length !== 3) { + throw new Error("Usage: record-digest ARCH DIGEST DIRECTORY"); + } + const [arch, digest, directory] = values; + assertArchitecture(arch); + validateDigest(digest); + await writeRecord(arch, directory, { arch, digest }); + return; + } + case "references": { + if (values.length !== 2) { + throw new Error("Usage: references IMAGE DIRECTORY"); + } + const [image, directory] = values; + const records = await readRecords(directory); + process.stdout.write(`${imageReferences(image, records).join("\n")}\n`); + return; + } + default: + throw new Error("Usage: cli-docker-artifacts.mjs ..."); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main(process.argv.slice(2)).catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/utils/cli-dockerfile-ci.unit.spec.ts b/utils/cli-dockerfile-ci.unit.spec.ts new file mode 100644 index 00000000..811e0e96 --- /dev/null +++ b/utils/cli-dockerfile-ci.unit.spec.ts @@ -0,0 +1,37 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const root = new URL("../", import.meta.url); +const dockerfile = readFileSync(new URL("src/apps/cli/Dockerfile", root), "utf8"); +const dockerfileLines = dockerfile.split(/\r?\n/); +const gitignoreLines = readFileSync(new URL(".gitignore", root), "utf8").split(/\r?\n/); + +function stageLine(stage: string): string | undefined { + return dockerfileLines.find((line) => /^FROM\b/i.test(line) && new RegExp(`\\bAS\\s+${stage}$`, "i").test(line)); +} + +describe("CLI Dockerfile bundle stages", () => { + it("keeps the default build self-contained from source", () => { + expect(dockerfileLines).toContain("ARG CLI_BUNDLE_STAGE=builder"); + expect(stageLine("builder")).toBe("FROM --platform=$BUILDPLATFORM node:22-slim AS builder"); + expect(dockerfileLines).toContain("COPY . ."); + expect(dockerfileLines).toContain("RUN cd src/apps/cli && npm run build"); + expect(dockerfileLines).toContain("FROM ${CLI_BUNDLE_STAGE} AS cli-bundle"); + expect(dockerfileLines).toContain("COPY --from=cli-bundle /build/src/apps/cli/dist ./dist"); + }); + + it("provides an explicit stage for a prebuilt bundle", () => { + expect(stageLine("prebuilt")).toBe("FROM scratch AS prebuilt"); + expect(dockerfileLines).toContain("COPY _cli-bundle/ /build/src/apps/cli/dist/"); + expect(dockerfileLines).toContain("FROM ${CLI_BUNDLE_STAGE} AS cli-bundle"); + }); + + it("builds source on the host platform and native dependencies in the target stages", () => { + expect(stageLine("runtime-deps")).toBe("FROM node:22-slim AS runtime-deps"); + expect(dockerfileLines).toContain("FROM node:22-slim"); + }); + + it("ignores the temporary prebuilt bundle directory", () => { + expect(gitignoreLines).toContain("_cli-bundle/"); + }); +}); From 51d16d06c57d41abf03ea60df815ac1877a0d5d7 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 14 Oct 2026 12:22:52 +0000 Subject: [PATCH 4/5] Use CLI release tags for Docker image versions --- .github/workflows/cli-docker.yml | 11 ++++++- src/apps/cli/README.md | 2 ++ updates.md | 7 +++++ utils/cli-ci.unit.spec.ts | 50 +++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-docker.yml b/.github/workflows/cli-docker.yml index 47347626..b8dd04a1 100644 --- a/.github/workflows/cli-docker.yml +++ b/.github/workflows/cli-docker.yml @@ -1,5 +1,5 @@ # Build and push the CLI Docker image to GitHub Container Registry (GHCR). -# Image tag format: -cli +# Release image tag format: -cli # Example: 1.0.36-cli # # Stable releases are also tagged with their major-minor version and 'latest'. @@ -72,6 +72,15 @@ jobs: DRY_RUN: ${{ inputs.dry_run }} run: | VERSION=$(jq -r '.version' manifest.json) + if [[ "${REF}" == refs/tags/*-cli ]]; then + # CLI releases use their Git tag independently of the plug-in version. + VERSION="${REF#refs/tags/}" + VERSION="${VERSION%-cli}" + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$ ]]; then + printf 'Invalid CLI release tag: %s\n' "$REF" >&2 + exit 1 + fi + fi MAJOR_MINOR=$(echo "${VERSION}" | cut -d. -f1,2) SHORT_SHA=$(git rev-parse --short HEAD) IMAGE="ghcr.io/${{ github.repository_owner }}/livesync-cli" diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 8761bb08..9f55fa63 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -193,6 +193,8 @@ The generated systemd unit uses the wrapper's absolute path and does not depend ### Docker +CLI-only Docker patches use their full release tag, such as `ghcr.io/vrtmrz/livesync-cli:1.0.36-patch1-cli`. Patch and RC images receive version and SHA-qualified tags. The `latest` and major-minor tags are updated by ordinary stable releases. CLI image versions come from the release Git tag, so a CLI-only patch does not require changing the Obsidian plug-in version. + A Docker image is provided for headless / server deployments. Build from the repository root: ```bash diff --git a/updates.md b/updates.md index 2a25ed9f..1776f24b 100644 --- a/updates.md +++ b/updates.md @@ -12,6 +12,13 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased +### CLI + +#### Fixed + +- We can select CLI-only Docker patches by their full release tag, such as `1.0.36-patch1-cli`, while keeping the Obsidian plug-in version. + - Patch and RC images receive version and SHA-qualified tags. The `latest` and major-minor tags remain reserved for ordinary stable releases. + ## 1.0.36 13th October, 2026 diff --git a/utils/cli-ci.unit.spec.ts b/utils/cli-ci.unit.spec.ts index c6841b57..500de60e 100644 --- a/utils/cli-ci.unit.spec.ts +++ b/utils/cli-ci.unit.spec.ts @@ -273,9 +273,51 @@ describe("CLI CI selection", () => { true, ["1.0.36-rc.1-cli", "1.0.36-rc.1-sha-deadbee-cli"], ], + [ + "1.0.36", + "refs/tags/1.0.36-patch1-cli", + "push", + "false", + true, + ["1.0.36-patch1-cli", "1.0.36-patch1-sha-deadbee-cli"], + ], + [ + "1.0.36", + "refs/tags/1.0.36-rc.1-cli", + "push", + "false", + true, + ["1.0.36-rc.1-cli", "1.0.36-rc.1-sha-deadbee-cli"], + ], + [ + "1.0.36", + "refs/tags/1.0.37-cli", + "push", + "false", + true, + ["1.0.37-cli", "1.0-cli", "latest", "1.0.37-sha-deadbee-cli"], + ], + [ + "1.0.36", + "refs/tags/1.0.36-patch2-cli", + "workflow_dispatch", + "true", + false, + ["1.0.36-patch2-cli", "1.0.36-patch2-sha-deadbee-cli"], + ], + [ + "1.0.36", + "refs/tags/1.0.36", + "workflow_dispatch", + "true", + false, + ["1.0.36-cli", "1.0-cli", "latest", "1.0.36-sha-deadbee-cli"], + ], + ["1.0.36", "refs/tags/not-a-version-cli", "workflow_dispatch", "true", false, null], + ["1.0.36", "refs/tags/1.0.36+build-cli", "push", "false", false, null], ["1.0.36", "refs/heads/main", "push", "false", true, ["edge"]], ["1.0.36", "refs/heads/test", "workflow_dispatch", "true", false, ["1.0.36-dev-sha-deadbee-cli"]], - ] as const)("preserves tag selection for %s on %s", (version, ref, event, dryRun, push, tags) => { + ] as const)("derives publication metadata for manifest %s on %s", (version, ref, event, dryRun, push, tags) => { const metadata = workflow("cli-docker.yml").jobs.metadata.steps?.find((step) => step.id === "meta"); expect(metadata?.run).toBeDefined(); const directory = mkdtempSync(join(tmpdir(), "livesync-cli-image-metadata-")); @@ -298,6 +340,12 @@ describe("CLI CI selection", () => { }, encoding: "utf8", }); + if (tags === null) { + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Invalid CLI release tag"); + expect(existsSync(output)).toBe(false); + return; + } expect(result.status, result.stderr).toBe(0); const image = "ghcr.io/test-owner/livesync-cli"; expect(readFileSync(output, "utf8").trim().split("\n")).toEqual([ From 759130fb39e9613cc0365abc2f6f5b93306c13bf Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Wed, 14 Oct 2026 12:36:32 +0000 Subject: [PATCH 5/5] Document the CLI-only patch release procedure --- devs.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/devs.md b/devs.md index 713eb6cc..bbeaf1be 100644 --- a/devs.md +++ b/devs.md @@ -311,11 +311,13 @@ Existing legacy Modules continue to register their handlers in `onBindFunction() ## Pre-release Policy +This policy governs plug-in pre-releases and their accompanying CLI images. For a correction needed only by the CLI, use [CLI-only patch releases](#cli-only-patch-releases). + - Use SemVer beta identifiers such as `1.0.0-beta.0` for immutable integration previews. Increment the beta number when a published preview needs a correction. Reserve `1.0.0-rc.0` for the first feature- and contract-frozen release candidate. Historical `-patchedN` releases remain unchanged in the release history. - Publish a pre-release from an immutable reviewed tag, mark its GitHub Release as a pre-release, and do not replace the latest stable release. - A plug-in review release may omit the CLI image when the CLI artefact is not part of the required validation. When a pre-release CLI image is published, it receives immutable version and SHA-qualified tags only; it must not advance `latest` or a stable major-minor tag. - Keep a hyphenated pre-release's release pull request in draft and unmerged after BRAT validation. Reconcile the published version's metadata into its base branch through a reviewed metadata-only commit, then close the release pull request only through a separate maintainer action. -- Stage a stable version for BRAT by publishing its exact `x.y.z` tag initially as a GitHub pre-release with `prerelease=true` and `publish_cli=false`. The stable manifest version would otherwise make the CLI workflow advance `latest` and the major-minor image tag before validation. +- Stage a stable version for BRAT by publishing its exact `x.y.z` tag initially as a GitHub pre-release with `prerelease=true` and `publish_cli=false`. A stable CLI release tag would otherwise make the CLI workflow advance `latest` and the major-minor image tag before validation. - After a staged stable version passes BRAT validation, remove the GitHub pre-release designation and make the exact release the latest stable release first. Confirm that promotion, then merge its exact release commit into the reviewed base branch and integrate it through the reviewed branch chain into the repository's default branch. This order prevents Community Directory review from inspecting a stable manifest version while the matching GitHub Release remains a pre-release. Publish stable CLI tags through a separate maintainer gate. - If validation fails, leave every published tag unchanged and prepare the next pre-release or patch version. @@ -333,6 +335,7 @@ Existing legacy Modules continue to register their handlers in `onBindFunction() This workflow is for maintainers. Contributors should update `## Unreleased` for user-facing feature or fix PRs, but do not need to run the release workflows. The `Finalise Release Tags` and `Release Obsidian Plugin` workflows use the `release` GitHub Environment. Configure Environment protection in the repository settings so tag creation and release publication require maintainer approval. +For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch-releases). The steps below describe plug-in releases. - Run the `Prepare Release PR` workflow with the target version and selected base branch. It creates the release branch, updates versions, confirms that Commonlib is locked to an immutable package version, moves the `## Unreleased` notes to the target version, commits the release preparation, pushes the branch, and opens a draft release PR. The base branch may already select the target development version; the workflow still runs the version lifecycle so that release-only metadata such as `versions.json` is recorded in the release commit. - Do not tag the release branch when the PR is first created. Polish the release PR first, especially `updates.md`. @@ -377,6 +380,60 @@ The `Finalise Release Tags` and `Release Obsidian Plugin` workflows use the `rel 11. After a stable version passes, remove the pre-release designation and make the exact release the latest stable release. Confirm that promotion, then mark the release PR ready, merge the exact release commit into the selected base branch, and integrate it through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata, then publish stable CLI tags through a separate maintainer gate if selected. 12. If validation fails, leave the PR in draft and do not move the published tags. Reconcile the published version's `updates.md` section and `versions.json` entry into the base branch as metadata only, then prepare the next patch or pre-release version from the remaining `## Unreleased` entries. +### CLI-only patch releases + +Use this route when a correction affects only CLI commands, CLI host behaviour, or the CLI Docker image. For a change which also affects the Obsidian plug-in, use the plug-in release workflow above. + +A CLI-only patch publishes a Docker image from a reviewed commit using `x.y.z-patchN-cli`, for example `1.0.36-patch1-cli`. Keep `x.y.z` as the underlying plug-in version and increment `N` for each CLI-only correction. The image version comes from the CLI Git tag. Keep the plug-in version metadata unchanged: assigning this tag does not require a version bump, a `versions.json` entry, a plug-in tag, or a GitHub Release. Record the fix under `## Unreleased`; leave the plug-in release-note lifecycle for the next plug-in release. + +1. Review and integrate the CLI fix, including its relevant CLI or Docker regression tests. Select a source branch whose reviewed commit contains both the fix and the CLI release workflow. Record that exact commit and confirm that the proposed patch tag is unused: + + ```bash + CLI_RELEASE_BRANCH=main + CLI_PATCH_TAG=1.0.36-patch1-cli + git fetch origin "$CLI_RELEASE_BRANCH" --tags + CLI_RELEASE_COMMIT=$(git rev-parse "origin/${CLI_RELEASE_BRANCH}^{commit}") + git show --no-patch --oneline "$CLI_RELEASE_COMMIT" + git ls-remote --tags origin "refs/tags/$CLI_PATCH_TAG" + ``` + + The last command must succeed without returning a tag. If the tag already exists, choose the next patch number; keep published tags unchanged. + +2. Run the Docker workflow without publication before pushing the tag: + + ```bash + gh workflow run cli-docker.yml --ref "$CLI_RELEASE_BRANCH" --field dry_run=true --field force=false + ``` + + Wait for success and verify that the run's head SHA is exactly `$CLI_RELEASE_COMMIT`. Both native amd64 and arm64 Docker E2E suites must pass. If the branch has advanced, review and validate the intended commit again. A branch dry run uses `edge` on `main` or a development tag on another branch; it validates the image and publication preparation without reserving the patch tag. + +3. After maintainer approval of the exact commit and tag, create and push only the CLI tag: + + ```bash + git tag "$CLI_PATCH_TAG" "$CLI_RELEASE_COMMIT" + git push origin "refs/tags/$CLI_PATCH_TAG" + ``` + + Pushing this tag with maintainer credentials starts `Build and Push CLI Docker Image` and publishes after both architecture validations pass. This route uses `cli-docker.yml`; the plug-in release preparation and finalisation workflows are not needed. + +4. If automation pushed the tag using `GITHUB_TOKEN`, explicitly dispatch publication against that existing tag. Such a push does not start another workflow ([GitHub workflow trigger rules](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow#triggering-a-workflow-from-a-workflow)): + + ```bash + gh workflow run cli-docker.yml --ref "$CLI_PATCH_TAG" --field dry_run=false --field force=false + ``` + + Use this dispatch when the tag push did not start publication, rather than starting a second publication alongside an existing run. + +5. Confirm that the workflow completed and the published image contains `linux/amd64` and `linux/arm64`: + + ```bash + CLI_IMAGE="ghcr.io/vrtmrz/livesync-cli:$CLI_PATCH_TAG" + docker buildx imagetools inspect "$CLI_IMAGE" + docker run --rm "$CLI_IMAGE" --help + ``` + + Retest the corrected CLI behaviour using this exact image tag. A patch receives its full version tag and a SHA-qualified tag, such as `1.0.36-patch1-sha--cli`. Publishing that patch tag does not update `latest`, `edge`, or the stable major-minor tag. Consumers must select the full patch tag explicitly. + ## Contribution Guidelines - Follow existing code style and conventions