From b5d0b7b2334dc94725669da73070c7d319599f34 Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 25 Jun 2026 20:08:23 +0800 Subject: [PATCH 01/21] Add open Shorebird CI and release tooling --- .github/workflows/open-shorebird-ci.yml | 1690 +++++++++++++ .gitignore | 2 + README.md | 91 + depot_tools | 2 +- docs/CI.md | 454 ++++ docs/PLATFORM_TESTING.md | 270 ++- flutter | 2 +- scripts/android_runtime_patch_smoke.sh | 249 ++ scripts/assemble_artifact_mirror.sh | 100 + scripts/check_ci_capacity.sh | 55 + scripts/free_ci_disk_linux.sh | 48 + scripts/linux_runtime_patch_smoke.sh | 238 ++ scripts/platform_test_common.sh | 12 +- scripts/safe_extract_tar.py | 117 + scripts/sync_open_sources.sh | 174 +- scripts/validate_artifact_mirror.py | 210 ++ scripts/validate_release_manifest.py | 249 ++ scripts/verify_artifact_mirror_validator.sh | 121 + ...erify_artifact_mirror_workflow_assembly.sh | 287 +++ scripts/verify_assemble_artifact_mirror.sh | 178 ++ scripts/verify_ci_capacity.sh | 26 + scripts/verify_ci_workflow.rb | 2124 +++++++++++++++++ scripts/verify_ci_workflow.sh | 59 + scripts/verify_dart_sdk_args.sh | 60 + .../verify_downloaded_release_artifacts.sh | 169 ++ scripts/verify_engine_args.sh | 84 + scripts/verify_hosted_full_sdk_build.sh | 209 ++ scripts/verify_ios_interpreter_route.sh | 151 +- .../verify_ios_interpreter_route_validator.sh | 119 + .../verify_open_infrastructure_defaults.sh | 121 + scripts/verify_powershell_open_defaults.sh | 87 + scripts/verify_release_manifest.sh | 238 ++ scripts/verify_sync_open_sources.sh | 125 + scripts/verify_upload_readiness.sh | 10 + scripts/verify_write_sha256.sh | 51 + scripts/write_artifact_manifest.py | 85 + scripts/write_gclient.sh | 9 +- scripts/write_release_manifest.py | 177 ++ scripts/write_sha256.py | 43 + scripts/write_sha256.sh | 15 + shorebird | 2 +- shorebird-server | 2 +- .../ios/Runner/SceneDelegate.swift | 34 + .../license_flavor_patch_test/shorebird.yaml | 1 - updater | 2 +- 45 files changed, 8451 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/open-shorebird-ci.yml create mode 100644 docs/CI.md create mode 100755 scripts/android_runtime_patch_smoke.sh create mode 100755 scripts/assemble_artifact_mirror.sh create mode 100755 scripts/check_ci_capacity.sh create mode 100755 scripts/free_ci_disk_linux.sh create mode 100755 scripts/linux_runtime_patch_smoke.sh create mode 100755 scripts/safe_extract_tar.py create mode 100755 scripts/validate_artifact_mirror.py create mode 100755 scripts/validate_release_manifest.py create mode 100755 scripts/verify_artifact_mirror_validator.sh create mode 100755 scripts/verify_artifact_mirror_workflow_assembly.sh create mode 100755 scripts/verify_assemble_artifact_mirror.sh create mode 100755 scripts/verify_ci_capacity.sh create mode 100755 scripts/verify_ci_workflow.rb create mode 100755 scripts/verify_ci_workflow.sh create mode 100755 scripts/verify_dart_sdk_args.sh create mode 100755 scripts/verify_downloaded_release_artifacts.sh create mode 100755 scripts/verify_engine_args.sh create mode 100755 scripts/verify_hosted_full_sdk_build.sh create mode 100755 scripts/verify_ios_interpreter_route_validator.sh create mode 100755 scripts/verify_open_infrastructure_defaults.sh create mode 100755 scripts/verify_powershell_open_defaults.sh create mode 100755 scripts/verify_release_manifest.sh create mode 100755 scripts/verify_sync_open_sources.sh create mode 100755 scripts/verify_upload_readiness.sh create mode 100755 scripts/verify_write_sha256.sh create mode 100755 scripts/write_artifact_manifest.py create mode 100755 scripts/write_release_manifest.py create mode 100755 scripts/write_sha256.py create mode 100755 scripts/write_sha256.sh diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml new file mode 100644 index 0000000..5c4fc35 --- /dev/null +++ b/.github/workflows/open-shorebird-ci.yml @@ -0,0 +1,1690 @@ +name: Open Shorebird CI + +on: + push: + branches: ["**"] + pull_request: + workflow_dispatch: + inputs: + full_sdk_build: + description: Build custom SDK and engine artifacts. + required: false + default: false + type: boolean + run_gclient_sync: + description: Run gclient sync before heavy SDK/engine builds. + required: false + default: true + type: boolean + base_flutter_engine_revision: + description: Optional upstream Flutter engine revision for artifact proxy fallback metadata. + required: false + default: "" + type: string + linux_heavy_runner: + description: Runner label for Linux SDK/Android/web heavy jobs. + required: false + default: open-shorebird-linux-heavy + type: string + macos_heavy_runner: + description: Runner label for macOS SDK and Apple engine heavy jobs. + required: false + default: open-shorebird-macos-heavy + type: string + sdk_min_free_disk_gb: + description: Minimum free disk GiB required before SDK-only heavy jobs. + required: false + default: 35 + type: number + engine_min_free_disk_gb: + description: Minimum free disk GiB required before engine heavy jobs. + required: false + default: 40 + type: number + run_runtime_smokes: + description: Run Android/Linux runtime patch smokes on provisioned runners. + required: false + default: false + type: boolean + linux_runtime_runner: + description: Runner label for the Linux runtime smoke job. + required: false + default: self-hosted + type: string + android_runtime_runner: + description: Runner label for the Android runtime smoke job. + required: false + default: self-hosted + type: string + android_serial: + description: Optional adb serial for the Android runtime smoke target. + required: false + default: "" + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + DART_SDK_CHANNEL: stable + GO_VERSION: "1.23.x" + JAVA_VERSION: "17" + DEPOT_TOOLS_UPDATE: "0" + PUB_CACHE: ${{ github.workspace }}/.pub-cache + BASE_FLUTTER_ENGINE_REVISION: ${{ inputs.base_flutter_engine_revision || '' }} + SDK_MIN_FREE_DISK_GB: ${{ inputs.sdk_min_free_disk_gb || 35 }} + ENGINE_MIN_FREE_DISK_GB: ${{ inputs.engine_min_free_disk_gb || 40 }} + +jobs: + source-checks: + name: Source checks + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout workspace + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set up Dart + uses: dart-lang/setup-dart@v1 + with: + sdk: ${{ env.DART_SDK_CHANNEL }} + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: shorebird-server/go.sum + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache pub packages + uses: actions/cache@v4 + with: + path: | + .pub-cache + key: pub-${{ runner.os }}-${{ hashFiles('shorebird/pubspec.lock', 'shorebird/**/pubspec.yaml') }} + restore-keys: | + pub-${{ runner.os }}- + + - name: Run source-level workspace checks + env: + SKIP_GCLIENT_SYNC: "1" + run: ./scripts/bootstrap_linux.sh + + - name: Verify CI scripts + run: | + bash -n scripts/*.sh shorebird/third_party/flutter/bin/internal/shared.sh flutter/bin/internal/update_dart_sdk.sh + python3 -m py_compile scripts/validate_artifact_mirror.py + python3 -m py_compile scripts/validate_release_manifest.py + python3 -m py_compile scripts/safe_extract_tar.py + python3 -m py_compile scripts/write_artifact_manifest.py + python3 -m py_compile scripts/write_release_manifest.py + python3 -m py_compile scripts/write_sha256.py + python3 - <<'PY' + from pathlib import Path + + compile(Path('flutter/DEPS').read_text(encoding='utf-8'), 'flutter/DEPS', 'exec') + PY + ./scripts/verify_ci_capacity.sh + ./scripts/verify_open_infrastructure_defaults.sh + ./scripts/verify_sync_open_sources.sh + ./scripts/verify_powershell_open_defaults.sh + ./scripts/verify_write_sha256.sh + ./scripts/verify_release_manifest.sh + ./scripts/verify_artifact_mirror_validator.sh + ./scripts/verify_assemble_artifact_mirror.sh + ./scripts/verify_artifact_mirror_workflow_assembly.sh + ./scripts/verify_hosted_full_sdk_build.sh --help >/dev/null 2>&1 + ./scripts/verify_ios_interpreter_route_validator.sh + ./scripts/verify_ci_workflow.sh + + cli-artifacts: + name: Build CLI artifacts + needs: source-checks + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + artifact_os: linux + artifact_arch: x64 + extension: "" + patch_zip: patch-linux-x64.zip + patch_entry: patch + - os: macos-15-intel + artifact_os: macos + artifact_arch: x64 + extension: "" + patch_zip: patch-darwin-x64.zip + patch_entry: patch + - os: macos-14 + artifact_os: macos + artifact_arch: arm64 + extension: "" + patch_zip: patch-darwin-arm64.zip + patch_entry: patch + - os: windows-latest + artifact_os: windows + artifact_arch: x64 + extension: ".exe" + patch_zip: patch-windows-x64.zip + patch_entry: patch.exe + steps: + - name: Checkout workspace + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set up Dart + uses: dart-lang/setup-dart@v1 + with: + sdk: ${{ env.DART_SDK_CHANNEL }} + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Verify Rust + run: | + rustc --version + cargo --version + + - name: Cache pub packages + uses: actions/cache@v4 + with: + path: | + .pub-cache + key: pub-${{ runner.os }}-${{ hashFiles('shorebird/pubspec.lock', 'shorebird/**/pubspec.yaml') }} + restore-keys: | + pub-${{ runner.os }}- + + - name: Compile Shorebird CLI and open patch tools + shell: bash + run: | + set -euo pipefail + rm -rf artifacts/cli + mkdir -p artifacts/cli/bin/internal + cp shorebird/bin/internal/flutter.version artifacts/cli/bin/internal/flutter.version + cp flutter/bin/internal/engine.version artifacts/cli/bin/internal/engine.version + cd shorebird + dart pub get + dart compile exe packages/shorebird_cli/bin/shorebird.dart \ + -o ../artifacts/cli/bin/shorebird${{ matrix.extension }} + dart compile exe packages/open_aot_patch_tools/bin/open_aot_patch_tools.dart \ + -o ../artifacts/cli/bin/open_aot_patch_tools${{ matrix.extension }} + dart compile exe packages/artifact_proxy/bin/server.dart \ + -o ../artifacts/cli/bin/artifact_proxy${{ matrix.extension }} + cd .. + flutter_revision="$(cat artifacts/cli/bin/internal/flutter.version)" + engine_revision="$(cat artifacts/cli/bin/internal/engine.version)" + cat > artifacts/cli/manifest.json </dev/null + version_output="$(HOME="$PWD/artifacts/cli-home" artifacts/cli/bin/shorebird${{ matrix.extension }} --version)" + printf '%s\n' "$version_output" + grep -q 'https://git.tonycloud.org/flutter/shorebird.git' <<<"$version_output" + if grep -q 'git@github.com:shorebirdtech/shorebird.git' <<<"$version_output"; then + echo "compiled CLI still reports the official Shorebird SSH remote" >&2 + exit 1 + fi + HOME="$PWD/artifacts/cli-home" artifacts/cli/bin/artifact_proxy${{ matrix.extension }} --health-check + tar -C artifacts/cli -czf open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz . + cli_extract_dir="$(mktemp -d)" + mkdir -p artifacts/cli-extracted-home + tar -C "$cli_extract_dir" -xzf open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz + test -f "$cli_extract_dir/manifest.json" + test -f "$cli_extract_dir/bin/internal/flutter.version" + test -f "$cli_extract_dir/bin/internal/engine.version" + HOME="$PWD/artifacts/cli-extracted-home" "$cli_extract_dir/bin/open_aot_patch_tools${{ matrix.extension }}" --help >/dev/null + extracted_version_output="$(HOME="$PWD/artifacts/cli-extracted-home" "$cli_extract_dir/bin/shorebird${{ matrix.extension }}" --version)" + printf '%s\n' "$extracted_version_output" + grep -q 'https://git.tonycloud.org/flutter/shorebird.git' <<<"$extracted_version_output" + if grep -q 'git@github.com:shorebirdtech/shorebird.git' <<<"$extracted_version_output"; then + echo "extracted CLI still reports the official Shorebird SSH remote" >&2 + exit 1 + fi + HOME="$PWD/artifacts/cli-extracted-home" "$cli_extract_dir/bin/artifact_proxy${{ matrix.extension }}" --health-check + ./scripts/write_sha256.sh \ + open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz \ + open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz.sha256 + + - name: Build mirror patch artifact + shell: bash + run: | + set -euo pipefail + cargo_target_dir="$PWD/artifacts/cargo-target/${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}" + CARGO_TARGET_DIR="$cargo_target_dir" \ + cargo build --release --manifest-path updater/patch/Cargo.toml --bin patch + + mkdir -p artifacts/mirror + patch_binary="$cargo_target_dir/release/patch${{ matrix.extension }}" + cp "$patch_binary" "artifacts/mirror/${{ matrix.patch_entry }}" + + python_bin=python3 + if ! command -v "$python_bin" >/dev/null 2>&1; then + python_bin=python + fi + "$python_bin" - \ + "artifacts/mirror/${{ matrix.patch_zip }}" \ + "artifacts/mirror/${{ matrix.patch_entry }}" \ + "${{ matrix.patch_entry }}" <<'PY' + import sys + import zipfile + + zip_path, input_path, archive_name = sys.argv[1:] + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as archive: + archive.write(input_path, archive_name) + PY + "$python_bin" - \ + "artifacts/mirror/${{ matrix.patch_zip }}" \ + "${{ matrix.patch_entry }}" <<'PY' + import sys + import zipfile + + zip_path, expected_entry = sys.argv[1:] + with zipfile.ZipFile(zip_path) as archive: + names = archive.namelist() + if names != [expected_entry]: + raise SystemExit( + f"expected only {expected_entry!r} in {zip_path}, got {names!r}" + ) + info = archive.getinfo(expected_entry) + if info.file_size <= 0: + raise SystemExit(f"{expected_entry} in {zip_path} is empty") + PY + ./scripts/write_sha256.sh \ + "artifacts/mirror/${{ matrix.patch_zip }}" \ + "artifacts/mirror/${{ matrix.patch_zip }}.sha256" + engine_revision="$(cat flutter/bin/internal/engine.version)" + mirror_patch_dir="artifacts/mirror/shorebird/$engine_revision" + mkdir -p "$mirror_patch_dir" + cp "artifacts/mirror/${{ matrix.patch_zip }}" "$mirror_patch_dir/${{ matrix.patch_zip }}" + cp "artifacts/mirror/${{ matrix.patch_zip }}.sha256" "$mirror_patch_dir/${{ matrix.patch_zip }}.sha256" + test -f "$mirror_patch_dir/${{ matrix.patch_zip }}" + test -f "$mirror_patch_dir/${{ matrix.patch_zip }}.sha256" + + - name: Build mirror metadata + if: ${{ matrix.artifact_os == 'linux' && matrix.artifact_arch == 'x64' }} + shell: bash + run: | + set -euo pipefail + engine_revision="$(cat flutter/bin/internal/engine.version)" + base_engine_revision="${BASE_FLUTTER_ENGINE_REVISION:-$engine_revision}" + manifest_path="artifacts/mirror/shorebird/$engine_revision/artifacts_manifest.yaml" + python3 scripts/write_artifact_manifest.py \ + --flutter-engine-revision "$base_engine_revision" \ + --output "$manifest_path" + ./scripts/write_sha256.sh "$manifest_path" "$manifest_path.sha256" + test -f "artifacts/mirror/shorebird/$engine_revision/artifacts_manifest.yaml.sha256" + + - name: Upload CLI artifacts + uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }} + path: | + open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz + open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz.sha256 + if-no-files-found: error + + - name: Upload mirror patch artifact + uses: actions/upload-artifact@v4 + with: + name: mirror-${{ matrix.patch_zip }} + path: | + artifacts/mirror/${{ matrix.patch_zip }} + artifacts/mirror/${{ matrix.patch_zip }}.sha256 + artifacts/mirror/shorebird/**/${{ matrix.patch_zip }} + artifacts/mirror/shorebird/**/${{ matrix.patch_zip }}.sha256 + if-no-files-found: error + + - name: Upload mirror metadata + if: ${{ matrix.artifact_os == 'linux' && matrix.artifact_arch == 'x64' }} + uses: actions/upload-artifact@v4 + with: + name: mirror-metadata + path: | + artifacts/mirror/shorebird/**/artifacts_manifest.yaml + artifacts/mirror/shorebird/**/artifacts_manifest.yaml.sha256 + if-no-files-found: error + + server-artifacts: + name: Build server artifacts + needs: source-checks + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - goos: linux + goarch: amd64 + extension: "" + - goos: linux + goarch: arm64 + extension: "" + - goos: darwin + goarch: amd64 + extension: "" + - goos: darwin + goarch: arm64 + extension: "" + - goos: windows + goarch: amd64 + extension: ".exe" + steps: + - name: Checkout workspace + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: shorebird-server/go.sum + + - name: Test server + working-directory: shorebird-server + run: go test ./... + + - name: Build server binary + working-directory: shorebird-server + env: + CGO_ENABLED: "0" + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + set -euo pipefail + mkdir -p ../artifacts/server-${GOOS}-${GOARCH} + go build -trimpath -ldflags="-s -w" \ + -o ../artifacts/server-${GOOS}-${GOARCH}/shorebird-server${{ matrix.extension }} \ + ./cmd/server + cp -R web ../artifacts/server-${GOOS}-${GOARCH}/ + cp .env.example ../artifacts/server-${GOOS}-${GOARCH}/ + cp README.md ../artifacts/server-${GOOS}-${GOARCH}/ + cp internal/api/handlers/openapi.yaml ../artifacts/server-${GOOS}-${GOARCH}/openapi.yaml + server_revision="$(git rev-parse HEAD)" + cat > ../artifacts/server-${GOOS}-${GOARCH}/manifest.json </dev/null; then + kill "$smoke_pid" 2>/dev/null || true + wait "$smoke_pid" 2>/dev/null || true + fi + rm -rf "$smoke_dir" + } + trap cleanup_server_smoke EXIT + + tar -C "$smoke_dir" -xzf ../shorebird-server-${GOOS}-${GOARCH}.tar.gz + cd "$smoke_dir/server-${GOOS}-${GOARCH}" + SERVER_HOST=127.0.0.1 \ + SERVER_PORT=18080 \ + SERVER_BASE_URL=http://127.0.0.1:18080 \ + DB_DRIVER=sqlite \ + DB_PATH="$smoke_dir/shorebird.db" \ + STORAGE_DRIVER=local \ + STORAGE_LOCAL_DIR="$smoke_dir/storage" \ + DEFAULT_ADMIN_PASSWORD=admin123456 \ + ./shorebird-server >"$smoke_dir/server.log" 2>&1 & + smoke_pid="$!" + + for _ in {1..40}; do + if curl -fsS http://127.0.0.1:18080/health >"$smoke_dir/health.json"; then + break + fi + sleep 0.25 + done + cat "$smoke_dir/health.json" + grep -q '"status":"ok"' "$smoke_dir/health.json" + curl -fsS http://127.0.0.1:18080/ >"$smoke_dir/index.html" + grep -q '' "$smoke_dir/index.html" + curl -fsS http://127.0.0.1:18080/openapi.yaml >"$smoke_dir/openapi.yaml" + grep -q 'openapi: 3.1.0' "$smoke_dir/openapi.yaml" + fi + + - name: Upload server artifact + uses: actions/upload-artifact@v4 + with: + name: shorebird-server-${{ matrix.goos }}-${{ matrix.goarch }} + path: | + shorebird-server-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz + shorebird-server-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz.sha256 + if-no-files-found: error + + custom-dart-sdk: + name: Build custom Dart SDK + needs: source-checks + runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + timeout-minutes: 180 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + steps: + - name: Checkout workspace + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Install depot_tools + run: | + if [ ! -d depot_tools ]; then + git clone --depth 1 \ + https://chromium.googlesource.com/chromium/tools/depot_tools.git \ + depot_tools + fi + echo "$GITHUB_WORKSPACE/depot_tools" >> "$GITHUB_PATH" + + - name: Verify depot_tools build tools + run: | + python3 --version + gclient help >/dev/null + ninja --version + + - name: Free Ubuntu runner disk + run: ./scripts/free_ci_disk_linux.sh + + - name: Check runner capacity + run: CI_MIN_FREE_DISK_GB="$SDK_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh + + - name: Write gclient config + run: ./scripts/write_gclient.sh linux + + - name: Sync open-source links + run: ./scripts/sync_open_sources.sh + + - name: gclient sync + if: ${{ inputs.run_gclient_sync }} + run: gclient sync --no-history + + - name: Generate patched Dart SDK build files + working-directory: dart-sdk-new + run: | + python3 tools/gn.py \ + --mode release \ + --arch x64 \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true' + + - name: Verify patched Dart SDK args + run: ./scripts/verify_dart_sdk_args.sh dart-sdk-new/out/ReleaseX64/args.gn + + - name: Build patched Dart SDK + working-directory: dart-sdk-new + run: | + ninja -C out/ReleaseX64 create_sdk runtime gen_snapshot dartaotruntime + + - name: Run focused patch API tests + working-directory: dart-sdk-new/out/ReleaseX64 + run: | + ./run_vm_tests DartAPI_AotPatchingConfiguration + ./run_vm_tests DartAPI_BytecodePatchReloadConfiguration + + - name: Package Dart SDK artifact + run: | + mkdir -p artifacts/custom-dart-sdk-linux-x64 + cp -R dart-sdk-new/out/ReleaseX64/dart-sdk artifacts/custom-dart-sdk-linux-x64/ + cp dart-sdk-new/out/ReleaseX64/gen_snapshot artifacts/custom-dart-sdk-linux-x64/ + cp dart-sdk-new/out/ReleaseX64/dartaotruntime artifacts/custom-dart-sdk-linux-x64/ + cp dart-sdk-new/out/ReleaseX64/args.gn artifacts/custom-dart-sdk-linux-x64/ + dart_sdk_revision="$(git -C dart-sdk-new rev-parse HEAD)" + cat > artifacts/custom-dart-sdk-linux-x64/manifest.json <> "$GITHUB_PATH" + + - name: Verify depot_tools build tools + run: | + python3 --version + gclient help >/dev/null + ninja --version + + - name: Check runner capacity + run: CI_MIN_FREE_DISK_GB="$SDK_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh + + - name: Write gclient config + run: ./scripts/write_gclient.sh macos + + - name: Sync open-source links + run: ./scripts/sync_open_sources.sh + + - name: gclient sync + if: ${{ inputs.run_gclient_sync }} + run: gclient sync --no-history + + - name: Generate patched Dart SDK build files + working-directory: dart-sdk-new + run: | + python3 tools/gn.py \ + --mode release \ + --arch arm64 \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true' + + - name: Verify patched Dart SDK args + run: ./scripts/verify_dart_sdk_args.sh dart-sdk-new/xcodebuild/ReleaseARM64/args.gn + + - name: Build patched Dart SDK + working-directory: dart-sdk-new + run: | + ninja -C xcodebuild/ReleaseARM64 create_sdk runtime gen_snapshot dartaotruntime + + - name: Run focused patch API tests + working-directory: dart-sdk-new/xcodebuild/ReleaseARM64 + run: | + ./run_vm_tests DartAPI_AotPatchingConfiguration + ./run_vm_tests DartAPI_BytecodePatchReloadConfiguration + + - name: Package Dart SDK artifact + run: | + mkdir -p artifacts/custom-dart-sdk-macos-arm64 + cp -R dart-sdk-new/xcodebuild/ReleaseARM64/dart-sdk artifacts/custom-dart-sdk-macos-arm64/ + cp dart-sdk-new/xcodebuild/ReleaseARM64/gen_snapshot artifacts/custom-dart-sdk-macos-arm64/ + cp dart-sdk-new/xcodebuild/ReleaseARM64/dartaotruntime artifacts/custom-dart-sdk-macos-arm64/ + cp dart-sdk-new/xcodebuild/ReleaseARM64/args.gn artifacts/custom-dart-sdk-macos-arm64/ + dart_sdk_revision="$(git -C dart-sdk-new rev-parse HEAD)" + cat > artifacts/custom-dart-sdk-macos-arm64/manifest.json <> "$GITHUB_PATH" + + - name: Verify depot_tools build tools + run: | + python3 --version + gclient help >/dev/null + ninja --version + + - name: Free Ubuntu runner disk + run: ./scripts/free_ci_disk_linux.sh + + - name: Check runner capacity + run: CI_MIN_FREE_DISK_GB="$ENGINE_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh + + - name: Install Linux engine dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + libgtk-3-dev \ + liblzma-dev \ + pkg-config + + - name: Write gclient config + env: + INCLUDE_ENGINE_DEPS: "1" + run: ./scripts/write_gclient.sh linux + + - name: Sync open-source links + run: ./scripts/sync_open_sources.sh + + - name: gclient sync + if: ${{ inputs.run_gclient_sync }} + run: gclient sync --no-history + + - name: gclient sync Flutter engine deps + if: ${{ inputs.run_gclient_sync }} + working-directory: flutter + run: gclient sync --no-history + + - name: Configure Linux engine + working-directory: flutter/engine/src + run: | + ./flutter/tools/gn \ + --runtime-mode=release \ + --target-dir=linux_release_x64 \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false' + + - name: Build Linux engine + working-directory: flutter/engine/src + run: | + ninja -C out/linux_release_x64 \ + flutter/shell/platform/linux:flutter_gtk \ + flutter/build/archives:flutter_patched_sdk \ + flutter/build/archives:artifacts + + - name: Verify Linux engine args + run: | + ./scripts/verify_engine_args.sh \ + flutter/engine/src/out/linux_release_x64/args.gn \ + dart_dynamic_modules=false \ + dart_enable_aot_patching=true \ + dart_enable_shorebird_interpreter=false \ + shorebird_enable_aot_patching=true \ + shorebird_use_interpreter=false + + - name: Package Linux engine artifacts + run: | + set -euo pipefail + out=flutter/engine/src/out/linux_release_x64 + mkdir -p artifacts/linux-engine + + gtk_zip="$(find "$out/zip_archives" -name 'linux-x64-flutter-gtk.zip' -type f -print -quit)" + patched_sdk_zip="$(find "$out/zip_archives" -name 'flutter_patched_sdk_product.zip' -type f -print -quit)" + artifacts_zip="$(find "$out/zip_archives" -path '*/linux-x64-release/artifacts.zip' -type f -print -quit)" + test -n "$gtk_zip" + test -n "$patched_sdk_zip" + test -n "$artifacts_zip" + + cp "$gtk_zip" artifacts/linux-engine/ + cp "$patched_sdk_zip" artifacts/linux-engine/ + cp "$artifacts_zip" artifacts/linux-engine/artifacts.zip + cp "$out/args.gn" artifacts/linux-engine/linux_release_x64.args.gn + + engine_revision="$(cat flutter/bin/internal/engine.version)" + mirror_root="artifacts/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision" + mkdir -p "$mirror_root/linux-x64-release" + cp "$gtk_zip" "$mirror_root/linux-x64-release/linux-x64-flutter-gtk.zip" + cp "$artifacts_zip" "$mirror_root/linux-x64-release/artifacts.zip" + cp "$patched_sdk_zip" "$mirror_root/flutter_patched_sdk_product.zip" + test -f "$mirror_root/linux-x64-release/linux-x64-flutter-gtk.zip" + test -f "$mirror_root/linux-x64-release/artifacts.zip" + test -f "$mirror_root/flutter_patched_sdk_product.zip" + cat > artifacts/linux-engine/manifest.json <> "$GITHUB_PATH" + + - name: Verify depot_tools build tools + run: | + python3 --version + gclient help >/dev/null + ninja --version + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + + - name: Free Ubuntu runner disk + run: ./scripts/free_ci_disk_linux.sh + + - name: Check runner capacity + run: CI_MIN_FREE_DISK_GB="$ENGINE_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh + + - name: Write gclient config + env: + INCLUDE_ENGINE_DEPS: "1" + run: ./scripts/write_gclient.sh linux + + - name: Sync open-source links + run: ./scripts/sync_open_sources.sh + + - name: gclient sync + if: ${{ inputs.run_gclient_sync }} + run: gclient sync --no-history + + - name: gclient sync Flutter engine deps + if: ${{ inputs.run_gclient_sync }} + working-directory: flutter + run: gclient sync --no-history + + - name: Configure Android engine + working-directory: flutter/engine/src + run: | + ./flutter/tools/gn \ + --android \ + --runtime-mode=release \ + --android-cpu=arm64 \ + --target-dir=android_release_arm64 \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false' + + - name: Build Android engine + working-directory: flutter/engine/src + run: | + ninja -C out/android_release_arm64 \ + flutter/shell/platform/android:flutter_jar_zip \ + flutter/shell/platform/android:android_symbols \ + flutter/shell/platform/android:gen_snapshot \ + flutter/shell/platform/android:analyze_snapshot + + - name: Verify Android engine args + run: | + ./scripts/verify_engine_args.sh \ + flutter/engine/src/out/android_release_arm64/args.gn \ + dart_dynamic_modules=false \ + dart_enable_aot_patching=true \ + dart_enable_shorebird_interpreter=false \ + shorebird_enable_aot_patching=true \ + shorebird_use_interpreter=false + + - name: Package Android engine artifacts + run: | + set -euo pipefail + out=flutter/engine/src/out/android_release_arm64 + mkdir -p artifacts/android-engine + + cp "$out/args.gn" artifacts/android-engine/android_release_arm64.args.gn + cp "$out/zip_archives/android-arm64-release/artifacts.zip" artifacts/android-engine/ + cp "$out/zip_archives/android-arm64-release/symbols.zip" artifacts/android-engine/ + cp "$out/flutter.jar" artifacts/android-engine/ + cp "$out/libflutter.so" artifacts/android-engine/ + if [ -f "$out/arm64_v8a_release.jar" ]; then + cp "$out/arm64_v8a_release.jar" artifacts/android-engine/ + fi + + gen_snapshot="$(find "$out" -path '*/gen_snapshot_arm64' -type f -print -quit)" + analyze_snapshot="$(find "$out" -path '*/analyze_snapshot_arm64' -type f -print -quit)" + test -n "$gen_snapshot" + test -n "$analyze_snapshot" + cp "$gen_snapshot" artifacts/android-engine/gen_snapshot_arm64 + cp "$analyze_snapshot" artifacts/android-engine/analyze_snapshot_arm64 + + engine_revision="$(cat flutter/bin/internal/engine.version)" + mirror_root="artifacts/android-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/android-arm64-release" + mkdir -p "$mirror_root" + cp "$out/zip_archives/android-arm64-release/artifacts.zip" "$mirror_root/artifacts.zip" + cp "$out/zip_archives/android-arm64-release/symbols.zip" "$mirror_root/symbols.zip" + test -f "$mirror_root/artifacts.zip" + test -f "$mirror_root/symbols.zip" + + cat > artifacts/android-engine/manifest.json <> "$GITHUB_PATH" + + - name: Verify depot_tools build tools + run: | + python3 --version + gclient help >/dev/null + ninja --version + + - name: Free Ubuntu runner disk + run: ./scripts/free_ci_disk_linux.sh + + - name: Check runner capacity + run: CI_MIN_FREE_DISK_GB="$ENGINE_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh + + - name: Write gclient config + env: + INCLUDE_ENGINE_DEPS: "1" + run: ./scripts/write_gclient.sh linux + + - name: Sync open-source links + run: ./scripts/sync_open_sources.sh + + - name: gclient sync + if: ${{ inputs.run_gclient_sync }} + run: gclient sync --no-history + + - name: gclient sync Flutter engine deps + if: ${{ inputs.run_gclient_sync }} + working-directory: flutter + run: gclient sync --no-history + + - name: Configure web SDK + working-directory: flutter/engine/src + run: | + ./flutter/tools/gn \ + --web \ + --runtime-mode=release \ + --target-dir=wasm_release \ + --gn-args='dart_dynamic_modules=false' + + - name: Build web SDK + working-directory: flutter/engine/src + run: ninja -C out/wasm_release flutter/web_sdk:flutter_web_sdk_archive + + - name: Verify web SDK args + run: | + ./scripts/verify_engine_args.sh \ + flutter/engine/src/out/wasm_release/args.gn \ + dart_dynamic_modules=false + + - name: Package web SDK artifact + run: | + set -euo pipefail + out=flutter/engine/src/out/wasm_release + mkdir -p artifacts/web-sdk + cp "$out/args.gn" artifacts/web-sdk/wasm_release.args.gn + cp "$out/zip_archives/flutter-web-sdk.zip" artifacts/web-sdk/ + + engine_revision="$(cat flutter/bin/internal/engine.version)" + mirror_root="artifacts/web-sdk/mirror/shorebird/flutter_infra_release/flutter/$engine_revision" + mkdir -p "$mirror_root" + cp "$out/zip_archives/flutter-web-sdk.zip" "$mirror_root/flutter-web-sdk.zip" + test -f "$mirror_root/flutter-web-sdk.zip" + cat > artifacts/web-sdk/manifest.json <> "$GITHUB_PATH" + + - name: Verify depot_tools build tools + run: | + python3 --version + gclient help >/dev/null + ninja --version + + - name: Check runner capacity + run: CI_MIN_FREE_DISK_GB="$ENGINE_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh + + - name: Write gclient config + env: + INCLUDE_ENGINE_DEPS: "1" + run: ./scripts/write_gclient.sh macos + + - name: Sync open-source links + run: ./scripts/sync_open_sources.sh + + - name: gclient sync + if: ${{ inputs.run_gclient_sync }} + run: gclient sync --no-history + + - name: gclient sync Flutter engine deps + if: ${{ inputs.run_gclient_sync }} + working-directory: flutter + run: gclient sync --no-history + + - name: Configure host snapshotter + working-directory: flutter/engine/src + run: | + ./flutter/tools/gn \ + --runtime-mode=release \ + --mac-cpu=arm64 \ + --target-dir=host_release_arm64 \ + --shorebird-interpreter \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true' + + - name: Build host snapshotter + working-directory: flutter/engine/src + run: ninja -C out/host_release_arm64 gen_snapshot + + - name: Configure iOS engine + working-directory: flutter/engine/src + run: | + ./flutter/tools/gn \ + --ios \ + --runtime-mode=release \ + --target-dir=ios_release \ + --shorebird-interpreter + + - name: Build iOS engine + working-directory: flutter/engine/src + run: | + ninja -C out/ios_release \ + flutter/shell/platform/darwin/ios:flutter_framework \ + create_macos_gen_snapshots + + - name: Verify iOS interpreter route + run: ./scripts/verify_ios_interpreter_route.sh + + - name: Configure macOS engine + working-directory: flutter/engine/src + run: | + ./flutter/tools/gn \ + --mac \ + --runtime-mode=release \ + --mac-cpu=arm64 \ + --target-dir=macos_release_arm64 \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false' + + - name: Build macOS engine + working-directory: flutter/engine/src + run: | + ninja -C out/macos_release_arm64 \ + flutter/shell/platform/darwin/macos:zip_macos_flutter_framework \ + flutter/build/archives:flutter_patched_sdk \ + create_macos_gen_snapshots + + - name: Verify macOS engine args + run: | + ./scripts/verify_engine_args.sh \ + flutter/engine/src/out/macos_release_arm64/args.gn \ + dart_dynamic_modules=false \ + dart_enable_aot_patching=true \ + dart_enable_shorebird_interpreter=false \ + shorebird_enable_aot_patching=true \ + shorebird_use_interpreter=false + + - name: Package iOS engine artifacts + run: | + set -euo pipefail + mkdir -p artifacts/ios-engine/host_release_arm64 artifacts/ios-engine/ios-release + test -x flutter/engine/src/out/host_release_arm64/gen_snapshot + test -d flutter/engine/src/out/ios_release/Flutter.xcframework + test -x flutter/engine/src/out/ios_release/universal/gen_snapshot_arm64 + test -x flutter/engine/src/out/ios_release/analyze_snapshot_arm64 + cp -R flutter/engine/src/out/ios_release/Flutter.framework artifacts/ios-engine/ + cp -R flutter/engine/src/out/ios_release/Flutter.xcframework artifacts/ios-engine/ + cp flutter/engine/src/out/host_release_arm64/gen_snapshot artifacts/ios-engine/host_release_arm64/ + cp flutter/engine/src/out/ios_release/args.gn artifacts/ios-engine/ios_release.args.gn + cp flutter/engine/src/out/host_release_arm64/args.gn artifacts/ios-engine/host_release_arm64.args.gn + + ios_zip_staging="$(mktemp -d)" + cp -R flutter/engine/src/out/ios_release/Flutter.xcframework "$ios_zip_staging/" + cp flutter/engine/src/out/ios_release/universal/gen_snapshot_arm64 "$ios_zip_staging/" + cp flutter/engine/src/out/ios_release/analyze_snapshot_arm64 "$ios_zip_staging/" + cat > "$ios_zip_staging/entitlements.txt" <<'EOF' + analyze_snapshot_arm64 + gen_snapshot_arm64 + EOF + cat > "$ios_zip_staging/without_entitlements.txt" <<'EOF' + Flutter.xcframework/ios-arm64/Flutter.framework/Flutter + EOF + : > "$ios_zip_staging/unsigned_binaries.txt" + python3 - "$ios_zip_staging" artifacts/ios-engine/ios-release/artifacts.zip <<'PY' + import os + from pathlib import Path + import stat + import sys + import zipfile + + root = Path(sys.argv[1]) + output = Path(sys.argv[2]) + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + for path in sorted(p for p in root.rglob("*") if p.is_file()): + archive_name = path.relative_to(root).as_posix() + info = zipfile.ZipInfo(archive_name) + info.external_attr = (path.stat().st_mode & 0xFFFF) << 16 + with path.open("rb") as file: + archive.writestr(info, file.read(), zipfile.ZIP_DEFLATED) + PY + + engine_revision="$(cat flutter/bin/internal/engine.version)" + mirror_root="artifacts/ios-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/ios-release" + mkdir -p "$mirror_root" + cp artifacts/ios-engine/ios-release/artifacts.zip "$mirror_root/artifacts.zip" + test -f "$mirror_root/artifacts.zip" + cat > artifacts/ios-engine/manifest.json < artifacts/macos-engine/manifest.json < ` +to merge those subtrees, reject conflicting artifact bytes, validate +`artifacts_manifest.yaml` overrides, verify all platform `patch-*.zip` files, +and write missing checksum sidecars. If your open engine revision differs from +the upstream Flutter engine revision used for unchanged artifacts, set the +workflow `base_flutter_engine_revision` input when generating mirror metadata. +Vended Flutter commands receive `FLUTTER_STORAGE_BASE_URL` from +`SHOREBIRD_FLUTTER_STORAGE_BASE_URL`, falling back to +`http://localhost:8080/download.flutter.io` for local self-hosted artifacts. +The Flutter fork itself uses that same open mirror as its default engine and +Android Maven artifact host when `FLUTTER_STORAGE_BASE_URL` is unset, including +its Dart SDK refresh scripts. + +The iOS App Store candidate route is the encrypted Dart bytecode interpreter +path, not native AOT patch loading. Real-device testing on June 25, 2026 showed +the patched iPad app displaying `license:pro` without enabling +`DART_DYNAMIC_MODULES`. The macOS native-AOT desktop route was also verified +locally the same day by launching a saved free app with a pro Flutter snapshot +seeded as patch `1`; the patched app reported `license:pro`. +CI builds Linux, Android, and macOS engine artifacts with the native AOT patch +runtime enabled and the interpreter route disabled; iOS is the only App +Store-safe interpreter patch artifact path. +Android release APK builds pass locally with the custom Android engine; use +`scripts/android_runtime_patch_smoke.sh` on a machine with an attached Android +device or emulator to run the seeded runtime patch proof. Use +`scripts/linux_runtime_patch_smoke.sh` on a Linux desktop runner to exercise the +same saved-free-app/pro-patch flow for the Linux embedder. + ## Local Test Fixture `testapps/license_flavor_patch_test` is kept in the root repository because it @@ -49,6 +138,8 @@ Generated build output under that app is ignored by this meta-repository. ## Documents +- `docs/CI.md` describes the GitHub Actions jobs for source checks, CLI/server + artifacts, manual custom SDK/engine builds, and opt-in runtime smokes. - `docs/REPOSITORIES.md` records the remote URL inventory and cleanup policy. - `docs/PLATFORM_TESTING.md` documents the Linux, macOS, and later iOS test flow. diff --git a/depot_tools b/depot_tools index a8c9ac9..226aa79 160000 --- a/depot_tools +++ b/depot_tools @@ -1 +1 @@ -Subproject commit a8c9ac983296908f8d662cb23ad6b8a0cae890b2 +Subproject commit 226aa79e9947adc1e9e0c79f96b58562516535d9 diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 0000000..16c0c91 --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,454 @@ +# GitHub CI + +The root workflow at `.github/workflows/open-shorebird-ci.yml` validates the +combined open Shorebird workspace rather than any single submodule in isolation. + +## Default CI + +Push and pull request runs perform source-level checks and build distributable +binaries: + +1. `source-checks` runs the bootstrap test path with `SKIP_GCLIENT_SYNC=1`, + including CLI, code push client, artifact proxy, updater library, open patch + tooling, and server tests. + This checks the Shorebird CLI focused tests, code push client open-server + default tests, open patch tools, server tests, and the root shell scripts + without downloading the full engine dependency graph. It also runs + `scripts/verify_ci_workflow.sh`, which parses the + workflow YAML, checks every workflow `run:` block with `bash -n`, and + verifies the expected open SDK/CLI/server artifact contract, including + archive creation and checksum sidecar generation. It also checks + `.gitmodules`, `docs/REPOSITORIES.md`, and generated `.gclient` files so a + fresh GitHub checkout uses open HTTPS remotes instead of local paths, SSH + remotes, or official closed Shorebird repositories. The verifier also scans + runtime/build-sensitive CLI, updater, artifact-proxy, Flutter tool, Gradle, + engine metadata, and web UI artifact-copy sources to reject hosted Shorebird + endpoints and official `shorebirdtech` GitHub dependencies. It also reads + the forked engine BUILD files and checks that the GN targets and archive + names referenced by the manual engine jobs still exist. + Before uploading the workspace, run `scripts/verify_upload_readiness.sh`. + It invokes the same workflow contract verifier in upload-ready mode, which + requires every listed CI support file to be tracked in its owning checkout + and requires the root plus required submodule checkouts to be clean. + `scripts/verify_sync_open_sources.sh` also exercises the source-link helper + in an isolated temporary workspace so clean generated engine checkouts are + replaced by the workspace Dart/updater submodules and dirty targets are not + overwritten. + `scripts/verify_open_infrastructure_defaults.sh` independently scans + build-sensitive Flutter, Shorebird CLI, artifact-proxy, updater, `.gclient`, + and submodule metadata for closed Shorebird endpoints, official + `shorebirdtech` dependency remotes, and private prebuilt buckets, while + requiring the open local server and artifact mirror defaults. + `scripts/verify_powershell_open_defaults.sh` checks the Windows launcher and + Flutter Dart SDK updater PowerShell scripts for open Flutter/artifact + defaults and parses them with `pwsh` when it is available. + `scripts/verify_write_sha256.sh` smoke-tests the portable checksum helper + used by every artifact job. + `scripts/verify_ios_interpreter_route_validator.sh` smoke-tests the iOS + App Store route gate with synthetic GN args and encrypted patch artifacts, + covering a valid interpreter artifact and rejecting dynamic-module, + non-iOS, malformed JSON, and Mach-O native patch inputs. +2. All artifact and runtime-smoke jobs declare `needs: source-checks`, so no + uploaded CLI, server, SDK, or engine artifact is produced before the + source-level open-replacement contract passes. +3. `cli-artifacts` compiles: + - `shorebird` for Linux x64, macOS x64, macOS arm64, and Windows x64 + - `open_aot_patch_tools` for Linux x64, macOS x64, macOS arm64, and + Windows x64 + - `artifact_proxy` for Linux x64, macOS x64, macOS arm64, and Windows x64 + - public updater `patch` binaries packaged as CLI-cache mirror artifacts: + `patch-linux-x64.zip`, `patch-darwin-x64.zip`, + `patch-darwin-arm64.zip`, and `patch-windows-x64.zip` + - `mirror-metadata`, containing + `shorebird//artifacts_manifest.yaml` + The job provisions stable Rust before building the updater `patch` binary, + so those mirror artifacts do not depend on preinstalled runner state. + CI opens each generated patch mirror ZIP before upload and verifies it has + exactly the cache-facing entry name (`patch` on Unix, `patch.exe` on + Windows) with a non-empty payload. + Each CLI upload is an `open-shorebird-cli--.tar.gz` archive with + `bin/shorebird`, `bin/open_aot_patch_tools`, `bin/artifact_proxy`, + `bin/internal` version metadata, `manifest.json`, and a `.sha256` sidecar. + The manifest records the Flutter revision and engine revision used by the + bundled CLI cache metadata. The macOS CLI split is intentional: macOS x64 + runs on `macos-15-intel`, and macOS arm64 runs on `macos-14`. Before upload, + CI extracts the archive and runs `shorebird --version`, + `open_aot_patch_tools --help`, and `artifact_proxy --health-check` from the + extracted layout. The `shorebird --version` smoke must report the open + `https://git.tonycloud.org/flutter/shorebird.git` fork and rejects the old + official Shorebird SSH remote. +4. `server-artifacts` runs `go test ./...` and cross-compiles + `shorebird-server` for Linux x64/arm64, macOS x64/arm64, and Windows x64 + targets. Each uploaded archive contains the server binary, `web/` dashboard + assets, `.env.example`, `README.md`, `openapi.yaml`, `manifest.json`, and a + `.sha256` sidecar. The manifest records both the root workflow commit and + the server submodule commit. Keep `web/` beside the executable when + unpacking; the server looks for dashboard assets relative to its working + directory and executable path. + The Linux x64 archive is extracted and smoke-tested in CI by starting the + packaged binary with SQLite/local storage, checking `/health`, fetching the + dashboard HTML from `/`, and verifying `/openapi.yaml`. + +The CLI/server artifacts are uploaded from each workflow run. CLI archives are +intended for installation or release attachments; patch mirror ZIPs are separate +because the Shorebird CLI cache expects the `patch-*.zip` names and a `patch` +or `patch.exe` entry inside each ZIP. The patch mirror ZIPs include `.sha256` +sidecars and can be copied under +`/shorebird//` on the host referenced by +`SHOREBIRD_ARTIFACT_BASE_URL`. Each `mirror-patch-*.zip` GitHub artifact also +includes a publish-ready `shorebird//patch-*.zip` copy and +matching `.sha256` sidecar, so mirror publication can copy the `shorebird/` +subtree directly. The `mirror-metadata` artifact provides the matching +`artifacts_manifest.yaml` for the artifact proxy. +Manual engine artifacts also include a `mirror/` subtree whose contents are +already laid out under `shorebird/flutter_infra_release/...`; copy that subtree +to the same artifact mirror root to satisfy the generated +`artifacts_manifest.yaml` overrides. +On `full_sdk_build=true` runs, `artifact-mirror` downloads the mirror metadata, +all platform patch ZIPs, and the Linux/Android/web/iOS/macOS engine artifacts, +then uploads `open-shorebird-artifact-mirror`. It also downloads the CLI, +server, and custom Dart SDK artifacts and uploads +`open-shorebird-release-manifest`, a JSON manifest that verifies every archived +release artifact has a matching checksum sidecar and records its SHA-256 digest, +size, relative path, downloaded artifact group, file name, and workflow commit. +Zero-byte release artifacts are rejected even when their sidecars match. +The manifest job also requires every +expected CLI, server, custom Dart SDK, engine, patch-tool, metadata, and mirror +artifact family to be present. The mirror archive is the checked publish-ready +mirror root for `SHOREBIRD_ARTIFACT_BASE_URL`; the release manifest is the +provenance index for the full SDK/CLI/server/engine output set. +The artifact-mirror job validates the generated manifest with +`scripts/validate_release_manifest.py` before upload. After downloading workflow +artifacts, the same validator can audit the manifest against the downloaded +artifact directory and will reject missing files, duplicate or unsafe paths, +checksum/size mismatches, sidecar mismatches, orphan sidecars, mismatched +artifact group or file-name provenance fields, and artifacts that are present on +disk but missing from the manifest. +Use the wrapper below after downloading a completed `full_sdk_build=true` run: + +```sh +gh run download --dir downloaded-artifacts +./scripts/verify_downloaded_release_artifacts.sh \ + --github-sha \ + downloaded-artifacts +``` + +It verifies the release manifest sidecar, validates the manifest against every +downloaded artifact, optionally requires the manifest commit to match the +expected run SHA, verifies the mirror archive sidecar, safely extracts +`open-shorebird-artifact-mirror.tar.gz` after rejecting unsafe archive members, +and validates the extracted mirror root before anything is published. + +After the workspace is uploaded to GitHub, the end-to-end proof can be driven +from a local checkout with: + +```sh +./scripts/verify_hosted_full_sdk_build.sh \ + --repo owner/repo \ + --ref main +``` + +That helper dispatches `open-shorebird-ci.yml` with `full_sdk_build=true`, +waits for the hosted workflow run to succeed, downloads all artifacts, and runs +`scripts/verify_downloaded_release_artifacts.sh` against the downloaded output +with the workflow run's `headSha`. +Use `--linux-heavy-runner`, `--macos-heavy-runner`, and the disk-threshold flags +when the repository uses custom larger/self-hosted runner labels. +For local release assembly or reassembly after downloading workflow artifacts, +run: + +```sh +./scripts/assemble_artifact_mirror.sh downloaded-artifacts public-mirror +``` + +The assembler scans direct artifact contents plus `*.tar.gz` engine archives, +validates archive member paths, rejects links/devices before extraction, copies +every `shorebird/` mirror subtree into `public-mirror`, rejects conflicting +files with different bytes, validates that each +`artifacts_manifest.yaml` override resolves to a copied non-empty file, verifies +every platform `patch-*.zip` contains exactly the cache-facing `patch` or +`patch.exe` entry, and writes missing `.sha256` sidecars. +`scripts/validate_artifact_mirror.py public-mirror` can be run independently +after safely extracting `open-shorebird-artifact-mirror.tar.gz`; the +`artifact-mirror` job safe-extracts and validates the assembled archive before +upload. +`scripts/verify_assemble_artifact_mirror.sh` keeps the low-level assembler +covered in default source checks. `scripts/verify_artifact_mirror_workflow_assembly.sh` +also dry-runs the full `artifact-mirror` aggregation flow with fake downloaded +artifacts, including the mirror-only input subset, extracted mirror validation, +and release manifest requirements. This catches path-layout regressions before +the hosted `full_sdk_build=true` run. +CI writes checksum sidecars through `scripts/write_sha256.sh` so the artifact +jobs do not depend on platform-specific checksum tools. +The workflow defaults all `run` steps to Bash, including Windows matrix jobs, +because the workspace scripts and packaging commands use Bash syntax. + +On June 25, 2026, the Darwin arm64 CLI artifact path was smoke-tested locally: +the workflow-equivalent commands compiled `shorebird`, `open_aot_patch_tools`, +and `artifact_proxy`; built the Rust updater `patch` binary; produced +`open-shorebird-cli-macos-arm64.tar.gz` and `patch-darwin-arm64.zip`; wrote both +checksum sidecars; extracted the CLI archive; and verified the compiled tools +could run from the extracted layout. +The patch mirror ZIP contained a single `patch` entry, matching the CLI cache +contract. + +The Darwin arm64 server artifact path was smoke-tested locally the same day: +the workflow-equivalent packaging command produced +`shorebird-server-darwin-arm64.tar.gz`, wrote a `.sha256` sidecar, extracted +the archive, started the packaged server with SQLite and local storage, and +verified `/health` returned `status=ok` while `/` served the packaged dashboard +HTML from the sibling `web/` directory. The packaged server also serves the +checked OpenAPI contract at `/openapi.yaml`. + +## Open Artifact Hosts + +The CLI can be used with a self-hosted server and open artifact mirror without +changing source code. When no project `base_url`, user config, or environment +override is present, the open CLI defaults API/auth traffic to the local +self-hosted server at `http://localhost:8080` instead of Shorebird's hosted +service: + +```sh +export SHOREBIRD_HOSTED_URL=https://updates.example.com +export SHOREBIRD_ARTIFACT_BASE_URL=https://artifacts.example.com/open +export SHOREBIRD_FLUTTER_STORAGE_BASE_URL=https://artifacts.example.com/flutter +export SHOREBIRD_FLUTTER_GIT_URL=https://github.com/example/open-flutter.git +``` + +`SHOREBIRD_HOSTED_URL` points API calls at the self-hosted server and is also +written into new `shorebird.yaml` files as `base_url`. The bundled updater +library also defaults to `http://localhost:8080` when a legacy app omits +`base_url`. The self-hosted server publishes its checked OpenAPI contract at +`http://localhost:8080/openapi.yaml`. The artifact mirror keeps the same path +layout as Shorebird's default storage bucket: + +```text +$SHOREBIRD_ARTIFACT_BASE_URL/shorebird//patch-linux-x64.zip +$SHOREBIRD_ARTIFACT_BASE_URL/shorebird//patch-darwin-x64.zip +$SHOREBIRD_ARTIFACT_BASE_URL/shorebird//patch-darwin-arm64.zip +$SHOREBIRD_ARTIFACT_BASE_URL/shorebird//patch-windows-x64.zip +$SHOREBIRD_ARTIFACT_BASE_URL/shorebird//artifacts_manifest.yaml +``` + +When `SHOREBIRD_ARTIFACT_BASE_URL` is not set, the open CLI defaults CLI-managed +artifact downloads to `http://localhost:8080/artifacts`. Use that local mirror +root for development, or set `SHOREBIRD_ARTIFACT_BASE_URL` to the public mirror +where the CI-produced `patch-*.zip` artifacts are hosted. + +If the mirror is a bucket-style host, `SHOREBIRD_STORAGE_BASE_URL` and +`SHOREBIRD_STORAGE_BUCKET` can be used instead of +`SHOREBIRD_ARTIFACT_BASE_URL`. `SHOREBIRD_FLUTTER_STORAGE_BASE_URL` controls the +`FLUTTER_STORAGE_BASE_URL` passed to vended Flutter commands; when unset it +defaults to `http://localhost:8080/download.flutter.io`. +`SHOREBIRD_FLUTTER_GIT_URL` controls where the CLI clones vended Flutter +revisions during cache installation. When unset, the open CLI defaults to this +workspace's open Flutter fork instead of `github.com/shorebirdtech/flutter.git`. +The Flutter fork also defaults its own engine downloads, Dart SDK refresh +scripts, doctor network check, Android Gradle Maven host, Android host-app +integration fixtures, docs artifact scripts, and Shorebird integration tests to +`http://localhost:8080/download.flutter.io` when `FLUTTER_STORAGE_BASE_URL` is +unset, so direct Flutter use does not fall back to Shorebird's closed artifact +host. + +The open CI currently does not publish `aot-tools.dill`. That file is a +different Shorebird linker artifact used by the legacy native-AOT iOS linker +path. The open App Store-safe iOS path uses encrypted Dart bytecode interpreter +artifacts instead, and Android/Linux/macOS/Windows patch creation uses the +public updater `patch` binaries above. +The CLI cache does not download `aot-tools.dill` by default. Set +`SHOREBIRD_ENABLE_LEGACY_AOT_TOOLS=1` only when deliberately validating the +development-only legacy native-AOT linker path. + +Runtime smoke scripts that need external targets are kept out of default CI: +`scripts/android_runtime_patch_smoke.sh` requires an attached Android +device/emulator with app-private storage access, and +`scripts/linux_runtime_patch_smoke.sh` requires a Linux desktop runtime or +`xvfb-run`. Use them on suitable self-hosted runners when extending runtime +coverage beyond the artifact build matrix. + +Use `workflow_dispatch` with `run_runtime_smokes=true` to run those external +runtime smokes from CI. The runtime jobs are intentionally separate from +`full_sdk_build=true`: they are for provisioned runners that already have the +matching local engine build outputs in the checkout workspace. Runtime checkout +uses `clean: false` so self-hosted runners can preserve those `out/` directories +between preparation and smoke runs. + +| Input | Default | Used by | +| --- | --- | --- | +| `run_runtime_smokes` | `false` | Enables `linux-runtime-smoke` and `android-runtime-smoke` | +| `linux_runtime_runner` | `self-hosted` | Runner label for `scripts/linux_runtime_patch_smoke.sh` | +| `android_runtime_runner` | `self-hosted` | Runner label for `scripts/android_runtime_patch_smoke.sh` | +| `android_serial` | empty | Optional `adb` serial passed as `ANDROID_SERIAL` | + +The Linux runtime runner must provide +`flutter/engine/src/out/linux_release_x64`. The Android runtime runner must provide +`flutter/engine/src/out/android_release_arm64` and +`flutter/engine/src/out/host_release_arm64`, plus an Android target where either +`adb shell run-as` works for the test package or `adb root` works. It must also +provide Java on `PATH` because the smoke builds APKs before seeding the patch. + +## Heavy SDK Builds + +Use `workflow_dispatch` with `full_sdk_build=true` to build the custom SDK +artifacts. These jobs are intentionally manual because they run `gclient sync` +and build large engine/Dart outputs. + +The heavy jobs install Chromium's `depot_tools` into the workflow workspace +before running `gclient`, so they can bootstrap from a clean runner. Full +Dart/Flutter engine builds are large enough that standard GitHub-hosted runners +are not a realistic default for `full_sdk_build=true`. The workflow therefore +defaults heavy jobs to custom labels: + +- `open-shorebird-linux-heavy` for Linux SDK, Linux engine, Android engine, and + web SDK builds +- `open-shorebird-macos-heavy` for macOS Dart SDK, iOS engine, and macOS engine + builds + +Register self-hosted or larger runners with those labels, or override +`linux_heavy_runner` / `macos_heavy_runner` when dispatching the workflow. +Every manual SDK/engine job also runs `scripts/check_ci_capacity.sh` before +`gclient sync` or `ninja`; SDK-only jobs require at least 35 GiB free and engine +jobs require at least 40 GiB free by default. Adjust the dispatch thresholds +only for runner images whose dependency caches make a lower threshold +intentional. +The hosted Android engine job also provisions Temurin Java 17 before building +Android JAR/APK-related engine artifacts. Every manual SDK/engine job verifies +`python3`, `gclient`, and `ninja` before generating build files, so +PATH/toolchain problems fail before a long build starts. CI sets +`DEPOT_TOOLS_UPDATE=0`, and the local bootstrap scripts default the same way, +so depot_tools uses the pinned submodule revision unless explicitly overridden. + +Manual heavy builds accept these workflow inputs: + +| Input | Default | Used by | +| --- | --- | --- | +| `linux_heavy_runner` | `open-shorebird-linux-heavy` | `custom-dart-sdk`, `linux-engine`, `android-engine`, `web-sdk` | +| `macos_heavy_runner` | `open-shorebird-macos-heavy` | `custom-dart-sdk-macos`, `ios-engine` / Apple engine artifacts | +| `sdk_min_free_disk_gb` | `35` | Minimum free disk GiB for `custom-dart-sdk` and `custom-dart-sdk-macos` | +| `engine_min_free_disk_gb` | `40` | Minimum free disk GiB for Linux, Android, web, iOS, and macOS engine builds | +| `base_flutter_engine_revision` | empty | Optional upstream Flutter engine revision recorded in `artifacts_manifest.yaml` for non-overridden artifact proxy fallbacks | + +Use these inputs to move the SDK/engine jobs onto different larger or +self-hosted runners without editing the workflow file. The Linux heavy jobs also run +`scripts/free_ci_disk_linux.sh`, which removes unrelated preinstalled toolchain +caches only when `GITHUB_ACTIONS=true` and `RUNNER_ENVIRONMENT=github-hosted`. +Self-hosted runners skip that cleanup by default. Set `CI_FREE_DISK_SPACE=0` to +skip cleanup anywhere, or `CI_FREE_DISK_SPACE_FORCE=1` to opt in on a +self-hosted runner. +When `base_flutter_engine_revision` is empty, `mirror-metadata` records the +same revision as `flutter/bin/internal/engine.version`. Set it when the open +Shorebird engine revision is a custom fork revision and unchanged artifacts +should be proxied back to a different upstream Flutter engine revision. + +The engine jobs perform two syncs when `run_gclient_sync=true`: the root +workspace sync for the Dart SDK checkout, then a second sync from `flutter/` for +Flutter engine dependencies written by `scripts/write_gclient.sh`. Linux engine +jobs use `INCLUDE_ENGINE_DEPS=1` to include Flutter engine dependencies, +Android dependencies, and emsdk; the Apple engine job uses the same flag to +include iOS, Android, and emsdk +dependencies on macOS. +Before building, `scripts/sync_open_sources.sh` links +`flutter/engine/src/flutter/third_party/dart` to the `dart-sdk-new` submodule +and `flutter/engine/src/flutter/third_party/updater` to the `updater` +submodule. It does not clone the official `shorebirdtech/updater` repository by +default; set `UPDATER_URL` explicitly only when testing a different updater +fork. The helper rejects known unpatched upstream Dart SDK remotes and official +Shorebird updater remotes so CI does not accidentally build from the wrong +component checkout. +The Flutter fork's `DEPS` file also points its Dart SDK and updater dependency +URLs at the open mirrors and does not reference Shorebird's private +`shorebird-dart-sdk-prebuilt` bucket; custom SDK jobs build and archive the +open Dart SDK fork directly. + +`custom-dart-sdk` and `custom-dart-sdk-macos` build the Dart SDK fork with: + +```gn +dart_dynamic_modules = false +dart_enable_aot_patching = true +dart_enable_shorebird_interpreter = true +``` + +They run `scripts/verify_dart_sdk_args.sh`, then the focused VM patch API +tests. The Linux job uploads `custom-dart-sdk-linux-x64`; the macOS job uploads +`custom-dart-sdk-macos-arm64`. Each archive contains the Dart SDK, +`gen_snapshot`, `dartaotruntime`, `args.gn`, and a `manifest.json` that records +the root workflow commit, Dart SDK source commit, and patch-related build +flags. The workflow also uploads `.sha256` sidecars for the SDK archives. +Before writing the checksum, each SDK job extracts the archive, verifies +`manifest.json`, `args.gn`, `gen_snapshot`, `dartaotruntime`, and +`dart-sdk/bin/dart`, then runs the extracted `dart --version`. + +`ios-engine` builds: + +- `host_release_arm64` with `--shorebird-interpreter`, + `dart_dynamic_modules=false`, `dart_enable_aot_patching=true`, and + `dart_enable_shorebird_interpreter=true` +- `ios_release` with `--shorebird-interpreter` +- `macos_release_arm64` with `dart_dynamic_modules=false`, + `dart_enable_aot_patching=true`, `shorebird_enable_aot_patching=true`, and + `shorebird_use_interpreter=false` + +It runs `scripts/verify_ios_interpreter_route.sh` before uploading the +`ios-interpreter-engine` artifact, so the archived build proves +`DART_DYNAMIC_MODULES` is off and the iOS route uses the bytecode interpreter +instead of native AOT patch loading. The archive includes `Flutter.framework`, +`Flutter.xcframework`, a mirror-ready `ios-release/artifacts.zip` containing +the xcframework plus `gen_snapshot_arm64`, `analyze_snapshot_arm64`, and the +code-sign configuration files Flutter expects, the +`host_release_arm64/gen_snapshot` binary built with `--shorebird-interpreter`, +the iOS and host `args.gn` files, and a `manifest.json` that records +`dart_dynamic_modules=false`, +`dart_enable_aot_patching=true`, `dart_enable_shorebird_interpreter=true`, +`shorebird_enable_aot_patching=false`, `shorebird_use_interpreter=true`, and +the Flutter engine revision; the workflow also uploads a `.sha256` sidecar for +the engine archive. +Before checksum upload, CI extracts the iOS engine archive and verifies +`Flutter.framework`, `Flutter.xcframework`, `ios-release/artifacts.zip`, the +mirror copy under `mirror/shorebird/flutter_infra_release/flutter/`, +both args files, `manifest.json`, and the executable host `gen_snapshot`. +When an app or IPA is supplied to the gate, strict mode also rejects bundled +patch payloads, executable-memory entitlements, and raw `aot_patch_key_hex` +material in `shorebird.yaml`. + +The same Apple job uploads a separate `macos-engine-arm64` artifact containing +`FlutterMacOS.framework.zip`, `flutter_patched_sdk_product.zip`, the macOS +`args.gn`, a mirror-ready copy of the macOS framework override, a manifest with +the Flutter engine revision, and a `.sha256` sidecar. The shared +`flutter_patched_sdk_product.zip` mirror override is published only by the Linux +engine job to avoid duplicate producers for the same mirror path. The macOS +build is checked with +`scripts/verify_engine_args.sh` so CI fails if `dart_dynamic_modules=true` +appears or if the native AOT patch runtime flags are missing from the generated +args. CI extracts the macOS engine archive before upload and verifies the +framework zip, patched SDK zip, args file, manifest, and framework mirror +subtree. + +`linux-engine` builds `linux_release_x64` on Ubuntu with +`dart_dynamic_modules=false`, `dart_enable_aot_patching=true`, +`shorebird_enable_aot_patching=true`, and `shorebird_use_interpreter=false`. It +uploads `linux-x64-flutter-gtk.zip`, `flutter_patched_sdk_product.zip`, +`artifacts.zip`, mirror-ready copies of those engine override files, `args.gn`, +a manifest with the Flutter engine revision, and a `.sha256` sidecar. CI +extracts the archive before upload and verifies the GTK zip, patched SDK zip, +artifacts zip, args file, manifest, and mirror subtree. + +`android-engine` builds `android_release_arm64` on Ubuntu with +`dart_dynamic_modules=false`, `dart_enable_aot_patching=true`, +`shorebird_enable_aot_patching=true`, and `shorebird_use_interpreter=false`. It +uploads `artifacts.zip`, `symbols.zip`, `flutter.jar`, `libflutter.so`, host +`gen_snapshot_arm64`, `analyze_snapshot_arm64`, mirror-ready copies of the +Android engine override files, `args.gn`, a manifest with the Flutter engine +revision, and a `.sha256` sidecar. CI extracts the archive before upload and +verifies the Android artifacts/symbols zips, `flutter.jar`, `libflutter.so`, +host snapshot/analyzer tools, args file, manifest, and mirror subtree. + +`web-sdk` builds the Flutter web SDK archive from `wasm_release` with +`dart_dynamic_modules=false` and uploads `flutter-web-sdk.zip`, a mirror-ready +copy of that SDK archive, `args.gn`, a manifest with the Flutter engine +revision, and a `.sha256` sidecar. Web is still not a Shorebird CodePush release +platform in this CLI/protocol; this job exists to keep the open Flutter SDK/web +artifacts buildable from the workspace. CI extracts the web SDK archive before +upload and verifies the SDK zip, args file, manifest, and mirror subtree. + +Set `run_gclient_sync=false` only for debugging a runner image that already has +all gclient-managed dependencies restored. diff --git a/docs/PLATFORM_TESTING.md b/docs/PLATFORM_TESTING.md index 95fb522..0eac664 100644 --- a/docs/PLATFORM_TESTING.md +++ b/docs/PLATFORM_TESTING.md @@ -1,4 +1,4 @@ -# Linux, macOS, and iOS Platform Testing +# Platform Testing The first complete runtime proof was built on Windows. The next platform pass should verify the same source layout on Linux and macOS, then use macOS to @@ -14,6 +14,9 @@ Linux: - Dart or Flutter on `PATH` - Go 1.23+ - Python and standard build dependencies required by Dart/Flutter +- Flutter Linux desktop build dependencies (`clang`, `cmake`, `ninja`, + `pkg-config`, GTK development headers) +- A graphical session or `xvfb-run` for headless runtime smoke tests macOS: @@ -24,6 +27,16 @@ macOS: - Xcode command line tools - Full Xcode for iOS simulator/device follow-up +Android runtime validation: + +- Android SDK platform tools (`adb`) +- Java/JDK on `PATH` for APK builds +- A connected Android device or emulator +- Either `adb shell run-as` access for the test package or a rooted + emulator/device. The seeded runtime smoke writes directly to the test app's + private updater directory; production installs normally receive patches from + the updater download flow. + Both platforms need network access for a fresh `git submodule update` and `gclient sync`. @@ -59,6 +72,11 @@ Useful environment variables: | `IOS_IPA=/path/to/App.ipa` | Optionally verify the `Payload/*.app` inside an IPA instead of a raw app bundle. | | `IOS_PATCH_ARTIFACT=/path/to/dlc.vmcode` | Optionally verify that the patch artifact is an encrypted iOS arm64 interpreter full-snapshot artifact. | | `APP_STORE_STRICT=1` | Fail the iOS route check when App Store-inappropriate signing state, such as `get-task-allow=true`, is present. | +| `ANDROID_SERIAL=` | Select the Android device used by `scripts/android_runtime_patch_smoke.sh` when more than one device is attached. | +| `SKIP_ANDROID_BUILDS=1` | Reuse APKs for the Android runtime smoke instead of rebuilding; set `ANDROID_FREE_APK` and `ANDROID_PRO_APK`. | +| `KEEP_ANDROID_RUNTIME_SMOKE_ARTIFACTS=1` | Keep the temporary APK/libapp extraction directory after the Android runtime smoke. | +| `LINUX_RUNTIME_SMOKE_XVFB=0` | Disable automatic `xvfb-run` wrapping in `scripts/linux_runtime_patch_smoke.sh`. | +| `KEEP_LINUX_RUNTIME_SMOKE_ARTIFACTS=1` | Keep the temporary Linux app copy and free/pro bundles after the Linux runtime smoke. | ## Expected Checks @@ -82,6 +100,187 @@ The bootstrap scripts run: the device engine and host snapshotter, and the native AOT patch path is off for iOS. +## Current Coverage Status + +| Platform | Open patch route | Current status | +| --- | --- | --- | +| iOS | Encrypted Dart bytecode interpreter artifact, `DART_DYNAMIC_MODULES=false` | Verified on a real iPad on June 25, 2026; the patched app displayed `license:pro`. | +| Android | Public updater binary diff `patch-*.zip` artifact with native AOT patch runtime enabled | CI builds mirror-ready patch binaries and a manual `android-engine-arm64` artifact with `dart_enable_aot_patching=true`, `shorebird_enable_aot_patching=true`, and interpreter mode off; local arm64 release APK build passed. Runtime smoke script is available, but this macOS host currently has no Android device or AVD attached. | +| macOS | Public updater binary diff `patch-*.zip` artifact with native AOT patch runtime enabled | Verified locally on June 25, 2026 with a native-AOT Flutter snapshot patch; the patched app displayed `license:pro`. CI builds macOS x64/arm64 patch binaries and a manual `macos-engine-arm64` artifact with native AOT patch runtime args verified. | +| Linux | Public updater binary diff `patch-*.zip` artifact with native AOT patch runtime enabled | CI builds the Linux x64 patch binary and a manual `linux-engine-x64` artifact with native AOT patch runtime args verified. The runtime smoke defaults to `linux_release_x64`; local runtime validation requires a Linux runner. | +| Windows | Public updater binary diff `patch-*.zip` artifact | Previously tested; CI still builds the Windows patch binary. | +| Web | Not a Shorebird CodePush release platform in this CLI/protocol | Manual CI builds `flutter-web-sdk`; local release web build passed with the matching local engine, but it is not counted as a Shorebird runtime patch test. | + +The open CI currently does not publish `aot-tools.dill`. That artifact belongs +to the legacy native-AOT iOS linker path. The App Store-safe iOS route proven +above does not use it; Android/Linux/macOS/Windows patch creation uses the +public updater `patch` binary instead. + +## macOS Host Smoke Results + +On June 25, 2026, the default source-level bootstrap path passed locally with +`SKIP_GCLIENT_SYNC=1`, the workspace Flutter/Dart SDK, and network access for +pub packages: + +```sh +SKIP_GCLIENT_SYNC=1 \ +DART_BIN=/Users/tonylu/git/shorebird-workspace/flutter/bin/dart \ +FLUTTER_BIN=/Users/tonylu/git/shorebird-workspace/flutter/bin/flutter \ + ./scripts/bootstrap_linux.sh +``` + +This ran the focused Shorebird CLI tests, open patch tool tests, self-hosted +server Go tests, and license/flavor AOT verifier. The AOT verifier reported: + +```text +AOT patch applied to license_flavor_patch_test successfully. +base: license:free / pro-feature:off +patch: license:pro / pro-feature:enabled +``` + +The same app also built successfully for macOS release with the local engine: + +```sh +flutter build macos --release \ + --local-engine-src-path=/Users/tonylu/git/shorebird-workspace/flutter/engine/src \ + --local-engine=host_release_arm64 \ + --local-engine-host=host_release_arm64 \ + --dart-define=LICENSE_TYPE=free +``` + +Output: + +```text +build/macos/Build/Products/Release/license_flavor_patch_test.app +``` + +On June 25, 2026, the macOS runtime patch path was also verified locally. The +test saved a free baseline app, rebuilt the same Flutter app with +`--dart-define=LICENSE_TYPE=pro`, copied the resolved pro +`App.framework/Versions/A/App` Mach-O into the sandboxed updater container as: + +```text +Library/Application Support/shorebird/shorebird_updater/ + license-flavor-patch-test/patches/1/dlc.vmcode +``` + +and seeded `pointers.json` plus `patches/1/state.json` for patch `1`. +Launching the saved free app selected that patch path and wrote: + +```text +license:pro +pro-feature:enabled +``` + +The seeded payload must be a Flutter app snapshot for this smoke. A standalone +Dart verifier snapshot is loadable by the updater handoff, but it is not a +valid Flutter application isolate and fails later with `dart:ui` missing. +When copying from `App.framework/App`, resolve the framework symlink first +(`Versions/A/App` or `cp -L`) so the updater validates the real Mach-O size +rather than the symlink size. + +Android arm64 release also built successfully with the local Android engine: + +```sh +flutter build apk --release \ + --target-platform android-arm64 \ + --local-engine-src-path=/Users/tonylu/git/shorebird-workspace/flutter/engine/src \ + --local-engine=android_release_arm64 \ + --local-engine-host=host_release_arm64 \ + --dart-define=LICENSE_TYPE=free +``` + +Output: + +```text +build/app/outputs/flutter-apk/app-release.apk +``` + +Android runtime validation is scripted but was not run to completion on this +macOS host because `adb devices -l` returned no Android devices and +`emulator -list-avds` returned no AVDs. When a target is available, run: + +```sh +ANDROID_SERIAL= \ + ./scripts/android_runtime_patch_smoke.sh +``` + +The same script can be run from GitHub Actions with `workflow_dispatch` and +`run_runtime_smokes=true` on a provisioned Android runner. Use +`android_runtime_runner` to choose the runner label and `android_serial` to +select the target device when needed. + +The script builds free and pro arm64 release APKs with the local Android +engine, installs the free APK, verifies the visible `license:free` and +`pro-feature:off` UI with `uiautomator`, extracts the pro `libapp.so`, seeds it +as patch `1` under the app-private `files/shorebird_updater` directory, +relaunches the free app, and verifies `license:pro` plus +`pro-feature:enabled`. + +The seed step needs access to app-private storage. Use an emulator/device where +`adb shell run-as com.example.licenseflavorpatchtest.license_flavor_patch_test` +works, or a rooted emulator/device where `adb root` works. This seeded smoke is +for local runtime proof; production Android patch installation should exercise +the updater download path through the self-hosted server. + +A normal web release build succeeds when pointed at the matching local engine: + +```sh +flutter build web --release \ + --local-engine-src-path=/Users/tonylu/git/shorebird-workspace/flutter/engine/src \ + --local-engine=host_release_arm64 \ + --local-engine-host=host_release_arm64 \ + --dart-define=LICENSE_TYPE=free +``` + +Output: + +```text +build/web/main.dart.js +``` + +Running the same web build without `--local-engine` currently fails because the +prebuilt `const_finder.dart.snapshot` in `flutter/bin/cache` expects a different +kernel binary format than this workspace Dart SDK emits. Use the local engine +when validating this workspace's web build artifacts. + +The self-hosted server bundle packaging was also smoke-tested locally for +`darwin/arm64`: the archive was extracted, the server started with SQLite and +local storage, `/health` returned `status=ok`, and `/` served the packaged +dashboard HTML from the sibling `web/` directory. The server serves the checked +OpenAPI contract at `/openapi.yaml`, and CI verifies that endpoint from the +packaged Linux x64 archive. CI server artifacts therefore package `web/` and +`openapi.yaml` next to the binary instead of uploading a bare executable. + +## Linux Runtime Smoke + +Linux runtime validation is scripted for a Linux host: + +```sh +./scripts/linux_runtime_patch_smoke.sh +``` + +The script copies `testapps/license_flavor_patch_test` into a temporary +directory, generates the missing Linux platform scaffold there with +`flutter create --platforms=linux`, builds free and pro release bundles with the +local Linux engine, launches the saved free bundle, verifies +`license:free`/`pro-feature:off`, seeds the pro bundle's `lib/libapp.so` as +patch `1` under: + +```text +~/.shorebird_cache/shorebird_updater/license-flavor-patch-test/ +``` + +then relaunches the saved free bundle and verifies `license:pro` plus +`pro-feature:enabled`. If no `DISPLAY` is set and `xvfb-run` exists, the script +uses it automatically. + +This macOS host cannot execute Linux desktop bundles, so this check remains a +Linux-runner task. The same check can be triggered in GitHub Actions with +`workflow_dispatch` and `run_runtime_smokes=true` on a provisioned Linux +runner. Use `linux_runtime_runner` to choose the runner label; the runner must +preserve the matching `flutter/engine/src/out/host_release` local engine output. + ## iOS Preparation Notes The macOS script writes this minimal root `.gclient` by default: @@ -127,8 +326,14 @@ request JIT or unsigned-executable-memory entitlements, and should not have `get-task-allow=true` in strict mode. When `IOS_PATCH_ARTIFACT` is set, the gate also rejects Mach-O and ELF patch files and requires the encrypted open artifact metadata to declare `runtime_mode=dart-bytecode-interpreter`, `target_os=ios`, -`target_arch=arm64`, and `payload_kind=full-snapshot`. Real patches should -arrive through the updater after the reviewed app is installed. +`target_arch=arm64`, and `payload_kind=full-snapshot`. It parses the artifact +as JSON and also requires the AES-256-GCM wrapper fields, base64 ciphertext, +nonce/tag, key id, payload digest, and AAD digest to be present and well +formed. In strict mode, the gate also rejects bundled `aot_patch_key_hex` +values; production apps should provide patch keys with +`FlutterDartProject.shorebirdAotPatchKeyProvider` or an equivalent app-owned +key source. Real patches should arrive through the updater after the reviewed +app is installed. ## iOS Device Results @@ -177,12 +382,14 @@ path when it is used to produce iOS AOT snapshots: ```sh flutter/tools/gn --runtime-mode=release --mac-cpu=arm64 \ - --target-dir=host_release_arm64 --shorebird-interpreter + --target-dir=host_release_arm64 --shorebird-interpreter \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true' ninja -C out/host_release_arm64 gen_snapshot ``` -Without the host-side interpreter flag, the generated app snapshot can bind AOT -static calls directly to base `Code` objects and omit the static-call metadata +Without the host-side interpreter and AOT patch flags, the generated app +snapshot can bind AOT static calls directly to base `Code` objects and omit the +static-call metadata that the iOS interpreter runtime uses to enter patched bytecode. The current open-source interpreter path wires the no-DDM artifact handoff and @@ -190,8 +397,10 @@ an initial product-AOT function replacement mapper. Interpreter-mode artifacts now force and validate `payload_kind=full-snapshot`; compact interpreter diffs are rejected until runtime reconstruction exists. The engine can hand an encrypted open patch artifact to `Dart_InstallAotPatch`, decrypt it with an -explicit `shorebird.yaml` AES key, verify the decrypted payload is Dart -bytecode, and install it through `Dart_ReloadBytecodePatch`. +app-owned AES key provider, verify the decrypted payload is Dart bytecode, and +install it through `Dart_ReloadBytecodePatch`. A development-only +`aot_patch_key_hex` field in `shorebird.yaml` is still supported when no app key +provider is configured, but strict App Store-route checks reject it. For product iOS AOT, `Dart_ReloadBytecodePatch` no longer uses stock VM reload. It reads patch bytecode as data, maps declarations to already-loaded @@ -255,9 +464,9 @@ IOS_PATCH_ARTIFACT=/private/tmp/shorebird-seed-main/shorebird_updater/patches/1/ This verified the generated iOS and host engine args, found no bundled `shorebird_updater` or `dlc.vmcode` payload in the app bundle, found no JIT or unsigned-executable-memory entitlement, confirmed `get-task-allow` is not true, -and verified that the seeded patch artifact is an encrypted interpreter -full-snapshot for iOS arm64. Passing a Mach-O binary as `IOS_PATCH_ARTIFACT` -correctly fails the route gate. +confirmed no `aot_patch_key_hex` is bundled, and verified that the seeded patch +artifact is an encrypted interpreter full-snapshot for iOS arm64. Passing a +Mach-O binary as `IOS_PATCH_ARTIFACT` correctly fails the route gate. The remaining App Store/TestFlight proof is a distribution test, not a local build test: @@ -284,12 +493,13 @@ must emit an interpreter payload in that compatible shape, and compact reconstruction still needs to move into the SDK before compact bytecode patches can be accepted. -For development-device encrypted interpreter tests, `shorebird.yaml` may include -these top-level fields: +For encrypted interpreter tests, `shorebird.yaml` may include these top-level +fields: ```yaml +aot_patch_runtime_mode: dart-bytecode-interpreter aot_patch_key_id: test-key -aot_patch_key_hex: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f +aot_patch_app_build_id: 1.2.3+4 aot_patch_base_flavor_id: free aot_patch_base_license_type: free aot_patch_flavor_id: pro @@ -298,8 +508,30 @@ aot_patch_sdk_hash: aot_patch_base_snapshot_hash: ``` -The YAML key path is only the first open bridge for local verification. A -production App Store implementation still needs an app-owned key callback or -equivalent secure key source, a verified compatible interpreter payload -compiler, compact interpreter reconstruction, and TestFlight/App Store review -validation. +`aot_patch_bytecode_path` is optional and should normally be omitted. When it +is omitted, `shorebird patch ios` compiles the patch target with the Dart SDK's +`dart2bytecode` snapshot and writes `build/ios_interpreter_patch.bytecode`. +The generated bytecode compile forwards user `--dart-define` values, Flutter's +`FLUTTER_APP_FLAVOR` define, and Flutter's standard version/revision/Dart SDK +defines from `bin/cache/flutter.version.json`. It also mirrors Flutter's +runtime feature flag define (`FLUTTER_ENABLED_FEATURE_FLAGS`) for enabled +runtime-id features. +Set `aot_patch_bytecode_path` or `SHOREBIRD_IOS_INTERPRETER_PATCH_PATH` only to +override that generated payload during local experiments. + +Development-only local tests may also include `aot_patch_key_hex`, but reviewed +iOS app bundles should not. The `license_flavor_patch_test` iOS app now provides +the test AES key through `FlutterDartProject.shorebirdAotPatchKeyProvider` +instead. + +`shorebird patch ios` now treats the interpreter route as the default iOS patch +artifact path. It packages the generated or overridden Dart bytecode payload as +an encrypted `open-aot-vmcode-encrypted-v1` full-snapshot artifact with +`runtime_mode=dart-bytecode-interpreter`, `target_os=ios`, and +`target_arch=arm64`. For production builds, provide the encryption key to the +CLI with `SHOREBIRD_AOT_PATCH_KEY_HEX` and keep `aot_patch_key_hex` out of the +bundled app. The legacy native-AOT iOS patch path is development-only and now +requires `SHOREBIRD_IOS_NATIVE_AOT_PATCH=1`. + +A production App Store implementation still needs compact interpreter +reconstruction and TestFlight/App Store review validation. diff --git a/flutter b/flutter index 542e6e8..5b96fd5 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 542e6e89c85435b9c0662ba5e89b4eae2c2e3e2b +Subproject commit 5b96fd59be2f00061cccaaa23a02e15e720c1161 diff --git a/scripts/android_runtime_patch_smoke.sh b/scripts/android_runtime_patch_smoke.sh new file mode 100755 index 0000000..513a13b --- /dev/null +++ b/scripts/android_runtime_patch_smoke.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +APP_DIR="${APP_DIR:-$ROOT/testapps/license_flavor_patch_test}" +FLUTTER_BIN="${FLUTTER_BIN:-$ROOT/flutter/bin/flutter}" +ADB_BIN="${ADB_BIN:-adb}" +PACKAGE="${ANDROID_PACKAGE:-com.example.licenseflavorpatchtest.license_flavor_patch_test}" +ACTIVITY="${ANDROID_ACTIVITY:-com.example.licenseflavorpatchtest.license_flavor_patch_test.MainActivity}" +RELEASE_VERSION="${ANDROID_RELEASE_VERSION:-1.0+1}" +LOCAL_ENGINE_SRC_PATH="${LOCAL_ENGINE_SRC_PATH:-$ROOT/flutter/engine/src}" +LOCAL_ENGINE="${LOCAL_ENGINE:-android_release_arm64}" +LOCAL_ENGINE_HOST="${LOCAL_ENGINE_HOST:-host_release_arm64}" +WORK_DIR="${ANDROID_RUNTIME_SMOKE_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-android-runtime.XXXXXX")}" +SEED_REMOTE_DIR="${ANDROID_SEED_REMOTE_DIR:-/data/local/tmp/open-shorebird-android-runtime-seed}" +TARGET_PLATFORM="${ANDROID_TARGET_PLATFORM:-android-arm64}" + +if [[ "${KEEP_ANDROID_RUNTIME_SMOKE_ARTIFACTS:-0}" != "1" ]]; then + trap 'rm -rf "$WORK_DIR"' EXIT +fi + +ADB=("$ADB_BIN") +if [[ -n "${ANDROID_SERIAL:-}" ]]; then + ADB+=("-s" "$ANDROID_SERIAL") +fi + +adb_cmd() { + "${ADB[@]}" "$@" +} + +adb_shell() { + adb_cmd shell "$@" +} + +python_bin() { + if command -v python3 >/dev/null 2>&1; then + printf '%s\n' python3 + else + printf '%s\n' python + fi +} + +require_tool() { + local tool="$1" + if ! command -v "$tool" >/dev/null 2>&1; then + echo "$tool is required" >&2 + exit 127 + fi +} + +build_apk() { + local license="$1" + local output="$2" + if [[ "${SKIP_ANDROID_BUILDS:-0}" == "1" ]]; then + local env_name + case "$license" in + free) env_name=ANDROID_FREE_APK ;; + pro) env_name=ANDROID_PRO_APK ;; + *) + echo "Unsupported license variant: $license" >&2 + exit 64 + ;; + esac + local existing="${!env_name:-}" + if [[ -z "$existing" || ! -f "$existing" ]]; then + echo "SKIP_ANDROID_BUILDS=1 requires $env_name to point at an APK" >&2 + exit 66 + fi + cp "$existing" "$output" + return + fi + + ( + cd "$APP_DIR" + "$FLUTTER_BIN" build apk --release \ + --target-platform "$TARGET_PLATFORM" \ + --local-engine-src-path="$LOCAL_ENGINE_SRC_PATH" \ + --local-engine="$LOCAL_ENGINE" \ + --local-engine-host="$LOCAL_ENGINE_HOST" \ + --dart-define="LICENSE_TYPE=$license" + ) + cp "$APP_DIR/build/app/outputs/flutter-apk/app-release.apk" "$output" +} + +extract_libapp() { + local apk="$1" + local output="$2" + local py + py="$(python_bin)" + "$py" - "$apk" "$output" <<'PY' +import pathlib +import sys +import zipfile + +apk_path = pathlib.Path(sys.argv[1]) +output_path = pathlib.Path(sys.argv[2]) +with zipfile.ZipFile(apk_path) as archive: + candidates = [ + name for name in archive.namelist() + if name.endswith("/libapp.so") and "arm64-v8a/" in name + ] + if not candidates: + raise SystemExit(f"missing arm64 libapp.so in {apk_path}") + output_path.write_bytes(archive.read(candidates[0])) +PY +} + +wait_for_text() { + local expected="$1" + local dump="$WORK_DIR/window.xml" + for _ in $(seq 1 "${ANDROID_UI_WAIT_ATTEMPTS:-80}"); do + adb_shell uiautomator dump /sdcard/open_shorebird_window.xml >/dev/null 2>&1 || true + adb_cmd exec-out cat /sdcard/open_shorebird_window.xml >"$dump" 2>/dev/null || true + if grep -q "$expected" "$dump"; then + return 0 + fi + sleep 0.25 + done + echo "Timed out waiting for Android UI text: $expected" >&2 + echo "Last uiautomator dump:" >&2 + sed -n '1,120p' "$dump" >&2 || true + return 70 +} + +start_app() { + adb_shell am force-stop "$PACKAGE" >/dev/null 2>&1 || true + adb_shell am start -W -n "$PACKAGE/$ACTIVITY" >/dev/null +} + +can_seed_with_run_as() { + adb_shell run-as "$PACKAGE" sh -c 'test -d files' >/dev/null 2>&1 +} + +can_seed_with_root() { + if adb_shell sh -c 'test "$(id -u)" = "0"' >/dev/null 2>&1; then + return 0 + fi + adb_cmd root >/dev/null 2>&1 || true + adb_cmd wait-for-device >/dev/null 2>&1 || true + adb_shell sh -c 'test "$(id -u)" = "0"' >/dev/null 2>&1 +} + +seed_with_run_as() { + adb_shell run-as "$PACKAGE" sh -c \ + "mkdir -p files && rm -rf files/shorebird_updater && cp -R '$SEED_REMOTE_DIR/shorebird_updater' files/" +} + +seed_with_root() { + local target="/data/data/$PACKAGE/files" + local owner + owner="$(adb_shell sh -c "stat -c '%u:%g' '$target'" | tr -d '\r')" + adb_shell sh -c \ + "rm -rf '$target/shorebird_updater' && cp -R '$SEED_REMOTE_DIR/shorebird_updater' '$target/' && chown -R '$owner' '$target/shorebird_updater'" +} + +prepare_seed() { + local libapp="$1" + local seed_root="$WORK_DIR/seed/shorebird_updater" + local size + size="$(wc -c <"$libapp" | tr -d ' ')" + rm -rf "$WORK_DIR/seed" + mkdir -p "$seed_root/patches/1" + cp "$libapp" "$seed_root/patches/1/dlc.vmcode" + cat >"$seed_root/state.json" <"$seed_root/pointers.json" <<'EOF' +{ + "next_boot_patch": 1, + "last_booted_patch": null, + "currently_booting_patch": null, + "boot_started_at": null +} +EOF + cat >"$seed_root/patches/1/state.json" </dev/null 2>&1 || true + adb_shell mkdir -p "$SEED_REMOTE_DIR" >/dev/null + adb_cmd push "$seed_root" "$SEED_REMOTE_DIR/" >/dev/null + echo "seeded_patch_size=$size" +} + +require_tool "$ADB_BIN" +require_tool "$FLUTTER_BIN" +require_tool java + +adb_cmd start-server >/dev/null +device_count="$(adb_cmd devices | awk 'NR > 1 && $2 == "device" { count++ } END { print count + 0 }')" +if [[ "$device_count" -lt 1 ]]; then + echo "No Android device or emulator is connected." >&2 + echo "Start an emulator/device, then rerun this script. Use ANDROID_SERIAL to pick a device." >&2 + exit 69 +fi +if [[ "$device_count" -gt 1 && -z "${ANDROID_SERIAL:-}" ]]; then + echo "Multiple Android devices are connected; set ANDROID_SERIAL." >&2 + adb_cmd devices -l >&2 + exit 64 +fi + +free_apk="$WORK_DIR/free.apk" +pro_apk="$WORK_DIR/pro.apk" +pro_libapp="$WORK_DIR/pro-libapp.so" + +build_apk free "$free_apk" +build_apk pro "$pro_apk" +extract_libapp "$pro_apk" "$pro_libapp" + +adb_cmd uninstall "$PACKAGE" >/dev/null 2>&1 || true +adb_cmd install -r "$free_apk" >/dev/null + +start_app +wait_for_text 'license:free' +wait_for_text 'pro-feature:off' +echo "android_base_status=license:free/pro-feature:off" + +prepare_seed "$pro_libapp" +if can_seed_with_run_as; then + seed_with_run_as + echo "android_seed_mode=run-as" +elif can_seed_with_root; then + seed_with_root + echo "android_seed_mode=root" +else + cat >&2 </dev/null 2>&1 || true +echo "android_runtime_patch_smoke=passed" diff --git a/scripts/assemble_artifact_mirror.sh b/scripts/assemble_artifact_mirror.sh new file mode 100755 index 0000000..c4f256c --- /dev/null +++ b/scripts/assemble_artifact_mirror.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: assemble_artifact_mirror.sh + +Copies every publish-ready shorebird/ mirror subtree from downloaded GitHub +Actions artifacts into . Engine archives are extracted and +scanned for nested mirror/shorebird trees. Existing files may be reused only +when their bytes match. +EOF +} + +if [[ "$#" -ne 2 ]]; then + usage + exit 64 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +INPUT_DIR="$1" +OUTPUT_DIR="$2" + +if [[ ! -d "$INPUT_DIR" ]]; then + echo "input artifact directory does not exist: $INPUT_DIR" >&2 + exit 66 +fi + +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-mirror.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +mkdir -p "$OUTPUT_DIR" + +FOUND_TREES=0 + +copy_shorebird_tree() { + local tree="$1" + local source_file rel_file target_file + + FOUND_TREES=$((FOUND_TREES + 1)) + while IFS= read -r -d '' source_file; do + rel_file="${source_file#"$tree"/}" + target_file="$OUTPUT_DIR/shorebird/$rel_file" + mkdir -p "$(dirname "$target_file")" + if [[ -e "$target_file" ]]; then + if ! cmp -s "$source_file" "$target_file"; then + echo "conflicting mirror file: shorebird/$rel_file" >&2 + echo " existing: $target_file" >&2 + echo " incoming: $source_file" >&2 + exit 70 + fi + continue + fi + cp -p "$source_file" "$target_file" + done < <(find "$tree" -type f -print0) +} + +scan_for_shorebird_trees() { + local search_root="$1" + local tree + + while IFS= read -r -d '' tree; do + copy_shorebird_tree "$tree" + done < <(find "$search_root" -type d -name shorebird -print0) +} + +scan_for_shorebird_trees "$INPUT_DIR" + +archive_index=0 +while IFS= read -r -d '' archive_path; do + archive_index=$((archive_index + 1)) + extract_dir="$TMP_DIR/archive-$archive_index" + mkdir -p "$extract_dir" + "$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" "$archive_path" "$extract_dir" + scan_for_shorebird_trees "$extract_dir" +done < <(find "$INPUT_DIR" -type f \( -name '*.tar.gz' -o -name '*.tgz' \) -print0) + +if [[ "$FOUND_TREES" -eq 0 ]]; then + echo "no shorebird/ mirror subtrees found under $INPUT_DIR" >&2 + exit 65 +fi + +while IFS= read -r -d '' mirror_file; do + if [[ "$mirror_file" == *.sha256 ]]; then + continue + fi + sidecar="$mirror_file.sha256" + if [[ ! -f "$sidecar" ]]; then + "$ROOT/scripts/write_sha256.sh" "$mirror_file" "$sidecar" + fi +done < <(find "$OUTPUT_DIR/shorebird" -type f -print0) + +"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$OUTPUT_DIR" + +echo "assembled artifact mirror at $OUTPUT_DIR" diff --git a/scripts/check_ci_capacity.sh b/scripts/check_ci_capacity.sh new file mode 100755 index 0000000..5f82bf6 --- /dev/null +++ b/scripts/check_ci_capacity.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +MIN_FREE_DISK_GB="${CI_MIN_FREE_DISK_GB:-0}" +CHECK_PATH="${CI_CAPACITY_PATH:-$PWD}" +AVAILABLE_DISK_KB_OVERRIDE="${CI_AVAILABLE_DISK_KB_OVERRIDE:-}" + +fail() { + echo "error: $*" >&2 + exit 70 +} + +case "$MIN_FREE_DISK_GB" in + ''|*[!0-9]*) + fail "CI_MIN_FREE_DISK_GB must be a non-negative integer, got '$MIN_FREE_DISK_GB'" + ;; +esac + +if [[ "$MIN_FREE_DISK_GB" == "0" ]]; then + echo "CI capacity check skipped because CI_MIN_FREE_DISK_GB=0" + exit 0 +fi + +if [[ ! -e "$CHECK_PATH" ]]; then + fail "capacity check path does not exist: $CHECK_PATH" +fi + +available_disk_kb() { + if [[ -n "$AVAILABLE_DISK_KB_OVERRIDE" ]]; then + case "$AVAILABLE_DISK_KB_OVERRIDE" in + *[!0-9]*) + fail "CI_AVAILABLE_DISK_KB_OVERRIDE must be an integer, got '$AVAILABLE_DISK_KB_OVERRIDE'" + ;; + esac + printf '%s\n' "$AVAILABLE_DISK_KB_OVERRIDE" + return + fi + df -Pk "$CHECK_PATH" | awk 'NR == 2 { print $4 }' +} + +available_kb="$(available_disk_kb)" +case "$available_kb" in + ''|*[!0-9]*) + fail "could not determine available disk space for $CHECK_PATH" + ;; +esac + +required_kb=$((MIN_FREE_DISK_GB * 1024 * 1024)) +available_gb=$((available_kb / 1024 / 1024)) + +echo "CI capacity: ${available_gb} GiB free at $CHECK_PATH; required: ${MIN_FREE_DISK_GB} GiB" + +if (( available_kb < required_kb )); then + fail "runner has insufficient free disk for the heavy SDK/engine build. Use a larger or self-hosted runner label, or free disk before this step." +fi diff --git a/scripts/free_ci_disk_linux.sh b/scripts/free_ci_disk_linux.sh new file mode 100755 index 0000000..5ff7a2b --- /dev/null +++ b/scripts/free_ci_disk_linux.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != "true" ]]; then + echo "Refusing to free disk outside GitHub Actions." >&2 + exit 64 +fi + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "Skipping Linux disk cleanup on $(uname -s)." + exit 0 +fi + +if [[ "${CI_FREE_DISK_SPACE:-1}" == "0" ]]; then + echo "Skipping disk cleanup because CI_FREE_DISK_SPACE=0." + df -h + exit 0 +fi + +if [[ "${RUNNER_ENVIRONMENT:-github-hosted}" != "github-hosted" && + "${CI_FREE_DISK_SPACE_FORCE:-0}" != "1" ]]; then + echo "Skipping disk cleanup on ${RUNNER_ENVIRONMENT} runner." + echo "Set CI_FREE_DISK_SPACE_FORCE=1 to opt in on non-hosted runners." + df -h + exit 0 +fi + +echo "Disk before cleanup:" +df -h + +# GitHub-hosted Ubuntu images include large toolchains that are unrelated to +# Dart SDK, Flutter engine, and updater builds. Remove only well-known cache +# directories on ephemeral GitHub Actions runners. +for path in \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /usr/local/.ghcup \ + /usr/local/lib/android/sdk \ + /usr/local/share/boost \ + /usr/share/dotnet; do + if [[ -e "$path" ]]; then + echo "Removing $path" + sudo rm -rf "$path" + fi +done + +echo "Disk after cleanup:" +df -h diff --git a/scripts/linux_runtime_patch_smoke.sh b/scripts/linux_runtime_patch_smoke.sh new file mode 100755 index 0000000..c6dd990 --- /dev/null +++ b/scripts/linux_runtime_patch_smoke.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SOURCE_APP_DIR="${SOURCE_APP_DIR:-$ROOT/testapps/license_flavor_patch_test}" +FLUTTER_BIN="${FLUTTER_BIN:-$ROOT/flutter/bin/flutter}" +APP_ID="${SHOREBIRD_APP_ID:-license-flavor-patch-test}" +LOCAL_ENGINE_SRC_PATH="${LOCAL_ENGINE_SRC_PATH:-$ROOT/flutter/engine/src}" +DEFAULT_LOCAL_ENGINE="linux_release_x64" +if [[ ! -d "$LOCAL_ENGINE_SRC_PATH/out/$DEFAULT_LOCAL_ENGINE" && + -d "$LOCAL_ENGINE_SRC_PATH/out/host_release" ]]; then + DEFAULT_LOCAL_ENGINE="host_release" +fi +LOCAL_ENGINE="${LOCAL_ENGINE:-$DEFAULT_LOCAL_ENGINE}" +LOCAL_ENGINE_HOST="${LOCAL_ENGINE_HOST:-$LOCAL_ENGINE}" +WORK_DIR="${LINUX_RUNTIME_SMOKE_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-linux-runtime.XXXXXX")}" +APP_COPY="$WORK_DIR/app" +HOME_DIR="$WORK_DIR/home" + +if [[ "${KEEP_LINUX_RUNTIME_SMOKE_ARTIFACTS:-0}" != "1" ]]; then + trap 'rm -rf "$WORK_DIR"' EXIT +fi + +require_tool() { + local tool="$1" + if ! command -v "$tool" >/dev/null 2>&1; then + echo "$tool is required" >&2 + exit 127 + fi +} + +python_bin() { + if command -v python3 >/dev/null 2>&1; then + printf '%s\n' python3 + else + printf '%s\n' python + fi +} + +copy_app_fixture() { + mkdir -p "$APP_COPY" + ( + cd "$SOURCE_APP_DIR" + tar \ + --exclude='./build' \ + --exclude='./.dart_tool' \ + --exclude='./android/.gradle' \ + --exclude='./ios/Pods' \ + --exclude='./macos/Flutter/ephemeral' \ + -cf - . + ) | tar -C "$APP_COPY" -xf - +} + +ensure_linux_platform() { + if [[ -d "$APP_COPY/linux" ]]; then + return + fi + ( + cd "$APP_COPY" + "$FLUTTER_BIN" create --platforms=linux --project-name=license_flavor_patch_test . + ) +} + +build_linux_bundle() { + local license="$1" + local output="$2" + ( + cd "$APP_COPY" + "$FLUTTER_BIN" build linux --release \ + --local-engine-src-path="$LOCAL_ENGINE_SRC_PATH" \ + --local-engine="$LOCAL_ENGINE" \ + --local-engine-host="$LOCAL_ENGINE_HOST" \ + --dart-define="LICENSE_TYPE=$license" + ) + local bundle + bundle="$(find "$APP_COPY/build/linux" -path '*/release/bundle' -type d -print -quit)" + if [[ -z "$bundle" || ! -x "$bundle/license_flavor_patch_test" ]]; then + echo "Failed to find Linux release bundle under $APP_COPY/build/linux" >&2 + exit 66 + fi + rm -rf "$output" + cp -a "$bundle" "$output" +} + +release_version_for_bundle() { + local bundle="$1" + local version_json="$bundle/data/flutter_assets/version.json" + local py + py="$(python_bin)" + "$py" - "$version_json" <<'PY' +import json +import pathlib +import sys + +version = json.loads(pathlib.Path(sys.argv[1]).read_text()) +build_name = str(version.get("version", "")) +build_number = str(version.get("build_number", "")) +if build_number: + print(f"{build_name}+{build_number}") +else: + print(build_name) +PY +} + +run_saved_app() { + local label="$1" + local bundle="$2" + local stdout="$WORK_DIR/$label.stdout" + local stderr="$WORK_DIR/$label.stderr" + local tmp_status="$WORK_DIR/tmp/license_flavor_patch_status.txt" + local home_status="$HOME_DIR/Library/Application Support/license_flavor_patch_status.txt" + + rm -f "$tmp_status" "$home_status" + mkdir -p "$WORK_DIR/tmp" "$HOME_DIR" + + local -a app_command=(env HOME="$HOME_DIR" TMPDIR="$WORK_DIR/tmp" "$bundle/license_flavor_patch_test") + if [[ "${LINUX_RUNTIME_SMOKE_XVFB:-auto}" != "0" && -z "${DISPLAY:-}" && "$(command -v xvfb-run || true)" != "" ]]; then + app_command=(xvfb-run -a "${app_command[@]}") + fi + + "${app_command[@]}" >"$stdout" 2>"$stderr" & + local pid=$! + for _ in $(seq 1 "${LINUX_RUNTIME_WAIT_ATTEMPTS:-120}"); do + if [[ -f "$tmp_status" || -f "$home_status" ]]; then + break + fi + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.25 + done + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + else + wait "$pid" 2>/dev/null || true + fi + + echo "${label}_stdout=$stdout" + echo "${label}_stderr=$stderr" + if [[ -f "$tmp_status" ]]; then + echo "${label}_status_file=$tmp_status" + cat "$tmp_status" + elif [[ -f "$home_status" ]]; then + echo "${label}_status_file=$home_status" + cat "$home_status" + else + echo "${label}_status_file_missing" >&2 + echo "--- $label stderr ---" >&2 + sed -n '1,180p' "$stderr" >&2 || true + return 70 + fi +} + +read_launch_status() { + local tmp_status="$WORK_DIR/tmp/license_flavor_patch_status.txt" + local home_status="$HOME_DIR/Library/Application Support/license_flavor_patch_status.txt" + if [[ -f "$tmp_status" ]]; then + cat "$tmp_status" + elif [[ -f "$home_status" ]]; then + cat "$home_status" + fi +} + +require_status() { + local label="$1" + local expected_license="$2" + local expected_feature="$3" + local status + status="$(read_launch_status)" + if ! grep -q "license:$expected_license" <<<"$status" || + ! grep -q "pro-feature:$expected_feature" <<<"$status"; then + echo "$label Linux app did not report expected status." >&2 + echo "Expected: license:$expected_license / pro-feature:$expected_feature" >&2 + echo "Actual:" >&2 + printf '%s\n' "$status" >&2 + exit 70 + fi +} + +seed_patch() { + local pro_bundle="$1" + local release_version="$2" + local patch_file="$pro_bundle/lib/libapp.so" + local state_root="$HOME_DIR/.shorebird_cache/shorebird_updater/$APP_ID" + local size + size="$(wc -c <"$patch_file" | tr -d ' ')" + + rm -rf "$state_root" + mkdir -p "$state_root/patches/1" + cp "$patch_file" "$state_root/patches/1/dlc.vmcode" + cat >"$state_root/state.json" <"$state_root/pointers.json" <<'EOF' +{ + "next_boot_patch": 1, + "last_booted_patch": null, + "currently_booting_patch": null, + "boot_started_at": null +} +EOF + cat >"$state_root/patches/1/state.json" </dev/null 2>&1; then @@ -37,9 +39,13 @@ run "$ROOT/scripts/write_gclient.sh" "$PLATFORM" run "$ROOT/scripts/sync_open_sources.sh" export PATH="$ROOT/depot_tools:$PATH" +export DEPOT_TOOLS_UPDATE="${DEPOT_TOOLS_UPDATE:-0}" if [[ "${SKIP_GCLIENT_SYNC:-0}" != "1" ]]; then require_command gclient run gclient sync --no-history + if [[ "${INCLUDE_ENGINE_DEPS:-0}" == "1" && -f "$ROOT/flutter/.gclient" ]]; then + run bash -lc "cd '$ROOT/flutter' && gclient sync --no-history" + fi else echo echo "==> skipping gclient sync because SKIP_GCLIENT_SYNC=1" @@ -51,8 +57,12 @@ if [[ "${SKIP_TESTS:-0}" == "1" ]]; then exit 0 fi -run bash -lc "cd '$ROOT/shorebird/packages/shorebird_cli' && '$DART_BIN' pub get && '$DART_BIN' test test/src/user_config_test.dart test/src/shorebird_env_test.dart test/src/shorebird_cli_command_runner_test.dart test/src/commands/init_command_test.dart" +run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' pub get" +run bash -lc "cd '$ROOT/shorebird/packages/shorebird_cli' && '$DART_BIN' pub get && '$DART_BIN' test test/src/user_config_test.dart test/src/shorebird_env_test.dart test/src/shorebird_cli_command_runner_test.dart test/src/commands/doctor_command_test.dart test/src/commands/init_command_test.dart test/src/cache_test.dart test/src/shorebird_process_test.dart test/src/network_checker_test.dart test/src/shorebird_web_console_test.dart test/src/auth/auth_test.dart test/src/commands/login_command_test.dart test/src/commands/login_ci_command_test.dart test/src/commands/release/aar_releaser_test.dart test/src/shorebird_validator_test.dart test/src/shorebird_flutter_test.dart test/src/shorebird_artifacts_test.dart test/src/artifact_builder/artifact_builder_test.dart test/src/config/shorebird_yaml_test.dart test/src/commands/patch/ios_patcher_test.dart" +run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' test packages/shorebird_code_push_client/test/src/code_push_client_test.dart" +run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' test packages/artifact_proxy/test/artifact_proxy_test.dart packages/artifact_proxy/test/server_bin_test.dart packages/artifact_proxy/test/src/artifact_manifest_client_test.dart" run bash -lc "cd '$ROOT/shorebird/packages/open_aot_patch_tools' && '$DART_BIN' pub get && '$DART_BIN' test" +run "$CARGO_BIN" test --manifest-path "$ROOT/updater/library/Cargo.toml" run bash -lc "cd '$ROOT/shorebird-server' && '$GO_BIN' test ./..." AOT_PATCH_BUILD_DIR="${AOT_PATCH_BUILD_DIR:-}" diff --git a/scripts/safe_extract_tar.py b/scripts/safe_extract_tar.py new file mode 100755 index 0000000..8bd2b4a --- /dev/null +++ b/scripts/safe_extract_tar.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from pathlib import Path, PurePosixPath +import shutil +import sys +import tarfile + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + raise SystemExit(70) + + +def safe_member_name(member: tarfile.TarInfo) -> str | None: + name = member.name + if member.isdir(): + name = name.rstrip("/") + if not name or name == ".": + return None + if "\\" in name or "\x00" in name: + return None + if any(ord(character) < 32 for character in name): + return None + + candidate = PurePosixPath(name) + if candidate.is_absolute(): + return None + if any(part in ("", ".", "..") for part in candidate.parts): + return None + if candidate.parts and ":" in candidate.parts[0]: + return None + return candidate.as_posix() + + +def validate_members(archive_path: Path, members: list[tarfile.TarInfo]) -> None: + member_types: dict[str, str] = {} + for member in members: + safe_name = safe_member_name(member) + if safe_name is None: + fail(f"{archive_path}: unsafe archive member path {member.name!r}") + if not (member.isdir() or member.isfile()): + fail(f"{archive_path}: unsupported archive member type {member.name!r}") + + member_type = "dir" if member.isdir() else "file" + previous_type = member_types.get(safe_name) + if previous_type is None: + member_types[safe_name] = member_type + continue + if previous_type != "dir" or member_type != "dir": + fail(f"{archive_path}: duplicate archive member path {member.name!r}") + + +def ensure_within_root(archive_path: Path, extract_root: Path, target: Path, name: str) -> None: + try: + target.relative_to(extract_root) + except ValueError: + fail(f"{archive_path}: archive member escapes extraction root {name!r}") + + +def extract_safe_tar_archive(archive_path: Path, extract_dir: Path) -> None: + extract_root = extract_dir.resolve() + extracted_files: set[Path] = set() + + try: + with tarfile.open(archive_path, "r:*") as archive: + members = archive.getmembers() + validate_members(archive_path, members) + + for member in members: + safe_name = safe_member_name(member) + assert safe_name is not None + target = (extract_root / safe_name).resolve() + ensure_within_root(archive_path, extract_root, target, member.name) + + if member.isdir(): + if target in extracted_files: + fail(f"{archive_path}: directory collides with file {member.name!r}") + target.mkdir(parents=True, exist_ok=True) + continue + + for parent in target.parents: + if parent == extract_root: + break + if parent in extracted_files: + fail(f"{archive_path}: file parent collides with file {member.name!r}") + + if target.exists() and not target.is_file(): + fail(f"{archive_path}: file collides with directory {member.name!r}") + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + if source is None: + fail(f"{archive_path}: unable to read archive member {member.name!r}") + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + target.chmod(member.mode & 0o777) + extracted_files.add(target) + except tarfile.TarError as error: + fail(f"{archive_path}: invalid tar archive: {error}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Safely extract a tar archive containing only files and directories.", + ) + parser.add_argument("archive", type=Path) + parser.add_argument("extract_dir", type=Path) + args = parser.parse_args() + + args.extract_dir.mkdir(parents=True, exist_ok=True) + extract_safe_tar_archive(args.archive, args.extract_dir) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sync_open_sources.sh b/scripts/sync_open_sources.sh index 41115f3..18f340f 100755 --- a/scripts/sync_open_sources.sh +++ b/scripts/sync_open_sources.sh @@ -5,13 +5,99 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" DART_SRC="${DART_SRC:-$ROOT/dart-sdk-new}" DART_TARGET="$ROOT/flutter/engine/src/flutter/third_party/dart" UPDATER_SRC="${UPDATER_SRC:-$ROOT/updater}" -UPDATER_URL="${UPDATER_URL:-https://github.com/shorebirdtech/updater.git}" +UPDATER_URL="${UPDATER_URL:-}" TARGET="$ROOT/flutter/engine/src/flutter/third_party/updater" is_git_checkout() { git -C "$1" rev-parse --git-dir >/dev/null 2>&1 } +relative_path() { + python3 - "$1" "$2" <<'PY' +import os +import sys +print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1]))) +PY +} + +real_path() { + python3 - "$1" <<'PY' +import os +import sys +print(os.path.realpath(sys.argv[1])) +PY +} + +link_checkout() { + local target="$1" + local source="$2" + local label="$3" + local rel_target + rel_target="$(relative_path "$target" "$source")" + ln -s "$rel_target" "$target" + echo "[open-source-sync] linked $label checkout into Flutter engine." +} + +is_clean_git_checkout() { + [[ -z "$(git -C "$1" status --porcelain)" ]] +} + +reject_forbidden_remotes() { + local source="$1" + local label="$2" + shift 2 + + local remotes + remotes="$(git -C "$source" remote -v 2>/dev/null || true)" + if [[ -z "$remotes" ]]; then + return + fi + + local forbidden + for forbidden in "$@"; do + if grep -Fq "$forbidden" <<<"$remotes"; then + echo "$label source checkout uses forbidden remote fragment '$forbidden': $source" >&2 + echo "$remotes" >&2 + exit 1 + fi + done +} + +ensure_source_link() { + local target="$1" + local source="$2" + local label="$3" + + if [[ ! -d "$source" ]] || ! is_git_checkout "$source"; then + echo "$label source checkout is missing: $source" >&2 + exit 1 + fi + + if [[ -L "$target" ]]; then + local target_real + local source_real + target_real="$(real_path "$target")" + source_real="$(real_path "$source")" + if [[ "$target_real" != "$source_real" ]]; then + echo "$label target symlink points at $target_real, expected $source_real" >&2 + exit 1 + fi + echo "[open-source-sync] $label target already links to the workspace checkout." + elif is_git_checkout "$target"; then + if ! is_clean_git_checkout "$target"; then + echo "$label target is a dirty git checkout and cannot be replaced: $target" >&2 + exit 1 + fi + rm -rf "$target" + link_checkout "$target" "$source" "$label" + elif [[ -e "$target" ]]; then + echo "$label target exists but is not a symlink or git checkout: $target" >&2 + exit 1 + else + link_checkout "$target" "$source" "$label" + fi +} + echo "[open-source-sync] dart source: $DART_SRC" echo "[open-source-sync] dart target: $DART_TARGET" echo "[open-source-sync] updater source: $UPDATER_SRC" @@ -20,60 +106,54 @@ echo "[open-source-sync] target: $TARGET" mkdir -p "$(dirname "$DART_TARGET")" mkdir -p "$(dirname "$TARGET")" -if [[ -L "$DART_TARGET" ]]; then - echo "[open-source-sync] Dart target is already a symlink." -elif is_git_checkout "$DART_TARGET"; then - echo "[open-source-sync] Dart target is already a git checkout." -elif [[ -e "$DART_TARGET" ]]; then - echo "Dart target exists but is not a symlink or git checkout: $DART_TARGET" >&2 - exit 1 -elif is_git_checkout "$DART_SRC"; then - rel_target="$(python3 - "$DART_TARGET" "$DART_SRC" <<'PY' -import os -import sys -print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1]))) -PY -)" - ln -s "$rel_target" "$DART_TARGET" - echo "[open-source-sync] linked Dart SDK checkout into Flutter engine." -else - echo "Dart source checkout is missing: $DART_SRC" >&2 - exit 1 -fi +ensure_source_link "$DART_TARGET" "$DART_SRC" "Dart SDK" +reject_forbidden_remotes \ + "$DART_SRC" \ + "Dart SDK" \ + "github.com/dart-lang/sdk" \ + "dart.googlesource.com/sdk" if [[ ! -f "$DART_TARGET/runtime/vm/dart_api_impl.h" ]]; then echo "Dart checkout is missing runtime/vm/dart_api_impl.h" >&2 exit 1 fi -if [[ -L "$TARGET" ]]; then - echo "[open-source-sync] updater target is already a symlink." - exit 0 -fi - -if is_git_checkout "$TARGET"; then - echo "[open-source-sync] updating existing updater checkout." - git -C "$TARGET" fetch --tags origin - git -C "$TARGET" checkout "${UPDATER_REVISION:-main}" - if [[ "${UPDATER_REVISION:-main}" == "main" ]]; then - git -C "$TARGET" pull --ff-only +if is_git_checkout "$UPDATER_SRC"; then + ensure_source_link "$TARGET" "$UPDATER_SRC" "updater submodule" + reject_forbidden_remotes \ + "$UPDATER_SRC" \ + "updater submodule" \ + "github.com/shorebirdtech/updater" \ + "github.com/shorebirdtech/shorebird-updater" +elif [[ -n "$UPDATER_URL" ]]; then + if [[ "$UPDATER_URL" == *github.com/shorebirdtech/updater* || + "$UPDATER_URL" == *github.com/shorebirdtech/shorebird-updater* ]]; then + echo "UPDATER_URL points at a forbidden official Shorebird updater remote: $UPDATER_URL" >&2 + exit 1 + fi + if [[ -L "$TARGET" ]]; then + rm "$TARGET" + fi + if is_git_checkout "$TARGET"; then + echo "[open-source-sync] updating existing updater checkout." + git -C "$TARGET" remote set-url origin "$UPDATER_URL" + git -C "$TARGET" fetch --tags origin + git -C "$TARGET" checkout "${UPDATER_REVISION:-main}" + if [[ "${UPDATER_REVISION:-main}" == "main" ]]; then + git -C "$TARGET" pull --ff-only + fi + elif [[ -e "$TARGET" ]]; then + echo "target exists but is not a symlink or git checkout: $TARGET" >&2 + exit 1 + else + echo "[open-source-sync] cloning updater checkout from explicit UPDATER_URL." + git clone "$UPDATER_URL" "$TARGET" + git -C "$TARGET" checkout "${UPDATER_REVISION:-main}" fi -elif [[ -e "$TARGET" ]]; then - echo "target exists but is not a symlink or git checkout: $TARGET" >&2 - exit 1 -elif is_git_checkout "$UPDATER_SRC"; then - rel_target="$(python3 - "$TARGET" "$UPDATER_SRC" <<'PY' -import os -import sys -print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1]))) -PY -)" - ln -s "$rel_target" "$TARGET" - echo "[open-source-sync] linked updater submodule into Flutter engine." else - echo "[open-source-sync] cloning public updater checkout." - git clone "$UPDATER_URL" "$TARGET" - git -C "$TARGET" checkout "${UPDATER_REVISION:-main}" + echo "updater source checkout is missing: $UPDATER_SRC" >&2 + echo "Set UPDATER_SRC to a local fork or set UPDATER_URL explicitly." >&2 + exit 1 fi if [[ ! -f "$TARGET/library/include/updater_engine.h" ]]; then diff --git a/scripts/validate_artifact_mirror.py b/scripts/validate_artifact_mirror.py new file mode 100755 index 0000000..07db531 --- /dev/null +++ b/scripts/validate_artifact_mirror.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Validate an assembled open Shorebird artifact mirror.""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path, PurePosixPath +import re +import sys +import zipfile + + +REQUIRED_PATCH_ZIPS = { + "patch-linux-x64.zip": "patch", + "patch-darwin-x64.zip": "patch", + "patch-darwin-arm64.zip": "patch", + "patch-windows-x64.zip": "patch.exe", +} + + +def digest_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_plain_file(path: Path) -> bool: + return path.is_file() and not path.is_symlink() + + +def parse_sidecar(path: Path) -> tuple[str, str]: + text = path.read_text(encoding="utf-8").strip() + parts = text.split() + if len(parts) != 2: + raise ValueError(f"expected ' ', got {text!r}") + digest, filename = parts + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + raise ValueError(f"invalid sha256 digest {digest!r}") + return digest, filename + + +def is_safe_relative_path(path: str) -> bool: + if not path or path == ".": + return False + if "\\" in path or "\x00" in path or path.endswith("/"): + return False + if any(ord(character) < 32 for character in path): + return False + + candidate = PurePosixPath(path) + if candidate.is_absolute(): + return False + if any(part in ("", ".", "..") for part in candidate.parts): + return False + if candidate.parts and ":" in candidate.parts[0]: + return False + return True + + +def validate_sidecars(shorebird_root: Path) -> list[str]: + errors: list[str] = [] + for artifact_path in sorted(shorebird_root.rglob("*")): + if artifact_path.is_symlink(): + errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: symlink entries are not allowed") + continue + if not artifact_path.is_file(): + continue + if artifact_path.suffix == ".sha256": + artifact_path_without_suffix = Path(str(artifact_path)[: -len(".sha256")]) + if not is_plain_file(artifact_path_without_suffix): + errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: orphan sidecar") + continue + + sidecar_path = Path(f"{artifact_path}.sha256") + if not is_plain_file(sidecar_path): + errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: missing sidecar") + continue + try: + sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path) + except ValueError as error: + errors.append(f"{sidecar_path.relative_to(shorebird_root.parent)}: {error}") + continue + actual_digest = digest_file(artifact_path) + if sidecar_digest != actual_digest: + errors.append( + f"{sidecar_path.relative_to(shorebird_root.parent)}: digest mismatch " + f"{sidecar_digest} != {actual_digest}" + ) + if sidecar_filename != artifact_path.name: + errors.append( + f"{sidecar_path.relative_to(shorebird_root.parent)}: filename mismatch " + f"{sidecar_filename!r} != {artifact_path.name!r}" + ) + return errors + + +def validate_manifest_overrides(mirror_root: Path, manifest_paths: list[Path]) -> list[str]: + override_pattern = re.compile(r"^\s*-\s*'?(?P[^'#\n]+?)'?\s*(?:#.*)?$") + shorebird_root = mirror_root / "shorebird" + errors: list[str] = [] + for manifest_path in manifest_paths: + engine_revision = manifest_path.parent.name + for line in manifest_path.read_text(encoding="utf-8").splitlines(): + match = override_pattern.match(line) + if not match: + continue + artifact_path = match.group("path").replace("$engine", engine_revision) + if not is_safe_relative_path(artifact_path): + errors.append( + f"{manifest_path.relative_to(mirror_root)} -> " + f"unsafe artifact override path: {artifact_path}" + ) + continue + resolved_artifact_path = shorebird_root / artifact_path + if not is_plain_file(resolved_artifact_path): + errors.append( + f"{manifest_path.relative_to(mirror_root)} -> shorebird/{artifact_path}" + ) + continue + if resolved_artifact_path.stat().st_size <= 0: + errors.append( + f"{manifest_path.relative_to(mirror_root)} -> " + f"shorebird/{artifact_path}: artifact override is empty" + ) + return errors + + +def validate_patch_zips(mirror_root: Path, manifest_paths: list[Path]) -> list[str]: + errors: list[str] = [] + for manifest_path in manifest_paths: + engine_dir = manifest_path.parent + for zip_name, expected_entry in REQUIRED_PATCH_ZIPS.items(): + zip_path = engine_dir / zip_name + display_path = zip_path.relative_to(mirror_root) + if not is_plain_file(zip_path): + errors.append(f"{display_path}: missing") + continue + try: + with zipfile.ZipFile(zip_path) as archive: + names = archive.namelist() + if names != [expected_entry]: + errors.append( + f"{display_path}: expected only {expected_entry!r}, got {names!r}" + ) + continue + if archive.getinfo(expected_entry).file_size <= 0: + errors.append(f"{display_path}: {expected_entry} is empty") + except zipfile.BadZipFile: + errors.append(f"{display_path}: invalid zip") + return errors + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("mirror_root", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + mirror_root = args.mirror_root + shorebird_root = mirror_root / "shorebird" + + if not mirror_root.is_dir(): + print(f"missing mirror root: {mirror_root}", file=sys.stderr) + return 66 + if not shorebird_root.is_dir(): + print(f"mirror root is missing shorebird/: {mirror_root}", file=sys.stderr) + return 70 + + manifest_paths = sorted(shorebird_root.glob("*/artifacts_manifest.yaml")) + if not manifest_paths: + print( + "artifact mirror is missing shorebird//artifacts_manifest.yaml", + file=sys.stderr, + ) + return 70 + + errors: list[str] = [] + sidecar_errors = validate_sidecars(shorebird_root) + if sidecar_errors: + errors.append("invalid checksum sidecars:\n" + "\n".join(f" {e}" for e in sidecar_errors)) + + override_errors = validate_manifest_overrides(mirror_root, manifest_paths) + if override_errors: + errors.append( + "invalid files referenced by artifacts_manifest.yaml:\n" + + "\n".join(f" {path}" for path in override_errors) + ) + + patch_errors = validate_patch_zips(mirror_root, manifest_paths) + if patch_errors: + errors.append( + "invalid CLI patch-tool artifacts:\n" + + "\n".join(f" {error}" for error in patch_errors) + ) + + if errors: + print("artifact mirror validation failed:\n" + "\n".join(errors), file=sys.stderr) + return 70 + + print(f"artifact mirror validated: {mirror_root}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_release_manifest.py b/scripts/validate_release_manifest.py new file mode 100755 index 0000000..110f749 --- /dev/null +++ b/scripts/validate_release_manifest.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Validate a JSON release manifest against downloaded CI artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath +import sys + + +def digest_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_plain_file(path: Path) -> bool: + return path.is_file() and not path.is_symlink() + + +def parse_sidecar(path: Path) -> tuple[str, str]: + text = path.read_text(encoding="utf-8").strip() + parts = text.split() + if len(parts) != 2: + raise ValueError(f"expected ' ', got {text!r}") + digest, filename = parts + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + raise ValueError(f"invalid sha256 digest {digest!r}") + return digest, filename + + +def is_safe_relative_path(path: str) -> bool: + if not path or path == ".": + return False + if "\\" in path or "\x00" in path or path.endswith("/"): + return False + if any(ord(character) < 32 for character in path): + return False + + candidate = PurePosixPath(path) + if candidate.is_absolute(): + return False + if any(part in ("", ".", "..") for part in candidate.parts): + return False + if candidate.parts and ":" in candidate.parts[0]: + return False + return True + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--github-sha", + default="", + help="Require the manifest github_sha field to match this commit SHA.", + ) + parser.add_argument("input_dir", type=Path) + parser.add_argument("manifest", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + input_dir = args.input_dir + manifest_path = args.manifest + + if not input_dir.is_dir(): + print(f"missing input directory: {input_dir}", file=sys.stderr) + return 66 + if not manifest_path.is_file(): + print(f"missing release manifest: {manifest_path}", file=sys.stderr) + return 66 + + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + print(f"invalid release manifest JSON: {error}", file=sys.stderr) + return 70 + + errors: list[str] = [] + if manifest.get("format_version") != 1: + errors.append(f"format_version is {manifest.get('format_version')!r}; expected 1") + if not isinstance(manifest.get("github_sha", ""), str): + errors.append("github_sha must be a string") + elif args.github_sha and manifest.get("github_sha") != args.github_sha: + errors.append( + f"github_sha is {manifest.get('github_sha')!r}; expected {args.github_sha!r}" + ) + + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list): + errors.append("artifacts must be a list") + artifacts = [] + + expected_count = manifest.get("artifact_count") + if expected_count != len(artifacts): + errors.append( + f"artifact_count is {expected_count!r}; expected {len(artifacts)}" + ) + + seen_paths: set[str] = set() + seen_sidecars: set[str] = set() + for index, artifact in enumerate(artifacts): + if not isinstance(artifact, dict): + errors.append(f"artifacts[{index}] must be an object") + continue + + artifact_path_text = artifact.get("path") + artifact_group = artifact.get("artifact_group") + filename = artifact.get("filename") + sidecar_path_text = artifact.get("sidecar") + expected_digest = artifact.get("sha256") + expected_size = artifact.get("size") + if not isinstance(artifact_path_text, str): + errors.append(f"artifacts[{index}].path must be a string") + continue + if artifact_path_text in seen_paths: + errors.append(f"{artifact_path_text}: duplicate artifact path") + seen_paths.add(artifact_path_text) + if not is_safe_relative_path(artifact_path_text): + errors.append(f"{artifact_path_text}: unsafe artifact path") + continue + artifact_relative = PurePosixPath(artifact_path_text) + + if not isinstance(artifact_group, str): + errors.append(f"{artifact_path_text}: artifact_group must be a string") + elif artifact_group != artifact_relative.parts[0]: + errors.append( + f"{artifact_path_text}: artifact_group {artifact_group!r} " + f"does not match path group {artifact_relative.parts[0]!r}" + ) + + if not isinstance(filename, str): + errors.append(f"{artifact_path_text}: filename must be a string") + elif filename != artifact_relative.name: + errors.append( + f"{artifact_path_text}: filename {filename!r} " + f"does not match path filename {artifact_relative.name!r}" + ) + + if not isinstance(sidecar_path_text, str): + errors.append(f"{artifact_path_text}: sidecar must be a string") + continue + if sidecar_path_text in seen_sidecars: + errors.append(f"{sidecar_path_text}: duplicate sidecar path") + seen_sidecars.add(sidecar_path_text) + if not is_safe_relative_path(sidecar_path_text): + errors.append(f"{sidecar_path_text}: unsafe sidecar path") + continue + + artifact_path = input_dir / artifact_path_text + sidecar_path = input_dir / sidecar_path_text + if not is_plain_file(artifact_path): + errors.append(f"{artifact_path_text}: missing artifact file") + continue + if not is_plain_file(sidecar_path): + errors.append(f"{sidecar_path_text}: missing sidecar file") + continue + actual_size = artifact_path.stat().st_size + if actual_size <= 0: + errors.append(f"{artifact_path_text}: empty artifacts are not allowed") + if sidecar_path != Path(f"{artifact_path}.sha256"): + errors.append( + f"{artifact_path_text}: sidecar path {sidecar_path_text!r} " + f"does not match sibling {artifact_path.name}.sha256" + ) + + actual_digest = digest_file(artifact_path) + if expected_digest != actual_digest: + errors.append( + f"{artifact_path_text}: digest mismatch " + f"{expected_digest!r} != {actual_digest}" + ) + if expected_size != actual_size: + errors.append( + f"{artifact_path_text}: size mismatch " + f"{expected_size!r} != {actual_size}" + ) + + try: + sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path) + except ValueError as error: + errors.append(f"{sidecar_path_text}: {error}") + continue + if sidecar_digest != actual_digest: + errors.append( + f"{sidecar_path_text}: sidecar digest mismatch " + f"{sidecar_digest} != {actual_digest}" + ) + if sidecar_filename != artifact_path.name: + errors.append( + f"{sidecar_path_text}: sidecar filename mismatch " + f"{sidecar_filename!r} != {artifact_path.name!r}" + ) + + manifest_resolved = manifest_path.resolve() + manifest_sidecar_resolved = Path(f"{manifest_path}.sha256").resolve() + actual_artifacts: set[str] = set() + actual_sidecars: set[str] = set() + for path in sorted(input_dir.rglob("*")): + if not path.is_file(): + continue + if path.is_symlink(): + relative = path.relative_to(input_dir).as_posix() + if path.suffix == ".sha256": + actual_sidecars.add(relative) + else: + actual_artifacts.add(relative) + continue + resolved = path.resolve() + if resolved == manifest_resolved or resolved == manifest_sidecar_resolved: + continue + relative = path.relative_to(input_dir).as_posix() + if path.suffix == ".sha256": + actual_sidecars.add(relative) + else: + actual_artifacts.add(relative) + + missing_from_manifest = sorted(actual_artifacts - seen_paths) + if missing_from_manifest: + errors.append( + "artifacts missing from release manifest: " + + ", ".join(missing_from_manifest) + ) + + orphan_sidecars = sorted(actual_sidecars - seen_sidecars) + if orphan_sidecars: + errors.append( + "sidecars missing from release manifest: " + ", ".join(orphan_sidecars) + ) + + if errors: + print( + "release manifest validation failed:\n" + + "\n".join(f" {error}" for error in errors), + file=sys.stderr, + ) + return 70 + + print(f"release manifest validated: {manifest_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_artifact_mirror_validator.sh b/scripts/verify_artifact_mirror_validator.sh new file mode 100755 index 0000000..6b4572a --- /dev/null +++ b/scripts/verify_artifact_mirror_validator.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-mirror-validator.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +"$ROOT/scripts/verify_assemble_artifact_mirror.sh" >/dev/null + +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +ENGINE_REVISION=engine123 +MIRROR_ROOT="$TMP_DIR/mirror" +mkdir -p "$MIRROR_ROOT/shorebird/$ENGINE_REVISION" +cat > "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" < "$MIRROR_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" + +while IFS= read -r -d '' mirror_file; do + if [[ "$mirror_file" == *.sha256 ]]; then + continue + fi + "$ROOT/scripts/write_sha256.sh" "$mirror_file" +done < <(find "$MIRROR_ROOT/shorebird" -type f -print0) + +"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$MIRROR_ROOT" >/dev/null + +UNSAFE_MIRROR_ROOT="$TMP_DIR/unsafe-mirror" +cp -R "$MIRROR_ROOT" "$UNSAFE_MIRROR_ROOT" +"$PYTHON_BIN" - "$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<'PY' +from pathlib import Path +import sys + +manifest_path = Path(sys.argv[1]) +manifest_path.write_text( + "\n".join( + [ + "flutter_engine_revision: 'base-engine'", + "storage_bucket: 'shorebird'", + "artifact_overrides:", + " - '../outside/artifacts.zip'", + ] + ) + + "\n", + encoding="utf-8", +) +PY +"$ROOT/scripts/write_sha256.sh" \ + "$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" \ + "$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml.sha256" +if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \ + "$UNSAFE_MIRROR_ROOT" >"$TMP_DIR/unsafe-mirror.log" 2>&1; then + echo "validate_artifact_mirror.py unexpectedly accepted an unsafe manifest override" >&2 + exit 70 +fi +grep -q "unsafe artifact override path" "$TMP_DIR/unsafe-mirror.log" + +SYMLINK_MIRROR_ROOT="$TMP_DIR/symlink-mirror" +cp -R "$MIRROR_ROOT" "$SYMLINK_MIRROR_ROOT" +rm "$SYMLINK_MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" +ln -s "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" \ + "$SYMLINK_MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" +if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \ + "$SYMLINK_MIRROR_ROOT" >"$TMP_DIR/symlink-mirror.log" 2>&1; then + echo "validate_artifact_mirror.py unexpectedly accepted a symlink artifact" >&2 + exit 70 +fi +grep -q "symlink entries are not allowed" "$TMP_DIR/symlink-mirror.log" + +EMPTY_OVERRIDE_ROOT="$TMP_DIR/empty-override-mirror" +cp -R "$MIRROR_ROOT" "$EMPTY_OVERRIDE_ROOT" +"$PYTHON_BIN" - "$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" <<'PY' +from pathlib import Path +import sys + +Path(sys.argv[1]).write_bytes(b"") +PY +"$ROOT/scripts/write_sha256.sh" \ + "$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" \ + "$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip.sha256" +if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \ + "$EMPTY_OVERRIDE_ROOT" >"$TMP_DIR/empty-override-mirror.log" 2>&1; then + echo "validate_artifact_mirror.py unexpectedly accepted an empty manifest override artifact" >&2 + exit 70 +fi +grep -q "artifact override is empty" "$TMP_DIR/empty-override-mirror.log" + +printf 'tampered\n' >> "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" +if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$MIRROR_ROOT" >/dev/null 2>&1; then + echo "validate_artifact_mirror.py unexpectedly accepted a stale sidecar" >&2 + exit 70 +fi + +echo "validate_artifact_mirror.py smoke test passed" diff --git a/scripts/verify_artifact_mirror_workflow_assembly.sh b/scripts/verify_artifact_mirror_workflow_assembly.sh new file mode 100755 index 0000000..96d5436 --- /dev/null +++ b/scripts/verify_artifact_mirror_workflow_assembly.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-artifact-job.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +ENGINE_REVISION=engine123 +DOWNLOADED="$TMP_DIR/downloaded-artifacts" +mkdir -p "$DOWNLOADED" + +write_artifact() { + local path="$1" + local content="$2" + mkdir -p "$(dirname "$path")" + printf '%s\n' "$content" > "$path" + "$ROOT/scripts/write_sha256.sh" "$path" +} + +write_zip() { + local zip_path="$1" + local entry_name="$2" + local content="$3" + mkdir -p "$(dirname "$zip_path")" + "$PYTHON_BIN" - "$zip_path" "$entry_name" "$content" <<'PY' +from pathlib import Path +import sys +import zipfile + +zip_path, entry_name, content = sys.argv[1:] +with zipfile.ZipFile(Path(zip_path), "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr(entry_name, content + "\n") +PY + "$ROOT/scripts/write_sha256.sh" "$zip_path" +} + +write_tgz() { + local archive_path="$1" + local staging_dir="$2" + local root_entry="$3" + mkdir -p "$(dirname "$archive_path")" + tar -C "$staging_dir" -czf "$archive_path" "$root_entry" + "$ROOT/scripts/write_sha256.sh" "$archive_path" +} + +for target in \ + cli-linux-x64/open-shorebird-cli-linux-x64.tar.gz \ + cli-macos-x64/open-shorebird-cli-macos-x64.tar.gz \ + cli-macos-arm64/open-shorebird-cli-macos-arm64.tar.gz \ + cli-windows-x64/open-shorebird-cli-windows-x64.tar.gz \ + shorebird-server-linux-amd64/shorebird-server-linux-amd64.tar.gz \ + shorebird-server-linux-arm64/shorebird-server-linux-arm64.tar.gz \ + shorebird-server-darwin-amd64/shorebird-server-darwin-amd64.tar.gz \ + shorebird-server-darwin-arm64/shorebird-server-darwin-arm64.tar.gz \ + shorebird-server-windows-amd64/shorebird-server-windows-amd64.tar.gz \ + custom-dart-sdk-linux-x64/custom-dart-sdk-linux-x64.tar.gz \ + custom-dart-sdk-macos-arm64/custom-dart-sdk-macos-arm64.tar.gz; do + write_artifact "$DOWNLOADED/$target" "$target" +done + +mkdir -p "$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION" +cat > "$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" </dev/null +tar -C "$TMP_DIR/artifacts" -czf "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" open-shorebird-artifact-mirror +mirror_extract_dir="$TMP_DIR/mirror-extract" +mkdir -p "$mirror_extract_dir" +"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" \ + "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" \ + "$mirror_extract_dir" +"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \ + "$mirror_extract_dir/open-shorebird-artifact-mirror" >/dev/null +"$ROOT/scripts/write_sha256.sh" "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" + +manifest_input="$TMP_DIR/manifest-input" +mkdir -p "$manifest_input" +cp -R "$DOWNLOADED"/. "$manifest_input/" +mkdir -p "$manifest_input/open-shorebird-artifact-mirror" +cp "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" "$manifest_input/open-shorebird-artifact-mirror/" +cp "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz.sha256" "$manifest_input/open-shorebird-artifact-mirror/" + +"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$manifest_input" \ + --github-sha test-sha \ + --require 'cli-linux-x64/*open-shorebird-cli-linux-x64.tar.gz' \ + --require 'cli-macos-x64/*open-shorebird-cli-macos-x64.tar.gz' \ + --require 'cli-macos-arm64/*open-shorebird-cli-macos-arm64.tar.gz' \ + --require 'cli-windows-x64/*open-shorebird-cli-windows-x64.tar.gz' \ + --require 'shorebird-server-linux-amd64/*shorebird-server-linux-amd64.tar.gz' \ + --require 'shorebird-server-linux-arm64/*shorebird-server-linux-arm64.tar.gz' \ + --require 'shorebird-server-darwin-amd64/*shorebird-server-darwin-amd64.tar.gz' \ + --require 'shorebird-server-darwin-arm64/*shorebird-server-darwin-arm64.tar.gz' \ + --require 'shorebird-server-windows-amd64/*shorebird-server-windows-amd64.tar.gz' \ + --require 'custom-dart-sdk-linux-x64/*custom-dart-sdk-linux-x64.tar.gz' \ + --require 'custom-dart-sdk-macos-arm64/*custom-dart-sdk-macos-arm64.tar.gz' \ + --require 'linux-engine-x64/*linux-engine-x64.tar.gz' \ + --require 'android-engine-arm64/*android-engine-arm64.tar.gz' \ + --require 'flutter-web-sdk/*flutter-web-sdk.tar.gz' \ + --require 'ios-interpreter-engine/*ios-interpreter-engine.tar.gz' \ + --require 'macos-engine-arm64/*macos-engine-arm64.tar.gz' \ + --require 'mirror-patch-linux-x64.zip/*patch-linux-x64.zip' \ + --require 'mirror-patch-darwin-x64.zip/*patch-darwin-x64.zip' \ + --require 'mirror-patch-darwin-arm64.zip/*patch-darwin-arm64.zip' \ + --require 'mirror-patch-windows-x64.zip/*patch-windows-x64.zip' \ + --require 'mirror-metadata/*artifacts_manifest.yaml' \ + --require 'open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz' \ + --output "$TMP_DIR/open-shorebird-release-manifest.json" + +"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$manifest_input" \ + "$TMP_DIR/open-shorebird-release-manifest.json" >/dev/null + +mkdir -p "$manifest_input/open-shorebird-release-manifest" +cp "$TMP_DIR/open-shorebird-release-manifest.json" \ + "$manifest_input/open-shorebird-release-manifest/" +"$ROOT/scripts/write_sha256.sh" \ + "$manifest_input/open-shorebird-release-manifest/open-shorebird-release-manifest.json" +"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha test-sha \ + "$manifest_input" >/dev/null +if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha wrong-sha \ + "$manifest_input" >"$TMP_DIR/wrong-download-sha.log" 2>&1; then + echo "unexpectedly accepted downloaded artifacts for the wrong github_sha" >&2 + exit 1 +fi +grep -q "github_sha is" "$TMP_DIR/wrong-download-sha.log" + +release_manifest_path="$manifest_input/open-shorebird-release-manifest/open-shorebird-release-manifest.json" +mirror_archive_path="$manifest_input/open-shorebird-artifact-mirror/open-shorebird-artifact-mirror.tar.gz" +printf '%064d open-shorebird-release-manifest.json\n' 0 > "$release_manifest_path.sha256" +if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha test-sha \ + "$manifest_input" >/dev/null 2>&1; then + echo "unexpectedly accepted a stale downloaded release manifest sidecar" >&2 + exit 1 +fi +"$ROOT/scripts/write_sha256.sh" "$release_manifest_path" + +printf '%064d open-shorebird-artifact-mirror.tar.gz\n' 0 > "$mirror_archive_path.sha256" +if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha test-sha \ + "$manifest_input" >/dev/null 2>&1; then + echo "unexpectedly accepted a stale downloaded mirror archive sidecar" >&2 + exit 1 +fi +"$ROOT/scripts/write_sha256.sh" "$mirror_archive_path" +"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha test-sha \ + "$manifest_input" >/dev/null + +unsafe_download="$TMP_DIR/unsafe-download" +cp -R "$manifest_input" "$unsafe_download" +unsafe_mirror_archive="$unsafe_download/open-shorebird-artifact-mirror/open-shorebird-artifact-mirror.tar.gz" +"$PYTHON_BIN" - "$unsafe_mirror_archive" <<'PY' +import io +import sys +import tarfile + +archive_path = sys.argv[1] +with tarfile.open(archive_path, "w:gz") as archive: + content = b"unsafe\n" + member = tarfile.TarInfo("../outside.txt") + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) +PY +"$ROOT/scripts/write_sha256.sh" "$unsafe_mirror_archive" +"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$unsafe_download" \ + --github-sha test-sha \ + --require 'open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz' \ + --output "$unsafe_download/open-shorebird-release-manifest/open-shorebird-release-manifest.json" +"$ROOT/scripts/write_sha256.sh" \ + "$unsafe_download/open-shorebird-release-manifest/open-shorebird-release-manifest.json" +if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha test-sha \ + "$unsafe_download" >"$TMP_DIR/unsafe-download.log" 2>&1; then + echo "unexpectedly accepted an unsafe downloaded mirror archive" >&2 + exit 1 +fi +grep -q "unsafe archive member path" "$TMP_DIR/unsafe-download.log" + +"$PYTHON_BIN" - "$TMP_DIR/open-shorebird-release-manifest.json" <<'PY' +import json +import sys + +manifest = json.load(open(sys.argv[1], encoding="utf-8")) +paths = {artifact["path"] for artifact in manifest["artifacts"]} +assert any(path.endswith("open-shorebird-artifact-mirror.tar.gz") for path in paths) +assert any(path.endswith("linux-engine-x64.tar.gz") for path in paths) +assert any(path.endswith("patch-windows-x64.zip") for path in paths) +assert any(path.endswith("artifacts_manifest.yaml") for path in paths) +PY + +echo "artifact-mirror workflow assembly smoke test passed" diff --git a/scripts/verify_assemble_artifact_mirror.sh b/scripts/verify_assemble_artifact_mirror.sh new file mode 100755 index 0000000..13ae2d4 --- /dev/null +++ b/scripts/verify_assemble_artifact_mirror.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-assemble.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +ENGINE_REVISION=engine123 +INPUT_DIR="$TMP_DIR/downloaded-artifacts" +OUTPUT_DIR="$TMP_DIR/mirror" +mkdir -p "$INPUT_DIR" + +mkdir -p "$INPUT_DIR/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION" +cat > "$INPUT_DIR/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" < "$engine_staging/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" +tar -C "$engine_staging" -czf "$INPUT_DIR/linux-engine-x64.tar.gz" linux-engine + +"$ROOT/scripts/assemble_artifact_mirror.sh" "$INPUT_DIR" "$OUTPUT_DIR" + +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml.sha256" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-linux-x64.zip.sha256" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-x64.zip" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-x64.zip.sha256" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-arm64.zip" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-arm64.zip.sha256" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-windows-x64.zip" +test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-windows-x64.zip.sha256" +test -f "$OUTPUT_DIR/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" +test -f "$OUTPUT_DIR/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip.sha256" + +CONFLICT_INPUT="$TMP_DIR/conflicting-artifacts" +mkdir -p "$CONFLICT_INPUT/conflict/artifacts/mirror/shorebird/$ENGINE_REVISION" +"$PYTHON_BIN" - "$CONFLICT_INPUT/conflict/artifacts/mirror/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" <<'PY' +from pathlib import Path +import sys +import zipfile + +with zipfile.ZipFile(Path(sys.argv[1]), "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("patch", "different-patch\n") +PY + +if "$ROOT/scripts/assemble_artifact_mirror.sh" "$CONFLICT_INPUT" "$OUTPUT_DIR" >/dev/null 2>&1; then + echo "assemble_artifact_mirror.sh unexpectedly allowed a conflicting mirror file" >&2 + exit 70 +fi + +UNSAFE_TAR_INPUT="$TMP_DIR/unsafe-tar" +UNSAFE_TAR_OUTPUT="$TMP_DIR/unsafe-tar-output" +mkdir -p "$UNSAFE_TAR_INPUT" +"$PYTHON_BIN" - "$UNSAFE_TAR_INPUT/unsafe-engine.tar.gz" <<'PY' +import io +import sys +import tarfile + +archive_path = sys.argv[1] +with tarfile.open(archive_path, "w:gz") as archive: + content = b"unsafe\n" + member = tarfile.TarInfo("../outside.txt") + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) +PY +if "$ROOT/scripts/assemble_artifact_mirror.sh" \ + "$UNSAFE_TAR_INPUT" \ + "$UNSAFE_TAR_OUTPUT" >"$TMP_DIR/unsafe-tar.log" 2>&1; then + echo "assemble_artifact_mirror.sh unexpectedly allowed an unsafe tar member" >&2 + exit 70 +fi +grep -q "unsafe archive member path" "$TMP_DIR/unsafe-tar.log" + +SYMLINK_TAR_INPUT="$TMP_DIR/symlink-tar" +SYMLINK_TAR_OUTPUT="$TMP_DIR/symlink-tar-output" +mkdir -p "$SYMLINK_TAR_INPUT" +"$PYTHON_BIN" - "$SYMLINK_TAR_INPUT/symlink-engine.tar.gz" <<'PY' +import sys +import tarfile + +archive_path = sys.argv[1] +with tarfile.open(archive_path, "w:gz") as archive: + member = tarfile.TarInfo("engine/mirror/shorebird/link") + member.type = tarfile.SYMTYPE + member.linkname = "/tmp/outside" + archive.addfile(member) +PY +if "$ROOT/scripts/assemble_artifact_mirror.sh" \ + "$SYMLINK_TAR_INPUT" \ + "$SYMLINK_TAR_OUTPUT" >"$TMP_DIR/symlink-tar.log" 2>&1; then + echo "assemble_artifact_mirror.sh unexpectedly allowed a tar symlink member" >&2 + exit 70 +fi +grep -q "unsupported archive member type" "$TMP_DIR/symlink-tar.log" + +DUPLICATE_TAR_INPUT="$TMP_DIR/duplicate-tar" +DUPLICATE_TAR_OUTPUT="$TMP_DIR/duplicate-tar-output" +mkdir -p "$DUPLICATE_TAR_INPUT" +"$PYTHON_BIN" - "$DUPLICATE_TAR_INPUT/duplicate-engine.tar.gz" <<'PY' +import io +import sys +import tarfile + +archive_path = sys.argv[1] +with tarfile.open(archive_path, "w:gz") as archive: + for content in (b"first\n", b"second\n"): + member = tarfile.TarInfo("engine/mirror/shorebird/duplicate.txt") + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) +PY +if "$ROOT/scripts/assemble_artifact_mirror.sh" \ + "$DUPLICATE_TAR_INPUT" \ + "$DUPLICATE_TAR_OUTPUT" >"$TMP_DIR/duplicate-tar.log" 2>&1; then + echo "assemble_artifact_mirror.sh unexpectedly allowed a duplicate tar member" >&2 + exit 70 +fi +grep -q "duplicate archive member path" "$TMP_DIR/duplicate-tar.log" + +BAD_ZIP_INPUT="$TMP_DIR/bad-zip" +BAD_ZIP_OUTPUT="$TMP_DIR/bad-zip-output" +mkdir -p "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION" +cat > "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" < "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" + +if "$ROOT/scripts/assemble_artifact_mirror.sh" "$BAD_ZIP_INPUT" "$BAD_ZIP_OUTPUT" >/dev/null 2>&1; then + echo "assemble_artifact_mirror.sh unexpectedly allowed an invalid patch zip" >&2 + exit 70 +fi + +MISSING_INPUT="$TMP_DIR/missing-override" +MISSING_OUTPUT="$TMP_DIR/missing-output" +mkdir -p "$MISSING_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION" +cat > "$MISSING_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" </dev/null 2>&1; then + echo "assemble_artifact_mirror.sh unexpectedly allowed a missing manifest override" >&2 + exit 70 +fi + +echo "assemble_artifact_mirror.sh smoke test passed" diff --git a/scripts/verify_ci_capacity.sh b/scripts/verify_ci_capacity.sh new file mode 100755 index 0000000..b936889 --- /dev/null +++ b/scripts/verify_ci_capacity.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +CI_MIN_FREE_DISK_GB=2 \ + CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \ + "$ROOT/scripts/check_ci_capacity.sh" >/dev/null + +if CI_MIN_FREE_DISK_GB=4 \ + CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \ + "$ROOT/scripts/check_ci_capacity.sh" >/dev/null 2>&1; then + echo "check_ci_capacity.sh unexpectedly accepted insufficient disk" >&2 + exit 70 +fi + +if CI_MIN_FREE_DISK_GB=not-a-number \ + CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \ + "$ROOT/scripts/check_ci_capacity.sh" >/dev/null 2>&1; then + echo "check_ci_capacity.sh unexpectedly accepted an invalid minimum" >&2 + exit 70 +fi + +CI_MIN_FREE_DISK_GB=0 "$ROOT/scripts/check_ci_capacity.sh" >/dev/null + +echo "check_ci_capacity.sh smoke test passed" diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb new file mode 100755 index 0000000..6ec42e6 --- /dev/null +++ b/scripts/verify_ci_workflow.rb @@ -0,0 +1,2124 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'tempfile' +require 'yaml' +require 'open3' +require 'pathname' + +args = ARGV.dup +require_tracked = false +require_clean = false +args.delete_if do |arg| + case arg + when '--require-tracked' + require_tracked = true + true + when '--require-clean' + require_clean = true + true + when '--require-upload-ready' + require_tracked = true + require_clean = true + true + else + false + end +end + +workflow_path = args.fetch(0) +repo_root = File.expand_path('../..', File.dirname(workflow_path)) +workflow_text = File.read(workflow_path) +workflow = YAML.load_file(workflow_path) +jobs = workflow.fetch('jobs') + +def fail!(message) + warn "workflow contract failed: #{message}" + exit 70 +end + +def assert!(condition, message) + fail!(message) unless condition +end + +def job_runs(job) + job.fetch('steps', []).each_with_object([]) do |step, runs| + runs << step['run'] if step.key?('run') + end +end + +def upload_names(job) + job.fetch('steps', []).each_with_object([]) do |step, names| + next unless step['uses'].to_s.start_with?('actions/upload-artifact@') + + names << step.fetch('with', {}).fetch('name') + end +end + +def step_uses(job, action) + job.fetch('steps', []).any? { |step| step['uses'].to_s == action } +end + +def matrix_include(job) + job.fetch('strategy').fetch('matrix').fetch('include') +end + +def repo_path(repo_root, path) + File.join(repo_root, path) +end + +def read_repo_file(repo_root, path) + File.read(repo_path(repo_root, path)) +end + +def capture_command(*argv) + stdout, _stderr, status = Open3.capture3(*argv) + [status.success?, stdout.strip] +end + +def git_toplevel_for(path) + dir = File.directory?(path) ? path : File.dirname(path) + success, stdout = capture_command('git', '-C', dir, 'rev-parse', '--show-toplevel') + return nil unless success + + File.expand_path(stdout) +end + +def relative_path(from, to) + Pathname.new(File.expand_path(to)).relative_path_from(Pathname.new(File.expand_path(from))).to_s +end + +def display_repo(repo_root, git_root) + return '.' if File.expand_path(git_root) == File.expand_path(repo_root) + + relative_path(repo_root, git_root) +end + +def required_git_repos(paths) + paths.each_with_object({}) do |path, repos| + git_root = git_toplevel_for(path) + repos[git_root] = true unless git_root.nil? + end.keys +end + +scripts = [] +jobs.each_value do |job| + job.fetch('steps', []).each do |step| + scripts << step['run'] if step.key?('run') + end +end + +scripts.each_with_index do |script, index| + Tempfile.create(["open-shorebird-ci-run-#{index}", '.sh']) do |file| + file.write(script.gsub(/\$\{\{[^}]+\}\}/, 'x')) + file.flush + system('bash', '-n', file.path, exception: true) + end +end + +required_files = %w[ + .gitmodules + README.md + docs/CI.md + docs/PLATFORM_TESTING.md + docs/REPOSITORIES.md + shorebird/README.md + shorebird/OPEN_SOURCE_REPLACEMENTS.md + shorebird/docs/account/api-keys/README.md + shorebird/docs/code-push/troubleshooting/README.md + shorebird/docs/getting-started/flutter-version/README.md + scripts/assemble_artifact_mirror.sh + scripts/android_runtime_patch_smoke.sh + scripts/bootstrap_linux.sh + scripts/bootstrap_macos.sh + scripts/check_ci_capacity.sh + scripts/free_ci_disk_linux.sh + scripts/linux_runtime_patch_smoke.sh + scripts/verify_open_infrastructure_defaults.sh + scripts/safe_extract_tar.py + scripts/sync_open_sources.sh + scripts/validate_artifact_mirror.py + scripts/validate_release_manifest.py + scripts/verify_assemble_artifact_mirror.sh + scripts/verify_artifact_mirror_validator.sh + scripts/verify_artifact_mirror_workflow_assembly.sh + scripts/verify_ci_workflow.rb + scripts/verify_ci_workflow.sh + scripts/verify_ci_capacity.sh + scripts/verify_dart_sdk_args.sh + scripts/verify_engine_args.sh + scripts/verify_hosted_full_sdk_build.sh + scripts/verify_ios_interpreter_route.sh + scripts/verify_ios_interpreter_route_validator.sh + scripts/verify_downloaded_release_artifacts.sh + scripts/verify_powershell_open_defaults.sh + scripts/verify_release_manifest.sh + scripts/verify_sync_open_sources.sh + scripts/verify_upload_readiness.sh + scripts/verify_write_sha256.sh + scripts/write_artifact_manifest.py + scripts/write_release_manifest.py + scripts/write_gclient.sh + scripts/write_sha256.py + scripts/write_sha256.sh + shorebird/bin/shorebird.ps1 + shorebird/third_party/flutter/bin/internal/shared.sh + shorebird/packages/shorebird_cli/lib/src/cache.dart + shorebird/packages/shorebird_cli/lib/src/commands/doctor_command.dart + shorebird/packages/shorebird_cli/lib/src/network_checker.dart + shorebird/packages/shorebird_cli/lib/src/shorebird_cli_command_runner.dart + shorebird/packages/shorebird_cli/lib/src/shorebird_documentation.dart + shorebird/packages/shorebird_cli/lib/src/shorebird_env.dart + shorebird/packages/shorebird_cli/lib/src/shorebird_flutter.dart + shorebird/packages/shorebird_cli/lib/src/shorebird_process.dart + shorebird/packages/shorebird_cli/lib/src/shorebird_web_console.dart + shorebird/packages/shorebird_code_push_client/lib/src/code_push_client.dart + shorebird/packages/shorebird_code_push_protocol/tool/gen.dart + shorebird/packages/shorebird_code_push_protocol/README.md + shorebird-server/internal/api/handlers/openapi.yaml + shorebird-server/internal/api/handlers/router.go + shorebird/packages/artifact_proxy/lib/src/artifact_manifest_client.dart + shorebird/packages/artifact_proxy/lib/src/artifact_proxy.dart + shorebird/packages/artifact_proxy/lib/config.dart + flutter/bin/internal/update_dart_sdk.ps1 + flutter/bin/internal/update_dart_sdk.sh + flutter/packages/flutter_tools/lib/src/cache.dart + flutter/packages/flutter_tools/lib/src/http_host_validator.dart + flutter/packages/flutter_tools/pubspec.yaml + flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt + flutter/packages/flutter_tools/gradle/aar_init_script.gradle + flutter/DEPS + flutter/engine/src/flutter/build/dart/BUILD.gn + flutter/dev/bots/post_process_docs.dart + flutter/dev/bots/unpublish_package.dart + flutter/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle + flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts + flutter/dev/tools/create_api_docs.dart + flutter/engine/src/flutter/build/zip_bundle.gni + flutter/engine/src/flutter/runtime/dart_isolate.cc + flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart + flutter/packages/shorebird_tests/test/shorebird_tests.dart + updater/library/src/config.rs +] +missing_files = required_files.reject { |path| File.file?(repo_path(repo_root, path)) } +assert!(missing_files.empty?, "missing CI support files: #{missing_files.join(', ')}") + +required_executables = required_files.select { |path| path.start_with?('scripts/') } +missing_executable_bits = required_executables.reject do |path| + File.executable?(repo_path(repo_root, path)) +end +assert!( + missing_executable_bits.empty?, + "missing executable bit on CI support scripts: #{missing_executable_bits.join(', ')}" +) + +required_git_paths = [workflow_path] + required_files.map { |path| repo_path(repo_root, path) } +if require_tracked + untracked_required_files = [] + required_git_paths.each do |path| + git_root = git_toplevel_for(path) + display_path = path.start_with?(repo_root) ? relative_path(repo_root, path) : path + if git_root.nil? + untracked_required_files << "#{display_path}: not inside a git checkout" + next + end + + path_in_repo = relative_path(git_root, path) + tracked, = capture_command( + 'git', + '-C', + git_root, + 'ls-files', + '--error-unmatch', + '--', + path_in_repo + ) + next if tracked + + untracked_required_files << "#{display_path}: not tracked in #{display_repo(repo_root, git_root)}" + end + assert!( + untracked_required_files.empty?, + "required upload files are not tracked: #{untracked_required_files.join(', ')}" + ) +end + +if require_clean + dirty_repos = [] + required_git_repos(required_git_paths).each do |git_root| + status_ok, status = capture_command('git', '-C', git_root, 'status', '--porcelain') + unless status_ok + dirty_repos << "#{display_repo(repo_root, git_root)}: unable to read git status" + next + end + next if status.empty? + + dirty_repos << "#{display_repo(repo_root, git_root)}: has uncommitted changes" + end + assert!( + dirty_repos.empty?, + "required upload repositories are dirty: #{dirty_repos.join(', ')}" + ) +end + +readme = read_repo_file(repo_root, 'README.md') +ci_doc = read_repo_file(repo_root, 'docs/CI.md') +platform_doc = read_repo_file(repo_root, 'docs/PLATFORM_TESTING.md') +repositories_doc = read_repo_file(repo_root, 'docs/REPOSITORIES.md') +open_replacements_doc = read_repo_file(repo_root, 'shorebird/OPEN_SOURCE_REPLACEMENTS.md') +shorebird_readme = read_repo_file(repo_root, 'shorebird/README.md') +gitmodules = read_repo_file(repo_root, '.gitmodules') +shorebird_flutter = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/shorebird_flutter.dart' +) +shorebird_env = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/shorebird_env.dart' +) +code_push_client = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_code_push_client/lib/src/code_push_client.dart' +) +code_push_protocol_gen = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_code_push_protocol/tool/gen.dart' +) +code_push_protocol_readme = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_code_push_protocol/README.md' +) +server_openapi = read_repo_file( + repo_root, + 'shorebird-server/internal/api/handlers/openapi.yaml' +) +server_router = read_repo_file( + repo_root, + 'shorebird-server/internal/api/handlers/router.go' +) +shorebird_cache = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/cache.dart' +) +shorebird_cli_command_runner = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/shorebird_cli_command_runner.dart' +) +shorebird_documentation = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/shorebird_documentation.dart' +) +doctor_command = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/commands/doctor_command.dart' +) +shorebird_process = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/shorebird_process.dart' +) +shorebird_powershell_launcher = read_repo_file(repo_root, 'shorebird/bin/shorebird.ps1') +shorebird_shell_launcher = read_repo_file( + repo_root, + 'shorebird/third_party/flutter/bin/internal/shared.sh' +) +network_checker = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/network_checker.dart' +) +shorebird_web_console = read_repo_file( + repo_root, + 'shorebird/packages/shorebird_cli/lib/src/shorebird_web_console.dart' +) +updater_config = read_repo_file( + repo_root, + 'updater/library/src/config.rs' +) +artifact_manifest_client = read_repo_file( + repo_root, + 'shorebird/packages/artifact_proxy/lib/src/artifact_manifest_client.dart' +) +artifact_proxy = read_repo_file( + repo_root, + 'shorebird/packages/artifact_proxy/lib/src/artifact_proxy.dart' +) +artifact_proxy_test = read_repo_file( + repo_root, + 'shorebird/packages/artifact_proxy/test/artifact_proxy_test.dart' +) +artifact_proxy_config = read_repo_file( + repo_root, + 'shorebird/packages/artifact_proxy/lib/config.dart' +) +write_gclient = read_repo_file(repo_root, 'scripts/write_gclient.sh') +free_ci_disk_linux = read_repo_file(repo_root, 'scripts/free_ci_disk_linux.sh') +check_ci_capacity = read_repo_file(repo_root, 'scripts/check_ci_capacity.sh') +verify_ci_capacity = read_repo_file(repo_root, 'scripts/verify_ci_capacity.sh') +flutter_tool_cache = read_repo_file( + repo_root, + 'flutter/packages/flutter_tools/lib/src/cache.dart' +) +flutter_tools_pubspec = read_repo_file( + repo_root, + 'flutter/packages/flutter_tools/pubspec.yaml' +) +flutter_http_host_validator = read_repo_file( + repo_root, + 'flutter/packages/flutter_tools/lib/src/http_host_validator.dart' +) +flutter_gradle_constants = read_repo_file( + repo_root, + 'flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt' +) +flutter_aar_init_script = read_repo_file( + repo_root, + 'flutter/packages/flutter_tools/gradle/aar_init_script.gradle' +) +flutter_deps = read_repo_file(repo_root, 'flutter/DEPS') +flutter_post_process_docs = read_repo_file(repo_root, 'flutter/dev/bots/post_process_docs.dart') +flutter_unpublish_package = read_repo_file(repo_root, 'flutter/dev/bots/unpublish_package.dart') +flutter_android_host_app_settings = read_repo_file( + repo_root, + 'flutter/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle' +) +flutter_android_host_app_kts_settings = read_repo_file( + repo_root, + 'flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts' +) +flutter_create_api_docs = read_repo_file(repo_root, 'flutter/dev/tools/create_api_docs.dart') +flutter_zip_bundle = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/build/zip_bundle.gni' +) +flutter_dart_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/build/dart/BUILD.gn' +) +flutter_engine_archives_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/build/archives/BUILD.gn' +) +flutter_linux_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/shell/platform/linux/BUILD.gn' +) +flutter_android_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/shell/platform/android/BUILD.gn' +) +flutter_web_sdk_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/web_sdk/BUILD.gn' +) +flutter_ios_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn' +) +flutter_macos_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn' +) +flutter_dart_isolate = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/runtime/dart_isolate.cc' +) +flutter_snapshot_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/lib/snapshot/BUILD.gn' +) +flutter_web_ui_copy_artifacts = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart' +) +flutter_shorebird_tests = read_repo_file( + repo_root, + 'flutter/packages/shorebird_tests/test/shorebird_tests.dart' +) +flutter_update_dart_sdk_ps1 = read_repo_file( + repo_root, + 'flutter/bin/internal/update_dart_sdk.ps1' +) +flutter_update_dart_sdk_sh = read_repo_file( + repo_root, + 'flutter/bin/internal/update_dart_sdk.sh' +) +generated_artifact_manifest = IO.popen( + [ + 'python3', + repo_path(repo_root, 'scripts/write_artifact_manifest.py'), + '--flutter-engine-revision', + 'flutter-base-revision', + ], + &:read +) +assert!($?.success?, 'artifact manifest helper must run successfully') +assert!( + generated_artifact_manifest.include?("flutter_engine_revision: 'flutter-base-revision'") && + generated_artifact_manifest.include?("storage_bucket: 'shorebird'") && + generated_artifact_manifest.include?('flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip') && + generated_artifact_manifest.include?('flutter_infra_release/flutter/$engine/linux-x64-release/artifacts.zip') && + generated_artifact_manifest.include?('flutter_infra_release/flutter/$engine/ios-release/artifacts.zip') && + generated_artifact_manifest.include?('flutter_infra_release/flutter/$engine/flutter-web-sdk.zip') && + generated_artifact_manifest.include?('flutter_infra_release/flutter/$engine/darwin-arm64-release/FlutterMacOS.framework.zip'), + 'artifact manifest helper must emit open mirror overrides for CI-produced engine artifacts' +) +generated_manifest_yaml = YAML.safe_load(generated_artifact_manifest) +generated_artifact_overrides = generated_manifest_yaml.fetch('artifact_overrides') +missing_mirror_paths = generated_artifact_overrides.map do |artifact_path| + "mirror/shorebird/#{artifact_path.gsub('$engine', '${engine_revision}')}" +end.reject { |mirror_path| workflow_text.include?(mirror_path) } +assert!( + missing_mirror_paths.empty?, + "workflow engine artifacts must package every generated mirror override: #{missing_mirror_paths.join(', ')}" +) +assert!( + workflow_text.scan('test -f "$mirror_root').length >= generated_artifact_overrides.length, + 'workflow engine artifact packaging must verify mirror-ready files exist before uploading' +) +closed_runtime_endpoint_patterns = %w[ + download.shorebird.dev + api.shorebird.dev + auth.shorebird.dev + console.shorebird.dev + cdn.shorebird.cloud +] +runtime_endpoint_matches = Dir[ + repo_path(repo_root, 'shorebird/packages/shorebird_cli/lib/**/*.dart'), + repo_path(repo_root, 'shorebird/packages/shorebird_code_push_client/lib/**/*.dart'), + repo_path(repo_root, 'shorebird/packages/artifact_proxy/lib/**/*.dart'), + repo_path(repo_root, 'updater/library/src/**/*.rs'), + repo_path(repo_root, 'flutter/packages/flutter_tools/lib/src/cache.dart'), + repo_path(repo_root, 'flutter/packages/flutter_tools/lib/src/http_host_validator.dart'), + repo_path(repo_root, 'flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt'), + repo_path(repo_root, 'flutter/packages/flutter_tools/gradle/aar_init_script.gradle'), + repo_path(repo_root, 'flutter/DEPS'), + repo_path(repo_root, 'flutter/dev/bots/post_process_docs.dart'), + repo_path(repo_root, 'flutter/dev/bots/unpublish_package.dart'), + repo_path(repo_root, 'flutter/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle'), + repo_path(repo_root, 'flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts'), + repo_path(repo_root, 'flutter/dev/tools/create_api_docs.dart'), + repo_path(repo_root, 'flutter/bin/internal/update_dart_sdk.ps1'), + repo_path(repo_root, 'flutter/bin/internal/update_dart_sdk.sh'), + repo_path(repo_root, 'flutter/engine/src/flutter/build/zip_bundle.gni'), + repo_path(repo_root, 'flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart'), + repo_path(repo_root, 'flutter/packages/shorebird_tests/test/shorebird_tests.dart') +].flat_map do |path| + text = File.read(path) + closed_runtime_endpoint_patterns.each_with_object([]) do |pattern, matches| + matches << "#{path.delete_prefix("#{repo_root}/")}: #{pattern}" if text.include?(pattern) + end +end +assert!( + runtime_endpoint_matches.empty?, + "runtime code must not hard-code official hosted endpoints: #{runtime_endpoint_matches.join(', ')}" +) +official_support_link_patterns = %w[ + docs.shorebird.dev + app.codecov.io/gh/shorebirdtech/shorebird + codecov.io/gh/shorebirdtech/shorebird + discord.gg/shorebird + handbook.shorebird.dev + producthunt.com/posts/shorebird-code-push + shorebird.dev/privacy + contact@shorebird.dev + github.com/shorebirdtech/shorebird + github.com/shorebirdtech/updater + github.com/shorebirdtech/flutter + git@github.com:shorebirdtech/shorebird.git +] +official_support_link_matches = Dir[ + repo_path(repo_root, 'shorebird/README.md'), + repo_path(repo_root, 'shorebird/bin/shorebird.ps1'), + repo_path(repo_root, 'shorebird/third_party/flutter/bin/internal/shared.sh'), + repo_path(repo_root, 'shorebird/packages/shorebird_cli/lib/src/**/*.dart'), + repo_path(repo_root, 'shorebird/packages/shorebird_code_push_client/lib/**/*.dart'), + repo_path(repo_root, 'shorebird/packages/artifact_proxy/lib/**/*.dart'), + repo_path(repo_root, 'updater/library/src/**/*.rs'), + repo_path(repo_root, 'flutter/packages/flutter_tools/pubspec.yaml'), + repo_path(repo_root, 'flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart') +].flat_map do |path| + text = File.read(path) + official_support_link_patterns.each_with_object([]) do |pattern, matches| + matches << "#{path.delete_prefix("#{repo_root}/")}: #{pattern}" if text.include?(pattern) + end +end +assert!( + official_support_link_matches.empty?, + "shorebird CLI source must direct users to open docs/issues, not official Shorebird support links: #{official_support_link_matches.join(', ')}" +) +official_package_metadata_matches = Dir[ + repo_path(repo_root, 'shorebird/packages/*/pubspec.yaml') +].flat_map do |path| + text = File.read(path) + [ + 'github.com/shorebirdtech/shorebird', + 'homepage: https://shorebird.dev', + 'repository: https://shorebird.dev', + ].each_with_object([]) do |pattern, matches| + matches << "#{path.delete_prefix("#{repo_root}/")}: #{pattern}" if text.include?(pattern) + end +end +assert!( + official_package_metadata_matches.empty?, + "package metadata must point at the open workspace, not official Shorebird metadata: #{official_package_metadata_matches.join(', ')}" +) +expected_submodules = { + 'dart-sdk-new' => ['https://git.tonycloud.org/dart-lang/sdk.git', 'tonycloud/dev'], + 'depot_tools' => ['https://chromium.googlesource.com/chromium/tools/depot_tools.git', 'main'], + 'flutter' => ['https://git.tonycloud.org/flutter/flutter.git', 'shorebird/dev'], + 'shorebird' => ['https://git.tonycloud.org/flutter/shorebird.git', 'main'], + 'shorebird-server' => ['https://git.tonycloud.org/flutter/shorebird-server.git', 'main'], + 'updater' => ['https://git.tonycloud.org/flutter/shorebird-updater.git', 'main'], +} +expected_submodules.each do |path, (url, branch)| + assert!( + gitmodules.include?("[submodule \"#{path}\"]") && + gitmodules.include?("path = #{path}") && + gitmodules.include?("url = #{url}") && + gitmodules.include?("branch = #{branch}"), + ".gitmodules must pin open HTTPS submodule #{path} to #{url} on #{branch}" + ) + assert!( + repositories_doc.include?("| `#{path}` | `#{url}` | `#{branch}` |"), + "docs/REPOSITORIES.md must document submodule #{path}" + ) +end +forbidden_gitmodule_fragments = [ + 'git@', + 'file://', + '/Users/', + '../', + 'github.com/shorebirdtech/', + 'github.com/shorebirdtech', +] +forbidden_gitmodule_fragments.each do |fragment| + assert!( + !gitmodules.include?(fragment), + ".gitmodules must not contain non-open or non-portable submodule URL fragment #{fragment}" + ) +end +assert!( + write_gclient.include?('"url": "https://git.tonycloud.org/dart-lang/sdk.git"') && + write_gclient.include?('"url": "https://git.tonycloud.org/flutter/flutter.git"') && + !write_gclient.include?('github.com/shorebirdtech'), + 'gclient generation must use the open Dart/Flutter forks and avoid official Shorebird remotes' +) +{ + 'README.md' => [ + '.github/workflows/open-shorebird-ci.yml', + 'full_sdk_build=true', + 'run_runtime_smokes=true', + 'SHOREBIRD_FLUTTER_GIT_URL', + 'SHOREBIRD_FLUTTER_STORAGE_BASE_URL', + 'scripts/assemble_artifact_mirror.sh', + 'scripts/verify_upload_readiness.sh', + 'open-shorebird-artifact-mirror', + 'open-shorebird-release-manifest', + 'http://localhost:8080', + 'http://localhost:8080/artifacts', + 'http://localhost:8080/download.flutter.io', + 'DART_DYNAMIC_MODULES', + 'artifacts_manifest.yaml', + ], + 'docs/CI.md' => [ + 'custom-dart-sdk-linux-x64', + 'custom-dart-sdk-macos-arm64', + 'ios-interpreter-engine', + 'scripts/write_sha256.sh', + 'scripts/verify_upload_readiness.sh', + 'SHOREBIRD_FLUTTER_GIT_URL', + 'SHOREBIRD_FLUTTER_STORAGE_BASE_URL', + 'scripts/assemble_artifact_mirror.sh', + 'open-shorebird-artifact-mirror', + 'open-shorebird-release-manifest', + 'http://localhost:8080', + 'http://localhost:8080/artifacts', + 'http://localhost:8080/download.flutter.io', + 'http://localhost:8080/openapi.yaml', + 'mirror-metadata', + 'artifacts_manifest.yaml', + 'verify_artifact_mirror_workflow_assembly.sh', + ], + 'docs/PLATFORM_TESTING.md' => [ + 'license:pro', + 'DART_DYNAMIC_MODULES=false', + 'real iPad', + '/openapi.yaml', + ], + 'docs/REPOSITORIES.md' => [ + 'https://git.tonycloud.org/dart-lang/sdk.git', + 'https://git.tonycloud.org/flutter/flutter.git', + 'https://git.tonycloud.org/flutter/shorebird.git', + 'https://git.tonycloud.org/flutter/shorebird-server.git', + 'https://git.tonycloud.org/flutter/shorebird-updater.git', + 'Only top-level project repositories are submodules', + ], + 'shorebird/OPEN_SOURCE_REPLACEMENTS.md' => [ + 'Local forks/submodules are the source of truth', + './scripts/sync_open_sources.sh', + 'http://localhost:8080', + 'http://localhost:8080/openapi.yaml', + 'http://localhost:8080/artifacts', + 'http://localhost:8080/download.flutter.io', + 'assemble_artifact_mirror.sh', + 'open-shorebird-artifact-mirror', + 'open-shorebird-release-manifest', + 'DART_DYNAMIC_MODULES=false', + 'verify_ci_workflow.sh', + ], + 'shorebird/README.md' => [ + 'OPEN_SOURCE_REPLACEMENTS.md', + '.github/workflows/open-shorebird-ci.yml', + 'self-hosted services', + 'custom Dart SDK', + ], + 'shorebird/docs/account/api-keys/README.md' => [ + 'SHOREBIRD_TOKEN', + 'http://localhost:8080/auth', + ], + 'shorebird/docs/code-push/troubleshooting/README.md' => [ + 'SHOREBIRD_ARTIFACT_BASE_URL', + 'SHOREBIRD_FLUTTER_STORAGE_BASE_URL', + 'Asset changes are not part of a Dart code patch', + ], + 'shorebird/docs/getting-started/flutter-version/README.md' => [ + 'https://git.tonycloud.org/flutter/flutter.git', + 'http://localhost:8080/artifacts', + ], +}.each do |path, required_texts| + text = { + 'README.md' => readme, + 'docs/CI.md' => ci_doc, + 'docs/PLATFORM_TESTING.md' => platform_doc, + 'docs/REPOSITORIES.md' => repositories_doc, + 'shorebird/README.md' => shorebird_readme, + 'shorebird/OPEN_SOURCE_REPLACEMENTS.md' => open_replacements_doc, + }.fetch(path) { read_repo_file(repo_root, path) } + missing_texts = required_texts.reject { |required_text| text.include?(required_text) } + assert!(missing_texts.empty?, "#{path} missing required text: #{missing_texts.join(', ')}") +end + +assert!( + shorebird_flutter.include?('SHOREBIRD_FLUTTER_GIT_URL') && + shorebird_flutter.include?('defaultFlutterGitUrl') && + shorebird_flutter.include?('https://git.tonycloud.org/flutter/flutter.git'), + 'shorebird Flutter installer must default to the open fork and allow SHOREBIRD_FLUTTER_GIT_URL override' +) +assert!( + shorebird_documentation.include?('openShorebirdRepositoryUrl') && + shorebird_documentation.include?('https://git.tonycloud.org/flutter/shorebird') && + shorebird_documentation.include?(%q{openShorebirdIssueUrl = '$openShorebirdRepositoryUrl/issues/new'}) && + shorebird_documentation.include?(%q{docsUrl = '$openShorebirdRepositoryUrl/src/branch/main/docs'}) && + !shorebird_documentation.include?('docs.shorebird.dev') && + !shorebird_documentation.include?('github.com/shorebirdtech/shorebird'), + 'shorebird documentation links must point to the open repository docs and issue tracker' +) +{ + 'shorebird_cli_command_runner.dart' => shorebird_cli_command_runner, + 'doctor_command.dart' => doctor_command, +}.each do |path, text| + assert!( + text.include?('https://git.tonycloud.org/flutter/shorebird.git') && + !text.include?('git@github.com:shorebirdtech/shorebird.git'), + "#{path} must print the open Shorebird fork in user-visible version banners" + ) +end +assert!( + !shorebird_flutter.include?('github.com/shorebirdtech/flutter.git'), + 'shorebird Flutter installer must not clone the closed official Shorebird Flutter fork' +) +assert!( + shorebird_env.include?("defaultHostedUrl = 'http://localhost:8080'") && + shorebird_env.include?("defaultAuthServiceUrl = '\$defaultHostedUrl/auth'") && + shorebird_env.include?("defaultJwtIssuer = 'shorebird-auth'"), + 'shorebird environment must default to the open self-hosted server' +) +assert!( + !shorebird_env.include?('https://auth.shorebird.dev'), + 'shorebird environment must not default to the official hosted auth service' +) +assert!( + code_push_client.include?("defaultHostedUri = Uri.parse('http://localhost:8080')") && + !code_push_client.include?("Uri.https('api.shorebird.dev')"), + 'code push client must default to the open self-hosted server, not api.shorebird.dev' +) +assert!( + code_push_protocol_gen.include?('http://localhost:8080/openapi.yaml') && + code_push_protocol_gen.include?('../shorebird-server/internal/api/handlers/openapi.yaml') && + code_push_protocol_gen.include?('Do not regenerate this package from Shorebird') && + !code_push_protocol_gen.include?('api.shorebird.dev') && + code_push_protocol_readme.include?('http://localhost:8080/openapi.yaml') && + code_push_protocol_readme.include?('../shorebird-server/internal/api/handlers/openapi.yaml') && + code_push_protocol_readme.include?("Shorebird's hosted API") && + !code_push_protocol_readme.include?('api.shorebird.dev'), + 'code push protocol regeneration docs must require an open/self-hosted spec, not the hosted Shorebird API' +) +assert!( + server_openapi.include?('openapi: 3.1.0') && + server_openapi.include?('/openapi.yaml:') && + server_openapi.include?('/api/v1/openapi.yaml:') && + server_openapi.include?('/api/v1/patches/check:'), + 'self-hosted server must include a checked OpenAPI contract for open clients' +) +assert!( + server_router.include?('go:embed openapi.yaml') && + server_router.include?('serveOpenAPISpec') && + server_router.include?('r.Get("/openapi.yaml", serveOpenAPISpec)') && + server_router.include?('r.Get("/api/v1/openapi.yaml", serveOpenAPISpec)'), + 'self-hosted server must serve the checked OpenAPI contract' +) +assert!( + shorebird_cache.include?('SHOREBIRD_ENABLE_LEGACY_AOT_TOOLS') && + shorebird_cache.include?('legacyAotToolsEnabled') && + shorebird_cache.include?('registerArtifact(AotToolsArtifact'), + 'cache must gate legacy aot-tools behind an explicit environment variable' +) +assert!( + shorebird_cache.include?("defaultArtifactBaseUrl = 'http://localhost:8080/artifacts'") && + !shorebird_cache.include?('https://storage.googleapis.com') && + !shorebird_cache.include?('download.shorebird.dev'), + 'cache must default artifact downloads to the open local mirror, not Shorebird hosted storage' +) +assert!( + shorebird_process.include?('defaultFlutterStorageBaseUrl') && + shorebird_process.include?('${ShorebirdEnv.defaultHostedUrl}/download.flutter.io') && + !shorebird_process.include?('https://download.shorebird.dev'), + 'Flutter process environment must default to the open local Flutter artifact mirror' +) +{ + 'shorebird/bin/shorebird.ps1' => shorebird_powershell_launcher, + 'shorebird/third_party/flutter/bin/internal/shared.sh' => shorebird_shell_launcher, +}.each do |path, text| + assert!( + text.include?('SHOREBIRD_FLUTTER_GIT_URL') && + text.include?('https://git.tonycloud.org/flutter/flutter.git') && + text.include?('SHOREBIRD_FLUTTER_STORAGE_BASE_URL') && + text.include?('FLUTTER_STORAGE_BASE_URL') && + text.include?('http://localhost:8080/download.flutter.io') && + !text.include?('github.com/shorebirdtech/flutter.git') && + !text.include?('https://download.shorebird.dev'), + "#{path} must bootstrap from open Flutter/artifact defaults with environment overrides" + ) +end +assert!( + shorebird_shell_launcher.include?('https://git.tonycloud.org/flutter/shorebird') && + !shorebird_shell_launcher.include?('github.com/shorebirdtech/shorebird'), + 'shell launcher missing-clone guidance must point at the open Shorebird repository' +) +assert!( + network_checker.include?('shorebirdEnv.hostedUri') && + network_checker.include?('shorebirdEnv.authServiceUri') && + !network_checker.include?('api.shorebird.dev') && + !network_checker.include?('console.shorebird.dev') && + !network_checker.include?('cdn.shorebird.cloud'), + 'network checker must inspect self-hosted endpoints instead of fixed Shorebird hosted URLs' +) +assert!( + shorebird_web_console.include?('shorebirdEnv.hostedUri') && + !shorebird_web_console.include?('console.shorebird.dev'), + 'web console links must use the configured self-hosted URL' +) +assert!( + updater_config.include?('const DEFAULT_BASE_URL: &str = "http://localhost:8080";') && + !updater_config.include?('https://api.shorebird.dev'), + 'updater runtime must default to the open self-hosted server' +) +assert!( + artifact_manifest_client.include?('defaultManifestBaseUri') && + artifact_manifest_client.include?('http://localhost:8080/artifacts') && + !artifact_manifest_client.include?('download.shorebird.dev'), + 'artifact manifest client must default to the open artifact mirror' +) +assert!( + artifact_proxy.include?('defaultShorebirdArtifactBaseUri') && + artifact_proxy.include?('http://localhost:8080/artifacts') && + !artifact_proxy.include?('download.shorebird.dev') && + !artifact_proxy.include?('docs.shorebird.dev') && + !artifact_proxy.include?('shorebird.dev/contact'), + 'artifact proxy must default Shorebird-specific redirects to the open artifact mirror' +) +assert!( + artifact_proxy_config.include?('linux-x64-release\/artifacts\.zip') && + artifact_proxy_config.include?('ios-release\/artifacts\.zip') && + artifact_proxy_config.include?('flutter-web-sdk\.zip') && + artifact_proxy_config.include?('flutter_patched_sdk_product\.zip') && + artifact_proxy_config.include?('darwin-arm64-release\/FlutterMacOS\.framework\.zip'), + 'artifact proxy must recognize the generated artifact paths produced by open CI' +) +generated_artifact_overrides.each do |artifact_path| + expected_proxy_pattern = artifact_path + .gsub('/', '\\/') + .gsub('.', '\\.') + .gsub('$engine', '(.*)') + assert!( + artifact_proxy_config.include?(expected_proxy_pattern), + "artifact proxy config must recognize generated override #{artifact_path}" + ) + assert!( + artifact_proxy_test.include?(artifact_path), + "artifact proxy tests must cover generated override #{artifact_path}" + ) +end +assert!( + artifact_proxy_test.include?('ios-release/artifacts.zip') && + artifact_proxy_test.include?('flutter-web-sdk.zip') && + artifact_proxy_test.include?('flutter_patched_sdk_product.zip'), + 'artifact proxy tests must cover iOS, web, and patched SDK mirror redirects' +) +assert!( + flutter_tool_cache.include?("kOpenFlutterStorageUrl = 'http://localhost:8080/download.flutter.io'") && + !flutter_tool_cache.include?('download.shorebird.dev') && + !flutter_tool_cache.include?('kShorebirdStorageUrl'), + 'Flutter tool cache must default engine artifact downloads to the open local Flutter mirror' +) +assert!( + flutter_http_host_validator.include?("kCloudHost = 'http://localhost:8080/download.flutter.io/'") && + !flutter_http_host_validator.include?('download.shorebird.dev'), + 'Flutter doctor network validator must check the open local Flutter mirror by default' +) +{ + 'flutter/bin/internal/update_dart_sdk.ps1' => flutter_update_dart_sdk_ps1, + 'flutter/bin/internal/update_dart_sdk.sh' => flutter_update_dart_sdk_sh, +}.each do |path, text| + assert!( + text.include?('FLUTTER_STORAGE_BASE_URL') && + text.include?('http://localhost:8080/download.flutter.io') && + !text.include?('download.shorebird.dev'), + "#{path} must download Dart SDK archives from FLUTTER_STORAGE_BASE_URL with the open local Flutter mirror fallback" + ) +end +assert!( + flutter_gradle_constants.include?('DEFAULT_MAVEN_HOST = "http://localhost:8080/download.flutter.io"') && + !flutter_gradle_constants.include?('download.shorebird.dev') && + flutter_aar_init_script.include?('?: "http://localhost:8080/download.flutter.io"') && + !flutter_aar_init_script.include?('download.shorebird.dev'), + 'Flutter Gradle plugin defaults must use the open local Flutter mirror' +) +assert!( + flutter_deps.include?('"dart_sdk_git": "https://git.tonycloud.org/dart-lang/sdk.git"') && + flutter_deps.include?('"updater_git": "https://git.tonycloud.org/flutter/shorebird-updater.git"') && + !flutter_deps.include?('git@github.com:shorebirdtech/dart-sdk.git') && + !flutter_deps.include?('github.com/shorebirdtech/updater.git') && + !flutter_deps.include?('shorebird-dart-sdk-prebuilt') && + !flutter_deps.include?('shorebirdtech/_build_engine'), + 'Flutter DEPS must point Dart/updater dependencies at open remotes and avoid Shorebird private prebuilt buckets' +) +assert!( + flutter_android_host_app_settings.include?('System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "http://localhost:8080"') && + flutter_android_host_app_settings.include?('$flutterStorageUrl/download.flutter.io') && + !flutter_android_host_app_settings.include?('download.shorebird.dev') && + flutter_android_host_app_kts_settings.include?('System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "http://localhost:8080"') && + flutter_android_host_app_kts_settings.include?('$flutterStorageUrl/download.flutter.io') && + !flutter_android_host_app_kts_settings.include?('download.shorebird.dev'), + 'Flutter Android host-app integration fixtures must default to the open local Flutter mirror' +) +assert!( + flutter_create_api_docs.include?("Platform.environment['FLUTTER_STORAGE_BASE_URL']") && + flutter_create_api_docs.include?('http://localhost:8080/download.flutter.io') && + !flutter_create_api_docs.include?('download.shorebird.dev') && + flutter_post_process_docs.include?("Platform.environment['FLUTTER_STORAGE_BASE_URL']") && + flutter_post_process_docs.include?('http://localhost:8080/download.flutter.io') && + !flutter_post_process_docs.include?('download.shorebird.dev') && + flutter_unpublish_package.include?('http://localhost:8080/download.flutter.io/flutter_infra_release') && + !flutter_unpublish_package.include?('download.shorebird.dev') && + flutter_shorebird_tests.include?("'FLUTTER_STORAGE_BASE_URL': 'http://localhost:8080/download.flutter.io'") && + !flutter_shorebird_tests.include?('download.shorebird.dev'), + 'Flutter docs/test artifact paths must default to the open local Flutter mirror' +) +assert!( + flutter_zip_bundle.include?('http://localhost:8080/download.flutter.io/flutter_infra_release/flutter/$engine_version/sky_engine.zip') && + !flutter_zip_bundle.include?('download.shorebird.dev'), + 'engine zip bundle metadata must use the open local Flutter mirror' +) +assert!( + flutter_dart_build.include?('import("$dart_src/build/dart/copy_tree.gni")') && + flutter_dart_build.include?('copy_tree("copy_dart_sdk")') && + flutter_dart_build.include?('source = prebuilt_dart_sdk') && + flutter_dart_build.include?('dest = "$root_out_dir/dart-sdk"') && + !flutter_dart_build.include?('copy_trees('), + 'engine Dart SDK BUILD.gn must use the Dart SDK copy_tree template available in this checkout' +) +assert!( + flutter_dart_isolate.include?('#include "flutter/shell/common/shorebird/updater.h"') && + flutter_dart_isolate.include?('std::string GetYamlValue(') && + flutter_dart_isolate.include?('return UnquoteYamlValue(GetYamlValue(config.yaml_config, key));') && + !flutter_dart_isolate.include?('#include "flutter/shell/common/shorebird/shorebird.h"'), + 'dart_isolate.cc must avoid depending on the full Shorebird wrapper target so GN header checks pass' +) +assert!( + flutter_web_ui_copy_artifacts.include?("io.Platform.environment['FLUTTER_STORAGE_BASE_URL']") && + flutter_web_ui_copy_artifacts.include?('http://localhost:8080/download.flutter.io') && + flutter_web_ui_copy_artifacts.include?('storageBaseUri.resolve') && + !flutter_web_ui_copy_artifacts.include?('download.shorebird.dev'), + 'web UI artifact downloads must use FLUTTER_STORAGE_BASE_URL with the open local Flutter mirror fallback' +) +assert!( + flutter_tools_pubspec.include?('https://git.tonycloud.org/flutter/shorebird.git') && + !flutter_tools_pubspec.include?('https://github.com/shorebirdtech/shorebird.git'), + 'Flutter tools pubspec must fetch Shorebird build-trace code from the open fork' +) +assert!( + flutter_engine_archives_build.include?('zip_bundle("artifacts")') && + flutter_engine_archives_build.include?('output = "$prefix/artifacts.zip"') && + flutter_engine_archives_build.include?('zip_bundle("flutter_patched_sdk")') && + flutter_engine_archives_build.include?('output = "flutter_patched_sdk${file_suffix}.zip"'), + 'engine archive BUILD.gn must still define artifacts.zip and flutter_patched_sdk_product.zip targets used by CI' +) +assert!( + flutter_linux_build.include?('zip_bundle("flutter_gtk")') && + flutter_linux_build.include?('output = "${prefix}${full_target_platform_name}-flutter-gtk.zip"'), + 'Linux engine BUILD.gn must still define the flutter_gtk archive used by CI' +) +assert!( + flutter_android_build.include?('zip_bundle("android_symbols")') && + flutter_android_build.include?('output = "$android_zip_archive_dir/symbols.zip"') && + flutter_android_build.include?('zip_bundle("flutter_jar_zip")') && + flutter_android_build.include?('output = "$android_zip_archive_dir/artifacts.zip"') && + flutter_android_build.include?('zip_bundle("gen_snapshot")') && + flutter_android_build.include?('zip_bundle("analyze_snapshot")'), + 'Android engine BUILD.gn must still define symbols/artifacts/gen_snapshot/analyze_snapshot archives used by CI' +) +assert!( + flutter_web_sdk_build.include?('zip_bundle_from_file("flutter_web_sdk_archive")') && + flutter_web_sdk_build.include?('output = "flutter-web-sdk.zip"'), + 'web SDK BUILD.gn must still define flutter-web-sdk.zip used by CI' +) +assert!( + flutter_ios_build.include?('action("flutter_framework")') && + flutter_ios_build.include?('outputs = [ "$root_out_dir/Flutter.xcframework" ]'), + 'iOS engine BUILD.gn must still define Flutter.xcframework used by CI' +) +assert!( + flutter_macos_build.include?('zip_bundle("zip_macos_flutter_framework")') && + flutter_macos_build.include?('output = "${prefix}FlutterMacOS.framework.zip"'), + 'macOS engine BUILD.gn must still define FlutterMacOS.framework.zip used by CI' +) +assert!( + flutter_snapshot_build.include?('action("create_macos_gen_snapshots")') && + flutter_snapshot_build.include?('action("create_macos_analyze_snapshots")'), + 'snapshot BUILD.gn must still define macOS/iOS host snapshotter targets used by CI' +) +assert!( + !shorebird_cache.include?('aot-tools.dill artifact.\n/// Used for linking and generating optimized AOT snapshots.'), + 'cache comments must not describe legacy closed aot-tools as a default optimized AOT path' +) +platform_test_common = read_repo_file(repo_root, 'scripts/platform_test_common.sh') +bootstrap_linux = read_repo_file(repo_root, 'scripts/bootstrap_linux.sh') +bootstrap_macos = read_repo_file(repo_root, 'scripts/bootstrap_macos.sh') +sync_open_sources = read_repo_file(repo_root, 'scripts/sync_open_sources.sh') +verify_sync_open_sources = read_repo_file( + repo_root, + 'scripts/verify_sync_open_sources.sh' +) +verify_open_infrastructure_defaults = read_repo_file( + repo_root, + 'scripts/verify_open_infrastructure_defaults.sh' +) +verify_powershell_open_defaults = read_repo_file( + repo_root, + 'scripts/verify_powershell_open_defaults.sh' +) +android_runtime_smoke = read_repo_file(repo_root, 'scripts/android_runtime_patch_smoke.sh') +verify_ios_interpreter_route = read_repo_file( + repo_root, + 'scripts/verify_ios_interpreter_route.sh' +) +verify_ios_interpreter_route_validator = read_repo_file( + repo_root, + 'scripts/verify_ios_interpreter_route_validator.sh' +) +verify_engine_args = read_repo_file(repo_root, 'scripts/verify_engine_args.sh') +verify_dart_sdk_args = read_repo_file(repo_root, 'scripts/verify_dart_sdk_args.sh') +assert!( + bootstrap_linux.include?('exec "$ROOT/scripts/platform_test_common.sh" linux') && + bootstrap_macos.include?('exec "$ROOT/scripts/platform_test_common.sh" macos'), + 'platform bootstrap entry points must delegate to the shared open replacement test script' +) +assert!( + platform_test_common.include?("cd '$ROOT/shorebird' && '$DART_BIN' pub get") && + platform_test_common.include?('test/src/shorebird_env_test.dart') && + platform_test_common.include?('test/src/commands/doctor_command_test.dart') && + platform_test_common.include?('packages/shorebird_code_push_client/test/src/code_push_client_test.dart') && + platform_test_common.include?('packages/artifact_proxy/test/artifact_proxy_test.dart') && + platform_test_common.include?('packages/artifact_proxy/test/server_bin_test.dart') && + platform_test_common.include?('CARGO_BIN="${CARGO_BIN:-cargo}"') && + platform_test_common.include?('--manifest-path "$ROOT/updater/library/Cargo.toml"'), + 'platform bootstrap must test CLI, code push client, artifact proxy, and updater open server defaults' +) +assert!( + step_uses(jobs.fetch('source-checks'), 'dtolnay/rust-toolchain@stable'), + 'source-checks must install Rust so updater runtime tests run in default CI' +) +source_bootstrap_step = jobs.fetch('source-checks').fetch('steps', []).find do |step| + step['run'].to_s.include?('./scripts/bootstrap_linux.sh') +end +assert!( + source_bootstrap_step && source_bootstrap_step.dig('env', 'SKIP_GCLIENT_SYNC') == '1', + 'source-checks must run bootstrap_linux.sh with SKIP_GCLIENT_SYNC=1 for source-level CI' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('python3 -m py_compile scripts/write_artifact_manifest.py'), + 'source-checks must compile the artifact manifest helper' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('python3 -m py_compile scripts/validate_artifact_mirror.py'), + 'source-checks must compile the artifact mirror validator' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('python3 -m py_compile scripts/validate_release_manifest.py'), + 'source-checks must compile the release manifest validator' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('python3 -m py_compile scripts/write_release_manifest.py'), + 'source-checks must compile the release manifest helper' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_ci_capacity.sh'), + 'source-checks must smoke-test heavy runner capacity checks' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_sync_open_sources.sh'), + 'source-checks must smoke-test source sync behavior in an isolated workspace' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_open_infrastructure_defaults.sh'), + 'source-checks must verify open infrastructure defaults independently of the workflow contract' +) +assert!( + verify_open_infrastructure_defaults.include?('BUILD_SENSITIVE_FILES') && + verify_open_infrastructure_defaults.include?('check_forbidden_in_tree "$ROOT/shorebird/packages/artifact_proxy/lib" dart') && + verify_open_infrastructure_defaults.include?('https://git.tonycloud.org/dart-lang/sdk.git') && + verify_open_infrastructure_defaults.include?('https://git.tonycloud.org/flutter/shorebird-updater.git') && + verify_open_infrastructure_defaults.include?('http://localhost:8080/download.flutter.io') && + verify_open_infrastructure_defaults.include?('http://localhost:8080/artifacts') && + verify_open_infrastructure_defaults.include?('defaultHostedUrl = ') && + verify_open_infrastructure_defaults.include?('download.shorebird.dev') && + verify_open_infrastructure_defaults.include?('shorebird-dart-sdk-prebuilt'), + 'open infrastructure defaults verifier must scan build-sensitive files and enforce open remotes/artifact hosts' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_powershell_open_defaults.sh'), + 'source-checks must verify Windows PowerShell launcher/updater open defaults' +) +assert!( + verify_powershell_open_defaults.include?('shorebird/bin/shorebird.ps1') && + verify_powershell_open_defaults.include?('flutter/bin/internal/update_dart_sdk.ps1') && + verify_powershell_open_defaults.include?('System.Management.Automation.Language.Parser') && + verify_powershell_open_defaults.include?('https://git.tonycloud.org/flutter/flutter.git') && + verify_powershell_open_defaults.include?('http://localhost:8080/download.flutter.io') && + verify_powershell_open_defaults.include?('download.shorebird.dev') && + verify_powershell_open_defaults.include?('github.com/shorebirdtech/flutter.git'), + 'PowerShell verifier must parse Windows scripts and enforce open Flutter/artifact defaults' +) +assert!( + verify_sync_open_sources.include?('expected forbidden Dart SDK remote to fail') && + verify_sync_open_sources.include?('expected forbidden updater source remote to fail') && + verify_sync_open_sources.include?('expected forbidden explicit UPDATER_URL to fail'), + 'source sync smoke test must reject upstream Dart SDK and official Shorebird updater remotes' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_release_manifest.sh'), + 'source-checks must smoke-test release manifest validation' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_artifact_mirror_validator.sh'), + 'source-checks must smoke-test standalone artifact mirror validation' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_assemble_artifact_mirror.sh'), + 'source-checks must smoke-test assembling downloaded CI artifacts into a publishable mirror' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_artifact_mirror_workflow_assembly.sh'), + 'source-checks must smoke-test the artifact-mirror aggregation job with fake downloaded artifacts' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_hosted_full_sdk_build.sh --help >/dev/null 2>&1'), + 'source-checks must smoke-test hosted full-SDK helper argument parsing without contacting GitHub' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_ios_interpreter_route_validator.sh'), + 'source-checks must smoke-test iOS interpreter route artifact validation' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('bash -n scripts/*.sh shorebird/third_party/flutter/bin/internal/shared.sh flutter/bin/internal/update_dart_sdk.sh'), + 'source-checks must syntax-check the patched Flutter launchers/updaters as well as repo scripts' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?('python3 -m py_compile scripts/safe_extract_tar.py'), + 'source-checks must syntax-check the shared safe tar extractor' +) +assert!( + job_runs(jobs.fetch('source-checks')).join("\n").include?("compile(Path('flutter/DEPS').read_text") && + job_runs(jobs.fetch('source-checks')).join("\n").include?("'flutter/DEPS', 'exec'"), + 'source-checks must syntax-check Flutter DEPS before heavy gclient sync' +) +assemble_artifact_mirror = read_repo_file(repo_root, 'scripts/assemble_artifact_mirror.sh') +validate_artifact_mirror = read_repo_file(repo_root, 'scripts/validate_artifact_mirror.py') +validate_release_manifest = read_repo_file(repo_root, 'scripts/validate_release_manifest.py') +safe_extract_tar = read_repo_file(repo_root, 'scripts/safe_extract_tar.py') +verify_assemble_artifact_mirror = read_repo_file( + repo_root, + 'scripts/verify_assemble_artifact_mirror.sh' +) +verify_artifact_mirror_validator = read_repo_file( + repo_root, + 'scripts/verify_artifact_mirror_validator.sh' +) +verify_artifact_mirror_workflow_assembly = read_repo_file( + repo_root, + 'scripts/verify_artifact_mirror_workflow_assembly.sh' +) +verify_downloaded_release_artifacts = read_repo_file( + repo_root, + 'scripts/verify_downloaded_release_artifacts.sh' +) +verify_hosted_full_sdk_build = read_repo_file( + repo_root, + 'scripts/verify_hosted_full_sdk_build.sh' +) +verify_upload_readiness = read_repo_file(repo_root, 'scripts/verify_upload_readiness.sh') +write_release_manifest = read_repo_file(repo_root, 'scripts/write_release_manifest.py') +verify_release_manifest = read_repo_file(repo_root, 'scripts/verify_release_manifest.sh') +assert!( + assemble_artifact_mirror.include?('scan_for_shorebird_trees "$INPUT_DIR"') && + assemble_artifact_mirror.include?("find \"$INPUT_DIR\" -type f \\( -name '*.tar.gz' -o -name '*.tgz' \\)") && + assemble_artifact_mirror.include?('scripts/safe_extract_tar.py') && + assemble_artifact_mirror.include?('conflicting mirror file') && + assemble_artifact_mirror.include?('validate_artifact_mirror.py') && + assemble_artifact_mirror.include?('write_sha256.sh'), + 'artifact mirror assembler must safely extract archives, merge direct and archived shorebird/ subtrees, reject conflicts, write checksum sidecars, and run the shared mirror validator' +) +assert!( + safe_extract_tar.include?('unsafe archive member path') && + safe_extract_tar.include?('unsupported archive member type') && + safe_extract_tar.include?('duplicate archive member path') && + safe_extract_tar.include?('archive member escapes extraction root') && + safe_extract_tar.include?('tarfile.open') && + safe_extract_tar.include?('extract_safe_tar_archive'), + 'shared safe tar extractor must reject unsafe paths, unsupported members, duplicate file entries, and extraction escapes' +) +assert!( + validate_artifact_mirror.include?('REQUIRED_PATCH_ZIPS') && + validate_artifact_mirror.include?('"patch-linux-x64.zip": "patch"') && + validate_artifact_mirror.include?('"patch-darwin-x64.zip": "patch"') && + validate_artifact_mirror.include?('"patch-darwin-arm64.zip": "patch"') && + validate_artifact_mirror.include?('"patch-windows-x64.zip": "patch.exe"') && + validate_artifact_mirror.include?('validate_sidecars') && + validate_artifact_mirror.include?('validate_manifest_overrides') && + validate_artifact_mirror.include?('validate_patch_zips') && + validate_artifact_mirror.include?('is_safe_relative_path') && + validate_artifact_mirror.include?('is_plain_file') && + validate_artifact_mirror.include?('unsafe artifact override path') && + validate_artifact_mirror.include?('artifact override is empty') && + validate_artifact_mirror.include?('symlink entries are not allowed') && + validate_artifact_mirror.include?('replace("$engine", engine_revision)') && + validate_artifact_mirror.include?('zipfile.ZipFile(zip_path)') && + validate_artifact_mirror.include?('names != [expected_entry]'), + 'artifact mirror validator must validate sidecars, manifest overrides, and patch-tool ZIP contracts' +) +assert!( + verify_assemble_artifact_mirror.include?('linux-engine-x64.tar.gz') && + verify_assemble_artifact_mirror.include?('patch-linux-x64.zip') && + verify_assemble_artifact_mirror.include?('patch-darwin-x64.zip') && + verify_assemble_artifact_mirror.include?('patch-darwin-arm64.zip') && + verify_assemble_artifact_mirror.include?('patch-windows-x64.zip') && + verify_assemble_artifact_mirror.include?('artifacts_manifest.yaml') && + verify_assemble_artifact_mirror.include?('unexpectedly allowed a conflicting mirror file') && + verify_assemble_artifact_mirror.include?('unexpectedly allowed an unsafe tar member') && + verify_assemble_artifact_mirror.include?('unexpectedly allowed a tar symlink member') && + verify_assemble_artifact_mirror.include?('unexpectedly allowed a duplicate tar member') && + verify_assemble_artifact_mirror.include?('unexpectedly allowed an invalid patch zip') && + verify_assemble_artifact_mirror.include?('unexpectedly allowed a missing manifest override'), + 'artifact mirror assembler smoke test must cover archived engine subtrees, patch artifacts, metadata, conflicts, unsafe tar members, duplicate tar members, invalid patch zips, and missing overrides' +) +assert!( + verify_artifact_mirror_validator.include?('validate_artifact_mirror.py') && + verify_artifact_mirror_validator.include?('unexpectedly accepted a stale sidecar') && + verify_artifact_mirror_validator.include?('unexpectedly accepted an unsafe manifest override') && + verify_artifact_mirror_validator.include?('unexpectedly accepted a symlink artifact') && + verify_artifact_mirror_validator.include?('unexpectedly accepted an empty manifest override artifact') && + verify_artifact_mirror_validator.include?('verify_assemble_artifact_mirror.sh'), + 'artifact mirror validator smoke test must cover valid mirrors, unsafe override paths, empty override artifacts, symlink artifacts, and stale sidecar rejection' +) +assert!( + verify_artifact_mirror_workflow_assembly.include?('mirror-input') && + verify_artifact_mirror_workflow_assembly.include?('open-shorebird-artifact-mirror.tar.gz') && + verify_artifact_mirror_workflow_assembly.include?('open-shorebird-release-manifest.json') && + verify_artifact_mirror_workflow_assembly.include?('write_release_manifest.py') && + verify_artifact_mirror_workflow_assembly.include?('validate_release_manifest.py') && + verify_artifact_mirror_workflow_assembly.include?('verify_downloaded_release_artifacts.sh') && + verify_artifact_mirror_workflow_assembly.include?('validate_artifact_mirror.py') && + verify_artifact_mirror_workflow_assembly.include?('safe_extract_tar.py') && + verify_artifact_mirror_workflow_assembly.include?('mirror-metadata/*artifacts_manifest.yaml') && + verify_artifact_mirror_workflow_assembly.include?('flutter_patched_sdk_product.zip') && + verify_artifact_mirror_workflow_assembly.include?('unexpectedly accepted downloaded artifacts for the wrong github_sha') && + verify_artifact_mirror_workflow_assembly.include?('unexpectedly accepted a stale downloaded release manifest sidecar') && + verify_artifact_mirror_workflow_assembly.include?('unexpectedly accepted a stale downloaded mirror archive sidecar') && + verify_artifact_mirror_workflow_assembly.include?('unexpectedly accepted an unsafe downloaded mirror archive'), + 'artifact-mirror workflow dry-run must exercise mirror-only input assembly, release manifest requirements, downloaded sidecar rejection, and unsafe downloaded mirror archive rejection' +) +assert!( + verify_downloaded_release_artifacts.include?('open-shorebird-release-manifest.json') && + verify_downloaded_release_artifacts.include?('open-shorebird-artifact-mirror.tar.gz') && + verify_downloaded_release_artifacts.include?('validate_release_manifest.py') && + verify_downloaded_release_artifacts.include?('validate_artifact_mirror.py') && + verify_downloaded_release_artifacts.include?('--github-sha') && + verify_downloaded_release_artifacts.include?('EXPECTED_GITHUB_SHA') && + verify_downloaded_release_artifacts.include?('validate_manifest_args') && + verify_downloaded_release_artifacts.include?('verify_sha256_sidecar') && + verify_downloaded_release_artifacts.include?('scripts/safe_extract_tar.py') && + verify_downloaded_release_artifacts.include?('manifest_sidecar') && + verify_downloaded_release_artifacts.include?('mirror_sidecar') && + verify_downloaded_release_artifacts.include?('digest mismatch') && + verify_downloaded_release_artifacts.include?('filename mismatch') && + verify_downloaded_release_artifacts.include?('expected exactly one') && + verify_downloaded_release_artifacts.include?('mirror archive did not contain open-shorebird-artifact-mirror/'), + 'downloaded release verifier must validate manifest and mirror checksum sidecars plus safely extracted mirror archive from downloaded GitHub artifacts' +) +assert!( + verify_hosted_full_sdk_build.include?('gh workflow run "$WORKFLOW"') && + verify_hosted_full_sdk_build.include?('-f full_sdk_build=true') && + verify_hosted_full_sdk_build.include?('-f run_runtime_smokes=false') && + verify_hosted_full_sdk_build.include?('gh run download "$run_id"') && + verify_hosted_full_sdk_build.include?('--json status,conclusion,url,headSha') && + verify_hosted_full_sdk_build.include?('--github-sha "$run_head_sha"') && + verify_hosted_full_sdk_build.include?('unable to read headSha') && + verify_hosted_full_sdk_build.include?('verify_downloaded_release_artifacts.sh') && + verify_hosted_full_sdk_build.include?('--linux-heavy-runner') && + verify_hosted_full_sdk_build.include?('--macos-heavy-runner') && + verify_hosted_full_sdk_build.include?('--skip-gclient-sync'), + 'hosted full SDK verifier must dispatch the manual full build, download artifacts, and run the release artifact verifier' +) +verify_ci_workflow_shell = read_repo_file(repo_root, 'scripts/verify_ci_workflow.sh') +assert!( + verify_ci_workflow_shell.include?('--require-tracked') && + verify_ci_workflow_shell.include?('--require-clean') && + verify_ci_workflow_shell.include?('--require-upload-ready') && + verify_ci_workflow_shell.include?('RUBY_ARGS'), + 'workflow verifier shell wrapper must expose tracked, clean, and upload-ready modes' +) +assert!( + verify_upload_readiness.include?('verify_ci_workflow.sh') && + verify_upload_readiness.include?('--require-upload-ready') && + verify_upload_readiness.include?('upload readiness check passed'), + 'upload readiness verifier must run the workflow contract in upload-ready mode' +) +assert!( + write_release_manifest.include?('artifact_count') && + write_release_manifest.include?('github_sha') && + write_release_manifest.include?('"artifact_group": artifact_relative.parts[0]') && + write_release_manifest.include?('"filename": artifact_path.name') && + write_release_manifest.include?('fnmatch.fnmatchcase') && + write_release_manifest.include?('missing required artifact matching') && + write_release_manifest.include?('is_safe_relative_path') && + write_release_manifest.include?('is_plain_file') && + write_release_manifest.include?('unsafe artifact path') && + write_release_manifest.include?('empty artifacts are not allowed') && + write_release_manifest.include?('symlink artifacts are not allowed') && + write_release_manifest.include?('parse_sidecar') && + write_release_manifest.include?('digest mismatch') && + write_release_manifest.include?('filename mismatch') && + write_release_manifest.include?('orphan .sha256 sidecars'), + 'release manifest helper must validate checksum sidecars and write artifact provenance' +) +assert!( + validate_release_manifest.include?('format_version') && + validate_release_manifest.include?('artifact_count') && + validate_release_manifest.include?('github_sha is') && + validate_release_manifest.include?('artifact_group must be a string') && + validate_release_manifest.include?('does not match path group') && + validate_release_manifest.include?('filename must be a string') && + validate_release_manifest.include?('does not match path filename') && + validate_release_manifest.include?('is_safe_relative_path') && + validate_release_manifest.include?('is_plain_file') && + validate_release_manifest.include?('PurePosixPath') && + validate_release_manifest.include?('unsafe artifact path') && + validate_release_manifest.include?('duplicate artifact path') && + validate_release_manifest.include?('missing artifact file') && + validate_release_manifest.include?('empty artifacts are not allowed') && + validate_release_manifest.include?('size mismatch') && + validate_release_manifest.include?('digest mismatch') && + validate_release_manifest.include?('sidecar digest mismatch') && + validate_release_manifest.include?('artifacts missing from release manifest') && + validate_release_manifest.include?('sidecars missing from release manifest'), + 'release manifest validator must verify manifest structure, digests, sizes, sidecars, and complete artifact coverage' +) +assert!( + verify_release_manifest.include?('open-shorebird-cli-linux-x64.tar.gz') && + verify_release_manifest.include?('shorebird-server-linux-amd64.tar.gz') && + verify_release_manifest.include?('validate_release_manifest.py') && + verify_release_manifest.include?('unexpectedly accepted a tampered digest') && + verify_release_manifest.include?('unexpectedly accepted the wrong github_sha') && + verify_release_manifest.include?('unexpectedly accepted an unlisted artifact') && + verify_release_manifest.include?('unexpectedly accepted an unsafe artifact path') && + verify_release_manifest.include?('unexpectedly accepted bad provenance fields') && + verify_release_manifest.include?('unexpectedly accepted a symlink artifact') && + verify_release_manifest.include?('unexpectedly accepted an empty artifact') && + verify_release_manifest.include?('unexpectedly accepted a missing sidecar') && + verify_release_manifest.include?('unexpectedly accepted a bad sidecar') && + verify_release_manifest.include?('unexpectedly accepted an orphan sidecar') && + verify_release_manifest.include?('unexpectedly accepted a missing required artifact'), + 'release manifest smoke test must cover valid artifacts, required globs, provenance fields, path safety, symlink rejection, and checksum failure modes' +) +assert!( + sync_open_sources.include?('real_path()') && + sync_open_sources.include?('ensure_source_link "$DART_TARGET" "$DART_SRC" "Dart SDK"') && + sync_open_sources.include?('reject_forbidden_remotes') && + sync_open_sources.include?('github.com/dart-lang/sdk') && + sync_open_sources.include?('dart.googlesource.com/sdk') && + sync_open_sources.include?('github.com/shorebirdtech/updater') && + sync_open_sources.include?('target symlink points at') && + sync_open_sources.include?('UPDATER_URL="${UPDATER_URL:-}"') && + sync_open_sources.include?('ensure_source_link "$TARGET" "$UPDATER_SRC" "updater submodule"') && + sync_open_sources.include?('link_checkout "$target" "$source" "$label"'), + 'open source sync must force engine Dart/updater dependencies to workspace submodules and reject known unpatched/official remotes' +) +assert!( + !sync_open_sources.include?('github.com/shorebirdtech/updater.git'), + 'open source sync must not default to the official Shorebird updater repository' +) +assert!( + !open_replacements_doc.include?('github.com/shorebirdtech/updater.git') && + !open_replacements_doc.include?('sync_open_sources.ps1') && + !open_replacements_doc.include?('public `shorebirdtech/updater`'), + 'open replacement audit must document local submodules, not official upstream clone instructions' +) + +required_jobs = %w[ + source-checks + cli-artifacts + server-artifacts + custom-dart-sdk + custom-dart-sdk-macos + linux-engine + android-engine + web-sdk + ios-engine + artifact-mirror + linux-runtime-smoke + android-runtime-smoke +] +missing_jobs = required_jobs.reject { |job_name| jobs.key?(job_name) } +assert!(missing_jobs.empty?, "missing jobs: #{missing_jobs.join(', ')}") + +(required_jobs - ['source-checks']).each do |job_name| + assert!( + Array(jobs.fetch(job_name).fetch('needs', [])).include?('source-checks'), + "#{job_name} must depend on source-checks before producing artifacts or running runtime smokes" + ) +end + +on_config = workflow['on'] || workflow[true] || {} +inputs = on_config.dig('workflow_dispatch', 'inputs') || {} +assert!( + workflow.dig('defaults', 'run', 'shell') == 'bash', + 'workflow must default run steps to bash for cross-platform script portability' +) +required_inputs = %w[ + full_sdk_build + run_gclient_sync + base_flutter_engine_revision + linux_heavy_runner + macos_heavy_runner + sdk_min_free_disk_gb + engine_min_free_disk_gb + run_runtime_smokes + linux_runtime_runner + android_runtime_runner + android_serial +] +missing_inputs = required_inputs.reject { |input| inputs.key?(input) } +assert!(missing_inputs.empty?, "missing workflow_dispatch inputs: #{missing_inputs.join(', ')}") +assert!( + workflow.dig('env', 'BASE_FLUTTER_ENGINE_REVISION').to_s.include?('base_flutter_engine_revision'), + 'workflow must expose base_flutter_engine_revision as BASE_FLUTTER_ENGINE_REVISION' +) +assert!( + workflow.dig('env', 'SDK_MIN_FREE_DISK_GB').to_s.include?('sdk_min_free_disk_gb') && + workflow.dig('env', 'ENGINE_MIN_FREE_DISK_GB').to_s.include?('engine_min_free_disk_gb'), + 'workflow must expose dispatch-configurable heavy job disk thresholds' +) +assert!( + workflow.dig('env', 'DEPOT_TOOLS_UPDATE').to_s == '0' && + platform_test_common.include?('DEPOT_TOOLS_UPDATE="${DEPOT_TOOLS_UPDATE:-0}"'), + 'workflow and bootstrap must keep depot_tools pinned unless explicitly overridden' +) + +%w[source-checks cli-artifacts server-artifacts].each do |job_name| + condition = jobs.fetch(job_name).fetch('if', '') + assert!( + !condition.to_s.include?('full_sdk_build'), + "#{job_name} must run by default, not only during full_sdk_build" + ) +end + +%w[custom-dart-sdk custom-dart-sdk-macos linux-engine android-engine web-sdk ios-engine].each do |job_name| + condition = jobs.fetch(job_name).fetch('if', '').to_s + assert!( + condition.include?('workflow_dispatch') && condition.include?('full_sdk_build'), + "#{job_name} must be gated by manual full_sdk_build dispatch" + ) + run_text = job_runs(jobs.fetch(job_name)).join("\n") + assert!( + run_text.include?('python3 --version') && + run_text.include?('gclient help >/dev/null') && + run_text.include?('ninja --version'), + "#{job_name} must verify depot_tools-provided build tools before heavy builds" + ) +end +{ + 'custom-dart-sdk' => 'SDK_MIN_FREE_DISK_GB', + 'custom-dart-sdk-macos' => 'SDK_MIN_FREE_DISK_GB', + 'linux-engine' => 'ENGINE_MIN_FREE_DISK_GB', + 'android-engine' => 'ENGINE_MIN_FREE_DISK_GB', + 'web-sdk' => 'ENGINE_MIN_FREE_DISK_GB', + 'ios-engine' => 'ENGINE_MIN_FREE_DISK_GB', +}.each do |job_name, disk_env_var| + run_text = job_runs(jobs.fetch(job_name)).join("\n") + assert!( + run_text.include?('scripts/check_ci_capacity.sh') && + run_text.include?("CI_MIN_FREE_DISK_GB=\"$#{disk_env_var}\""), + "#{job_name} must fail early on runners with insufficient free disk" + ) +end +artifact_mirror_condition = jobs.fetch('artifact-mirror').fetch('if', '').to_s +assert!( + artifact_mirror_condition.include?('workflow_dispatch') && + artifact_mirror_condition.include?('full_sdk_build'), + 'artifact-mirror must be gated by manual full_sdk_build dispatch' +) +%w[cli-artifacts linux-engine android-engine web-sdk ios-engine].each do |dependency| + assert!( + Array(jobs.fetch('artifact-mirror').fetch('needs', [])).include?(dependency), + "artifact-mirror must depend on #{dependency}" + ) +end +%w[server-artifacts custom-dart-sdk custom-dart-sdk-macos].each do |dependency| + assert!( + Array(jobs.fetch('artifact-mirror').fetch('needs', [])).include?(dependency), + "artifact-mirror must depend on #{dependency} so release manifest covers it" + ) +end + +%w[linux-runtime-smoke android-runtime-smoke].each do |job_name| + condition = jobs.fetch(job_name).fetch('if', '').to_s + assert!( + condition.include?('workflow_dispatch') && condition.include?('run_runtime_smokes'), + "#{job_name} must be gated by manual run_runtime_smokes dispatch" + ) + assert!( + !condition.include?('full_sdk_build'), + "#{job_name} must not require full_sdk_build" + ) +end + +cli_matrix = matrix_include(jobs.fetch('cli-artifacts')) +expected_cli_targets = [ + ['ubuntu-latest', 'linux', 'x64', 'patch-linux-x64.zip', 'patch'], + ['macos-15-intel', 'macos', 'x64', 'patch-darwin-x64.zip', 'patch'], + ['macos-14', 'macos', 'arm64', 'patch-darwin-arm64.zip', 'patch'], + ['windows-latest', 'windows', 'x64', 'patch-windows-x64.zip', 'patch.exe'], +] +expected_cli_targets.each do |runner_os, artifact_os, artifact_arch, patch_zip, patch_entry| + assert!( + cli_matrix.any? do |entry| + entry['os'] == runner_os && + entry['artifact_os'] == artifact_os && + entry['artifact_arch'] == artifact_arch && + entry['patch_zip'] == patch_zip && + entry['patch_entry'] == patch_entry + end, + "missing CLI target #{artifact_os}/#{artifact_arch} on #{runner_os} with #{patch_zip}" + ) +end +assert!( + upload_names(jobs.fetch('cli-artifacts')).include?('cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}'), + 'CLI artifacts must be uploaded' +) +assert!( + upload_names(jobs.fetch('cli-artifacts')).include?('mirror-${{ matrix.patch_zip }}'), + 'mirror patch artifacts must be uploaded' +) +assert!( + upload_names(jobs.fetch('cli-artifacts')).include?('mirror-metadata'), + 'mirror artifact metadata must be uploaded' +) +assert!( + step_uses(jobs.fetch('cli-artifacts'), 'dtolnay/rust-toolchain@stable'), + 'CLI artifact job must install a stable Rust toolchain' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('cargo build --release --manifest-path updater/patch/Cargo.toml --bin patch'), + 'CLI artifact job must build the public updater patch binary' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('names = archive.namelist()') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('names != [expected_entry]') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('info.file_size <= 0') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('${{ matrix.patch_entry }}'), + 'CLI artifact job must verify patch mirror ZIPs contain the exact patch entry expected by the cache' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('mirror_patch_dir="artifacts/mirror/shorebird/$engine_revision"') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('cp "artifacts/mirror/${{ matrix.patch_zip }}" "$mirror_patch_dir/${{ matrix.patch_zip }}"') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('test -f "$mirror_patch_dir/${{ matrix.patch_zip }}"') && + jobs.fetch('cli-artifacts').fetch('steps').any? do |step| + step.dig('with', 'path').to_s.include?('artifacts/mirror/shorebird/**/${{ matrix.patch_zip }}') && + step.dig('with', 'path').to_s.include?('artifacts/mirror/shorebird/**/${{ matrix.patch_zip }}.sha256') + end, + 'CLI artifact job must upload a publish-ready shorebird// patch mirror subtree' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('scripts/write_artifact_manifest.py') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('artifacts_manifest.yaml') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('artifacts/mirror/shorebird/$engine_revision') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('base_engine_revision="${BASE_FLUTTER_ENGINE_REVISION:-$engine_revision}"') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('--flutter-engine-revision "$base_engine_revision"'), + 'CLI artifact job must generate open artifact mirror metadata' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('packages/artifact_proxy/bin/server.dart') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('bin/artifact_proxy${{ matrix.extension }}') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('artifact_proxy${{ matrix.extension }} --health-check'), + 'CLI artifact job must compile, package, and smoke-test the open artifact proxy binary' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?("grep -q 'https://git.tonycloud.org/flutter/shorebird.git' <<<\"$version_output\"") && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?("grep -q 'https://git.tonycloud.org/flutter/shorebird.git' <<<\"$extracted_version_output\"") && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?("git@github.com:shorebirdtech/shorebird.git") && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('compiled CLI still reports the official Shorebird SSH remote') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('extracted CLI still reports the official Shorebird SSH remote'), + 'CLI artifact job must assert compiled and extracted version banners report the open fork' +) +assert!( + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('cli_extract_dir="$(mktemp -d)"') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('tar -C "$cli_extract_dir" -xzf open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('test -f "$cli_extract_dir/manifest.json"') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('"$cli_extract_dir/bin/open_aot_patch_tools${{ matrix.extension }}" --help') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('"$cli_extract_dir/bin/shorebird${{ matrix.extension }}" --version') && + job_runs(jobs.fetch('cli-artifacts')).join("\n").include?('"$cli_extract_dir/bin/artifact_proxy${{ matrix.extension }}" --health-check'), + 'CLI artifact job must extract and smoke-test the downloadable archive layout' +) + +server_matrix = matrix_include(jobs.fetch('server-artifacts')) +expected_server_targets = [ + ['linux', 'amd64'], + ['linux', 'arm64'], + ['darwin', 'amd64'], + ['darwin', 'arm64'], + ['windows', 'amd64'], +] +expected_server_targets.each do |goos, goarch| + assert!( + server_matrix.any? { |entry| entry['goos'] == goos && entry['goarch'] == goarch }, + "missing server target #{goos}/#{goarch}" + ) +end +assert!( + upload_names(jobs.fetch('server-artifacts')).include?('shorebird-server-${{ matrix.goos }}-${{ matrix.goarch }}'), + 'server artifacts must be uploaded' +) +assert!( + inputs.dig('linux_heavy_runner', 'default') == 'open-shorebird-linux-heavy' && + inputs.dig('macos_heavy_runner', 'default') == 'open-shorebird-macos-heavy', + 'heavy SDK/engine workflow inputs must default to custom large-runner labels' +) +assert!( + jobs.fetch('custom-dart-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && + jobs.fetch('android-engine').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && + jobs.fetch('web-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && + jobs.fetch('custom-dart-sdk-macos').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'open-shorebird-macos-heavy'") && + jobs.fetch('ios-engine').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'open-shorebird-macos-heavy'"), + 'heavy SDK/engine jobs must default to custom large-runner labels, not standard hosted runners' +) +assert!( + workflow.dig('env', 'JAVA_VERSION').to_s == '17', + 'workflow must pin the Java version used by Android builds' +) +assert!( + free_ci_disk_linux.include?('GITHUB_ACTIONS') && + free_ci_disk_linux.include?('RUNNER_ENVIRONMENT:-github-hosted') && + free_ci_disk_linux.include?('CI_FREE_DISK_SPACE_FORCE') && + free_ci_disk_linux.include?('CI_FREE_DISK_SPACE:-1') && + free_ci_disk_linux.include?('sudo rm -rf "$path"'), + 'Linux disk cleanup must be explicitly gated for GitHub-hosted runners and overridable on custom runners' +) +assert!( + check_ci_capacity.include?('CI_MIN_FREE_DISK_GB') && + check_ci_capacity.include?('CI_CAPACITY_PATH') && + check_ci_capacity.include?('CI_AVAILABLE_DISK_KB_OVERRIDE') && + check_ci_capacity.include?('df -Pk "$CHECK_PATH"') && + check_ci_capacity.include?('runner has insufficient free disk'), + 'heavy runner capacity check must validate free disk with portable df output and a test override' +) +assert!( + verify_ci_capacity.include?('CI_AVAILABLE_DISK_KB_OVERRIDE') && + verify_ci_capacity.include?('unexpectedly accepted insufficient disk') && + verify_ci_capacity.include?('unexpectedly accepted an invalid minimum') && + verify_ci_capacity.include?('CI_MIN_FREE_DISK_GB=0'), + 'capacity check smoke test must cover pass, fail, invalid input, and disabled modes' +) +assert!( + step_uses(jobs.fetch('android-engine'), 'actions/setup-java@v4') && + workflow_text.include?('distribution: temurin') && + workflow_text.include?('java-version: ${{ env.JAVA_VERSION }}') && + job_runs(jobs.fetch('android-runtime-smoke')).join("\n").include?('java -version') && + android_runtime_smoke.include?('require_tool java'), + 'Android engine and runtime smoke paths must provision or verify Java explicitly' +) +server_artifact_run_text = job_runs(jobs.fetch('server-artifacts')).join("\n") +assert!( + server_artifact_run_text.include?('GOOS" == "linux"') && + server_artifact_run_text.include?('GOARCH" == "amd64"') && + server_artifact_run_text.include?('cp internal/api/handlers/openapi.yaml') && + server_artifact_run_text.include?('"openapi.yaml"') && + server_artifact_run_text.include?('curl -fsS http://127.0.0.1:18080/health') && + server_artifact_run_text.include?('grep -q \'"status":"ok"\'') && + server_artifact_run_text.include?('grep -q \'\'') && + server_artifact_run_text.include?('curl -fsS http://127.0.0.1:18080/openapi.yaml') && + server_artifact_run_text.include?("grep -q 'openapi: 3.1.0'"), + 'server artifact job must package and smoke-test the native Linux package from its archive' +) + +required_uploads = { + 'custom-dart-sdk' => ['custom-dart-sdk-linux-x64'], + 'custom-dart-sdk-macos' => ['custom-dart-sdk-macos-arm64'], + 'linux-engine' => ['linux-engine-x64'], + 'android-engine' => ['android-engine-arm64'], + 'web-sdk' => ['flutter-web-sdk'], + 'ios-engine' => ['ios-interpreter-engine', 'macos-engine-arm64'], + 'artifact-mirror' => ['open-shorebird-artifact-mirror', 'open-shorebird-release-manifest'], +} +required_uploads.each do |job_name, expected_names| + actual_names = upload_names(jobs.fetch(job_name)) + expected_names.each do |expected_name| + assert!(actual_names.include?(expected_name), "#{job_name} must upload #{expected_name}") + end +end + +jobs.each do |job_name, job| + job.fetch('steps', []).each do |step| + next unless step['uses'].to_s.start_with?('actions/upload-artifact@') + + assert!( + step.dig('with', 'if-no-files-found') == 'error', + "#{job_name} upload #{step.dig('with', 'name')} must fail on missing files" + ) + assert!( + step.dig('with', 'path').to_s.include?('.sha256'), + "#{job_name} upload #{step.dig('with', 'name')} must include a checksum sidecar" + ) + end +end + +run_text_by_job = jobs.transform_values { |job| job_runs(job).join("\n") } +assert!( + run_text_by_job.fetch('cli-artifacts').include?('"flutter_revision": "${flutter_revision}"') && + run_text_by_job.fetch('cli-artifacts').include?('"engine_revision": "${engine_revision}"'), + 'CLI artifact manifest must record Flutter and engine revisions' +) +assert!( + run_text_by_job.fetch('server-artifacts').include?('server_revision="$(git rev-parse HEAD)"') && + run_text_by_job.fetch('server-artifacts').include?('"server_git_sha": "${server_revision}"'), + 'server artifact manifest must record the server source revision' +) +%w[custom-dart-sdk custom-dart-sdk-macos].each do |job_name| + assert!( + run_text_by_job.fetch(job_name).include?('dart_sdk_revision="$(git -C dart-sdk-new rev-parse HEAD)"') && + run_text_by_job.fetch(job_name).include?('"dart_sdk_git_sha": "${dart_sdk_revision}"'), + "#{job_name} manifest must record the Dart SDK source revision" + ) +end +{ + 'custom-dart-sdk' => 'custom-dart-sdk-linux-x64', + 'custom-dart-sdk-macos' => 'custom-dart-sdk-macos-arm64', +}.each do |job_name, artifact_name| + run_text = run_text_by_job.fetch(job_name) + assert!( + run_text.include?('sdk_extract_dir="$(mktemp -d)"') && + run_text.include?("tar -C \"$sdk_extract_dir\" -xzf #{artifact_name}.tar.gz") && + run_text.include?("test -f \"$sdk_extract_dir/#{artifact_name}/manifest.json\"") && + run_text.include?("test -f \"$sdk_extract_dir/#{artifact_name}/args.gn\"") && + run_text.include?("test -x \"$sdk_extract_dir/#{artifact_name}/dart-sdk/bin/dart\"") && + run_text.include?("test -x \"$sdk_extract_dir/#{artifact_name}/gen_snapshot\"") && + run_text.include?("test -x \"$sdk_extract_dir/#{artifact_name}/dartaotruntime\"") && + run_text.include?("\"$sdk_extract_dir/#{artifact_name}/dart-sdk/bin/dart\" --version"), + "#{job_name} must extract and smoke-test the downloadable Dart SDK archive layout" + ) +end +%w[linux-engine android-engine web-sdk ios-engine].each do |job_name| + assert!( + run_text_by_job.fetch(job_name).include?('engine_revision="$(cat flutter/bin/internal/engine.version)"') && + run_text_by_job.fetch(job_name).include?('"engine_revision": "${engine_revision}"'), + "#{job_name} manifest must record the Flutter engine revision" + ) +end +engine_archive_checks = { + 'linux-engine' => [ + 'engine_extract_dir="$(mktemp -d)"', + 'tar -C "$engine_extract_dir" -xzf linux-engine-x64.tar.gz', + 'test -f "$engine_extract_dir/linux-engine/manifest.json"', + 'test -f "$engine_extract_dir/linux-engine/linux_release_x64.args.gn"', + 'test -f "$engine_extract_dir/linux-engine/linux-x64-flutter-gtk.zip"', + 'test -f "$engine_extract_dir/linux-engine/flutter_patched_sdk_product.zip"', + 'test -f "$engine_extract_dir/linux-engine/artifacts.zip"', + 'test -f "$engine_extract_dir/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/linux-x64-release/linux-x64-flutter-gtk.zip"', + 'test -f "$engine_extract_dir/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/linux-x64-release/artifacts.zip"', + 'test -f "$engine_extract_dir/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/flutter_patched_sdk_product.zip"', + ], + 'android-engine' => [ + 'engine_extract_dir="$(mktemp -d)"', + 'tar -C "$engine_extract_dir" -xzf android-engine-arm64.tar.gz', + 'test -f "$engine_extract_dir/android-engine/manifest.json"', + 'test -f "$engine_extract_dir/android-engine/android_release_arm64.args.gn"', + 'test -f "$engine_extract_dir/android-engine/artifacts.zip"', + 'test -f "$engine_extract_dir/android-engine/symbols.zip"', + 'test -f "$engine_extract_dir/android-engine/flutter.jar"', + 'test -f "$engine_extract_dir/android-engine/libflutter.so"', + 'test -f "$engine_extract_dir/android-engine/gen_snapshot_arm64"', + 'test -f "$engine_extract_dir/android-engine/analyze_snapshot_arm64"', + 'test -f "$engine_extract_dir/android-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/android-arm64-release/artifacts.zip"', + 'test -f "$engine_extract_dir/android-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/android-arm64-release/symbols.zip"', + ], + 'web-sdk' => [ + 'sdk_extract_dir="$(mktemp -d)"', + 'tar -C "$sdk_extract_dir" -xzf flutter-web-sdk.tar.gz', + 'test -f "$sdk_extract_dir/web-sdk/manifest.json"', + 'test -f "$sdk_extract_dir/web-sdk/wasm_release.args.gn"', + 'test -f "$sdk_extract_dir/web-sdk/flutter-web-sdk.zip"', + 'test -f "$sdk_extract_dir/web-sdk/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/flutter-web-sdk.zip"', + ], + 'ios-engine' => [ + 'engine_extract_dir="$(mktemp -d)"', + 'tar -C "$engine_extract_dir" -xzf ios-interpreter-engine.tar.gz', + 'test -f "$engine_extract_dir/ios-engine/manifest.json"', + 'test -d "$engine_extract_dir/ios-engine/Flutter.framework"', + 'test -d "$engine_extract_dir/ios-engine/Flutter.xcframework"', + 'test -f "$engine_extract_dir/ios-engine/ios-release/artifacts.zip"', + 'test -f "$engine_extract_dir/ios-engine/ios_release.args.gn"', + 'test -f "$engine_extract_dir/ios-engine/host_release_arm64.args.gn"', + 'test -x "$engine_extract_dir/ios-engine/host_release_arm64/gen_snapshot"', + 'test -f "$engine_extract_dir/ios-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/ios-release/artifacts.zip"', + '"Flutter.xcframework/Info.plist"', + '"Flutter.xcframework/ios-arm64/Flutter.framework/Flutter"', + '"gen_snapshot_arm64"', + '"analyze_snapshot_arm64"', + '"entitlements.txt"', + '"without_entitlements.txt"', + '"unsigned_binaries.txt"', + 'tar -C "$engine_extract_dir" -xzf macos-engine-arm64.tar.gz', + 'test -f "$engine_extract_dir/macos-engine/manifest.json"', + 'test -f "$engine_extract_dir/macos-engine/macos_release_arm64.args.gn"', + 'test -f "$engine_extract_dir/macos-engine/FlutterMacOS.framework.zip"', + 'test -f "$engine_extract_dir/macos-engine/flutter_patched_sdk_product.zip"', + 'test -f "$engine_extract_dir/macos-engine/mirror/shorebird/flutter_infra_release/flutter/$engine_revision/darwin-arm64-release/FlutterMacOS.framework.zip"', + ], +} +engine_archive_checks.each do |job_name, expected_texts| + run_text = run_text_by_job.fetch(job_name) + missing_texts = expected_texts.reject { |expected_text| run_text.include?(expected_text) } + assert!( + missing_texts.empty?, + "#{job_name} must extract and smoke-test downloadable engine archive layout: #{missing_texts.join(', ')}" + ) +end +expected_outputs = { + 'cli-artifacts' => [ + 'open-shorebird-cli-${{ matrix.artifact_os }}-${{ matrix.artifact_arch }}.tar.gz', + 'artifacts/mirror/${{ matrix.patch_zip }}', + 'artifacts_manifest.yaml', + ], + 'server-artifacts' => [ + 'shorebird-server-${GOOS}-${GOARCH}.tar.gz', + ], + 'custom-dart-sdk' => [ + 'custom-dart-sdk-linux-x64.tar.gz', + ], + 'custom-dart-sdk-macos' => [ + 'custom-dart-sdk-macos-arm64.tar.gz', + ], + 'linux-engine' => [ + 'linux-engine-x64.tar.gz', + ], + 'android-engine' => [ + 'android-engine-arm64.tar.gz', + ], + 'web-sdk' => [ + 'flutter-web-sdk.tar.gz', + ], + 'ios-engine' => [ + 'ios-interpreter-engine.tar.gz', + 'macos-engine-arm64.tar.gz', + ], + 'artifact-mirror' => [ + 'open-shorebird-artifact-mirror.tar.gz', + 'open-shorebird-release-manifest.json', + ], +} +expected_outputs.each do |job_name, outputs| + run_text = run_text_by_job.fetch(job_name) + outputs.each do |output| + assert!(run_text.include?(output), "#{job_name} must create #{output}") + assert!( + run_text.include?("#{output}.sha256"), + "#{job_name} must create checksum sidecar for #{output}" + ) + end +end +%w[ + cli-artifacts + server-artifacts + custom-dart-sdk + custom-dart-sdk-macos + linux-engine + android-engine + web-sdk + ios-engine + artifact-mirror +].each do |job_name| + assert!( + run_text_by_job.fetch(job_name).include?('write_sha256.sh'), + "#{job_name} must use scripts/write_sha256.sh for checksum sidecars" + ) +end +assert!( + run_text_by_job.fetch('custom-dart-sdk').include?('verify_dart_sdk_args.sh dart-sdk-new/out/ReleaseX64/args.gn'), + 'Linux Dart SDK job must verify patched SDK args' +) +assert!( + run_text_by_job.fetch('custom-dart-sdk-macos').include?('verify_dart_sdk_args.sh dart-sdk-new/xcodebuild/ReleaseARM64/args.gn'), + 'macOS Dart SDK job must verify patched SDK args' +) +%w[custom-dart-sdk custom-dart-sdk-macos].each do |job_name| + run_text = run_text_by_job.fetch(job_name) + assert!(run_text.include?('dart_dynamic_modules=false'), "#{job_name} must disable DDM") + assert!(run_text.include?('dart_enable_aot_patching=true'), "#{job_name} must enable AOT patching") + assert!(run_text.include?('dart_enable_shorebird_interpreter=true'), "#{job_name} must enable the interpreter route") + assert!(run_text.include?('DartAPI_AotPatchingConfiguration'), "#{job_name} must run AOT patching VM tests") + assert!(run_text.include?('DartAPI_BytecodePatchReloadConfiguration'), "#{job_name} must run bytecode reload VM tests") +end + +assert!( + run_text_by_job.fetch('ios-engine').include?('--shorebird-interpreter') && + run_text_by_job.fetch('ios-engine').include?("--gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true'") && + run_text_by_job.fetch('ios-engine').include?('verify_ios_interpreter_route.sh') && + run_text_by_job.fetch('ios-engine').include?('test -x flutter/engine/src/out/host_release_arm64/gen_snapshot') && + run_text_by_job.fetch('ios-engine').include?('host_release_arm64/gen_snapshot') && + run_text_by_job.fetch('ios-engine').include?('ios-release/artifacts.zip') && + run_text_by_job.fetch('ios-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/ios-release/artifacts.zip') && + run_text_by_job.fetch('ios-engine').include?('"dart_enable_aot_patching": true') && + run_text_by_job.fetch('ios-engine').include?('"dart_enable_shorebird_interpreter": true') && + run_text_by_job.fetch('ios-engine').include?('"shorebird_enable_aot_patching": false') && + run_text_by_job.fetch('ios-engine').include?('"shorebird_use_interpreter": true'), + 'Apple engine job must build, verify, package, and describe the iOS interpreter host snapshotter' +) +assert!( + verify_ios_interpreter_route.include?('json.load(file)') && + verify_ios_interpreter_route.include?('metadata = artifact.get("metadata")') && + verify_ios_interpreter_route.scan('require_gn_value "$args_file" dart_enable_aot_patching true').length >= 2 && + verify_ios_interpreter_route.include?('sub("[[:space:]]*$", "", value)') && + verify_ios_interpreter_route.include?('require(metadata, "runtime_mode", "dart-bytecode-interpreter", "metadata")') && + verify_ios_interpreter_route.include?('require(metadata, "target_os", "ios", "metadata")') && + verify_ios_interpreter_route.include?('require(metadata, "target_arch", "arm64", "metadata")') && + verify_ios_interpreter_route.include?('require(artifact, "payload_kind", "full-snapshot", "artifact")') && + verify_ios_interpreter_route.include?('require(encryption, "algorithm", "AES-256-GCM", "encryption")') && + verify_ios_interpreter_route.include?('require_base64(artifact, "encrypted_payload_base64", "artifact")') && + verify_ios_interpreter_route.include?('payload_sha256') && + verify_ios_interpreter_route.include?('aad_sha256'), + 'iOS route gate must parse and validate encrypted interpreter artifact metadata, not grep raw strings' +) +assert!( + verify_ios_interpreter_route_validator.include?('write_artifact "$valid_artifact" "dart-bytecode-interpreter"') && + verify_ios_interpreter_route_validator.scan('dart_enable_aot_patching = true').length >= 2 && + verify_ios_interpreter_route_validator.include?('dart_enable_aot_patching = false') && + verify_ios_interpreter_route_validator.include?('shorebird_use_interpreter = false') && + verify_ios_interpreter_route_validator.include?('write_artifact "$bad_runtime" "dart-dynamic-modules"') && + verify_ios_interpreter_route_validator.include?('write_artifact "$bad_target" "dart-bytecode-interpreter" "android"') && + verify_ios_interpreter_route_validator.include?('unexpectedly accepted malformed JSON') && + verify_ios_interpreter_route_validator.include?('unexpectedly accepted a Mach-O patch artifact'), + 'iOS route validator smoke test must cover valid interpreter artifacts and invalid native/DDM/malformed cases' +) +assert!( + [verify_engine_args, verify_dart_sdk_args, verify_ios_interpreter_route].all? do |script| + script.include?('read_gn_value()') && + script.include?('sub("[[:space:]]*$", "", value)') && + script.include?('print value') + end, + 'GN arg verifiers must read the final assignment so default and explicit args match GN semantics' +) +assert!( + run_text_by_job.fetch('linux-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('linux-engine').include?('flutter/engine/src/out/linux_release_x64/args.gn') && + run_text_by_job.fetch('linux-engine').include?('dart_enable_aot_patching=true') && + run_text_by_job.fetch('linux-engine').include?('dart_enable_shorebird_interpreter=false') && + run_text_by_job.fetch('linux-engine').include?('shorebird_enable_aot_patching=true') && + run_text_by_job.fetch('linux-engine').include?('shorebird_use_interpreter=false') && + run_text_by_job.fetch('linux-engine').include?('linux-x64-flutter-gtk.zip') && + run_text_by_job.fetch('linux-engine').include?('flutter_patched_sdk_product.zip') && + run_text_by_job.fetch('linux-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/linux-x64-release/artifacts.zip') && + run_text_by_job.fetch('linux-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/flutter_patched_sdk_product.zip'), + 'Linux engine job must build and verify the native AOT patch runtime and package GTK/patched SDK artifacts' +) +assert!( + run_text_by_job.fetch('android-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('android-engine').include?('flutter/engine/src/out/android_release_arm64/args.gn') && + run_text_by_job.fetch('android-engine').include?('dart_enable_aot_patching=true') && + run_text_by_job.fetch('android-engine').include?('dart_enable_shorebird_interpreter=false') && + run_text_by_job.fetch('android-engine').include?('shorebird_enable_aot_patching=true') && + run_text_by_job.fetch('android-engine').include?('shorebird_use_interpreter=false') && + run_text_by_job.fetch('android-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/android-arm64-release/artifacts.zip') && + run_text_by_job.fetch('android-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/android-arm64-release/symbols.zip'), + 'Android engine job must build and verify the native AOT patch runtime without DDM or interpreter mode' +) +assert!( + run_text_by_job.fetch('web-sdk').include?('verify_engine_args.sh') && + run_text_by_job.fetch('web-sdk').include?('flutter/engine/src/out/wasm_release/args.gn') && + run_text_by_job.fetch('web-sdk').include?('dart_dynamic_modules=false') && + run_text_by_job.fetch('web-sdk').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/flutter-web-sdk.zip'), + 'web SDK job must explicitly disable and verify DDM' +) +assert!( + run_text_by_job.fetch('ios-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('ios-engine').include?('flutter/engine/src/out/macos_release_arm64/args.gn') && + run_text_by_job.fetch('ios-engine').include?('dart_enable_aot_patching=true') && + run_text_by_job.fetch('ios-engine').include?('dart_enable_shorebird_interpreter=false') && + run_text_by_job.fetch('ios-engine').include?('shorebird_enable_aot_patching=true') && + run_text_by_job.fetch('ios-engine').include?('shorebird_use_interpreter=false') && + run_text_by_job.fetch('ios-engine').include?('FlutterMacOS.framework.zip') && + run_text_by_job.fetch('ios-engine').include?('flutter_patched_sdk_product.zip') && + run_text_by_job.fetch('ios-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/darwin-arm64-release/FlutterMacOS.framework.zip'), + 'macOS engine artifact must build and verify the native AOT patch runtime without DDM or interpreter mode, while leaving the shared patched SDK mirror path to linux-engine' +) +assert!( + run_text_by_job.fetch('linux-runtime-smoke').include?('test -d flutter/engine/src/out/linux_release_x64') && + run_text_by_job.fetch('linux-runtime-smoke').include?('scripts/linux_runtime_patch_smoke.sh'), + 'Linux runtime smoke job must require linux_release_x64 and run scripts/linux_runtime_patch_smoke.sh' +) +assert!( + run_text_by_job.fetch('android-runtime-smoke').include?('scripts/android_runtime_patch_smoke.sh'), + 'Android runtime smoke job must run scripts/android_runtime_patch_smoke.sh' +) + +artifact_mirror_run_text = run_text_by_job.fetch('artifact-mirror') +assert!( + jobs.fetch('artifact-mirror').fetch('steps').count do |step| + step['uses'].to_s == 'actions/download-artifact@v4' + end >= 6, + 'artifact-mirror must download mirror patch/metadata and engine artifact producers' +) +%w[ + mirror-* + cli-* + shorebird-server-* + custom-dart-sdk-linux-x64 + custom-dart-sdk-macos-arm64 + linux-engine-x64 + android-engine-arm64 + flutter-web-sdk + ios-interpreter-engine + macos-engine-arm64 +].each do |download_name| + assert!( + jobs.fetch('artifact-mirror').fetch('steps').any? do |step| + step['uses'].to_s == 'actions/download-artifact@v4' && + step.fetch('with', {}).values.any? { |value| value.to_s.include?(download_name) } + end, + "artifact-mirror must download #{download_name}" + ) +end +assert!( + artifact_mirror_run_text.include?('./scripts/assemble_artifact_mirror.sh') && + artifact_mirror_run_text.include?('downloaded-artifacts') && + artifact_mirror_run_text.include?('mkdir -p mirror-input') && + artifact_mirror_run_text.include?('cp -R downloaded-artifacts/mirror-* mirror-input/') && + artifact_mirror_run_text.include?('cp -R downloaded-artifacts/linux-engine-x64 mirror-input/') && + artifact_mirror_run_text.include?('cp -R downloaded-artifacts/android-engine-arm64 mirror-input/') && + artifact_mirror_run_text.include?('cp -R downloaded-artifacts/flutter-web-sdk mirror-input/') && + artifact_mirror_run_text.include?('cp -R downloaded-artifacts/ios-interpreter-engine mirror-input/') && + artifact_mirror_run_text.include?('cp -R downloaded-artifacts/macos-engine-arm64 mirror-input/') && + artifact_mirror_run_text.match?( + %r{\./scripts/assemble_artifact_mirror\.sh\s+\\\n\s+mirror-input\s+\\\n\s+artifacts/open-shorebird-artifact-mirror} + ) && + artifact_mirror_run_text.include?('artifacts/open-shorebird-artifact-mirror') && + artifact_mirror_run_text.include?('tar -C artifacts -czf open-shorebird-artifact-mirror.tar.gz open-shorebird-artifact-mirror') && + artifact_mirror_run_text.include?('python3 scripts/safe_extract_tar.py open-shorebird-artifact-mirror.tar.gz "$mirror_extract_dir"') && + artifact_mirror_run_text.include?('open-shorebird-artifact-mirror.tar.gz.sha256') && + artifact_mirror_run_text.include?('validate_artifact_mirror.py "$mirror_extract_dir/open-shorebird-artifact-mirror"') && + artifact_mirror_run_text.include?('scripts/write_release_manifest.py') && + artifact_mirror_run_text.include?('scripts/validate_release_manifest.py') && + artifact_mirror_run_text.include?('"$manifest_input"') && + artifact_mirror_run_text.include?('open-shorebird-release-manifest.json') && + artifact_mirror_run_text.include?('open-shorebird-release-manifest.json.sha256') && + artifact_mirror_run_text.include?('--github-sha "${{ github.sha }}"'), + 'artifact-mirror must assemble, archive, checksum, and manifest a publishable mirror root' +) +%w[ + cli-linux-x64/*open-shorebird-cli-linux-x64.tar.gz + cli-macos-x64/*open-shorebird-cli-macos-x64.tar.gz + cli-macos-arm64/*open-shorebird-cli-macos-arm64.tar.gz + cli-windows-x64/*open-shorebird-cli-windows-x64.tar.gz + shorebird-server-linux-amd64/*shorebird-server-linux-amd64.tar.gz + shorebird-server-linux-arm64/*shorebird-server-linux-arm64.tar.gz + shorebird-server-darwin-amd64/*shorebird-server-darwin-amd64.tar.gz + shorebird-server-darwin-arm64/*shorebird-server-darwin-arm64.tar.gz + shorebird-server-windows-amd64/*shorebird-server-windows-amd64.tar.gz + custom-dart-sdk-linux-x64/*custom-dart-sdk-linux-x64.tar.gz + custom-dart-sdk-macos-arm64/*custom-dart-sdk-macos-arm64.tar.gz + linux-engine-x64/*linux-engine-x64.tar.gz + android-engine-arm64/*android-engine-arm64.tar.gz + flutter-web-sdk/*flutter-web-sdk.tar.gz + ios-interpreter-engine/*ios-interpreter-engine.tar.gz + macos-engine-arm64/*macos-engine-arm64.tar.gz + mirror-patch-linux-x64.zip/*patch-linux-x64.zip + mirror-patch-darwin-x64.zip/*patch-darwin-x64.zip + mirror-patch-darwin-arm64.zip/*patch-darwin-arm64.zip + mirror-patch-windows-x64.zip/*patch-windows-x64.zip + mirror-metadata/*artifacts_manifest.yaml + open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz +].each do |required_glob| + assert!( + artifact_mirror_run_text.include?("--require '#{required_glob}'"), + "artifact-mirror release manifest must require #{required_glob}" + ) +end +%w[ + artifacts_manifest.yaml + patch-linux-x64.zip + patch-darwin-x64.zip + patch-darwin-arm64.zip + patch-windows-x64.zip + android-arm64-release/artifacts.zip + android-arm64-release/symbols.zip + linux-x64-release/artifacts.zip + linux-x64-release/linux-x64-flutter-gtk.zip + ios-release/artifacts.zip + flutter-web-sdk.zip + darwin-arm64-release/FlutterMacOS.framework.zip + flutter_patched_sdk_product.zip +].each do |required_path| + assert!( + artifact_mirror_run_text.include?(required_path), + "artifact-mirror archive smoke must verify #{required_path}" + ) +end + +assert!( + !workflow_text.match?(/\bdart_dynamic_modules\s*=\s*true\b/), + 'workflow must not enable DART_DYNAMIC_MODULES' +) +assert!( + !workflow_text.include?('aot-tools.dill'), + 'workflow must not publish the legacy closed/native AOT tools artifact' +) +assert!( + !workflow_text.include?('shasum'), + 'workflow must use scripts/write_sha256.sh instead of runner-specific shasum' +) +assert!( + workflow_text.include?('scripts/write_sha256.sh'), + 'workflow must use the workspace checksum helper' +) + +puts "workflow yaml ok: #{workflow_path}" +puts "workflow run blocks ok: #{scripts.length} checked" +puts "workflow artifact contract ok" diff --git a/scripts/verify_ci_workflow.sh b/scripts/verify_ci_workflow.sh new file mode 100755 index 0000000..b31023d --- /dev/null +++ b/scripts/verify_ci_workflow.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORKFLOW="$ROOT/.github/workflows/open-shorebird-ci.yml" +RUBY_ARGS=() + +usage() { + cat >&2 <<'EOF' +usage: verify_ci_workflow.sh [--require-tracked] [--require-clean] [--require-upload-ready] [workflow.yml] + +Validates the Open Shorebird GitHub Actions workflow contract. The upload-ready +mode additionally requires every required CI support file to be tracked in its +own git checkout and every required owning checkout to be clean. +EOF +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --require-tracked|--require-clean|--require-upload-ready) + RUBY_ARGS+=("$1") + shift + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "unknown argument: $1" >&2 + usage + exit 64 + ;; + *) + WORKFLOW="$1" + shift + if [[ "$#" -gt 0 ]]; then + echo "unexpected extra argument: $1" >&2 + usage + exit 64 + fi + ;; + esac +done + +if [[ ! -f "$WORKFLOW" ]]; then + echo "missing workflow: $WORKFLOW" >&2 + exit 66 +fi + +command -v ruby >/dev/null 2>&1 || { + echo "ruby is required to validate GitHub workflow YAML" >&2 + exit 127 +} + +if [[ "${#RUBY_ARGS[@]}" -eq 0 ]]; then + ruby "$ROOT/scripts/verify_ci_workflow.rb" "$WORKFLOW" +else + ruby "$ROOT/scripts/verify_ci_workflow.rb" "${RUBY_ARGS[@]}" "$WORKFLOW" +fi diff --git a/scripts/verify_dart_sdk_args.sh b/scripts/verify_dart_sdk_args.sh new file mode 100755 index 0000000..6d32e3d --- /dev/null +++ b/scripts/verify_dart_sdk_args.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -eq 0 ]]; then + echo "usage: $0 ..." >&2 + exit 64 +fi + +require_gn_value() { + local args_file="$1" + local key="$2" + local value="$3" + local actual + + if ! actual="$(read_gn_value "$args_file" "$key")"; then + echo "expected $key = $value in $args_file, but $key is missing" >&2 + exit 70 + fi + if [[ "$actual" != "$value" ]]; then + echo "expected $key = $value in $args_file, found $actual" >&2 + exit 70 + fi +} + +read_gn_value() { + local file="$1" + local key="$2" + awk -v key="$key" ' + $1 == key && $2 == "=" { + value = $0 + sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value) + sub("[[:space:]]*$", "", value) + found = 1 + } + END { + if (!found) { + exit 1 + } + print value + } + ' "$file" +} + +for args_file in "$@"; do + if [[ ! -f "$args_file" ]]; then + echo "missing args.gn: $args_file" >&2 + exit 66 + fi + + if [[ "$(read_gn_value "$args_file" dart_dynamic_modules)" == "true" ]]; then + echo "DART_DYNAMIC_MODULES must not be enabled: $args_file" >&2 + exit 70 + fi + + require_gn_value "$args_file" dart_dynamic_modules false + require_gn_value "$args_file" dart_enable_aot_patching true + require_gn_value "$args_file" dart_enable_shorebird_interpreter true + + echo "Verified $args_file: patched Dart SDK flags are enabled without DDM" +done diff --git a/scripts/verify_downloaded_release_artifacts.sh b/scripts/verify_downloaded_release_artifacts.sh new file mode 100755 index 0000000..8cff2d2 --- /dev/null +++ b/scripts/verify_downloaded_release_artifacts.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +usage() { + cat >&2 <<'EOF' +usage: verify_downloaded_release_artifacts.sh [--github-sha sha] downloaded-artifacts + +Verifies a downloaded full SDK build artifact set. The release manifest and +artifact mirror archive must have valid checksum sidecars, the manifest must +cover every downloaded artifact, and the mirror archive must safely extract to a +valid open Shorebird artifact mirror. +EOF +} + +DOWNLOAD_DIR="" +EXPECTED_GITHUB_SHA="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --github-sha) + if [[ "$#" -lt 2 || -z "${2:-}" ]]; then + echo "--github-sha value is required" >&2 + usage + exit 64 + fi + EXPECTED_GITHUB_SHA="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "unknown argument: $1" >&2 + usage + exit 64 + ;; + *) + if [[ -n "$DOWNLOAD_DIR" ]]; then + echo "unexpected extra argument: $1" >&2 + usage + exit 64 + fi + DOWNLOAD_DIR="$1" + shift + ;; + esac +done + +if [[ -z "$DOWNLOAD_DIR" ]]; then + usage + exit 64 +fi + +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-downloaded-release.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +fail() { + echo "error: $*" >&2 + exit 70 +} + +verify_sha256_sidecar() { + local artifact_path="$1" + local sidecar_path="$2" + + [[ -f "$sidecar_path" ]] || fail "missing checksum sidecar: $sidecar_path" + + "$PYTHON_BIN" - "$artifact_path" "$sidecar_path" <<'PY' +import hashlib +from pathlib import Path +import sys + +artifact_path = Path(sys.argv[1]) +sidecar_path = Path(sys.argv[2]) + +try: + text = sidecar_path.read_text(encoding="utf-8").strip() +except UnicodeDecodeError as error: + print(f"error: {sidecar_path}: invalid UTF-8: {error}", file=sys.stderr) + sys.exit(70) + +parts = text.split() +if len(parts) != 2: + print( + f"error: {sidecar_path}: expected ' ', got {text!r}", + file=sys.stderr, + ) + sys.exit(70) + +expected_digest, expected_filename = parts +if len(expected_digest) != 64 or any( + char not in "0123456789abcdef" for char in expected_digest +): + print(f"error: {sidecar_path}: invalid sha256 digest {expected_digest!r}", file=sys.stderr) + sys.exit(70) + +if expected_filename != artifact_path.name: + print( + f"error: {sidecar_path}: filename mismatch " + f"{expected_filename!r} != {artifact_path.name!r}", + file=sys.stderr, + ) + sys.exit(70) + +digest = hashlib.sha256() +with artifact_path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + +actual_digest = digest.hexdigest() +if expected_digest != actual_digest: + print( + f"error: {sidecar_path}: digest mismatch " + f"{expected_digest} != {actual_digest}", + file=sys.stderr, + ) + sys.exit(70) +PY +} + +[[ -d "$DOWNLOAD_DIR" ]] || fail "missing downloaded artifacts directory: $DOWNLOAD_DIR" + +manifest_paths=() +while IFS= read -r path; do + manifest_paths+=("$path") +done < <(find "$DOWNLOAD_DIR" -type f -name open-shorebird-release-manifest.json | sort) +if [[ "${#manifest_paths[@]}" -ne 1 ]]; then + fail "expected exactly one open-shorebird-release-manifest.json, found ${#manifest_paths[@]}" +fi +manifest_path="${manifest_paths[0]}" +manifest_sidecar="$manifest_path.sha256" + +verify_sha256_sidecar "$manifest_path" "$manifest_sidecar" + +validate_manifest_args=() +if [[ -n "$EXPECTED_GITHUB_SHA" ]]; then + validate_manifest_args+=(--github-sha "$EXPECTED_GITHUB_SHA") +fi +"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "${validate_manifest_args[@]}" \ + "$DOWNLOAD_DIR" \ + "$manifest_path" + +mirror_archives=() +while IFS= read -r path; do + mirror_archives+=("$path") +done < <(find "$DOWNLOAD_DIR" -type f -name open-shorebird-artifact-mirror.tar.gz | sort) +if [[ "${#mirror_archives[@]}" -ne 1 ]]; then + fail "expected exactly one open-shorebird-artifact-mirror.tar.gz, found ${#mirror_archives[@]}" +fi +mirror_archive="${mirror_archives[0]}" + +mirror_sidecar="$mirror_archive.sha256" +verify_sha256_sidecar "$mirror_archive" "$mirror_sidecar" + +"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" "$mirror_archive" "$TMP_DIR" +mirror_root="$TMP_DIR/open-shorebird-artifact-mirror" +[[ -d "$mirror_root" ]] || fail "mirror archive did not contain open-shorebird-artifact-mirror/" + +"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$mirror_root" + +echo "downloaded release artifacts verified: $DOWNLOAD_DIR" diff --git a/scripts/verify_engine_args.sh b/scripts/verify_engine_args.sh new file mode 100755 index 0000000..2260e34 --- /dev/null +++ b/scripts/verify_engine_args.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -eq 0 ]]; then + echo "usage: $0 ... | key=value ..." >&2 + exit 64 +fi + +verify_no_ddm() { + local args_file="$1" + + if [[ ! -f "$args_file" ]]; then + echo "missing args.gn: $args_file" >&2 + exit 66 + fi + + local actual + if ! actual="$(read_gn_value "$args_file" dart_dynamic_modules)"; then + return 0 + fi + if [[ "$actual" == "true" ]]; then + echo "DART_DYNAMIC_MODULES must not be enabled: $args_file" >&2 + exit 70 + fi +} + +read_gn_value() { + local file="$1" + local key="$2" + awk -v key="$key" ' + $1 == key && $2 == "=" { + value = $0 + sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value) + sub("[[:space:]]*$", "", value) + found = 1 + } + END { + if (!found) { + exit 1 + } + print value + } + ' "$file" +} + +has_expectations=0 +for arg in "$@"; do + if [[ "$arg" == *=* ]]; then + has_expectations=1 + break + fi +done + +if [[ "$has_expectations" == "0" ]]; then + for args_file in "$@"; do + verify_no_ddm "$args_file" + echo "Verified $args_file: dart_dynamic_modules is not true" + done + exit 0 +fi + +args_file="$1" +shift +verify_no_ddm "$args_file" + +for expectation in "$@"; do + if [[ "$expectation" != *=* ]]; then + echo "invalid expectation: $expectation; expected key=value" >&2 + exit 64 + fi + + key="${expectation%%=*}" + value="${expectation#*=}" + if ! actual="$(read_gn_value "$args_file" "$key")"; then + echo "expected $key = $value in $args_file, but $key is missing" >&2 + exit 70 + fi + if [[ "$actual" != "$value" ]]; then + echo "expected $key = $value in $args_file, found $actual" >&2 + exit 70 + fi +done + +echo "Verified $args_file: dart_dynamic_modules is not true and expected flags are present" diff --git a/scripts/verify_hosted_full_sdk_build.sh b/scripts/verify_hosted_full_sdk_build.sh new file mode 100755 index 0000000..af6eb74 --- /dev/null +++ b/scripts/verify_hosted_full_sdk_build.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: verify_hosted_full_sdk_build.sh --repo owner/name [options] + +Dispatches the Open Shorebird full SDK build on GitHub Actions, waits for the +workflow run to finish, downloads all artifacts, and verifies the release +manifest plus assembled artifact mirror. + +Options: + --repo owner/name GitHub repository to run against. + --ref branch-or-sha Ref to dispatch. Defaults to current branch. + --workflow file Workflow file. Defaults to open-shorebird-ci.yml. + --download-dir path Artifact download directory. Defaults to hosted-full-sdk-artifacts. + --timeout-minutes minutes Maximum wait time. Defaults to 720. + --poll-seconds seconds Poll interval. Defaults to 30. + --linux-heavy-runner label Override linux_heavy_runner. + --macos-heavy-runner label Override macos_heavy_runner. + --sdk-min-free-disk-gb value Override sdk_min_free_disk_gb. + --engine-min-free-disk-gb value Override engine_min_free_disk_gb. + --base-flutter-engine-revision v Override base_flutter_engine_revision. + --skip-gclient-sync Dispatch with run_gclient_sync=false. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO="" +REF="" +WORKFLOW="open-shorebird-ci.yml" +DOWNLOAD_DIR="hosted-full-sdk-artifacts" +TIMEOUT_MINUTES=720 +POLL_SECONDS=30 +LINUX_HEAVY_RUNNER="" +MACOS_HEAVY_RUNNER="" +SDK_MIN_FREE_DISK_GB="" +ENGINE_MIN_FREE_DISK_GB="" +BASE_FLUTTER_ENGINE_REVISION="" +RUN_GCLIENT_SYNC=true + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --repo) + REPO="${2:-}" + shift 2 + ;; + --ref) + REF="${2:-}" + shift 2 + ;; + --workflow) + WORKFLOW="${2:-}" + shift 2 + ;; + --download-dir) + DOWNLOAD_DIR="${2:-}" + shift 2 + ;; + --timeout-minutes) + TIMEOUT_MINUTES="${2:-}" + shift 2 + ;; + --poll-seconds) + POLL_SECONDS="${2:-}" + shift 2 + ;; + --linux-heavy-runner) + LINUX_HEAVY_RUNNER="${2:-}" + shift 2 + ;; + --macos-heavy-runner) + MACOS_HEAVY_RUNNER="${2:-}" + shift 2 + ;; + --sdk-min-free-disk-gb) + SDK_MIN_FREE_DISK_GB="${2:-}" + shift 2 + ;; + --engine-min-free-disk-gb) + ENGINE_MIN_FREE_DISK_GB="${2:-}" + shift 2 + ;; + --base-flutter-engine-revision) + BASE_FLUTTER_ENGINE_REVISION="${2:-}" + shift 2 + ;; + --skip-gclient-sync) + RUN_GCLIENT_SYNC=false + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 64 + ;; + esac +done + +if [[ -z "$REPO" ]]; then + echo "--repo owner/name is required" >&2 + usage + exit 64 +fi + +if ! command -v gh >/dev/null 2>&1; then + echo "GitHub CLI 'gh' is required" >&2 + exit 69 +fi + +if [[ -z "$REF" ]]; then + REF="$(git -C "$ROOT" branch --show-current 2>/dev/null || true)" +fi +if [[ -z "$REF" ]]; then + echo "--ref is required when the current checkout is detached" >&2 + exit 64 +fi + +run_fields=( + -f full_sdk_build=true + -f run_gclient_sync="$RUN_GCLIENT_SYNC" + -f run_runtime_smokes=false +) +[[ -z "$LINUX_HEAVY_RUNNER" ]] || run_fields+=(-f linux_heavy_runner="$LINUX_HEAVY_RUNNER") +[[ -z "$MACOS_HEAVY_RUNNER" ]] || run_fields+=(-f macos_heavy_runner="$MACOS_HEAVY_RUNNER") +[[ -z "$SDK_MIN_FREE_DISK_GB" ]] || run_fields+=(-f sdk_min_free_disk_gb="$SDK_MIN_FREE_DISK_GB") +[[ -z "$ENGINE_MIN_FREE_DISK_GB" ]] || run_fields+=(-f engine_min_free_disk_gb="$ENGINE_MIN_FREE_DISK_GB") +[[ -z "$BASE_FLUTTER_ENGINE_REVISION" ]] || run_fields+=(-f base_flutter_engine_revision="$BASE_FLUTTER_ENGINE_REVISION") + +echo "Dispatching $WORKFLOW on $REPO@$REF with full_sdk_build=true" +start_epoch="$(date +%s)" +start_iso="$(date -u -r "$start_epoch" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "@$start_epoch" +"%Y-%m-%dT%H:%M:%SZ")" +gh workflow run "$WORKFLOW" \ + --repo "$REPO" \ + --ref "$REF" \ + "${run_fields[@]}" + +run_id="" +for _ in {1..40}; do + run_list_args=( + --repo "$REPO" + --workflow "$WORKFLOW" + --event workflow_dispatch + --json databaseId,createdAt + --limit 20 + ) + if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then + run_list_args+=(--commit "$REF") + else + run_list_args+=(--branch "$REF") + fi + run_id="$( + gh run list \ + "${run_list_args[@]}" \ + --jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \ + | + head -n 1 + )" + [[ -z "$run_id" ]] || break + sleep 3 +done + +if [[ -z "$run_id" ]]; then + echo "unable to find dispatched workflow run for $WORKFLOW on $REF" >&2 + exit 70 +fi + +echo "Waiting for hosted full SDK run: $run_id" +deadline=$((start_epoch + TIMEOUT_MINUTES * 60)) +run_head_sha="" +while true; do + IFS=$'\t' read -r status conclusion url run_head_sha < <( + gh run view "$run_id" \ + --repo "$REPO" \ + --json status,conclusion,url,headSha \ + --jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv' + ) + echo "run $run_id status=$status conclusion=${conclusion:-null} url=$url" + + if [[ "$status" == "completed" ]]; then + if [[ "$conclusion" != "success" ]]; then + echo "hosted full SDK build failed: conclusion=$conclusion" >&2 + exit 70 + fi + break + fi + if [[ "$(date +%s)" -ge "$deadline" ]]; then + echo "timed out waiting for hosted full SDK build after $TIMEOUT_MINUTES minutes" >&2 + exit 70 + fi + sleep "$POLL_SECONDS" +done + +rm -rf "$DOWNLOAD_DIR" +mkdir -p "$DOWNLOAD_DIR" +gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR" +if [[ -z "$run_head_sha" ]]; then + echo "unable to read headSha for workflow run $run_id" >&2 + exit 70 +fi +"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \ + --github-sha "$run_head_sha" \ + "$DOWNLOAD_DIR" + +echo "hosted full SDK build verified: $run_id" diff --git a/scripts/verify_ios_interpreter_route.sh b/scripts/verify_ios_interpreter_route.sh index 7a34eff..c7cbfed 100755 --- a/scripts/verify_ios_interpreter_route.sh +++ b/scripts/verify_ios_interpreter_route.sh @@ -14,6 +14,7 @@ CHECKED_IPA="" CHECKED_PATCH_ARTIFACT="" ENTITLEMENTS_CHECKED=0 APP_STORE_STRICT_CHECKED=0 +NO_BUNDLED_KEY_CHECKED=0 CLEANUP_DIR="" cleanup() { @@ -35,14 +36,14 @@ read_gn_value() { $1 == key && $2 == "=" { value = $0 sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value) - print value + sub("[[:space:]]*$", "", value) found = 1 - exit } END { if (!found) { exit 1 } + print value } ' "$file" } @@ -66,6 +67,7 @@ verify_ios_engine_args() { require_gn_value "$args_file" target_os '"ios"' require_gn_value "$args_file" dart_dynamic_modules false + require_gn_value "$args_file" dart_enable_aot_patching true require_gn_value "$args_file" dart_enable_shorebird_interpreter true require_gn_value "$args_file" shorebird_use_interpreter true require_gn_value "$args_file" shorebird_enable_aot_patching false @@ -77,6 +79,7 @@ verify_host_engine_args() { require_gn_value "$args_file" target_os '"mac"' require_gn_value "$args_file" dart_dynamic_modules false + require_gn_value "$args_file" dart_enable_aot_patching true require_gn_value "$args_file" dart_enable_shorebird_interpreter true require_gn_value "$args_file" shorebird_use_interpreter true } @@ -143,6 +146,24 @@ verify_entitlements() { fi } +verify_no_bundled_patch_key() { + local app_bundle="$1" + [[ "$APP_STORE_STRICT" == "1" ]] || return 0 + + local shorebird_yaml + shorebird_yaml="$( + find "$app_bundle" -name "shorebird.yaml" -type f -print -quit + )" + if [[ -z "$shorebird_yaml" ]]; then + return 0 + fi + + if grep -Eq '^[[:space:]]*aot_patch_key_hex[[:space:]]*:' "$shorebird_yaml"; then + fail "APP_STORE_STRICT=1 rejects bundled aot_patch_key_hex in $shorebird_yaml" + fi + NO_BUNDLED_KEY_CHECKED=1 +} + verify_app_bundle() { if [[ -n "$IOS_APP_BUNDLE" && -n "$IOS_IPA" ]]; then fail "set only one of IOS_APP_BUNDLE or IOS_IPA" @@ -165,6 +186,7 @@ verify_app_bundle() { fi verify_entitlements "$IOS_APP_BUNDLE" + verify_no_bundled_patch_key "$IOS_APP_BUNDLE" } file_magic_hex() { @@ -186,31 +208,105 @@ verify_patch_artifact() { ;; esac - local compact_json - compact_json="$(LC_ALL=C tr -d '[:space:]' < "$IOS_PATCH_ARTIFACT")" + local python_bin + python_bin=python3 + if ! command -v "$python_bin" >/dev/null 2>&1; then + python_bin=python + fi + command -v "$python_bin" >/dev/null 2>&1 || + fail "python3 or python is required to inspect IOS_PATCH_ARTIFACT" - if [[ "$compact_json" != \{* ]]; then - fail "iOS patch artifact must be the encrypted open JSON wrapper, not a raw native/code payload" - fi - if ! grep -Fq '"format":"open-aot-vmcode-encrypted-v1"' <<<"$compact_json"; then - fail "iOS patch artifact is not an open encrypted VM code artifact" - fi - if grep -Fq '"runtime_mode":"dart-dynamic-modules"' <<<"$compact_json" || - grep -Fq '"runtime_mode":"dynamic-modules"' <<<"$compact_json"; then - fail "iOS patch artifact uses DART_DYNAMIC_MODULES runtime mode" - fi - if ! grep -Fq '"runtime_mode":"dart-bytecode-interpreter"' <<<"$compact_json"; then - fail "iOS patch artifact must declare runtime_mode dart-bytecode-interpreter" - fi - if ! grep -Fq '"target_os":"ios"' <<<"$compact_json"; then - fail "iOS patch artifact must target iOS" - fi - if ! grep -Fq '"target_arch":"arm64"' <<<"$compact_json"; then - fail "iOS patch artifact must target arm64" - fi - if ! grep -Fq '"payload_kind":"full-snapshot"' <<<"$compact_json"; then - fail "current iOS interpreter mapper requires payload_kind full-snapshot" - fi + "$python_bin" - "$IOS_PATCH_ARTIFACT" <<'PY' +import base64 +import binascii +import json +import re +import sys + +path = sys.argv[1] + +try: + with open(path, encoding="utf-8") as file: + artifact = json.load(file) +except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise SystemExit( + "iOS patch artifact must be the encrypted open JSON wrapper, " + f"not a raw native/code payload: {error}" + ) + +if not isinstance(artifact, dict): + raise SystemExit("iOS patch artifact JSON must be an object") + +metadata = artifact.get("metadata") +if not isinstance(metadata, dict): + raise SystemExit("iOS patch artifact must contain metadata object") + + +def require(mapping, key, expected, scope): + actual = mapping.get(key) + if actual != expected: + raise SystemExit( + f"iOS patch artifact {scope}.{key} is {actual!r}; " + f"expected {expected!r}" + ) + + +require(artifact, "format", "open-aot-vmcode-encrypted-v1", "artifact") +runtime_mode = metadata.get("runtime_mode") +if runtime_mode in {"dart-dynamic-modules", "dynamic-modules"}: + raise SystemExit("iOS patch artifact uses DART_DYNAMIC_MODULES runtime mode") +require(metadata, "runtime_mode", "dart-bytecode-interpreter", "metadata") +require(metadata, "target_os", "ios", "metadata") +require(metadata, "target_arch", "arm64", "metadata") +require(artifact, "payload_kind", "full-snapshot", "artifact") + +encryption = artifact.get("encryption") +if not isinstance(encryption, dict): + raise SystemExit("iOS patch artifact must contain encryption object") +require(encryption, "algorithm", "AES-256-GCM", "encryption") + + +def require_base64(mapping, key, scope): + value = mapping.get(key) + if not isinstance(value, str) or not value: + raise SystemExit(f"iOS patch artifact {scope}.{key} must be non-empty") + try: + decoded = base64.b64decode(value, validate=True) + except (binascii.Error, ValueError) as error: + raise SystemExit( + f"iOS patch artifact {scope}.{key} is not valid base64: {error}" + ) + if not decoded: + raise SystemExit(f"iOS patch artifact {scope}.{key} decodes to empty bytes") + return decoded + + +require_base64(artifact, "encrypted_payload_base64", "artifact") +require_base64(encryption, "nonce_base64", "encryption") +require_base64(encryption, "tag_base64", "encryption") + +key_id = encryption.get("key_id") +if not isinstance(key_id, str) or not key_id: + raise SystemExit("iOS patch artifact encryption.key_id must be non-empty") + +hex_pattern = re.compile(r"^[0-9a-f]{64}$") +for scope, mapping, key in ( + ("artifact", artifact, "payload_sha256"), + ("encryption", encryption, "aad_sha256"), +): + value = mapping.get(key) + if not isinstance(value, str) or not hex_pattern.fullmatch(value): + raise SystemExit( + f"iOS patch artifact {scope}.{key} must be a lowercase SHA-256 hex digest" + ) + +reconstructed_size = artifact.get("reconstructed_size") +if reconstructed_size is not None: + if not isinstance(reconstructed_size, int) or reconstructed_size <= 0: + raise SystemExit( + "iOS patch artifact reconstructed_size must be a positive integer" + ) +PY CHECKED_PATCH_ARTIFACT="$IOS_PATCH_ARTIFACT" } @@ -240,6 +336,9 @@ fi if [[ "$APP_STORE_STRICT_CHECKED" == "1" ]]; then echo " App Store strict: get-task-allow is not true" fi +if [[ "$NO_BUNDLED_KEY_CHECKED" == "1" ]]; then + echo " key material: no bundled aot_patch_key_hex" +fi if [[ -n "$CHECKED_PATCH_ARTIFACT" ]]; then echo " patch artifact: encrypted interpreter full-snapshot for ios/arm64" fi diff --git a/scripts/verify_ios_interpreter_route_validator.sh b/scripts/verify_ios_interpreter_route_validator.sh new file mode 100755 index 0000000..64f7947 --- /dev/null +++ b/scripts/verify_ios_interpreter_route_validator.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/ios-route-validator.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +IOS_ENGINE_DIR="$TMP_DIR/ios_release" +HOST_ENGINE_DIR="$TMP_DIR/host_release_arm64" +mkdir -p "$IOS_ENGINE_DIR" "$HOST_ENGINE_DIR" + +cat > "$IOS_ENGINE_DIR/args.gn" <<'EOF' +target_os = "android" +target_os = "ios" +dart_dynamic_modules = false +dart_enable_aot_patching = false +dart_enable_aot_patching = true +dart_enable_shorebird_interpreter = false +dart_enable_shorebird_interpreter = true +shorebird_use_interpreter = false +shorebird_use_interpreter = true +shorebird_enable_aot_patching = true +shorebird_enable_aot_patching = false +EOF + +cat > "$HOST_ENGINE_DIR/args.gn" <<'EOF' +target_os = "linux" +target_os = "mac" +dart_dynamic_modules = false +dart_enable_aot_patching = false +dart_enable_aot_patching = true +dart_enable_shorebird_interpreter = false +dart_enable_shorebird_interpreter = true +shorebird_use_interpreter = false +shorebird_use_interpreter = true +EOF + +write_artifact() { + local path="$1" + local runtime_mode="$2" + local target_os="${3:-ios}" + cat > "$path" </dev/null + +bad_runtime="$TMP_DIR/bad-runtime.vmcode" +write_artifact "$bad_runtime" "dart-dynamic-modules" +if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \ + HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \ + IOS_PATCH_ARTIFACT="$bad_runtime" \ + "$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then + echo "iOS route validator unexpectedly accepted DART_DYNAMIC_MODULES metadata" >&2 + exit 70 +fi + +bad_target="$TMP_DIR/bad-target.vmcode" +write_artifact "$bad_target" "dart-bytecode-interpreter" "android" +if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \ + HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \ + IOS_PATCH_ARTIFACT="$bad_target" \ + "$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then + echo "iOS route validator unexpectedly accepted a non-iOS patch artifact" >&2 + exit 70 +fi + +bad_json="$TMP_DIR/bad-json.vmcode" +printf '{"format":"open-aot-vmcode-encrypted-v1"\n' > "$bad_json" +if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \ + HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \ + IOS_PATCH_ARTIFACT="$bad_json" \ + "$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then + echo "iOS route validator unexpectedly accepted malformed JSON" >&2 + exit 70 +fi + +bad_native="$TMP_DIR/bad-native.vmcode" +printf '\xcf\xfa\xed\xfe' > "$bad_native" +if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \ + HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \ + IOS_PATCH_ARTIFACT="$bad_native" \ + "$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then + echo "iOS route validator unexpectedly accepted a Mach-O patch artifact" >&2 + exit 70 +fi + +echo "iOS interpreter route validator smoke test passed" diff --git a/scripts/verify_open_infrastructure_defaults.sh b/scripts/verify_open_infrastructure_defaults.sh new file mode 100755 index 0000000..2448502 --- /dev/null +++ b/scripts/verify_open_infrastructure_defaults.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +fail() { + echo "error: $*" >&2 + exit 70 +} + +require_contains() { + local path="$1" + local needle="$2" + grep -Fq "$needle" "$path" || fail "$path is missing required text: $needle" +} + +reject_contains() { + local path="$1" + local needle="$2" + if grep -Fq "$needle" "$path"; then + fail "$path contains forbidden text: $needle" + fi +} + +check_forbidden_in_file() { + local path="$1" + local pattern + + [[ -f "$path" ]] || fail "missing open-infrastructure check input: $path" + for pattern in "${FORBIDDEN_PATTERNS[@]}"; do + reject_contains "$path" "$pattern" + done +} + +check_forbidden_in_tree() { + local tree="$1" + local extension="$2" + local path + + [[ -d "$tree" ]] || fail "missing open-infrastructure check tree: $tree" + while IFS= read -r -d '' path; do + check_forbidden_in_file "$path" + done < <(find "$tree" -type f -name "*.$extension" -print0) +} + +FORBIDDEN_PATTERNS=( + "https://download.shorebird.dev" + "download.shorebird.dev" + "api.shorebird.dev" + "auth.shorebird.dev" + "console.shorebird.dev" + "cdn.shorebird.cloud" + "git@github.com:shorebirdtech/dart-sdk.git" + "github.com/shorebirdtech/updater.git" + "github.com/shorebirdtech/flutter.git" + "shorebird-dart-sdk-prebuilt" + "shorebirdtech/_build_engine" +) + +BUILD_SENSITIVE_FILES=( + "$ROOT/.gitmodules" + "$ROOT/flutter/DEPS" + "$ROOT/flutter/bin/internal/update_dart_sdk.ps1" + "$ROOT/flutter/bin/internal/update_dart_sdk.sh" + "$ROOT/flutter/dev/bots/post_process_docs.dart" + "$ROOT/flutter/dev/bots/unpublish_package.dart" + "$ROOT/flutter/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle" + "$ROOT/flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts" + "$ROOT/flutter/dev/tools/create_api_docs.dart" + "$ROOT/flutter/engine/src/flutter/build/zip_bundle.gni" + "$ROOT/flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart" + "$ROOT/flutter/packages/flutter_tools/gradle/aar_init_script.gradle" + "$ROOT/flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt" + "$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart" + "$ROOT/flutter/packages/flutter_tools/lib/src/http_host_validator.dart" + "$ROOT/flutter/packages/flutter_tools/pubspec.yaml" + "$ROOT/flutter/packages/shorebird_tests/test/shorebird_tests.dart" + "$ROOT/scripts/write_gclient.sh" + "$ROOT/shorebird/bin/shorebird.ps1" + "$ROOT/shorebird/third_party/flutter/bin/internal/shared.sh" + "$ROOT/updater/library/src/config.rs" +) + +for path in "${BUILD_SENSITIVE_FILES[@]}"; do + check_forbidden_in_file "$path" +done + +check_forbidden_in_tree "$ROOT/shorebird/packages/artifact_proxy/lib" dart +check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_cli/lib" dart +check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_code_push_client/lib" dart + +require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/dart-lang/sdk.git" +require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/flutter.git" +require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird.git" +require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-server.git" +require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-updater.git" + +require_contains "$ROOT/flutter/DEPS" '"dart_sdk_git": "https://git.tonycloud.org/dart-lang/sdk.git"' +require_contains "$ROOT/flutter/DEPS" '"updater_git": "https://git.tonycloud.org/flutter/shorebird-updater.git"' + +require_contains "$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart" \ + "kOpenFlutterStorageUrl = 'http://localhost:8080/download.flutter.io'" +require_contains "$ROOT/flutter/bin/internal/update_dart_sdk.sh" \ + "http://localhost:8080/download.flutter.io" +require_contains "$ROOT/flutter/bin/internal/update_dart_sdk.ps1" \ + "http://localhost:8080/download.flutter.io" +require_contains "$ROOT/flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt" \ + 'DEFAULT_MAVEN_HOST = "http://localhost:8080/download.flutter.io"' +require_contains "$ROOT/flutter/dev/tools/create_api_docs.dart" \ + "Platform.environment['FLUTTER_STORAGE_BASE_URL']" +require_contains "$ROOT/flutter/packages/shorebird_tests/test/shorebird_tests.dart" \ + "'FLUTTER_STORAGE_BASE_URL': 'http://localhost:8080/download.flutter.io'" + +require_contains "$ROOT/shorebird/packages/shorebird_cli/lib/src/shorebird_env.dart" \ + "defaultHostedUrl = 'http://localhost:8080'" +require_contains "$ROOT/shorebird/packages/shorebird_cli/lib/src/cache.dart" \ + "defaultArtifactBaseUrl = 'http://localhost:8080/artifacts'" +require_contains "$ROOT/updater/library/src/config.rs" \ + 'const DEFAULT_BASE_URL: &str = "http://localhost:8080";' + +echo "open infrastructure defaults check passed" diff --git a/scripts/verify_powershell_open_defaults.sh b/scripts/verify_powershell_open_defaults.sh new file mode 100755 index 0000000..0494e49 --- /dev/null +++ b/scripts/verify_powershell_open_defaults.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +fail() { + echo "error: $*" >&2 + exit 70 +} + +require_contains() { + local path="$1" + local needle="$2" + grep -Fq "$needle" "$path" || fail "$path is missing required text: $needle" +} + +reject_contains() { + local path="$1" + local needle="$2" + if grep -Fq "$needle" "$path"; then + fail "$path contains forbidden text: $needle" + fi +} + +shorebird_launcher="$ROOT/shorebird/bin/shorebird.ps1" +flutter_dart_updater="$ROOT/flutter/bin/internal/update_dart_sdk.ps1" + +[[ -f "$shorebird_launcher" ]] || fail "missing PowerShell launcher: $shorebird_launcher" +[[ -f "$flutter_dart_updater" ]] || fail "missing Flutter Dart SDK updater: $flutter_dart_updater" + +require_contains "$shorebird_launcher" \ + '$defaultFlutterGitUrl = "https://git.tonycloud.org/flutter/flutter.git"' +require_contains "$shorebird_launcher" \ + '$defaultFlutterStorageBaseUrl = "http://localhost:8080/download.flutter.io"' +require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_GIT_URL' +require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_STORAGE_BASE_URL' +require_contains "$shorebird_launcher" 'FLUTTER_STORAGE_BASE_URL' + +require_contains "$flutter_dart_updater" '$Env:FLUTTER_STORAGE_BASE_URL' +require_contains "$flutter_dart_updater" \ + '$dartSdkBaseUrl = "http://localhost:8080/download.flutter.io"' + +for path in "$shorebird_launcher" "$flutter_dart_updater"; do + reject_contains "$path" 'download.shorebird.dev' + reject_contains "$path" 'api.shorebird.dev' + reject_contains "$path" 'auth.shorebird.dev' + reject_contains "$path" 'console.shorebird.dev' + reject_contains "$path" 'docs.shorebird.dev' + reject_contains "$path" 'github.com/shorebirdtech/flutter.git' + reject_contains "$path" 'github.com/shorebirdtech/shorebird' + reject_contains "$path" 'git@github.com:shorebirdtech' +done + +if command -v pwsh >/dev/null 2>&1; then + SHOREBIRD_POWERSHELL_LAUNCHER="$shorebird_launcher" \ + FLUTTER_DART_SDK_POWERSHELL_UPDATER="$flutter_dart_updater" \ + pwsh -NoProfile -NonInteractive -Command ' + $paths = @( + $env:SHOREBIRD_POWERSHELL_LAUNCHER, + $env:FLUTTER_DART_SDK_POWERSHELL_UPDATER + ) + $failed = $false + foreach ($path in $paths) { + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile( + $path, + [ref]$tokens, + [ref]$errors + ) | Out-Null + if ($errors.Count -gt 0) { + Write-Error "$path has PowerShell parse errors:" + foreach ($errorRecord in $errors) { + Write-Error " $($errorRecord.Message)" + } + $failed = $true + } + } + if ($failed) { + exit 70 + } + ' +else + echo "warning: pwsh not found; skipped PowerShell parse check" >&2 +fi + +echo "PowerShell open-default checks passed" diff --git a/scripts/verify_release_manifest.sh b/scripts/verify_release_manifest.sh new file mode 100755 index 0000000..32551a6 --- /dev/null +++ b/scripts/verify_release_manifest.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-release-manifest.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +INPUT_DIR="$TMP_DIR/artifacts" +mkdir -p "$INPUT_DIR/cli-linux" "$INPUT_DIR/server-linux" "$INPUT_DIR/mirror-patch/artifacts/mirror" +printf 'cli archive\n' > "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz" +printf 'server archive\n' > "$INPUT_DIR/server-linux/shorebird-server-linux-amd64.tar.gz" +printf 'patch archive\n' > "$INPUT_DIR/mirror-patch/artifacts/mirror/patch-linux-x64.zip" + +"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz" +"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/server-linux/shorebird-server-linux-amd64.tar.gz" +"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/mirror-patch/artifacts/mirror/patch-linux-x64.zip" + +"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$INPUT_DIR" \ + --github-sha test-sha \ + --require 'cli-linux/*open-shorebird-cli-linux-x64.tar.gz' \ + --require 'server-linux/*shorebird-server-linux-amd64.tar.gz' \ + --require 'mirror-patch/*patch-linux-x64.zip' \ + --output "$TMP_DIR/release-manifest.json" + +"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + --github-sha test-sha \ + "$INPUT_DIR" \ + "$TMP_DIR/release-manifest.json" + +"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" <<'PY' +import json +import sys + +manifest = json.load(open(sys.argv[1], encoding="utf-8")) +assert manifest["format_version"] == 1 +assert manifest["github_sha"] == "test-sha" +assert manifest["artifact_count"] == 3 +paths = {artifact["path"] for artifact in manifest["artifacts"]} +assert "cli-linux/open-shorebird-cli-linux-x64.tar.gz" in paths +assert "server-linux/shorebird-server-linux-amd64.tar.gz" in paths +assert "mirror-patch/artifacts/mirror/patch-linux-x64.zip" in paths +for artifact in manifest["artifacts"]: + assert artifact["artifact_group"] in { + "cli-linux", + "server-linux", + "mirror-patch", + } + assert artifact["filename"] == artifact["path"].split("/")[-1] + assert len(artifact["sha256"]) == 64 + assert artifact["size"] > 0 + assert artifact["sidecar"].endswith(".sha256") +PY + +TAMPERED_MANIFEST="$TMP_DIR/tampered-release-manifest.json" +"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$TAMPERED_MANIFEST" <<'PY' +import json +import sys + +source, output = sys.argv[1:] +manifest = json.load(open(source, encoding="utf-8")) +manifest["artifacts"][0]["sha256"] = "0" * 64 +json.dump(manifest, open(output, "w", encoding="utf-8")) +PY +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$INPUT_DIR" \ + "$TAMPERED_MANIFEST" >/dev/null 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted a tampered digest" >&2 + exit 70 +fi + +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + --github-sha wrong-sha \ + "$INPUT_DIR" \ + "$TMP_DIR/release-manifest.json" >"$TMP_DIR/wrong-sha.log" 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted the wrong github_sha" >&2 + exit 70 +fi +grep -q "github_sha is" "$TMP_DIR/wrong-sha.log" + +UNLISTED_ARTIFACTS="$TMP_DIR/unlisted-artifacts" +cp -R "$INPUT_DIR" "$UNLISTED_ARTIFACTS" +printf 'extra artifact\n' > "$UNLISTED_ARTIFACTS/extra.tar.gz" +"$ROOT/scripts/write_sha256.sh" "$UNLISTED_ARTIFACTS/extra.tar.gz" +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$UNLISTED_ARTIFACTS" \ + "$TMP_DIR/release-manifest.json" >/dev/null 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted an unlisted artifact" >&2 + exit 70 +fi + +UNSAFE_MANIFEST="$TMP_DIR/unsafe-release-manifest.json" +"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$UNSAFE_MANIFEST" <<'PY' +import json +import sys + +source, output = sys.argv[1:] +manifest = json.load(open(source, encoding="utf-8")) +manifest["artifacts"][0]["path"] = r"cli-linux\open-shorebird-cli-linux-x64.tar.gz" +manifest["artifacts"][0]["sidecar"] = ( + r"cli-linux\open-shorebird-cli-linux-x64.tar.gz.sha256" +) +json.dump(manifest, open(output, "w", encoding="utf-8")) +PY +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$INPUT_DIR" \ + "$UNSAFE_MANIFEST" >/dev/null 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted an unsafe artifact path" >&2 + exit 70 +fi + +BAD_PROVENANCE_MANIFEST="$TMP_DIR/bad-provenance-release-manifest.json" +"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$BAD_PROVENANCE_MANIFEST" <<'PY' +import json +import sys + +source, output = sys.argv[1:] +manifest = json.load(open(source, encoding="utf-8")) +manifest["artifacts"][0]["artifact_group"] = "wrong-group" +manifest["artifacts"][1]["filename"] = "wrong-name.tar.gz" +json.dump(manifest, open(output, "w", encoding="utf-8")) +PY +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$INPUT_DIR" \ + "$BAD_PROVENANCE_MANIFEST" >/dev/null 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted bad provenance fields" >&2 + exit 70 +fi + +SYMLINK_LISTED_ARTIFACT="$TMP_DIR/symlink-listed-artifact" +cp -R "$INPUT_DIR" "$SYMLINK_LISTED_ARTIFACT" +rm "$SYMLINK_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz" +ln -s "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz" \ + "$SYMLINK_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz" +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$SYMLINK_LISTED_ARTIFACT" \ + "$TMP_DIR/release-manifest.json" >/dev/null 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted a symlink artifact" >&2 + exit 70 +fi + +EMPTY_LISTED_ARTIFACT="$TMP_DIR/empty-listed-artifact" +cp -R "$INPUT_DIR" "$EMPTY_LISTED_ARTIFACT" +printf '' > "$EMPTY_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz" +"$ROOT/scripts/write_sha256.sh" \ + "$EMPTY_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz" +EMPTY_LISTED_MANIFEST="$TMP_DIR/empty-listed-release-manifest.json" +"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$EMPTY_LISTED_MANIFEST" <<'PY' +import hashlib +import json +import sys + +source, output = sys.argv[1:] +manifest = json.load(open(source, encoding="utf-8")) +empty_digest = hashlib.sha256(b"").hexdigest() +for artifact in manifest["artifacts"]: + if artifact["path"] == "cli-linux/open-shorebird-cli-linux-x64.tar.gz": + artifact["sha256"] = empty_digest + artifact["size"] = 0 +json.dump(manifest, open(output, "w", encoding="utf-8")) +PY +if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \ + "$EMPTY_LISTED_ARTIFACT" \ + "$EMPTY_LISTED_MANIFEST" >"$TMP_DIR/empty-listed.log" 2>&1; then + echo "validate_release_manifest.py unexpectedly accepted an empty artifact" >&2 + exit 70 +fi +grep -q "empty artifacts are not allowed" "$TMP_DIR/empty-listed.log" + +SYMLINK_INPUT="$TMP_DIR/symlink-input" +mkdir -p "$SYMLINK_INPUT" +printf 'symlink target\n' > "$SYMLINK_INPUT/target.tar.gz" +"$ROOT/scripts/write_sha256.sh" "$SYMLINK_INPUT/target.tar.gz" +ln -s target.tar.gz "$SYMLINK_INPUT/link.tar.gz" +if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$SYMLINK_INPUT" \ + --output "$TMP_DIR/symlink-input.json" >/dev/null 2>&1; then + echo "write_release_manifest.py unexpectedly accepted a symlink artifact" >&2 + exit 70 +fi + +MISSING_SIDECAR="$TMP_DIR/missing-sidecar" +mkdir -p "$MISSING_SIDECAR" +printf 'missing sidecar\n' > "$MISSING_SIDECAR/artifact.tar.gz" +if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$MISSING_SIDECAR" \ + --output "$TMP_DIR/missing.json" >/dev/null 2>&1; then + echo "write_release_manifest.py unexpectedly accepted a missing sidecar" >&2 + exit 70 +fi + +BAD_SIDECAR="$TMP_DIR/bad-sidecar" +mkdir -p "$BAD_SIDECAR" +printf 'bad sidecar\n' > "$BAD_SIDECAR/artifact.tar.gz" +printf '%064d artifact.tar.gz\n' 0 > "$BAD_SIDECAR/artifact.tar.gz.sha256" +if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$BAD_SIDECAR" \ + --output "$TMP_DIR/bad.json" >/dev/null 2>&1; then + echo "write_release_manifest.py unexpectedly accepted a bad sidecar" >&2 + exit 70 +fi + +EMPTY_ARTIFACT="$TMP_DIR/empty-artifact" +mkdir -p "$EMPTY_ARTIFACT" +printf '' > "$EMPTY_ARTIFACT/artifact.tar.gz" +"$ROOT/scripts/write_sha256.sh" "$EMPTY_ARTIFACT/artifact.tar.gz" +if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$EMPTY_ARTIFACT" \ + --output "$TMP_DIR/empty.json" >"$TMP_DIR/empty-artifact.log" 2>&1; then + echo "write_release_manifest.py unexpectedly accepted an empty artifact" >&2 + exit 70 +fi +grep -q "empty artifacts are not allowed" "$TMP_DIR/empty-artifact.log" + +ORPHAN_SIDECAR="$TMP_DIR/orphan-sidecar" +mkdir -p "$ORPHAN_SIDECAR" +printf '%064d missing.tar.gz\n' 0 > "$ORPHAN_SIDECAR/missing.tar.gz.sha256" +if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$ORPHAN_SIDECAR" \ + --output "$TMP_DIR/orphan.json" >/dev/null 2>&1; then + echo "write_release_manifest.py unexpectedly accepted an orphan sidecar" >&2 + exit 70 +fi + +if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \ + "$INPUT_DIR" \ + --require 'missing-artifact/*.tar.gz' \ + --output "$TMP_DIR/missing-required.json" >/dev/null 2>&1; then + echo "write_release_manifest.py unexpectedly accepted a missing required artifact" >&2 + exit 70 +fi + +echo "write_release_manifest.py smoke test passed" diff --git a/scripts/verify_sync_open_sources.sh b/scripts/verify_sync_open_sources.sh new file mode 100755 index 0000000..7d73502 --- /dev/null +++ b/scripts/verify_sync_open_sources.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-source-sync.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +init_git_checkout() { + local path="$1" + mkdir -p "$path" + git -C "$path" init -q +} + +real_path() { + python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$1" +} + +make_workspace() { + local workspace="$1" + + mkdir -p "$workspace/scripts" + cp "$ROOT/scripts/sync_open_sources.sh" "$workspace/scripts/sync_open_sources.sh" + chmod +x "$workspace/scripts/sync_open_sources.sh" + + init_git_checkout "$workspace/dart-sdk-new" + mkdir -p "$workspace/dart-sdk-new/runtime/vm" + : > "$workspace/dart-sdk-new/runtime/vm/dart_api_impl.h" + + init_git_checkout "$workspace/updater" + mkdir -p "$workspace/updater/library/include" + : > "$workspace/updater/library/include/updater_engine.h" + + mkdir -p "$workspace/flutter/engine/src/flutter/third_party" +} + +run_sync() { + local workspace="$1" + DART_SRC="$workspace/dart-sdk-new" \ + UPDATER_SRC="$workspace/updater" \ + "$workspace/scripts/sync_open_sources.sh" +} + +assert_links_to_workspace_sources() { + local workspace="$1" + local dart_target="$workspace/flutter/engine/src/flutter/third_party/dart" + local updater_target="$workspace/flutter/engine/src/flutter/third_party/updater" + + test -L "$dart_target" + test -L "$updater_target" + [[ "$(real_path "$dart_target")" == "$(real_path "$workspace/dart-sdk-new")" ]] + [[ "$(real_path "$updater_target")" == "$(real_path "$workspace/updater")" ]] + test -f "$dart_target/runtime/vm/dart_api_impl.h" + test -f "$updater_target/library/include/updater_engine.h" +} + +clean_checkout_workspace="$TMP_DIR/clean-checkouts" +make_workspace "$clean_checkout_workspace" +init_git_checkout "$clean_checkout_workspace/flutter/engine/src/flutter/third_party/dart" +init_git_checkout "$clean_checkout_workspace/flutter/engine/src/flutter/third_party/updater" +run_sync "$clean_checkout_workspace" +assert_links_to_workspace_sources "$clean_checkout_workspace" +run_sync "$clean_checkout_workspace" +assert_links_to_workspace_sources "$clean_checkout_workspace" + +stale_link_workspace="$TMP_DIR/stale-link" +make_workspace "$stale_link_workspace" +mkdir -p "$stale_link_workspace/other-dart" +ln -s "../../../../../other-dart" \ + "$stale_link_workspace/flutter/engine/src/flutter/third_party/dart" +if run_sync "$stale_link_workspace" >"$TMP_DIR/stale-link.log" 2>&1; then + echo "expected stale Dart symlink to fail" >&2 + exit 1 +fi +grep -q "Dart SDK target symlink points at" "$TMP_DIR/stale-link.log" + +dirty_checkout_workspace="$TMP_DIR/dirty-checkout" +make_workspace "$dirty_checkout_workspace" +dirty_dart="$dirty_checkout_workspace/flutter/engine/src/flutter/third_party/dart" +init_git_checkout "$dirty_dart" +: > "$dirty_dart/untracked.txt" +if run_sync "$dirty_checkout_workspace" >"$TMP_DIR/dirty-checkout.log" 2>&1; then + echo "expected dirty Dart checkout to fail" >&2 + exit 1 +fi +grep -q "Dart SDK target is a dirty git checkout" "$TMP_DIR/dirty-checkout.log" +test -d "$dirty_dart/.git" +test -f "$dirty_dart/untracked.txt" + +forbidden_dart_remote_workspace="$TMP_DIR/forbidden-dart-remote" +make_workspace "$forbidden_dart_remote_workspace" +git -C "$forbidden_dart_remote_workspace/dart-sdk-new" remote add origin \ + https://github.com/dart-lang/sdk.git +if run_sync "$forbidden_dart_remote_workspace" >"$TMP_DIR/forbidden-dart-remote.log" 2>&1; then + echo "expected forbidden Dart SDK remote to fail" >&2 + exit 1 +fi +grep -q "Dart SDK source checkout uses forbidden remote fragment" \ + "$TMP_DIR/forbidden-dart-remote.log" + +forbidden_updater_remote_workspace="$TMP_DIR/forbidden-updater-remote" +make_workspace "$forbidden_updater_remote_workspace" +git -C "$forbidden_updater_remote_workspace/updater" remote add origin \ + https://github.com/shorebirdtech/updater.git +if run_sync "$forbidden_updater_remote_workspace" >"$TMP_DIR/forbidden-updater-remote.log" 2>&1; then + echo "expected forbidden updater source remote to fail" >&2 + exit 1 +fi +grep -q "updater submodule source checkout uses forbidden remote fragment" \ + "$TMP_DIR/forbidden-updater-remote.log" + +forbidden_updater_url_workspace="$TMP_DIR/forbidden-updater-url" +make_workspace "$forbidden_updater_url_workspace" +rm -rf "$forbidden_updater_url_workspace/updater" +if DART_SRC="$forbidden_updater_url_workspace/dart-sdk-new" \ + UPDATER_SRC="$forbidden_updater_url_workspace/missing-updater" \ + UPDATER_URL=https://github.com/shorebirdtech/updater.git \ + "$forbidden_updater_url_workspace/scripts/sync_open_sources.sh" \ + >"$TMP_DIR/forbidden-updater-url.log" 2>&1; then + echo "expected forbidden explicit UPDATER_URL to fail" >&2 + exit 1 +fi +grep -q "UPDATER_URL points at a forbidden official Shorebird updater remote" \ + "$TMP_DIR/forbidden-updater-url.log" + +echo "sync_open_sources.sh smoke test passed" diff --git a/scripts/verify_upload_readiness.sh b/scripts/verify_upload_readiness.sh new file mode 100755 index 0000000..e0cef74 --- /dev/null +++ b/scripts/verify_upload_readiness.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +"$ROOT/scripts/verify_ci_workflow.sh" \ + --require-upload-ready \ + "$ROOT/.github/workflows/open-shorebird-ci.yml" + +echo "upload readiness check passed" diff --git a/scripts/verify_write_sha256.sh b/scripts/verify_write_sha256.sh new file mode 100755 index 0000000..b387006 --- /dev/null +++ b/scripts/verify_write_sha256.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-sha256.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +ARTIFACT="$TMP_DIR/artifact.txt" +SIDECAR="$TMP_DIR/artifact.txt.sha256" +CUSTOM_SIDECAR="$TMP_DIR/custom.sha256" + +printf 'open-shorebird\n' > "$ARTIFACT" + +"$ROOT/scripts/write_sha256.sh" "$ARTIFACT" + +EXPECTED_HASH="$("$PYTHON_BIN" - "$ARTIFACT" <<'PY' +import hashlib +import pathlib +import sys + +print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest()) +PY +)" +EXPECTED_LINE="$EXPECTED_HASH artifact.txt" +ACTUAL_LINE="$(cat "$SIDECAR")" + +if [[ "$ACTUAL_LINE" != "$EXPECTED_LINE" ]]; then + echo "unexpected sha256 sidecar: $ACTUAL_LINE" >&2 + echo "expected: $EXPECTED_LINE" >&2 + exit 70 +fi + +"$ROOT/scripts/write_sha256.sh" "$ARTIFACT" "$CUSTOM_SIDECAR" +CUSTOM_LINE="$(cat "$CUSTOM_SIDECAR")" + +if [[ "$CUSTOM_LINE" != "$EXPECTED_LINE" ]]; then + echo "unexpected custom sha256 sidecar: $CUSTOM_LINE" >&2 + echo "expected: $EXPECTED_LINE" >&2 + exit 70 +fi + +if "$ROOT/scripts/write_sha256.sh" "$TMP_DIR/missing.txt" >/dev/null 2>&1; then + echo "write_sha256.sh unexpectedly succeeded for a missing artifact" >&2 + exit 70 +fi + +echo "write_sha256.sh smoke test passed" diff --git a/scripts/write_artifact_manifest.py b/scripts/write_artifact_manifest.py new file mode 100755 index 0000000..43f8b1e --- /dev/null +++ b/scripts/write_artifact_manifest.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Write an open Shorebird artifact proxy manifest. + +The artifact proxy expects this file at: + + /shorebird//artifacts_manifest.yaml + +It maps a custom Shorebird engine revision back to the upstream Flutter engine +revision for unchanged artifacts, and lists the artifact paths that should be +served from the open Shorebird mirror. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +DEFAULT_ARTIFACT_OVERRIDES = ( + "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip", + "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip", + "flutter_infra_release/flutter/$engine/linux-x64-release/artifacts.zip", + "flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip", + "flutter_infra_release/flutter/$engine/ios-release/artifacts.zip", + "flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip", + "flutter_infra_release/flutter/$engine/flutter-web-sdk.zip", + "flutter_infra_release/flutter/$engine/darwin-arm64-release/FlutterMacOS.framework.zip", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--flutter-engine-revision", + required=True, + help="Upstream Flutter engine revision used for non-overridden artifacts.", + ) + parser.add_argument( + "--storage-bucket", + default="shorebird", + help="Bucket/path prefix under the Shorebird artifact mirror.", + ) + parser.add_argument( + "--output", + type=Path, + help="Output path. Writes to stdout when omitted.", + ) + return parser.parse_args() + + +def yaml_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def build_manifest(flutter_engine_revision: str, storage_bucket: str) -> str: + lines = [ + f"flutter_engine_revision: {yaml_quote(flutter_engine_revision)}", + f"storage_bucket: {yaml_quote(storage_bucket)}", + "artifact_overrides:", + ] + lines.extend( + f" - {yaml_quote(override)}" for override in DEFAULT_ARTIFACT_OVERRIDES + ) + return "\n".join(lines) + "\n" + + +def main() -> int: + args = parse_args() + manifest = build_manifest( + flutter_engine_revision=args.flutter_engine_revision, + storage_bucket=args.storage_bucket, + ) + + if args.output is None: + sys.stdout.write(manifest) + return 0 + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(manifest, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/write_gclient.sh b/scripts/write_gclient.sh index a68474c..907c987 100755 --- a/scripts/write_gclient.sh +++ b/scripts/write_gclient.sh @@ -7,8 +7,13 @@ INCLUDE_ENGINE_DEPS="${INCLUDE_ENGINE_DEPS:-0}" case "$PLATFORM" in linux) - TARGET_OS='["linux"]' - DART_DOWNLOAD_ANDROID_DEPS="False" + if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then + TARGET_OS='["linux", "android"]' + DART_DOWNLOAD_ANDROID_DEPS="True" + else + TARGET_OS='["linux"]' + DART_DOWNLOAD_ANDROID_DEPS="False" + fi ;; macos) if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then diff --git a/scripts/write_release_manifest.py b/scripts/write_release_manifest.py new file mode 100755 index 0000000..771820e --- /dev/null +++ b/scripts/write_release_manifest.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Write a JSON manifest for CI release artifacts. + +Every non-sidecar file under the input directory must have a sibling +`.sha256` sidecar in the format written by scripts/write_sha256.py: + + +""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +from pathlib import Path, PurePosixPath +import sys + + +def digest_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_plain_file(path: Path) -> bool: + return path.is_file() and not path.is_symlink() + + +def parse_sidecar(path: Path) -> tuple[str, str]: + text = path.read_text(encoding="utf-8").strip() + parts = text.split() + if len(parts) != 2: + raise ValueError(f"expected ' ', got {text!r}") + digest, filename = parts + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + raise ValueError(f"invalid sha256 digest {digest!r}") + return digest, filename + + +def is_safe_relative_path(path: str) -> bool: + if not path or path == ".": + return False + if "\\" in path or "\x00" in path or path.endswith("/"): + return False + if any(ord(character) < 32 for character in path): + return False + + candidate = PurePosixPath(path) + if candidate.is_absolute(): + return False + if any(part in ("", ".", "..") for part in candidate.parts): + return False + if candidate.parts and ":" in candidate.parts[0]: + return False + return True + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("input_dir", type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--github-sha", default="") + parser.add_argument( + "--require", + action="append", + default=[], + metavar="GLOB", + help="Require at least one artifact path matching this glob. May be repeated.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + input_dir = args.input_dir + output_path = args.output + + if not input_dir.is_dir(): + print(f"missing input directory: {input_dir}", file=sys.stderr) + return 66 + + artifacts = [] + errors = [] + for artifact_path in sorted(path for path in input_dir.rglob("*") if path.is_file()): + if artifact_path.is_symlink(): + errors.append(f"{artifact_path.relative_to(input_dir).as_posix()}: symlink artifacts are not allowed") + continue + if artifact_path.suffix == ".sha256": + continue + if artifact_path.resolve() == output_path.resolve(): + continue + + artifact_relative_path = artifact_path.relative_to(input_dir).as_posix() + if not is_safe_relative_path(artifact_relative_path): + errors.append(f"{artifact_relative_path}: unsafe artifact path") + continue + if artifact_path.stat().st_size <= 0: + errors.append(f"{artifact_relative_path}: empty artifacts are not allowed") + continue + artifact_relative = PurePosixPath(artifact_relative_path) + + sidecar_path = Path(f"{artifact_path}.sha256") + if not is_plain_file(sidecar_path): + errors.append(f"{artifact_relative_path}: missing .sha256 sidecar") + continue + + actual_digest = digest_file(artifact_path) + try: + sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path) + except ValueError as error: + errors.append(f"{sidecar_path.relative_to(input_dir).as_posix()}: {error}") + continue + + if sidecar_digest != actual_digest: + errors.append( + f"{sidecar_path.relative_to(input_dir).as_posix()}: digest mismatch " + f"{sidecar_digest} != {actual_digest}" + ) + if sidecar_filename != artifact_path.name: + errors.append( + f"{sidecar_path.relative_to(input_dir).as_posix()}: filename mismatch " + f"{sidecar_filename!r} != {artifact_path.name!r}" + ) + + artifacts.append( + { + "path": artifact_relative_path, + "artifact_group": artifact_relative.parts[0], + "filename": artifact_path.name, + "sha256": actual_digest, + "size": artifact_path.stat().st_size, + "sidecar": sidecar_path.relative_to(input_dir).as_posix(), + } + ) + + orphan_sidecars = [] + for sidecar_path in sorted(input_dir.rglob("*.sha256")): + artifact_path = Path(str(sidecar_path)[: -len(".sha256")]) + if sidecar_path.is_symlink(): + orphan_sidecars.append(sidecar_path.relative_to(input_dir).as_posix()) + elif not is_plain_file(artifact_path): + orphan_sidecars.append(sidecar_path.relative_to(input_dir).as_posix()) + if orphan_sidecars: + errors.append(f"orphan .sha256 sidecars: {', '.join(orphan_sidecars)}") + + artifact_paths = [artifact["path"] for artifact in artifacts] + for required_glob in args.require: + if not any(fnmatch.fnmatchcase(path, required_glob) for path in artifact_paths): + errors.append(f"missing required artifact matching {required_glob!r}") + + if errors: + print( + "release manifest validation failed:\n" + + "\n".join(f" {error}" for error in errors), + file=sys.stderr, + ) + return 70 + + manifest = { + "format_version": 1, + "github_sha": args.github_sha, + "artifact_count": len(artifacts), + "artifacts": artifacts, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/write_sha256.py b/scripts/write_sha256.py new file mode 100755 index 0000000..dc77fb0 --- /dev/null +++ b/scripts/write_sha256.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Write a sha256 sidecar in the common `digest filename` format.""" + +from __future__ import annotations + +import hashlib +import pathlib +import sys + + +def digest_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + if len(sys.argv) not in (2, 3): + print("usage: write_sha256.py [output]", file=sys.stderr) + return 64 + + artifact_path = pathlib.Path(sys.argv[1]) + output_path = ( + pathlib.Path(sys.argv[2]) + if len(sys.argv) == 3 + else pathlib.Path(f"{artifact_path}.sha256") + ) + + if not artifact_path.is_file(): + print(f"missing artifact: {artifact_path}", file=sys.stderr) + return 66 + + output_path.write_text( + f"{digest_file(artifact_path)} {artifact_path.name}\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/write_sha256.sh b/scripts/write_sha256.sh new file mode 100755 index 0000000..a406f0a --- /dev/null +++ b/scripts/write_sha256.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 && "$#" -ne 2 ]]; then + echo "usage: $0 [output]" >&2 + exit 64 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN=python3 +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + PYTHON_BIN=python +fi + +"$PYTHON_BIN" "$ROOT/scripts/write_sha256.py" "$@" diff --git a/shorebird b/shorebird index 08b8f07..07a654e 160000 --- a/shorebird +++ b/shorebird @@ -1 +1 @@ -Subproject commit 08b8f07bfbeafc8ffcf8ddd152f0c251fcb8ee6f +Subproject commit 07a654e7717a20d685c3eadf4b55c2e6731e96fa diff --git a/shorebird-server b/shorebird-server index 774954f..d6e5a39 160000 --- a/shorebird-server +++ b/shorebird-server @@ -1 +1 @@ -Subproject commit 774954fce70181846995ebcad842e9c54e57f1a2 +Subproject commit d6e5a39546905b85e2345b8058c4350ed236184c diff --git a/testapps/license_flavor_patch_test/ios/Runner/SceneDelegate.swift b/testapps/license_flavor_patch_test/ios/Runner/SceneDelegate.swift index b9ce8ea..b9e5f02 100644 --- a/testapps/license_flavor_patch_test/ios/Runner/SceneDelegate.swift +++ b/testapps/license_flavor_patch_test/ios/Runner/SceneDelegate.swift @@ -2,5 +2,39 @@ import Flutter import UIKit class SceneDelegate: FlutterSceneDelegate { + override func scene( + _ scene: UIScene, + willConnectTo session: UISceneSession, + options connectionOptions: UIScene.ConnectionOptions + ) { + guard let windowScene = scene as? UIWindowScene else { + return + } + let project = FlutterDartProject() + project.shorebirdAotPatchKeyProvider = { keyIdentifier in + guard keyIdentifier == "test-key" else { + return nil + } + let bytes: [UInt8] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + ] + return Data(bytes) + } + + let flutterViewController = FlutterViewController( + project: project, + nibName: nil, + bundle: nil + ) + GeneratedPluginRegistrant.register(with: flutterViewController.engine) + + let window = UIWindow(windowScene: windowScene) + window.rootViewController = flutterViewController + self.window = window + window.makeKeyAndVisible() + } } diff --git a/testapps/license_flavor_patch_test/shorebird.yaml b/testapps/license_flavor_patch_test/shorebird.yaml index 1122e72..acfa47d 100644 --- a/testapps/license_flavor_patch_test/shorebird.yaml +++ b/testapps/license_flavor_patch_test/shorebird.yaml @@ -1,6 +1,5 @@ app_id: license-flavor-patch-test aot_patch_key_id: test-key -aot_patch_key_hex: "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" aot_patch_base_flavor_id: free aot_patch_base_license_type: free aot_patch_flavor_id: pro diff --git a/updater b/updater index 3ac748f..5d4e9c3 160000 --- a/updater +++ b/updater @@ -1 +1 @@ -Subproject commit 3ac748ff28eb4ebb53630c66230b9b1be5df73c2 +Subproject commit 5d4e9c339636fc2f67cbe9892026d9e10b0888bc From 6be198f3abdf971ca3e557faab2cdec1067d29e8 Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 25 Jun 2026 20:11:17 +0800 Subject: [PATCH 02/21] Update repository inventory pins --- docs/REPOSITORIES.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/REPOSITORIES.md b/docs/REPOSITORIES.md index 1c2f8b9..1c6f86a 100644 --- a/docs/REPOSITORIES.md +++ b/docs/REPOSITORIES.md @@ -1,17 +1,17 @@ # Repository Inventory -Last verified: 2026-06-24. +Last verified: 2026-06-25. ## Active Top-Level Repositories | Path | Remote URL | Branch | Pinned commit | | --- | --- | --- | --- | -| `dart-sdk-new` | `https://git.tonycloud.org/dart-lang/sdk.git` | `tonycloud/dev` | `08139af589d0d32fbdec64b127c075ea3427cde8` | -| `depot_tools` | `https://chromium.googlesource.com/chromium/tools/depot_tools.git` | `main` | `90f5d4d14de66a5aae4a55b554ad4565111169af` | -| `flutter` | `https://git.tonycloud.org/flutter/flutter.git` | `shorebird/dev` | `9754ce03d76fb06c23a72ddbd8b7137bfa85446c` | -| `shorebird` | `https://git.tonycloud.org/flutter/shorebird.git` | `main` | `c824005c2dc65f56b0fa078984fadb649876fce7` | -| `shorebird-server` | `https://git.tonycloud.org/flutter/shorebird-server.git` | `main` | `774954fce70181846995ebcad842e9c54e57f1a2` | -| `updater` | `https://git.tonycloud.org/flutter/shorebird-updater.git` | `main` | `3ac748ff28eb4ebb53630c66230b9b1be5df73c2` | +| `dart-sdk-new` | `https://git.tonycloud.org/dart-lang/sdk.git` | `tonycloud/dev` | `57b27a7b44a9112c7cb3a1cb3c73636fc149bd16` | +| `depot_tools` | `https://chromium.googlesource.com/chromium/tools/depot_tools.git` | `main` | `226aa79e9947adc1e9e0c79f96b58562516535d9` | +| `flutter` | `https://git.tonycloud.org/flutter/flutter.git` | `shorebird/dev` | `5b96fd59be2f00061cccaaa23a02e15e720c1161` | +| `shorebird` | `https://git.tonycloud.org/flutter/shorebird.git` | `main` | `07a654e7717a20d685c3eadf4b55c2e6731e96fa` | +| `shorebird-server` | `https://git.tonycloud.org/flutter/shorebird-server.git` | `main` | `d6e5a39546905b85e2345b8058c4350ed236184c` | +| `updater` | `https://git.tonycloud.org/flutter/shorebird-updater.git` | `main` | `5d4e9c339636fc2f67cbe9892026d9e10b0888bc` | ## Removed Top-Level Repositories From 3445aa0636ea5a1b4f02474a16b544e22de794ac Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 25 Jun 2026 20:19:40 +0800 Subject: [PATCH 03/21] Support hosted verification without gh --- README.md | 3 +- docs/CI.md | 2 + scripts/verify_ci_workflow.rb | 4 + scripts/verify_hosted_full_sdk_build.sh | 242 ++++++++++++++++++++---- 4 files changed, 215 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index bf7ba1d..1a37485 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,8 @@ is untracked or if a required root/submodule checkout still has uncommitted changes. After upload, `scripts/verify_hosted_full_sdk_build.sh --repo owner/repo --ref main` dispatches the hosted full SDK workflow, waits for it, downloads artifacts, and -runs the downloaded-release verifier. +runs the downloaded-release verifier. It uses `gh` when available, or the +GitHub REST API with `GITHUB_TOKEN`/`GH_TOKEN`, `curl`, `jq`, and `unzip`. The heavy SDK/engine jobs default to custom runner labels `open-shorebird-linux-heavy` and `open-shorebird-macos-heavy`, then run an early disk-capacity preflight. Register larger/self-hosted runners with those labels diff --git a/docs/CI.md b/docs/CI.md index 16c0c91..a1b3e80 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -152,6 +152,8 @@ That helper dispatches `open-shorebird-ci.yml` with `full_sdk_build=true`, waits for the hosted workflow run to succeed, downloads all artifacts, and runs `scripts/verify_downloaded_release_artifacts.sh` against the downloaded output with the workflow run's `headSha`. +It uses GitHub CLI when `gh` is installed; otherwise it uses the GitHub REST API +with `GITHUB_TOKEN` or `GH_TOKEN` plus `curl`, `jq`, and `unzip`. Use `--linux-heavy-runner`, `--macos-heavy-runner`, and the disk-threshold flags when the repository uses custom larger/self-hosted runner labels. For local release assembly or reassembly after downloading workflow artifacts, diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 6ec42e6..e92e8e9 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -1280,6 +1280,10 @@ assert!( verify_hosted_full_sdk_build.include?('-f full_sdk_build=true') && verify_hosted_full_sdk_build.include?('-f run_runtime_smokes=false') && verify_hosted_full_sdk_build.include?('gh run download "$run_id"') && + verify_hosted_full_sdk_build.include?('GITHUB_TOKEN or GH_TOKEN is required') && + verify_hosted_full_sdk_build.include?('api_request POST "/actions/workflows/$WORKFLOW/dispatches"') && + verify_hosted_full_sdk_build.include?('archive_download_url') && + verify_hosted_full_sdk_build.include?('unzip -q "$zip_path" -d "$artifact_dir"') && verify_hosted_full_sdk_build.include?('--json status,conclusion,url,headSha') && verify_hosted_full_sdk_build.include?('--github-sha "$run_head_sha"') && verify_hosted_full_sdk_build.include?('unable to read headSha') && diff --git a/scripts/verify_hosted_full_sdk_build.sh b/scripts/verify_hosted_full_sdk_build.sh index af6eb74..2081ca4 100755 --- a/scripts/verify_hosted_full_sdk_build.sh +++ b/scripts/verify_hosted_full_sdk_build.sh @@ -9,6 +9,10 @@ Dispatches the Open Shorebird full SDK build on GitHub Actions, waits for the workflow run to finish, downloads all artifacts, and verifies the release manifest plus assembled artifact mirror. +Uses GitHub CLI when `gh` is available. Otherwise uses the GitHub REST API with +`GITHUB_TOKEN` or `GH_TOKEN`; that fallback also requires `curl`, `jq`, and +`unzip`. + Options: --repo owner/name GitHub repository to run against. --ref branch-or-sha Ref to dispatch. Defaults to current branch. @@ -38,6 +42,7 @@ SDK_MIN_FREE_DISK_GB="" ENGINE_MIN_FREE_DISK_GB="" BASE_FLUTTER_ENGINE_REVISION="" RUN_GCLIENT_SYNC=true +GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}" while [[ "$#" -gt 0 ]]; do case "$1" in @@ -107,11 +112,6 @@ if [[ -z "$REPO" ]]; then exit 64 fi -if ! command -v gh >/dev/null 2>&1; then - echo "GitHub CLI 'gh' is required" >&2 - exit 69 -fi - if [[ -z "$REF" ]]; then REF="$(git -C "$ROOT" branch --show-current 2>/dev/null || true)" fi @@ -131,35 +131,195 @@ run_fields=( [[ -z "$ENGINE_MIN_FREE_DISK_GB" ]] || run_fields+=(-f engine_min_free_disk_gb="$ENGINE_MIN_FREE_DISK_GB") [[ -z "$BASE_FLUTTER_ENGINE_REVISION" ]] || run_fields+=(-f base_flutter_engine_revision="$BASE_FLUTTER_ENGINE_REVISION") +api_token() { + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + printf '%s' "$GITHUB_TOKEN" + elif [[ -n "${GH_TOKEN:-}" ]]; then + printf '%s' "$GH_TOKEN" + else + echo "GITHUB_TOKEN or GH_TOKEN is required when gh is not installed" >&2 + exit 69 + fi +} + +api_require_tool() { + local tool="$1" + if ! command -v "$tool" >/dev/null 2>&1; then + echo "$tool is required when gh is not installed" >&2 + exit 69 + fi +} + +api_request() { + local method="$1" + local path="$2" + local data="${3:-}" + local token + token="$(api_token)" + + local curl_args=( + -fsSL + -X "$method" + -H "Accept: application/vnd.github+json" + -H "Authorization: Bearer $token" + -H "X-GitHub-Api-Version: 2022-11-28" + ) + if [[ -n "$data" ]]; then + curl_args+=(-H "Content-Type: application/json" -d "$data") + fi + + curl "${curl_args[@]}" "$GITHUB_API_URL/repos/$REPO$path" +} + +api_dispatch_payload() { + local inputs_filter + inputs_filter='{full_sdk_build: "true", run_gclient_sync: $run_gclient_sync, run_runtime_smokes: "false"}' + local jq_args=( + --arg ref "$REF" + --arg run_gclient_sync "$RUN_GCLIENT_SYNC" + ) + if [[ -n "$LINUX_HEAVY_RUNNER" ]]; then + inputs_filter="$inputs_filter + {linux_heavy_runner: \$linux_heavy_runner}" + jq_args+=(--arg linux_heavy_runner "$LINUX_HEAVY_RUNNER") + fi + if [[ -n "$MACOS_HEAVY_RUNNER" ]]; then + inputs_filter="$inputs_filter + {macos_heavy_runner: \$macos_heavy_runner}" + jq_args+=(--arg macos_heavy_runner "$MACOS_HEAVY_RUNNER") + fi + if [[ -n "$SDK_MIN_FREE_DISK_GB" ]]; then + inputs_filter="$inputs_filter + {sdk_min_free_disk_gb: \$sdk_min_free_disk_gb}" + jq_args+=(--arg sdk_min_free_disk_gb "$SDK_MIN_FREE_DISK_GB") + fi + if [[ -n "$ENGINE_MIN_FREE_DISK_GB" ]]; then + inputs_filter="$inputs_filter + {engine_min_free_disk_gb: \$engine_min_free_disk_gb}" + jq_args+=(--arg engine_min_free_disk_gb "$ENGINE_MIN_FREE_DISK_GB") + fi + if [[ -n "$BASE_FLUTTER_ENGINE_REVISION" ]]; then + inputs_filter="$inputs_filter + {base_flutter_engine_revision: \$base_flutter_engine_revision}" + jq_args+=(--arg base_flutter_engine_revision "$BASE_FLUTTER_ENGINE_REVISION") + fi + + jq -n "${jq_args[@]}" "{ref: \$ref, inputs: ($inputs_filter)}" +} + +api_download_artifacts() { + rm -rf "$DOWNLOAD_DIR" + mkdir -p "$DOWNLOAD_DIR" + + local page=1 + local downloaded_count=0 + while true; do + local response + response="$(api_request GET "/actions/runs/$run_id/artifacts?per_page=100&page=$page")" + local artifact_count + artifact_count="$(jq '.artifacts | length' <<<"$response")" + [[ "$artifact_count" == "0" ]] && break + + while IFS= read -r artifact; do + local name + local url + name="$(jq -r '.name' <<<"$artifact")" + url="$(jq -r '.archive_download_url' <<<"$artifact")" + case "$name" in + ""|*/*|*..*) + echo "unsafe GitHub artifact name: $name" >&2 + exit 70 + ;; + esac + if [[ -z "$url" || "$url" == "null" ]]; then + echo "missing archive_download_url for artifact: $name" >&2 + exit 70 + fi + + local artifact_dir + local zip_path + artifact_dir="$DOWNLOAD_DIR/$name" + zip_path="$(mktemp "${TMPDIR:-/tmp}/github-artifact.XXXXXX")" + curl \ + -fsSL \ + -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(api_token)" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -o "$zip_path" \ + "$url" + rm -rf "$artifact_dir" + mkdir -p "$artifact_dir" + unzip -q "$zip_path" -d "$artifact_dir" + rm -f "$zip_path" + downloaded_count=$((downloaded_count + 1)) + done < <(jq -c '.artifacts[] | select(.expired | not)' <<<"$response") + + page=$((page + 1)) + done + + if [[ "$downloaded_count" -eq 0 ]]; then + echo "no non-expired artifacts were available for workflow run $run_id" >&2 + exit 70 + fi +} + +use_gh=0 +if command -v gh >/dev/null 2>&1; then + use_gh=1 +else + api_require_tool curl + api_require_tool jq + api_require_tool unzip + api_token >/dev/null +fi + echo "Dispatching $WORKFLOW on $REPO@$REF with full_sdk_build=true" start_epoch="$(date +%s)" start_iso="$(date -u -r "$start_epoch" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "@$start_epoch" +"%Y-%m-%dT%H:%M:%SZ")" -gh workflow run "$WORKFLOW" \ - --repo "$REPO" \ - --ref "$REF" \ - "${run_fields[@]}" +if [[ "$use_gh" == "1" ]]; then + gh workflow run "$WORKFLOW" \ + --repo "$REPO" \ + --ref "$REF" \ + "${run_fields[@]}" +else + api_request POST "/actions/workflows/$WORKFLOW/dispatches" "$(api_dispatch_payload)" >/dev/null +fi run_id="" for _ in {1..40}; do - run_list_args=( - --repo "$REPO" - --workflow "$WORKFLOW" - --event workflow_dispatch - --json databaseId,createdAt - --limit 20 - ) - if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then - run_list_args+=(--commit "$REF") + if [[ "$use_gh" == "1" ]]; then + run_list_args=( + --repo "$REPO" + --workflow "$WORKFLOW" + --event workflow_dispatch + --json databaseId,createdAt + --limit 20 + ) + if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then + run_list_args+=(--commit "$REF") + else + run_list_args+=(--branch "$REF") + fi + run_id="$( + gh run list \ + "${run_list_args[@]}" \ + --jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \ + | + head -n 1 + )" else - run_list_args+=(--branch "$REF") + runs_response="$(api_request GET "/actions/workflows/$WORKFLOW/runs?event=workflow_dispatch&per_page=20")" + run_id="$( + jq -r \ + --arg ref "$REF" \ + --arg start_iso "$start_iso" \ + ' + .workflow_runs + | map(select(.created_at >= $start_iso)) + | map(select(.head_branch == $ref or .head_sha == $ref)) + | sort_by(.created_at) + | reverse + | .[0].id // "" + ' \ + <<<"$runs_response" + )" fi - run_id="$( - gh run list \ - "${run_list_args[@]}" \ - --jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \ - | - head -n 1 - )" [[ -z "$run_id" ]] || break sleep 3 done @@ -173,12 +333,20 @@ echo "Waiting for hosted full SDK run: $run_id" deadline=$((start_epoch + TIMEOUT_MINUTES * 60)) run_head_sha="" while true; do - IFS=$'\t' read -r status conclusion url run_head_sha < <( - gh run view "$run_id" \ - --repo "$REPO" \ - --json status,conclusion,url,headSha \ - --jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv' - ) + if [[ "$use_gh" == "1" ]]; then + IFS=$'\t' read -r status conclusion url run_head_sha < <( + gh run view "$run_id" \ + --repo "$REPO" \ + --json status,conclusion,url,headSha \ + --jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv' + ) + else + run_response="$(api_request GET "/actions/runs/$run_id")" + IFS=$'\t' read -r status conclusion url run_head_sha < <( + jq -r '[.status, (.conclusion // ""), .html_url, (.head_sha // "")] | @tsv' \ + <<<"$run_response" + ) + fi echo "run $run_id status=$status conclusion=${conclusion:-null} url=$url" if [[ "$status" == "completed" ]]; then @@ -195,9 +363,13 @@ while true; do sleep "$POLL_SECONDS" done -rm -rf "$DOWNLOAD_DIR" -mkdir -p "$DOWNLOAD_DIR" -gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR" +if [[ "$use_gh" == "1" ]]; then + rm -rf "$DOWNLOAD_DIR" + mkdir -p "$DOWNLOAD_DIR" + gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR" +else + api_download_artifacts +fi if [[ -z "$run_head_sha" ]]; then echo "unable to read headSha for workflow run $run_id" >&2 exit 70 From c49761a76503cb76fbe98a2e1d411161ce63949f Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 25 Jun 2026 23:18:59 +0800 Subject: [PATCH 04/21] Use hosted runners for full SDK CI by default --- .github/workflows/open-shorebird-ci.yml | 42 ++++++++++++------------- README.md | 29 ++++++++--------- docs/CI.md | 42 +++++++++++-------------- scripts/verify_ci_workflow.rb | 30 ++++++++++-------- 4 files changed, 72 insertions(+), 71 deletions(-) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 5c4fc35..e0d3d6e 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -7,9 +7,9 @@ on: workflow_dispatch: inputs: full_sdk_build: - description: Build custom SDK and engine artifacts. + description: Build custom SDK and engine artifacts during manual dispatch. required: false - default: false + default: true type: boolean run_gclient_sync: description: Run gclient sync before heavy SDK/engine builds. @@ -24,22 +24,22 @@ on: linux_heavy_runner: description: Runner label for Linux SDK/Android/web heavy jobs. required: false - default: open-shorebird-linux-heavy + default: ubuntu-latest type: string macos_heavy_runner: description: Runner label for macOS SDK and Apple engine heavy jobs. required: false - default: open-shorebird-macos-heavy + default: macos-latest type: string sdk_min_free_disk_gb: description: Minimum free disk GiB required before SDK-only heavy jobs. required: false - default: 35 + default: 8 type: number engine_min_free_disk_gb: description: Minimum free disk GiB required before engine heavy jobs. required: false - default: 40 + default: 8 type: number run_runtime_smokes: description: Run Android/Linux runtime patch smokes on provisioned runners. @@ -76,8 +76,8 @@ env: DEPOT_TOOLS_UPDATE: "0" PUB_CACHE: ${{ github.workspace }}/.pub-cache BASE_FLUTTER_ENGINE_REVISION: ${{ inputs.base_flutter_engine_revision || '' }} - SDK_MIN_FREE_DISK_GB: ${{ inputs.sdk_min_free_disk_gb || 35 }} - ENGINE_MIN_FREE_DISK_GB: ${{ inputs.engine_min_free_disk_gb || 40 }} + SDK_MIN_FREE_DISK_GB: ${{ inputs.sdk_min_free_disk_gb || 8 }} + ENGINE_MIN_FREE_DISK_GB: ${{ inputs.engine_min_free_disk_gb || 8 }} jobs: source-checks: @@ -505,9 +505,9 @@ jobs: custom-dart-sdk: name: Build custom Dart SDK needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 180 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -618,9 +618,9 @@ jobs: custom-dart-sdk-macos: name: Build custom Dart SDK (macOS arm64) needs: source-checks - runs-on: ${{ inputs.macos_heavy_runner || 'open-shorebird-macos-heavy' }} + runs-on: ${{ inputs.macos_heavy_runner || 'macos-latest' }} timeout-minutes: 180 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -728,9 +728,9 @@ jobs: linux-engine: name: Build Linux engine artifacts needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 240 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -888,9 +888,9 @@ jobs: android-engine: name: Build Android engine artifacts needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 240 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -1056,9 +1056,9 @@ jobs: web-sdk: name: Build web SDK artifact needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 180 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -1175,9 +1175,9 @@ jobs: ios-engine: name: Build Apple engine artifacts needs: source-checks - runs-on: ${{ inputs.macos_heavy_runner || 'open-shorebird-macos-heavy' }} + runs-on: ${{ inputs.macos_heavy_runner || 'macos-latest' }} timeout-minutes: 300 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -1475,7 +1475,7 @@ jobs: - ios-engine runs-on: ubuntu-latest timeout-minutes: 30 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 diff --git a/README.md b/README.md index 1a37485..67f95b8 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,16 @@ execute source-level checks and build distributable CLI/server artifacts: - `mirror-metadata` with `artifacts_manifest.yaml` for the artifact proxy - self-hosted `shorebird-server` archives for Linux, macOS, and Windows -Manual `workflow_dispatch` runs with `full_sdk_build=true` build the large SDK -and engine outputs: patched Dart SDK archives for Linux x64 and macOS arm64, -Linux x64 desktop engine artifacts, Android arm64 engine artifacts, Flutter web -SDK artifacts, and Apple iOS/macOS engine artifacts. A successful full SDK run -also uploads `open-shorebird-artifact-mirror`, a publish-ready mirror archive -assembled from the produced patch-tool, metadata, engine, and web artifacts, -plus `open-shorebird-release-manifest`, a checksum-verified provenance index -for the CLI, server, SDK, engine, and mirror archives. +Default push and pull request runs also build the large SDK and engine outputs: +patched Dart SDK archives for Linux x64 and macOS arm64, Linux x64 desktop +engine artifacts, Android arm64 engine artifacts, Flutter web SDK artifacts, +and Apple iOS/macOS engine artifacts. A successful full SDK run also uploads +`open-shorebird-artifact-mirror`, a publish-ready mirror archive assembled from +the produced patch-tool, metadata, engine, and web artifacts, plus +`open-shorebird-release-manifest`, a checksum-verified provenance index for the +CLI, server, SDK, engine, and mirror archives. Manual `workflow_dispatch` runs +keep `full_sdk_build=true` by default; set it to `false` only when you want a +source/CLI/server-only run. Use `scripts/validate_release_manifest.py` to audit a downloaded manifest against the downloaded workflow artifacts before publishing or mirroring them. The wrapper `scripts/verify_downloaded_release_artifacts.sh` runs that manifest @@ -74,12 +76,11 @@ After upload, `scripts/verify_hosted_full_sdk_build.sh --repo owner/repo --ref m dispatches the hosted full SDK workflow, waits for it, downloads artifacts, and runs the downloaded-release verifier. It uses `gh` when available, or the GitHub REST API with `GITHUB_TOKEN`/`GH_TOKEN`, `curl`, `jq`, and `unzip`. -The heavy SDK/engine jobs default to custom runner labels -`open-shorebird-linux-heavy` and `open-shorebird-macos-heavy`, then run an early -disk-capacity preflight. Register larger/self-hosted runners with those labels -or override `linux_heavy_runner` / `macos_heavy_runner` at dispatch time. The -dispatch inputs `sdk_min_free_disk_gb` and `engine_min_free_disk_gb` control -the preflight thresholds. +The heavy SDK/engine jobs default to managed GitHub-hosted runners +`ubuntu-latest` and `macos-latest`, then run an early disk-capacity preflight. +Override `linux_heavy_runner` / `macos_heavy_runner` only when you want larger +or self-hosted runners. The dispatch inputs `sdk_min_free_disk_gb` and +`engine_min_free_disk_gb` control the preflight thresholds. The CI contract is validated by `scripts/verify_ci_workflow.sh`; it rejects `dart_dynamic_modules=true`, legacy `aot-tools.dill` publishing, missing checksum sidecars, and missing diff --git a/docs/CI.md b/docs/CI.md index a1b3e80..a923548 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -291,43 +291,39 @@ provide Java on `PATH` because the smoke builds APKs before seeding the patch. ## Heavy SDK Builds -Use `workflow_dispatch` with `full_sdk_build=true` to build the custom SDK -artifacts. These jobs are intentionally manual because they run `gclient sync` -and build large engine/Dart outputs. +Default push and pull request runs build the custom SDK and engine artifacts. +Manual `workflow_dispatch` runs also build them by default because +`full_sdk_build=true` is the default input value; set `full_sdk_build=false` +only for source/CLI/server-only manual runs. The heavy jobs install Chromium's `depot_tools` into the workflow workspace -before running `gclient`, so they can bootstrap from a clean runner. Full -Dart/Flutter engine builds are large enough that standard GitHub-hosted runners -are not a realistic default for `full_sdk_build=true`. The workflow therefore -defaults heavy jobs to custom labels: +before running `gclient`, so they can bootstrap from a clean runner. The +workflow defaults heavy jobs to managed GitHub-hosted runners: -- `open-shorebird-linux-heavy` for Linux SDK, Linux engine, Android engine, and - web SDK builds -- `open-shorebird-macos-heavy` for macOS Dart SDK, iOS engine, and macOS engine +- `ubuntu-latest` for Linux SDK, Linux engine, Android engine, and web SDK builds +- `macos-latest` for macOS Dart SDK, iOS engine, and macOS engine builds -Register self-hosted or larger runners with those labels, or override -`linux_heavy_runner` / `macos_heavy_runner` when dispatching the workflow. -Every manual SDK/engine job also runs `scripts/check_ci_capacity.sh` before -`gclient sync` or `ninja`; SDK-only jobs require at least 35 GiB free and engine -jobs require at least 40 GiB free by default. Adjust the dispatch thresholds -only for runner images whose dependency caches make a lower threshold -intentional. +Override `linux_heavy_runner` / `macos_heavy_runner` when a repository wants +larger or self-hosted runners. Every SDK/engine job also runs +`scripts/check_ci_capacity.sh` before `gclient sync` or `ninja`; SDK-only jobs +and engine jobs require at least 8 GiB free by default. Raise the dispatch +thresholds for runner images where a larger preflight budget should be enforced. The hosted Android engine job also provisions Temurin Java 17 before building -Android JAR/APK-related engine artifacts. Every manual SDK/engine job verifies +Android JAR/APK-related engine artifacts. Every SDK/engine job verifies `python3`, `gclient`, and `ninja` before generating build files, so PATH/toolchain problems fail before a long build starts. CI sets `DEPOT_TOOLS_UPDATE=0`, and the local bootstrap scripts default the same way, so depot_tools uses the pinned submodule revision unless explicitly overridden. -Manual heavy builds accept these workflow inputs: +Manual dispatch accepts these workflow inputs: | Input | Default | Used by | | --- | --- | --- | -| `linux_heavy_runner` | `open-shorebird-linux-heavy` | `custom-dart-sdk`, `linux-engine`, `android-engine`, `web-sdk` | -| `macos_heavy_runner` | `open-shorebird-macos-heavy` | `custom-dart-sdk-macos`, `ios-engine` / Apple engine artifacts | -| `sdk_min_free_disk_gb` | `35` | Minimum free disk GiB for `custom-dart-sdk` and `custom-dart-sdk-macos` | -| `engine_min_free_disk_gb` | `40` | Minimum free disk GiB for Linux, Android, web, iOS, and macOS engine builds | +| `linux_heavy_runner` | `ubuntu-latest` | `custom-dart-sdk`, `linux-engine`, `android-engine`, `web-sdk` | +| `macos_heavy_runner` | `macos-latest` | `custom-dart-sdk-macos`, `ios-engine` / Apple engine artifacts | +| `sdk_min_free_disk_gb` | `8` | Minimum free disk GiB for `custom-dart-sdk` and `custom-dart-sdk-macos` | +| `engine_min_free_disk_gb` | `8` | Minimum free disk GiB for Linux, Android, web, iOS, and macOS engine builds | | `base_flutter_engine_revision` | empty | Optional upstream Flutter engine revision recorded in `artifacts_manifest.yaml` for non-overridden artifact proxy fallbacks | Use these inputs to move the SDK/engine jobs onto different larger or diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index e92e8e9..83bcf42 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -1459,8 +1459,9 @@ end %w[custom-dart-sdk custom-dart-sdk-macos linux-engine android-engine web-sdk ios-engine].each do |job_name| condition = jobs.fetch(job_name).fetch('if', '').to_s assert!( - condition.include?('workflow_dispatch') && condition.include?('full_sdk_build'), - "#{job_name} must be gated by manual full_sdk_build dispatch" + condition.include?("github.event_name != 'workflow_dispatch'") && + condition.include?('inputs.full_sdk_build'), + "#{job_name} must run on default push/PR CI and allow manual full_sdk_build opt-out" ) run_text = job_runs(jobs.fetch(job_name)).join("\n") assert!( @@ -1487,9 +1488,9 @@ end end artifact_mirror_condition = jobs.fetch('artifact-mirror').fetch('if', '').to_s assert!( - artifact_mirror_condition.include?('workflow_dispatch') && + artifact_mirror_condition.include?("github.event_name != 'workflow_dispatch'") && artifact_mirror_condition.include?('full_sdk_build'), - 'artifact-mirror must be gated by manual full_sdk_build dispatch' + 'artifact-mirror must run on default push/PR CI and allow manual full_sdk_build opt-out' ) %w[cli-artifacts linux-engine android-engine web-sdk ios-engine].each do |dependency| assert!( @@ -1623,17 +1624,20 @@ assert!( 'server artifacts must be uploaded' ) assert!( - inputs.dig('linux_heavy_runner', 'default') == 'open-shorebird-linux-heavy' && - inputs.dig('macos_heavy_runner', 'default') == 'open-shorebird-macos-heavy', - 'heavy SDK/engine workflow inputs must default to custom large-runner labels' + inputs.dig('full_sdk_build', 'default') == true && + inputs.dig('linux_heavy_runner', 'default') == 'ubuntu-latest' && + inputs.dig('macos_heavy_runner', 'default') == 'macos-latest' && + inputs.dig('sdk_min_free_disk_gb', 'default') == 8 && + inputs.dig('engine_min_free_disk_gb', 'default') == 8, + 'heavy SDK/engine workflow inputs must default to managed GitHub-hosted runners and enabled full builds' ) assert!( - jobs.fetch('custom-dart-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && - jobs.fetch('android-engine').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && - jobs.fetch('web-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && - jobs.fetch('custom-dart-sdk-macos').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'open-shorebird-macos-heavy'") && - jobs.fetch('ios-engine').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'open-shorebird-macos-heavy'"), - 'heavy SDK/engine jobs must default to custom large-runner labels, not standard hosted runners' + jobs.fetch('custom-dart-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'ubuntu-latest'") && + jobs.fetch('android-engine').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'ubuntu-latest'") && + jobs.fetch('web-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'ubuntu-latest'") && + jobs.fetch('custom-dart-sdk-macos').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'macos-latest'") && + jobs.fetch('ios-engine').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'macos-latest'"), + 'heavy SDK/engine jobs must default to managed GitHub-hosted runners' ) assert!( workflow.dig('env', 'JAVA_VERSION').to_s == '17', From 256d4007d255b9efaffa4051f48e0546e9c9595b Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 01:42:06 +0800 Subject: [PATCH 05/21] Move Dart SDK and Flutter forks to GitHub --- .github/workflows/open-shorebird-ci.yml | 40 ++++----- .gitmodules | 10 +-- README.md | 6 +- dart-sdk-new => dart-sdk | 0 docs/CI.md | 2 +- docs/PLATFORM_TESTING.md | 6 +- docs/REPOSITORIES.md | 90 +++++++++---------- flutter | 2 +- scripts/platform_test_common.sh | 6 +- scripts/sync_open_sources.sh | 2 +- scripts/verify_ci_workflow.rb | 30 +++---- .../verify_open_infrastructure_defaults.sh | 6 +- scripts/verify_powershell_open_defaults.sh | 2 +- scripts/verify_sync_open_sources.sh | 14 +-- scripts/write_gclient.sh | 8 +- shorebird | 2 +- .../tool/verify_aot_patch.dart | 2 +- updater | 2 +- 18 files changed, 115 insertions(+), 115 deletions(-) rename dart-sdk-new => dart-sdk (100%) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index e0d3d6e..7d01bc6 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -547,7 +547,7 @@ jobs: run: gclient sync --no-history - name: Generate patched Dart SDK build files - working-directory: dart-sdk-new + working-directory: dart-sdk run: | python3 tools/gn.py \ --mode release \ @@ -555,15 +555,15 @@ jobs: --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true' - name: Verify patched Dart SDK args - run: ./scripts/verify_dart_sdk_args.sh dart-sdk-new/out/ReleaseX64/args.gn + run: ./scripts/verify_dart_sdk_args.sh dart-sdk/out/ReleaseX64/args.gn - name: Build patched Dart SDK - working-directory: dart-sdk-new + working-directory: dart-sdk run: | ninja -C out/ReleaseX64 create_sdk runtime gen_snapshot dartaotruntime - name: Run focused patch API tests - working-directory: dart-sdk-new/out/ReleaseX64 + working-directory: dart-sdk/out/ReleaseX64 run: | ./run_vm_tests DartAPI_AotPatchingConfiguration ./run_vm_tests DartAPI_BytecodePatchReloadConfiguration @@ -571,11 +571,11 @@ jobs: - name: Package Dart SDK artifact run: | mkdir -p artifacts/custom-dart-sdk-linux-x64 - cp -R dart-sdk-new/out/ReleaseX64/dart-sdk artifacts/custom-dart-sdk-linux-x64/ - cp dart-sdk-new/out/ReleaseX64/gen_snapshot artifacts/custom-dart-sdk-linux-x64/ - cp dart-sdk-new/out/ReleaseX64/dartaotruntime artifacts/custom-dart-sdk-linux-x64/ - cp dart-sdk-new/out/ReleaseX64/args.gn artifacts/custom-dart-sdk-linux-x64/ - dart_sdk_revision="$(git -C dart-sdk-new rev-parse HEAD)" + cp -R dart-sdk/out/ReleaseX64/dart-sdk artifacts/custom-dart-sdk-linux-x64/ + cp dart-sdk/out/ReleaseX64/gen_snapshot artifacts/custom-dart-sdk-linux-x64/ + cp dart-sdk/out/ReleaseX64/dartaotruntime artifacts/custom-dart-sdk-linux-x64/ + cp dart-sdk/out/ReleaseX64/args.gn artifacts/custom-dart-sdk-linux-x64/ + dart_sdk_revision="$(git -C dart-sdk rev-parse HEAD)" cat > artifacts/custom-dart-sdk-linux-x64/manifest.json < artifacts/custom-dart-sdk-macos-arm64/manifest.json < ['https://git.tonycloud.org/dart-lang/sdk.git', 'tonycloud/dev'], + 'dart-sdk' => ['https://github.com/tony-cloud/dart-sdk.git', 'tonycloud/dev'], 'depot_tools' => ['https://chromium.googlesource.com/chromium/tools/depot_tools.git', 'main'], - 'flutter' => ['https://git.tonycloud.org/flutter/flutter.git', 'shorebird/dev'], + 'flutter' => ['https://github.com/tony-cloud/flutter.git', 'tonycloud/dev'], 'shorebird' => ['https://git.tonycloud.org/flutter/shorebird.git', 'main'], 'shorebird-server' => ['https://git.tonycloud.org/flutter/shorebird-server.git', 'main'], 'updater' => ['https://git.tonycloud.org/flutter/shorebird-updater.git', 'main'], @@ -597,8 +597,8 @@ forbidden_gitmodule_fragments.each do |fragment| ) end assert!( - write_gclient.include?('"url": "https://git.tonycloud.org/dart-lang/sdk.git"') && - write_gclient.include?('"url": "https://git.tonycloud.org/flutter/flutter.git"') && + write_gclient.include?('"url": "https://github.com/tony-cloud/dart-sdk.git"') && + write_gclient.include?('"url": "https://github.com/tony-cloud/flutter.git"') && !write_gclient.include?('github.com/shorebirdtech'), 'gclient generation must use the open Dart/Flutter forks and avoid official Shorebird remotes' ) @@ -645,8 +645,8 @@ assert!( '/openapi.yaml', ], 'docs/REPOSITORIES.md' => [ - 'https://git.tonycloud.org/dart-lang/sdk.git', - 'https://git.tonycloud.org/flutter/flutter.git', + 'https://github.com/tony-cloud/dart-sdk.git', + 'https://github.com/tony-cloud/flutter.git', 'https://git.tonycloud.org/flutter/shorebird.git', 'https://git.tonycloud.org/flutter/shorebird-server.git', 'https://git.tonycloud.org/flutter/shorebird-updater.git', @@ -681,7 +681,7 @@ assert!( 'Asset changes are not part of a Dart code patch', ], 'shorebird/docs/getting-started/flutter-version/README.md' => [ - 'https://git.tonycloud.org/flutter/flutter.git', + 'https://github.com/tony-cloud/flutter.git', 'http://localhost:8080/artifacts', ], }.each do |path, required_texts| @@ -700,7 +700,7 @@ end assert!( shorebird_flutter.include?('SHOREBIRD_FLUTTER_GIT_URL') && shorebird_flutter.include?('defaultFlutterGitUrl') && - shorebird_flutter.include?('https://git.tonycloud.org/flutter/flutter.git'), + shorebird_flutter.include?('https://github.com/tony-cloud/flutter.git'), 'shorebird Flutter installer must default to the open fork and allow SHOREBIRD_FLUTTER_GIT_URL override' ) assert!( @@ -790,7 +790,7 @@ assert!( }.each do |path, text| assert!( text.include?('SHOREBIRD_FLUTTER_GIT_URL') && - text.include?('https://git.tonycloud.org/flutter/flutter.git') && + text.include?('https://github.com/tony-cloud/flutter.git') && text.include?('SHOREBIRD_FLUTTER_STORAGE_BASE_URL') && text.include?('FLUTTER_STORAGE_BASE_URL') && text.include?('http://localhost:8080/download.flutter.io') && @@ -894,7 +894,7 @@ assert!( 'Flutter Gradle plugin defaults must use the open local Flutter mirror' ) assert!( - flutter_deps.include?('"dart_sdk_git": "https://git.tonycloud.org/dart-lang/sdk.git"') && + flutter_deps.include?('"dart_sdk_git": "https://github.com/tony-cloud/dart-sdk.git"') && flutter_deps.include?('"updater_git": "https://git.tonycloud.org/flutter/shorebird-updater.git"') && !flutter_deps.include?('git@github.com:shorebirdtech/dart-sdk.git') && !flutter_deps.include?('github.com/shorebirdtech/updater.git') && @@ -1086,7 +1086,7 @@ assert!( assert!( verify_open_infrastructure_defaults.include?('BUILD_SENSITIVE_FILES') && verify_open_infrastructure_defaults.include?('check_forbidden_in_tree "$ROOT/shorebird/packages/artifact_proxy/lib" dart') && - verify_open_infrastructure_defaults.include?('https://git.tonycloud.org/dart-lang/sdk.git') && + verify_open_infrastructure_defaults.include?('https://github.com/tony-cloud/dart-sdk.git') && verify_open_infrastructure_defaults.include?('https://git.tonycloud.org/flutter/shorebird-updater.git') && verify_open_infrastructure_defaults.include?('http://localhost:8080/download.flutter.io') && verify_open_infrastructure_defaults.include?('http://localhost:8080/artifacts') && @@ -1103,7 +1103,7 @@ assert!( verify_powershell_open_defaults.include?('shorebird/bin/shorebird.ps1') && verify_powershell_open_defaults.include?('flutter/bin/internal/update_dart_sdk.ps1') && verify_powershell_open_defaults.include?('System.Management.Automation.Language.Parser') && - verify_powershell_open_defaults.include?('https://git.tonycloud.org/flutter/flutter.git') && + verify_powershell_open_defaults.include?('https://github.com/tony-cloud/flutter.git') && verify_powershell_open_defaults.include?('http://localhost:8080/download.flutter.io') && verify_powershell_open_defaults.include?('download.shorebird.dev') && verify_powershell_open_defaults.include?('github.com/shorebirdtech/flutter.git'), @@ -1732,7 +1732,7 @@ assert!( ) %w[custom-dart-sdk custom-dart-sdk-macos].each do |job_name| assert!( - run_text_by_job.fetch(job_name).include?('dart_sdk_revision="$(git -C dart-sdk-new rev-parse HEAD)"') && + run_text_by_job.fetch(job_name).include?('dart_sdk_revision="$(git -C dart-sdk rev-parse HEAD)"') && run_text_by_job.fetch(job_name).include?('"dart_sdk_git_sha": "${dart_sdk_revision}"'), "#{job_name} manifest must record the Dart SDK source revision" ) @@ -1890,11 +1890,11 @@ end ) end assert!( - run_text_by_job.fetch('custom-dart-sdk').include?('verify_dart_sdk_args.sh dart-sdk-new/out/ReleaseX64/args.gn'), + run_text_by_job.fetch('custom-dart-sdk').include?('verify_dart_sdk_args.sh dart-sdk/out/ReleaseX64/args.gn'), 'Linux Dart SDK job must verify patched SDK args' ) assert!( - run_text_by_job.fetch('custom-dart-sdk-macos').include?('verify_dart_sdk_args.sh dart-sdk-new/xcodebuild/ReleaseARM64/args.gn'), + run_text_by_job.fetch('custom-dart-sdk-macos').include?('verify_dart_sdk_args.sh dart-sdk/xcodebuild/ReleaseARM64/args.gn'), 'macOS Dart SDK job must verify patched SDK args' ) %w[custom-dart-sdk custom-dart-sdk-macos].each do |job_name| diff --git a/scripts/verify_open_infrastructure_defaults.sh b/scripts/verify_open_infrastructure_defaults.sh index 2448502..94adb02 100755 --- a/scripts/verify_open_infrastructure_defaults.sh +++ b/scripts/verify_open_infrastructure_defaults.sh @@ -89,13 +89,13 @@ check_forbidden_in_tree "$ROOT/shorebird/packages/artifact_proxy/lib" dart check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_cli/lib" dart check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_code_push_client/lib" dart -require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/dart-lang/sdk.git" -require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/flutter.git" +require_contains "$ROOT/.gitmodules" "https://github.com/tony-cloud/dart-sdk.git" +require_contains "$ROOT/.gitmodules" "https://github.com/tony-cloud/flutter.git" require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird.git" require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-server.git" require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-updater.git" -require_contains "$ROOT/flutter/DEPS" '"dart_sdk_git": "https://git.tonycloud.org/dart-lang/sdk.git"' +require_contains "$ROOT/flutter/DEPS" '"dart_sdk_git": "https://github.com/tony-cloud/dart-sdk.git"' require_contains "$ROOT/flutter/DEPS" '"updater_git": "https://git.tonycloud.org/flutter/shorebird-updater.git"' require_contains "$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart" \ diff --git a/scripts/verify_powershell_open_defaults.sh b/scripts/verify_powershell_open_defaults.sh index 0494e49..0db671b 100755 --- a/scripts/verify_powershell_open_defaults.sh +++ b/scripts/verify_powershell_open_defaults.sh @@ -29,7 +29,7 @@ flutter_dart_updater="$ROOT/flutter/bin/internal/update_dart_sdk.ps1" [[ -f "$flutter_dart_updater" ]] || fail "missing Flutter Dart SDK updater: $flutter_dart_updater" require_contains "$shorebird_launcher" \ - '$defaultFlutterGitUrl = "https://git.tonycloud.org/flutter/flutter.git"' + '$defaultFlutterGitUrl = "https://github.com/tony-cloud/flutter.git"' require_contains "$shorebird_launcher" \ '$defaultFlutterStorageBaseUrl = "http://localhost:8080/download.flutter.io"' require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_GIT_URL' diff --git a/scripts/verify_sync_open_sources.sh b/scripts/verify_sync_open_sources.sh index 7d73502..a3c4c74 100755 --- a/scripts/verify_sync_open_sources.sh +++ b/scripts/verify_sync_open_sources.sh @@ -22,9 +22,9 @@ make_workspace() { cp "$ROOT/scripts/sync_open_sources.sh" "$workspace/scripts/sync_open_sources.sh" chmod +x "$workspace/scripts/sync_open_sources.sh" - init_git_checkout "$workspace/dart-sdk-new" - mkdir -p "$workspace/dart-sdk-new/runtime/vm" - : > "$workspace/dart-sdk-new/runtime/vm/dart_api_impl.h" + init_git_checkout "$workspace/dart-sdk" + mkdir -p "$workspace/dart-sdk/runtime/vm" + : > "$workspace/dart-sdk/runtime/vm/dart_api_impl.h" init_git_checkout "$workspace/updater" mkdir -p "$workspace/updater/library/include" @@ -35,7 +35,7 @@ make_workspace() { run_sync() { local workspace="$1" - DART_SRC="$workspace/dart-sdk-new" \ + DART_SRC="$workspace/dart-sdk" \ UPDATER_SRC="$workspace/updater" \ "$workspace/scripts/sync_open_sources.sh" } @@ -47,7 +47,7 @@ assert_links_to_workspace_sources() { test -L "$dart_target" test -L "$updater_target" - [[ "$(real_path "$dart_target")" == "$(real_path "$workspace/dart-sdk-new")" ]] + [[ "$(real_path "$dart_target")" == "$(real_path "$workspace/dart-sdk")" ]] [[ "$(real_path "$updater_target")" == "$(real_path "$workspace/updater")" ]] test -f "$dart_target/runtime/vm/dart_api_impl.h" test -f "$updater_target/library/include/updater_engine.h" @@ -88,7 +88,7 @@ test -f "$dirty_dart/untracked.txt" forbidden_dart_remote_workspace="$TMP_DIR/forbidden-dart-remote" make_workspace "$forbidden_dart_remote_workspace" -git -C "$forbidden_dart_remote_workspace/dart-sdk-new" remote add origin \ +git -C "$forbidden_dart_remote_workspace/dart-sdk" remote add origin \ https://github.com/dart-lang/sdk.git if run_sync "$forbidden_dart_remote_workspace" >"$TMP_DIR/forbidden-dart-remote.log" 2>&1; then echo "expected forbidden Dart SDK remote to fail" >&2 @@ -111,7 +111,7 @@ grep -q "updater submodule source checkout uses forbidden remote fragment" \ forbidden_updater_url_workspace="$TMP_DIR/forbidden-updater-url" make_workspace "$forbidden_updater_url_workspace" rm -rf "$forbidden_updater_url_workspace/updater" -if DART_SRC="$forbidden_updater_url_workspace/dart-sdk-new" \ +if DART_SRC="$forbidden_updater_url_workspace/dart-sdk" \ UPDATER_SRC="$forbidden_updater_url_workspace/missing-updater" \ UPDATER_URL=https://github.com/shorebirdtech/updater.git \ "$forbidden_updater_url_workspace/scripts/sync_open_sources.sh" \ diff --git a/scripts/write_gclient.sh b/scripts/write_gclient.sh index 907c987..17d18bb 100755 --- a/scripts/write_gclient.sh +++ b/scripts/write_gclient.sh @@ -33,13 +33,13 @@ esac cat > "$ROOT/.gclient" < main() async { final appDir = File.fromUri(Platform.script).parent.parent; final workspaceRoot = appDir.parent.parent; - final dartRoot = Directory('${workspaceRoot.path}/dart-sdk-new'); + final dartRoot = Directory('${workspaceRoot.path}/dart-sdk'); final patchBuildDir = _resolvePatchBuildDir(dartRoot); final toolWorkspace = Directory('${workspaceRoot.path}/shorebird'); final workDir = Directory('${appDir.path}/build/open_aot_patch_verify'); diff --git a/updater b/updater index 5d4e9c3..6e9aab2 160000 --- a/updater +++ b/updater @@ -1 +1 @@ -Subproject commit 5d4e9c339636fc2f67cbe9892026d9e10b0888bc +Subproject commit 6e9aab2ce7ede48b38472b7cfe9c8b53b5a98e5c From 6451962a83a58668093bc66e2f69c32b4240d221 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 02:01:17 +0800 Subject: [PATCH 06/21] Run gclient sync by default in hosted CI --- .github/workflows/open-shorebird-ci.yml | 20 ++++++++++---------- scripts/verify_ci_workflow.rb | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 7d01bc6..e22b9cf 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -543,7 +543,7 @@ jobs: run: ./scripts/sync_open_sources.sh - name: gclient sync - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} run: gclient sync --no-history - name: Generate patched Dart SDK build files @@ -653,7 +653,7 @@ jobs: run: ./scripts/sync_open_sources.sh - name: gclient sync - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} run: gclient sync --no-history - name: Generate patched Dart SDK build files @@ -776,11 +776,11 @@ jobs: run: ./scripts/sync_open_sources.sh - name: gclient sync - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} run: gclient sync --no-history - name: gclient sync Flutter engine deps - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} working-directory: flutter run: gclient sync --no-history @@ -934,11 +934,11 @@ jobs: run: ./scripts/sync_open_sources.sh - name: gclient sync - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} run: gclient sync --no-history - name: gclient sync Flutter engine deps - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} working-directory: flutter run: gclient sync --no-history @@ -1096,11 +1096,11 @@ jobs: run: ./scripts/sync_open_sources.sh - name: gclient sync - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} run: gclient sync --no-history - name: gclient sync Flutter engine deps - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} working-directory: flutter run: gclient sync --no-history @@ -1212,11 +1212,11 @@ jobs: run: ./scripts/sync_open_sources.sh - name: gclient sync - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} run: gclient sync --no-history - name: gclient sync Flutter engine deps - if: ${{ inputs.run_gclient_sync }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_gclient_sync }} working-directory: flutter run: gclient sync --no-history diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 2b97ba1..0469650 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -1470,6 +1470,21 @@ end run_text.include?('ninja --version'), "#{job_name} must verify depot_tools-provided build tools before heavy builds" ) + gclient_sync_steps = jobs.fetch(job_name).fetch('steps', []).select do |step| + step.fetch('name', '').start_with?('gclient sync') + end + assert!( + !gclient_sync_steps.empty?, + "#{job_name} must run gclient sync before heavy builds" + ) + gclient_sync_steps.each do |step| + condition = step.fetch('if', '').to_s + assert!( + condition.include?("github.event_name != 'workflow_dispatch'") && + condition.include?('inputs.run_gclient_sync'), + "#{job_name} #{step.fetch('name')} must run on push/PR CI and allow manual run_gclient_sync opt-out" + ) + end end { 'custom-dart-sdk' => 'SDK_MIN_FREE_DISK_GB', From 6804b8e9a87a374b1664e88a1626d13c6420f9d5 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 02:31:32 +0800 Subject: [PATCH 07/21] Build engine artifacts without Flutter prebuilt Dart SDK --- .github/workflows/open-shorebird-ci.yml | 31 ++++++++++++------ docs/CI.md | 15 +++++---- flutter | 2 +- scripts/verify_ci_workflow.rb | 32 ++++++++++++++++++- scripts/verify_ios_interpreter_route.sh | 2 ++ .../verify_ios_interpreter_route_validator.sh | 2 ++ 6 files changed, 66 insertions(+), 18 deletions(-) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index e22b9cf..37911b5 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -790,7 +790,8 @@ jobs: ./flutter/tools/gn \ --runtime-mode=release \ --target-dir=linux_release_x64 \ - --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false' + --no-prebuilt-dart-sdk \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false flutter_prebuilt_dart_sdk=false' - name: Build Linux engine working-directory: flutter/engine/src @@ -808,7 +809,8 @@ jobs: dart_enable_aot_patching=true \ dart_enable_shorebird_interpreter=false \ shorebird_enable_aot_patching=true \ - shorebird_use_interpreter=false + shorebird_use_interpreter=false \ + flutter_prebuilt_dart_sdk=false - name: Package Linux engine artifacts run: | @@ -950,7 +952,8 @@ jobs: --runtime-mode=release \ --android-cpu=arm64 \ --target-dir=android_release_arm64 \ - --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false' + --no-prebuilt-dart-sdk \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false flutter_prebuilt_dart_sdk=false' - name: Build Android engine working-directory: flutter/engine/src @@ -969,7 +972,8 @@ jobs: dart_enable_aot_patching=true \ dart_enable_shorebird_interpreter=false \ shorebird_enable_aot_patching=true \ - shorebird_use_interpreter=false + shorebird_use_interpreter=false \ + flutter_prebuilt_dart_sdk=false - name: Package Android engine artifacts run: | @@ -1111,7 +1115,8 @@ jobs: --web \ --runtime-mode=release \ --target-dir=wasm_release \ - --gn-args='dart_dynamic_modules=false' + --no-prebuilt-dart-sdk \ + --gn-args='dart_dynamic_modules=false flutter_prebuilt_dart_sdk=false' - name: Build web SDK working-directory: flutter/engine/src @@ -1121,7 +1126,8 @@ jobs: run: | ./scripts/verify_engine_args.sh \ flutter/engine/src/out/wasm_release/args.gn \ - dart_dynamic_modules=false + dart_dynamic_modules=false \ + flutter_prebuilt_dart_sdk=false - name: Package web SDK artifact run: | @@ -1228,7 +1234,8 @@ jobs: --mac-cpu=arm64 \ --target-dir=host_release_arm64 \ --shorebird-interpreter \ - --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true' + --no-prebuilt-dart-sdk \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true flutter_prebuilt_dart_sdk=false' - name: Build host snapshotter working-directory: flutter/engine/src @@ -1241,7 +1248,9 @@ jobs: --ios \ --runtime-mode=release \ --target-dir=ios_release \ - --shorebird-interpreter + --shorebird-interpreter \ + --no-prebuilt-dart-sdk \ + --gn-args='flutter_prebuilt_dart_sdk=false' - name: Build iOS engine working-directory: flutter/engine/src @@ -1261,7 +1270,8 @@ jobs: --runtime-mode=release \ --mac-cpu=arm64 \ --target-dir=macos_release_arm64 \ - --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false' + --no-prebuilt-dart-sdk \ + --gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=false shorebird_enable_aot_patching=true shorebird_use_interpreter=false flutter_prebuilt_dart_sdk=false' - name: Build macOS engine working-directory: flutter/engine/src @@ -1279,7 +1289,8 @@ jobs: dart_enable_aot_patching=true \ dart_enable_shorebird_interpreter=false \ shorebird_enable_aot_patching=true \ - shorebird_use_interpreter=false + shorebird_use_interpreter=false \ + flutter_prebuilt_dart_sdk=false - name: Package iOS engine artifacts run: | diff --git a/docs/CI.md b/docs/CI.md index c31cee7..4bd2490 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -441,12 +441,15 @@ verifies the Android artifacts/symbols zips, `flutter.jar`, `libflutter.so`, host snapshot/analyzer tools, args file, manifest, and mirror subtree. `web-sdk` builds the Flutter web SDK archive from `wasm_release` with -`dart_dynamic_modules=false` and uploads `flutter-web-sdk.zip`, a mirror-ready -copy of that SDK archive, `args.gn`, a manifest with the Flutter engine -revision, and a `.sha256` sidecar. Web is still not a Shorebird CodePush release -platform in this CLI/protocol; this job exists to keep the open Flutter SDK/web -artifacts buildable from the workspace. CI extracts the web SDK archive before -upload and verifies the SDK zip, args file, manifest, and mirror subtree. +`dart_dynamic_modules=false` and `flutter_prebuilt_dart_sdk=false`, then uploads +`flutter-web-sdk.zip`, a mirror-ready copy of that SDK archive, `args.gn`, a +manifest with the Flutter engine revision, and a `.sha256` sidecar. The engine +and web jobs use `--no-prebuilt-dart-sdk` because the open workspace disables +Flutter's private prebuilt Dart SDK download path and builds from the linked +Dart checkout instead. Web is still not a Shorebird CodePush release platform +in this CLI/protocol; this job exists to keep the open Flutter SDK/web artifacts +buildable from the workspace. CI extracts the web SDK archive before upload and +verifies the SDK zip, args file, manifest, and mirror subtree. Set `run_gclient_sync=false` only for debugging a runner image that already has all gclient-managed dependencies restored. diff --git a/flutter b/flutter index 3f70a0e..4b30ef3 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 3f70a0e1761f266329163dc972b8ffdfb7c5e7b4 +Subproject commit 4b30ef34f812e355cd217a3e4670f7e1bcd9aa39 diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 0469650..95a04ed 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -196,6 +196,8 @@ required_files = %w[ flutter/dev/tools/create_api_docs.dart flutter/engine/src/flutter/build/zip_bundle.gni flutter/engine/src/flutter/runtime/dart_isolate.cc + flutter/engine/src/flutter/runtime/shorebird/BUILD.gn + flutter/engine/src/flutter/shell/platform/embedder/BUILD.gn flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart flutter/packages/shorebird_tests/test/shorebird_tests.dart updater/library/src/config.rs @@ -397,6 +399,14 @@ flutter_engine_archives_build = read_repo_file( repo_root, 'flutter/engine/src/flutter/build/archives/BUILD.gn' ) +flutter_runtime_shorebird_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/runtime/shorebird/BUILD.gn' +) +flutter_embedder_build = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/shell/platform/embedder/BUILD.gn' +) flutter_linux_build = read_repo_file( repo_root, 'flutter/engine/src/flutter/shell/platform/linux/BUILD.gn' @@ -944,6 +954,14 @@ assert!( !flutter_dart_isolate.include?('#include "flutter/shell/common/shorebird/shorebird.h"'), 'dart_isolate.cc must avoid depending on the full Shorebird wrapper target so GN header checks pass' ) +assert!( + flutter_runtime_shorebird_build.include?('$dart_src/runtime/bin:shared_object_loaders') && + !flutter_runtime_shorebird_build.include?('$dart_src/runtime/bin:elf_loader') && + flutter_embedder_build.include?(%q("$dart_src/runtime/bin:common_embedder_dart_io", + "$dart_src/runtime/bin:shared_object_loaders")) && + !flutter_embedder_build.include?('if (is_ios || is_mac)'), + 'Shorebird patch cache and embedder targets must depend on shared_object_loaders so GN header checks allow both ELF and Mach-O loader includes' +) assert!( flutter_web_ui_copy_artifacts.include?("io.Platform.environment['FLUTTER_STORAGE_BASE_URL']") && flutter_web_ui_copy_artifacts.include?('http://localhost:8080/download.flutter.io') && @@ -1923,7 +1941,9 @@ end assert!( run_text_by_job.fetch('ios-engine').include?('--shorebird-interpreter') && - run_text_by_job.fetch('ios-engine').include?("--gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true'") && + run_text_by_job.fetch('ios-engine').include?('--no-prebuilt-dart-sdk') && + run_text_by_job.fetch('ios-engine').include?("--gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true flutter_prebuilt_dart_sdk=false'") && + run_text_by_job.fetch('ios-engine').include?("--gn-args='flutter_prebuilt_dart_sdk=false'") && run_text_by_job.fetch('ios-engine').include?('verify_ios_interpreter_route.sh') && run_text_by_job.fetch('ios-engine').include?('test -x flutter/engine/src/out/host_release_arm64/gen_snapshot') && run_text_by_job.fetch('ios-engine').include?('host_release_arm64/gen_snapshot') && @@ -1939,6 +1959,7 @@ assert!( verify_ios_interpreter_route.include?('json.load(file)') && verify_ios_interpreter_route.include?('metadata = artifact.get("metadata")') && verify_ios_interpreter_route.scan('require_gn_value "$args_file" dart_enable_aot_patching true').length >= 2 && + verify_ios_interpreter_route.scan('require_gn_value "$args_file" flutter_prebuilt_dart_sdk false').length >= 2 && verify_ios_interpreter_route.include?('sub("[[:space:]]*$", "", value)') && verify_ios_interpreter_route.include?('require(metadata, "runtime_mode", "dart-bytecode-interpreter", "metadata")') && verify_ios_interpreter_route.include?('require(metadata, "target_os", "ios", "metadata")') && @@ -1955,6 +1976,7 @@ assert!( verify_ios_interpreter_route_validator.scan('dart_enable_aot_patching = true').length >= 2 && verify_ios_interpreter_route_validator.include?('dart_enable_aot_patching = false') && verify_ios_interpreter_route_validator.include?('shorebird_use_interpreter = false') && + verify_ios_interpreter_route_validator.scan('flutter_prebuilt_dart_sdk = false').length >= 2 && verify_ios_interpreter_route_validator.include?('write_artifact "$bad_runtime" "dart-dynamic-modules"') && verify_ios_interpreter_route_validator.include?('write_artifact "$bad_target" "dart-bytecode-interpreter" "android"') && verify_ios_interpreter_route_validator.include?('unexpectedly accepted malformed JSON') && @@ -1971,11 +1993,13 @@ assert!( ) assert!( run_text_by_job.fetch('linux-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('linux-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('linux-engine').include?('flutter/engine/src/out/linux_release_x64/args.gn') && run_text_by_job.fetch('linux-engine').include?('dart_enable_aot_patching=true') && run_text_by_job.fetch('linux-engine').include?('dart_enable_shorebird_interpreter=false') && run_text_by_job.fetch('linux-engine').include?('shorebird_enable_aot_patching=true') && run_text_by_job.fetch('linux-engine').include?('shorebird_use_interpreter=false') && + run_text_by_job.fetch('linux-engine').include?('flutter_prebuilt_dart_sdk=false') && run_text_by_job.fetch('linux-engine').include?('linux-x64-flutter-gtk.zip') && run_text_by_job.fetch('linux-engine').include?('flutter_patched_sdk_product.zip') && run_text_by_job.fetch('linux-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/linux-x64-release/artifacts.zip') && @@ -1984,29 +2008,35 @@ assert!( ) assert!( run_text_by_job.fetch('android-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('android-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('android-engine').include?('flutter/engine/src/out/android_release_arm64/args.gn') && run_text_by_job.fetch('android-engine').include?('dart_enable_aot_patching=true') && run_text_by_job.fetch('android-engine').include?('dart_enable_shorebird_interpreter=false') && run_text_by_job.fetch('android-engine').include?('shorebird_enable_aot_patching=true') && run_text_by_job.fetch('android-engine').include?('shorebird_use_interpreter=false') && + run_text_by_job.fetch('android-engine').include?('flutter_prebuilt_dart_sdk=false') && run_text_by_job.fetch('android-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/android-arm64-release/artifacts.zip') && run_text_by_job.fetch('android-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/android-arm64-release/symbols.zip'), 'Android engine job must build and verify the native AOT patch runtime without DDM or interpreter mode' ) assert!( run_text_by_job.fetch('web-sdk').include?('verify_engine_args.sh') && + run_text_by_job.fetch('web-sdk').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('web-sdk').include?('flutter/engine/src/out/wasm_release/args.gn') && run_text_by_job.fetch('web-sdk').include?('dart_dynamic_modules=false') && + run_text_by_job.fetch('web-sdk').include?('flutter_prebuilt_dart_sdk=false') && run_text_by_job.fetch('web-sdk').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/flutter-web-sdk.zip'), 'web SDK job must explicitly disable and verify DDM' ) assert!( run_text_by_job.fetch('ios-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('ios-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('ios-engine').include?('flutter/engine/src/out/macos_release_arm64/args.gn') && run_text_by_job.fetch('ios-engine').include?('dart_enable_aot_patching=true') && run_text_by_job.fetch('ios-engine').include?('dart_enable_shorebird_interpreter=false') && run_text_by_job.fetch('ios-engine').include?('shorebird_enable_aot_patching=true') && run_text_by_job.fetch('ios-engine').include?('shorebird_use_interpreter=false') && + run_text_by_job.fetch('ios-engine').include?('flutter_prebuilt_dart_sdk=false') && run_text_by_job.fetch('ios-engine').include?('FlutterMacOS.framework.zip') && run_text_by_job.fetch('ios-engine').include?('flutter_patched_sdk_product.zip') && run_text_by_job.fetch('ios-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/darwin-arm64-release/FlutterMacOS.framework.zip'), diff --git a/scripts/verify_ios_interpreter_route.sh b/scripts/verify_ios_interpreter_route.sh index c7cbfed..cf8c131 100755 --- a/scripts/verify_ios_interpreter_route.sh +++ b/scripts/verify_ios_interpreter_route.sh @@ -71,6 +71,7 @@ verify_ios_engine_args() { require_gn_value "$args_file" dart_enable_shorebird_interpreter true require_gn_value "$args_file" shorebird_use_interpreter true require_gn_value "$args_file" shorebird_enable_aot_patching false + require_gn_value "$args_file" flutter_prebuilt_dart_sdk false } verify_host_engine_args() { @@ -82,6 +83,7 @@ verify_host_engine_args() { require_gn_value "$args_file" dart_enable_aot_patching true require_gn_value "$args_file" dart_enable_shorebird_interpreter true require_gn_value "$args_file" shorebird_use_interpreter true + require_gn_value "$args_file" flutter_prebuilt_dart_sdk false } app_bundle_from_ipa() { diff --git a/scripts/verify_ios_interpreter_route_validator.sh b/scripts/verify_ios_interpreter_route_validator.sh index 64f7947..cef0330 100755 --- a/scripts/verify_ios_interpreter_route_validator.sh +++ b/scripts/verify_ios_interpreter_route_validator.sh @@ -21,6 +21,7 @@ shorebird_use_interpreter = false shorebird_use_interpreter = true shorebird_enable_aot_patching = true shorebird_enable_aot_patching = false +flutter_prebuilt_dart_sdk = false EOF cat > "$HOST_ENGINE_DIR/args.gn" <<'EOF' @@ -33,6 +34,7 @@ dart_enable_shorebird_interpreter = false dart_enable_shorebird_interpreter = true shorebird_use_interpreter = false shorebird_use_interpreter = true +flutter_prebuilt_dart_sdk = false EOF write_artifact() { From 228a23fa6b901e9d88ec3ec557942828eeda9adb Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 02:50:49 +0800 Subject: [PATCH 08/21] Fix hosted engine Rust and web SDK setup --- .github/workflows/open-shorebird-ci.yml | 14 ++++- docs/CI.md | 18 +++--- scripts/sync_flutter_prebuilt_dart_sdk.sh | 67 +++++++++++++++++++++++ scripts/verify_ci_workflow.rb | 20 ++++++- 4 files changed, 106 insertions(+), 13 deletions(-) create mode 100755 scripts/sync_flutter_prebuilt_dart_sdk.sh diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 37911b5..46b2b24 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -921,6 +921,9 @@ jobs: distribution: temurin java-version: ${{ env.JAVA_VERSION }} + - name: Install Android Rust target + run: rustup target add aarch64-linux-android + - name: Free Ubuntu runner disk run: ./scripts/free_ci_disk_linux.sh @@ -1108,6 +1111,9 @@ jobs: working-directory: flutter run: gclient sync --no-history + - name: Sync Flutter prebuilt Dart SDK + run: ./scripts/sync_flutter_prebuilt_dart_sdk.sh linux-x64 + - name: Configure web SDK working-directory: flutter/engine/src run: | @@ -1115,8 +1121,7 @@ jobs: --web \ --runtime-mode=release \ --target-dir=wasm_release \ - --no-prebuilt-dart-sdk \ - --gn-args='dart_dynamic_modules=false flutter_prebuilt_dart_sdk=false' + --gn-args='dart_dynamic_modules=false' - name: Build web SDK working-directory: flutter/engine/src @@ -1127,7 +1132,7 @@ jobs: ./scripts/verify_engine_args.sh \ flutter/engine/src/out/wasm_release/args.gn \ dart_dynamic_modules=false \ - flutter_prebuilt_dart_sdk=false + flutter_prebuilt_dart_sdk=true - name: Package web SDK artifact run: | @@ -1206,6 +1211,9 @@ jobs: gclient help >/dev/null ninja --version + - name: Install Apple Rust targets + run: rustup target add aarch64-apple-ios aarch64-apple-darwin + - name: Check runner capacity run: CI_MIN_FREE_DISK_GB="$ENGINE_MIN_FREE_DISK_GB" ./scripts/check_ci_capacity.sh diff --git a/docs/CI.md b/docs/CI.md index 4bd2490..f276e0a 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -441,15 +441,17 @@ verifies the Android artifacts/symbols zips, `flutter.jar`, `libflutter.so`, host snapshot/analyzer tools, args file, manifest, and mirror subtree. `web-sdk` builds the Flutter web SDK archive from `wasm_release` with -`dart_dynamic_modules=false` and `flutter_prebuilt_dart_sdk=false`, then uploads +`dart_dynamic_modules=false` and `flutter_prebuilt_dart_sdk=true`, then uploads `flutter-web-sdk.zip`, a mirror-ready copy of that SDK archive, `args.gn`, a -manifest with the Flutter engine revision, and a `.sha256` sidecar. The engine -and web jobs use `--no-prebuilt-dart-sdk` because the open workspace disables -Flutter's private prebuilt Dart SDK download path and builds from the linked -Dart checkout instead. Web is still not a Shorebird CodePush release platform -in this CLI/protocol; this job exists to keep the open Flutter SDK/web artifacts -buildable from the workspace. CI extracts the web SDK archive before upload and -verifies the SDK zip, args file, manifest, and mirror subtree. +manifest with the Flutter engine revision, and a `.sha256` sidecar. The web job +does not download Flutter's private prebuilt Dart SDK; it links +`dart-sdk/tools/sdks/dart-sdk` into Flutter's expected prebuilt path before +running GN. Native engine jobs use `--no-prebuilt-dart-sdk` because the open +workspace builds those SDK artifacts from the linked Dart checkout instead. Web +is still not a Shorebird CodePush release platform in this CLI/protocol; this +job exists to keep the open Flutter SDK/web artifacts buildable from the +workspace. CI extracts the web SDK archive before upload and verifies the SDK +zip, args file, manifest, and mirror subtree. Set `run_gclient_sync=false` only for debugging a runner image that already has all gclient-managed dependencies restored. diff --git a/scripts/sync_flutter_prebuilt_dart_sdk.sh b/scripts/sync_flutter_prebuilt_dart_sdk.sh new file mode 100755 index 0000000..1c85f29 --- /dev/null +++ b/scripts/sync_flutter_prebuilt_dart_sdk.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HOST_CONFIG="${1:-linux-x64}" +DART_SDK_SOURCE="${DART_SDK_SOURCE:-$ROOT/dart-sdk/tools/sdks/dart-sdk}" +TARGET="$ROOT/flutter/engine/src/flutter/prebuilts/$HOST_CONFIG/dart-sdk" + +relative_path() { + python3 - "$1" "$2" <<'PY' +import os +import sys +print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1]))) +PY +} + +real_path() { + python3 - "$1" <<'PY' +import os +import sys +print(os.path.realpath(sys.argv[1])) +PY +} + +case "$HOST_CONFIG" in + linux-x64|macos-x64|macos-arm64) + ;; + *) + echo "unsupported Flutter prebuilt Dart SDK host config: $HOST_CONFIG" >&2 + exit 64 + ;; +esac + +if [[ ! -d "$DART_SDK_SOURCE" ]]; then + echo "missing Dart tool SDK source: $DART_SDK_SOURCE" >&2 + exit 66 +fi + +for required in \ + bin/dart \ + bin/dartaotruntime \ + bin/snapshots/dartdevc_aot.dart.snapshot \ + bin/snapshots/kernel_worker_aot.dart.snapshot +do + if [[ ! -e "$DART_SDK_SOURCE/$required" ]]; then + echo "Dart tool SDK is missing $required: $DART_SDK_SOURCE" >&2 + exit 66 + fi +done + +mkdir -p "$(dirname "$TARGET")" + +if [[ -L "$TARGET" ]]; then + source_real="$(real_path "$DART_SDK_SOURCE")" + target_real="$(real_path "$TARGET")" + if [[ "$source_real" != "$target_real" ]]; then + echo "Flutter prebuilt Dart SDK link points at $target_real, expected $source_real" >&2 + exit 70 + fi +elif [[ -e "$TARGET" ]]; then + echo "Flutter prebuilt Dart SDK target exists but is not the workspace Dart tool SDK symlink: $TARGET" >&2 + exit 70 +else + ln -s "$(relative_path "$TARGET" "$DART_SDK_SOURCE")" "$TARGET" +fi + +echo "[open-source-sync] Flutter $HOST_CONFIG prebuilt Dart SDK uses $DART_SDK_SOURCE" diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 95a04ed..99ae446 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -136,6 +136,7 @@ required_files = %w[ scripts/linux_runtime_patch_smoke.sh scripts/verify_open_infrastructure_defaults.sh scripts/safe_extract_tar.py + scripts/sync_flutter_prebuilt_dart_sdk.sh scripts/sync_open_sources.sh scripts/validate_artifact_mirror.py scripts/validate_release_manifest.py @@ -1023,6 +1024,10 @@ platform_test_common = read_repo_file(repo_root, 'scripts/platform_test_common.s bootstrap_linux = read_repo_file(repo_root, 'scripts/bootstrap_linux.sh') bootstrap_macos = read_repo_file(repo_root, 'scripts/bootstrap_macos.sh') sync_open_sources = read_repo_file(repo_root, 'scripts/sync_open_sources.sh') +sync_flutter_prebuilt_dart_sdk = read_repo_file( + repo_root, + 'scripts/sync_flutter_prebuilt_dart_sdk.sh' +) verify_sync_open_sources = read_repo_file( repo_root, 'scripts/verify_sync_open_sources.sh' @@ -1133,6 +1138,15 @@ assert!( verify_sync_open_sources.include?('expected forbidden explicit UPDATER_URL to fail'), 'source sync smoke test must reject upstream Dart SDK and official Shorebird updater remotes' ) +assert!( + sync_flutter_prebuilt_dart_sdk.include?('dart-sdk/tools/sdks/dart-sdk') && + sync_flutter_prebuilt_dart_sdk.include?('flutter/engine/src/flutter/prebuilts/$HOST_CONFIG/dart-sdk') && + sync_flutter_prebuilt_dart_sdk.include?('bin/dartaotruntime') && + sync_flutter_prebuilt_dart_sdk.include?('bin/snapshots/kernel_worker_aot.dart.snapshot') && + sync_flutter_prebuilt_dart_sdk.include?('ln -s') && + !sync_flutter_prebuilt_dart_sdk.include?('shorebird-dart-sdk-prebuilt'), + 'Flutter web prebuilt Dart SDK sync must link the open Dart tool SDK into Flutter prebuilts' +) assert!( job_runs(jobs.fetch('source-checks')).join("\n").include?('./scripts/verify_release_manifest.sh'), 'source-checks must smoke-test release manifest validation' @@ -1941,6 +1955,7 @@ end assert!( run_text_by_job.fetch('ios-engine').include?('--shorebird-interpreter') && + run_text_by_job.fetch('ios-engine').include?('rustup target add aarch64-apple-ios aarch64-apple-darwin') && run_text_by_job.fetch('ios-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('ios-engine').include?("--gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true flutter_prebuilt_dart_sdk=false'") && run_text_by_job.fetch('ios-engine').include?("--gn-args='flutter_prebuilt_dart_sdk=false'") && @@ -2008,6 +2023,7 @@ assert!( ) assert!( run_text_by_job.fetch('android-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('android-engine').include?('rustup target add aarch64-linux-android') && run_text_by_job.fetch('android-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('android-engine').include?('flutter/engine/src/out/android_release_arm64/args.gn') && run_text_by_job.fetch('android-engine').include?('dart_enable_aot_patching=true') && @@ -2021,10 +2037,10 @@ assert!( ) assert!( run_text_by_job.fetch('web-sdk').include?('verify_engine_args.sh') && - run_text_by_job.fetch('web-sdk').include?('--no-prebuilt-dart-sdk') && + run_text_by_job.fetch('web-sdk').include?('sync_flutter_prebuilt_dart_sdk.sh linux-x64') && run_text_by_job.fetch('web-sdk').include?('flutter/engine/src/out/wasm_release/args.gn') && run_text_by_job.fetch('web-sdk').include?('dart_dynamic_modules=false') && - run_text_by_job.fetch('web-sdk').include?('flutter_prebuilt_dart_sdk=false') && + run_text_by_job.fetch('web-sdk').include?('flutter_prebuilt_dart_sdk=true') && run_text_by_job.fetch('web-sdk').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/flutter-web-sdk.zip'), 'web SDK job must explicitly disable and verify DDM' ) From b0f8294205c97ffdde1ecd877356f1bf9bff927a Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 03:12:47 +0800 Subject: [PATCH 09/21] Fix iOS updater GN dependency --- flutter | 2 +- scripts/verify_ci_workflow.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/flutter b/flutter index 4b30ef3..e7248b3 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 4b30ef34f812e355cd217a3e4670f7e1bcd9aa39 +Subproject commit e7248b36030ba87facf9813f51318e35c373ac10 diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 99ae446..6583139 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -963,6 +963,10 @@ assert!( !flutter_embedder_build.include?('if (is_ios || is_mac)'), 'Shorebird patch cache and embedder targets must depend on shared_object_loaders so GN header checks allow both ELF and Mach-O loader includes' ) +assert!( + flutter_ios_build.include?('"//flutter/shell/common/shorebird:updater"'), + 'iOS Flutter framework source must directly depend on the Shorebird updater target because FlutterDartProject.mm includes updater.h' +) assert!( flutter_web_ui_copy_artifacts.include?("io.Platform.environment['FLUTTER_STORAGE_BASE_URL']") && flutter_web_ui_copy_artifacts.include?('http://localhost:8080/download.flutter.io') && From 6cbc3afddb1d282834fe7a4ab05326f1e258ff0d Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 03:45:07 +0800 Subject: [PATCH 10/21] Merge Dart upstream kernel fixes --- dart-sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dart-sdk b/dart-sdk index 57b27a7..bb9acb5 160000 --- a/dart-sdk +++ b/dart-sdk @@ -1 +1 @@ -Subproject commit 57b27a7b44a9112c7cb3a1cb3c73636fc149bd16 +Subproject commit bb9acb510b99427d6a30a45cfdf689a073e1e343 From 2e3bc6dde7035fadebcf27216bce621ae304ecb6 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 04:01:11 +0800 Subject: [PATCH 11/21] Sync Dart webdev package for engine pub get --- scripts/verify_ci_workflow.rb | 11 +++++++++++ scripts/write_gclient.sh | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 6583139..fc21fd4 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -196,6 +196,7 @@ required_files = %w[ flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts flutter/dev/tools/create_api_docs.dart flutter/engine/src/flutter/build/zip_bundle.gni + flutter/engine/src/flutter/pubspec.yaml flutter/engine/src/flutter/runtime/dart_isolate.cc flutter/engine/src/flutter/runtime/shorebird/BUILD.gn flutter/engine/src/flutter/shell/platform/embedder/BUILD.gn @@ -377,6 +378,10 @@ flutter_aar_init_script = read_repo_file( 'flutter/packages/flutter_tools/gradle/aar_init_script.gradle' ) flutter_deps = read_repo_file(repo_root, 'flutter/DEPS') +flutter_engine_pubspec = read_repo_file( + repo_root, + 'flutter/engine/src/flutter/pubspec.yaml' +) flutter_post_process_docs = read_repo_file(repo_root, 'flutter/dev/bots/post_process_docs.dart') flutter_unpublish_package = read_repo_file(repo_root, 'flutter/dev/bots/unpublish_package.dart') flutter_android_host_app_settings = read_repo_file( @@ -613,6 +618,12 @@ assert!( !write_gclient.include?('github.com/shorebirdtech'), 'gclient generation must use the open Dart/Flutter forks and avoid official Shorebird remotes' ) +assert!( + flutter_engine_pubspec.include?('frontend_server_client:') && + flutter_engine_pubspec.include?('./third_party/dart/third_party/pkg/webdev/frontend_server_client') && + !write_gclient.include?('"engine/src/flutter/third_party/dart/third_party/pkg/webdev": None'), + 'Flutter engine pub get needs Dart third_party/pkg/webdev for frontend_server_client, so write_gclient must not suppress it' +) { 'README.md' => [ '.github/workflows/open-shorebird-ci.yml', diff --git a/scripts/write_gclient.sh b/scripts/write_gclient.sh index 17d18bb..5bbbf4a 100755 --- a/scripts/write_gclient.sh +++ b/scripts/write_gclient.sh @@ -82,7 +82,6 @@ solutions = [ "engine/src/flutter/third_party/dart/third_party/pkg/tools": None, "engine/src/flutter/third_party/dart/third_party/pkg/vector_math": None, "engine/src/flutter/third_party/dart/third_party/pkg/web": None, - "engine/src/flutter/third_party/dart/third_party/pkg/webdev": None, "engine/src/flutter/third_party/dart/third_party/pkg/webdriver": None, "engine/src/flutter/third_party/dart/third_party/pkg/webkit_inspection_protocol": None, "engine/src/flutter/third_party/dart/tools/sdks/dart-sdk": None, From 3c00f0cf0b613954362ef8d68eba59e06f4030fc Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 04:42:09 +0800 Subject: [PATCH 12/21] Verify Dart tool SDK before engine builds --- .github/workflows/open-shorebird-ci.yml | 12 +++++++ flutter | 2 +- scripts/verify_ci_workflow.rb | 29 ++++++++++++++++ scripts/verify_dart_tool_sdk.sh | 45 +++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100755 scripts/verify_dart_tool_sdk.sh diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 46b2b24..a486796 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -784,6 +784,9 @@ jobs: working-directory: flutter run: gclient sync --no-history + - name: Verify Dart tool SDK + run: ./scripts/verify_dart_tool_sdk.sh + - name: Configure Linux engine working-directory: flutter/engine/src run: | @@ -947,6 +950,9 @@ jobs: working-directory: flutter run: gclient sync --no-history + - name: Verify Dart tool SDK + run: ./scripts/verify_dart_tool_sdk.sh + - name: Configure Android engine working-directory: flutter/engine/src run: | @@ -1111,6 +1117,9 @@ jobs: working-directory: flutter run: gclient sync --no-history + - name: Verify Dart tool SDK + run: ./scripts/verify_dart_tool_sdk.sh + - name: Sync Flutter prebuilt Dart SDK run: ./scripts/sync_flutter_prebuilt_dart_sdk.sh linux-x64 @@ -1234,6 +1243,9 @@ jobs: working-directory: flutter run: gclient sync --no-history + - name: Verify Dart tool SDK + run: ./scripts/verify_dart_tool_sdk.sh + - name: Configure host snapshotter working-directory: flutter/engine/src run: | diff --git a/flutter b/flutter index e7248b3..48bd78b 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit e7248b36030ba87facf9813f51318e35c373ac10 +Subproject commit 48bd78b4440938ce8736328af0f9854220fd96e4 diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index fc21fd4..9877c92 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -122,6 +122,7 @@ required_files = %w[ docs/CI.md docs/PLATFORM_TESTING.md docs/REPOSITORIES.md + dart-sdk/DEPS shorebird/README.md shorebird/OPEN_SOURCE_REPLACEMENTS.md shorebird/docs/account/api-keys/README.md @@ -146,6 +147,7 @@ required_files = %w[ scripts/verify_ci_workflow.rb scripts/verify_ci_workflow.sh scripts/verify_ci_capacity.sh + scripts/verify_dart_tool_sdk.sh scripts/verify_dart_sdk_args.sh scripts/verify_engine_args.sh scripts/verify_hosted_full_sdk_build.sh @@ -378,6 +380,7 @@ flutter_aar_init_script = read_repo_file( 'flutter/packages/flutter_tools/gradle/aar_init_script.gradle' ) flutter_deps = read_repo_file(repo_root, 'flutter/DEPS') +dart_deps = read_repo_file(repo_root, 'dart-sdk/DEPS') flutter_engine_pubspec = read_repo_file( repo_root, 'flutter/engine/src/flutter/pubspec.yaml' @@ -924,6 +927,21 @@ assert!( !flutter_deps.include?('shorebirdtech/_build_engine'), 'Flutter DEPS must point Dart/updater dependencies at open remotes and avoid Shorebird private prebuilt buckets' ) +dart_revision_ok, dart_revision = capture_command( + 'git', + '-C', + repo_path(repo_root, 'dart-sdk'), + 'rev-parse', + 'HEAD' +) +assert!(dart_revision_ok, 'must be able to read the Dart SDK submodule revision') +dart_tool_sdk_tag = dart_deps[/"sdk_tag": "([^"]+)"/, 1] +assert!(dart_tool_sdk_tag, 'Dart DEPS must declare sdk_tag for the bootstrap tool SDK') +assert!( + flutter_deps.include?("\"dart_sdk_revision\": \"#{dart_revision}\"") && + flutter_deps.include?("'version': '#{dart_tool_sdk_tag}'"), + 'Flutter DEPS must keep its Dart source/tool SDK pins aligned with the workspace Dart SDK' +) assert!( flutter_android_host_app_settings.include?('System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "http://localhost:8080"') && flutter_android_host_app_settings.include?('$flutterStorageUrl/download.flutter.io') && @@ -1066,6 +1084,7 @@ verify_ios_interpreter_route_validator = read_repo_file( ) verify_engine_args = read_repo_file(repo_root, 'scripts/verify_engine_args.sh') verify_dart_sdk_args = read_repo_file(repo_root, 'scripts/verify_dart_sdk_args.sh') +verify_dart_tool_sdk = read_repo_file(repo_root, 'scripts/verify_dart_tool_sdk.sh') assert!( bootstrap_linux.include?('exec "$ROOT/scripts/platform_test_common.sh" linux') && bootstrap_macos.include?('exec "$ROOT/scripts/platform_test_common.sh" macos'), @@ -1153,6 +1172,12 @@ assert!( verify_sync_open_sources.include?('expected forbidden explicit UPDATER_URL to fail'), 'source sync smoke test must reject upstream Dart SDK and official Shorebird updater remotes' ) +assert!( + verify_dart_tool_sdk.include?('pkg/front_end/pubspec.yaml') && + verify_dart_tool_sdk.include?('Dart tool SDK version does not satisfy front_end SDK constraint') && + verify_dart_tool_sdk.include?('Flutter engine Dart checkout points at'), + 'Dart tool SDK verifier must reject stale bootstrap SDKs and broken Flutter engine Dart links' +) assert!( sync_flutter_prebuilt_dart_sdk.include?('dart-sdk/tools/sdks/dart-sdk') && sync_flutter_prebuilt_dart_sdk.include?('flutter/engine/src/flutter/prebuilts/$HOST_CONFIG/dart-sdk') && @@ -2023,6 +2048,7 @@ assert!( ) assert!( run_text_by_job.fetch('linux-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('linux-engine').include?('verify_dart_tool_sdk.sh') && run_text_by_job.fetch('linux-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('linux-engine').include?('flutter/engine/src/out/linux_release_x64/args.gn') && run_text_by_job.fetch('linux-engine').include?('dart_enable_aot_patching=true') && @@ -2038,6 +2064,7 @@ assert!( ) assert!( run_text_by_job.fetch('android-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('android-engine').include?('verify_dart_tool_sdk.sh') && run_text_by_job.fetch('android-engine').include?('rustup target add aarch64-linux-android') && run_text_by_job.fetch('android-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('android-engine').include?('flutter/engine/src/out/android_release_arm64/args.gn') && @@ -2052,6 +2079,7 @@ assert!( ) assert!( run_text_by_job.fetch('web-sdk').include?('verify_engine_args.sh') && + run_text_by_job.fetch('web-sdk').include?('verify_dart_tool_sdk.sh') && run_text_by_job.fetch('web-sdk').include?('sync_flutter_prebuilt_dart_sdk.sh linux-x64') && run_text_by_job.fetch('web-sdk').include?('flutter/engine/src/out/wasm_release/args.gn') && run_text_by_job.fetch('web-sdk').include?('dart_dynamic_modules=false') && @@ -2061,6 +2089,7 @@ assert!( ) assert!( run_text_by_job.fetch('ios-engine').include?('verify_engine_args.sh') && + run_text_by_job.fetch('ios-engine').include?('verify_dart_tool_sdk.sh') && run_text_by_job.fetch('ios-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('ios-engine').include?('flutter/engine/src/out/macos_release_arm64/args.gn') && run_text_by_job.fetch('ios-engine').include?('dart_enable_aot_patching=true') && diff --git a/scripts/verify_dart_tool_sdk.sh b/scripts/verify_dart_tool_sdk.sh new file mode 100755 index 0000000..97af790 --- /dev/null +++ b/scripts/verify_dart_tool_sdk.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DART_SRC="${DART_SRC:-$ROOT/dart-sdk}" +TOOL_SDK="$DART_SRC/tools/sdks/dart-sdk" +ENGINE_DART="$ROOT/flutter/engine/src/flutter/third_party/dart" + +required_constraint="$( + ruby -ryaml -e 'puts YAML.load_file(ARGV[0]).fetch("environment").fetch("sdk")' \ + "$DART_SRC/pkg/front_end/pubspec.yaml" +)" +required_major_minor="$( + ruby -e 'ARGV[0] =~ /([0-9]+)\.([0-9]+)/ or abort "unable to parse SDK constraint"; puts "#{$1}.#{$2}"' \ + "$required_constraint" +)" + +if [[ ! -x "$TOOL_SDK/bin/dart" ]]; then + echo "missing executable Dart tool SDK: $TOOL_SDK/bin/dart" >&2 + exit 66 +fi + +version_output="$("$TOOL_SDK/bin/dart" --version 2>&1)" +actual_major_minor="$( + ruby -e 'ARGV[0] =~ /Dart SDK version: ([0-9]+)\.([0-9]+)/ or abort "unable to parse Dart version"; puts "#{$1}.#{$2}"' \ + "$version_output" +)" + +if [[ "$actual_major_minor" != "$required_major_minor" ]]; then + echo "Dart tool SDK version does not satisfy front_end SDK constraint." >&2 + echo " required major.minor: $required_major_minor from $required_constraint" >&2 + echo " actual: $version_output" >&2 + exit 70 +fi + +if [[ -e "$ENGINE_DART" ]]; then + engine_real="$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$ENGINE_DART")" + dart_real="$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$DART_SRC")" + if [[ "$engine_real" != "$dart_real" ]]; then + echo "Flutter engine Dart checkout points at $engine_real, expected $dart_real" >&2 + exit 70 + fi +fi + +echo "[open-source-sync] Dart tool SDK is compatible: $version_output" From 95ac9ea699be00f4544034ae6e62d264b0dfed7a Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 05:43:17 +0800 Subject: [PATCH 13/21] Fix Android engine artifact packaging --- .github/workflows/open-shorebird-ci.yml | 29 +++++++++++++++++++++---- scripts/verify_ci_workflow.rb | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index a486796..6bd1705 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -999,10 +999,31 @@ jobs: cp "$out/arm64_v8a_release.jar" artifacts/android-engine/ fi - gen_snapshot="$(find "$out" -path '*/gen_snapshot_arm64' -type f -print -quit)" - analyze_snapshot="$(find "$out" -path '*/analyze_snapshot_arm64' -type f -print -quit)" - test -n "$gen_snapshot" - test -n "$analyze_snapshot" + gen_snapshot="" + for candidate in "$out/gen_snapshot_arm64" "$out/clang_x64/gen_snapshot" "$out/universal/gen_snapshot"; do + if [ -f "$candidate" ]; then + gen_snapshot="$candidate" + break + fi + done + if [ -z "$gen_snapshot" ]; then + echo "missing Android gen_snapshot tool under $out" >&2 + find "$out" -maxdepth 3 -type f -name 'gen_snapshot*' -print >&2 + exit 66 + fi + + analyze_snapshot="" + for candidate in "$out/analyze_snapshot_arm64" "$out/clang_x64/analyze_snapshot" "$out/universal/analyze_snapshot"; do + if [ -f "$candidate" ]; then + analyze_snapshot="$candidate" + break + fi + done + if [ -z "$analyze_snapshot" ]; then + echo "missing Android analyze_snapshot tool under $out" >&2 + find "$out" -maxdepth 3 -type f -name 'analyze_snapshot*' -print >&2 + exit 66 + fi cp "$gen_snapshot" artifacts/android-engine/gen_snapshot_arm64 cp "$analyze_snapshot" artifacts/android-engine/analyze_snapshot_arm64 diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 9877c92..d440def 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -2073,6 +2073,8 @@ assert!( run_text_by_job.fetch('android-engine').include?('shorebird_enable_aot_patching=true') && run_text_by_job.fetch('android-engine').include?('shorebird_use_interpreter=false') && run_text_by_job.fetch('android-engine').include?('flutter_prebuilt_dart_sdk=false') && + run_text_by_job.fetch('android-engine').include?('$out/clang_x64/gen_snapshot') && + run_text_by_job.fetch('android-engine').include?('$out/clang_x64/analyze_snapshot') && run_text_by_job.fetch('android-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/android-arm64-release/artifacts.zip') && run_text_by_job.fetch('android-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/android-arm64-release/symbols.zip'), 'Android engine job must build and verify the native AOT patch runtime without DDM or interpreter mode' From 283bef2a810ca0d2755af2e83b7dfa31816bcff5 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 05:50:48 +0800 Subject: [PATCH 14/21] Avoid Linux tester build in release artifacts --- .github/workflows/open-shorebird-ci.yml | 20 ++++++++++++++++++-- scripts/verify_ci_workflow.rb | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 6bd1705..33f132c 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -801,8 +801,7 @@ jobs: run: | ninja -C out/linux_release_x64 \ flutter/shell/platform/linux:flutter_gtk \ - flutter/build/archives:flutter_patched_sdk \ - flutter/build/archives:artifacts + flutter/build/archives:flutter_patched_sdk - name: Verify Linux engine args run: | @@ -824,6 +823,23 @@ jobs: gtk_zip="$(find "$out/zip_archives" -name 'linux-x64-flutter-gtk.zip' -type f -print -quit)" patched_sdk_zip="$(find "$out/zip_archives" -name 'flutter_patched_sdk_product.zip' -type f -print -quit)" artifacts_zip="$(find "$out/zip_archives" -path '*/linux-x64-release/artifacts.zip' -type f -print -quit)" + if [ -z "$artifacts_zip" ]; then + test -x "$out/gen_snapshot" + artifacts_zip="$out/zip_archives/linux-x64-release/artifacts.zip" + mkdir -p "$(dirname "$artifacts_zip")" + python3 - "$out/gen_snapshot" "$artifacts_zip" <<'PY' + from pathlib import Path + import sys + import zipfile + + source = Path(sys.argv[1]) + output = Path(sys.argv[2]) + info = zipfile.ZipInfo('gen_snapshot') + info.external_attr = (source.stat().st_mode & 0xFFFF) << 16 + with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED) as archive: + archive.writestr(info, source.read_bytes(), zipfile.ZIP_DEFLATED) + PY + fi test -n "$gtk_zip" test -n "$patched_sdk_zip" test -n "$artifacts_zip" diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index d440def..33a9220 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -2056,6 +2056,8 @@ assert!( run_text_by_job.fetch('linux-engine').include?('shorebird_enable_aot_patching=true') && run_text_by_job.fetch('linux-engine').include?('shorebird_use_interpreter=false') && run_text_by_job.fetch('linux-engine').include?('flutter_prebuilt_dart_sdk=false') && + !run_text_by_job.fetch('linux-engine').include?('flutter/build/archives:artifacts') && + run_text_by_job.fetch('linux-engine').include?("zipfile.ZipInfo('gen_snapshot')") && run_text_by_job.fetch('linux-engine').include?('linux-x64-flutter-gtk.zip') && run_text_by_job.fetch('linux-engine').include?('flutter_patched_sdk_product.zip') && run_text_by_job.fetch('linux-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/linux-x64-release/artifacts.zip') && From 2d5f61d6f10f4a55f7d6f94a93a806ffb4fe1595 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 07:50:25 +0800 Subject: [PATCH 15/21] Build iOS analyze snapshot in Apple CI --- .github/workflows/open-shorebird-ci.yml | 37 +++++++++++++++++++++---- scripts/verify_ci_workflow.rb | 7 ++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 33f132c..58e9b8f 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -1314,7 +1314,8 @@ jobs: run: | ninja -C out/ios_release \ flutter/shell/platform/darwin/ios:flutter_framework \ - create_macos_gen_snapshots + create_macos_gen_snapshots \ + create_macos_analyze_snapshots - name: Verify iOS interpreter route run: ./scripts/verify_ios_interpreter_route.sh @@ -1353,10 +1354,36 @@ jobs: run: | set -euo pipefail mkdir -p artifacts/ios-engine/host_release_arm64 artifacts/ios-engine/ios-release - test -x flutter/engine/src/out/host_release_arm64/gen_snapshot - test -d flutter/engine/src/out/ios_release/Flutter.xcframework - test -x flutter/engine/src/out/ios_release/universal/gen_snapshot_arm64 - test -x flutter/engine/src/out/ios_release/analyze_snapshot_arm64 + + require_path() { + local path="$1" + if [ ! -e "$path" ]; then + echo "missing required path: $path" >&2 + find flutter/engine/src/out/ios_release -maxdepth 3 \( -name 'Flutter*' -o -name '*snapshot*' \) -print >&2 || true + exit 1 + fi + } + + require_directory() { + require_path "$1" + if [ ! -d "$1" ]; then + echo "required path is not a directory: $1" >&2 + exit 1 + fi + } + + require_executable() { + require_path "$1" + if [ ! -x "$1" ]; then + echo "required path is not executable: $1" >&2 + exit 1 + fi + } + + require_executable flutter/engine/src/out/host_release_arm64/gen_snapshot + require_directory flutter/engine/src/out/ios_release/Flutter.xcframework + require_executable flutter/engine/src/out/ios_release/universal/gen_snapshot_arm64 + require_executable flutter/engine/src/out/ios_release/analyze_snapshot_arm64 cp -R flutter/engine/src/out/ios_release/Flutter.framework artifacts/ios-engine/ cp -R flutter/engine/src/out/ios_release/Flutter.xcframework artifacts/ios-engine/ cp flutter/engine/src/out/host_release_arm64/gen_snapshot artifacts/ios-engine/host_release_arm64/ diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 33a9220..98b3031 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -1999,8 +1999,13 @@ assert!( run_text_by_job.fetch('ios-engine').include?('--no-prebuilt-dart-sdk') && run_text_by_job.fetch('ios-engine').include?("--gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true flutter_prebuilt_dart_sdk=false'") && run_text_by_job.fetch('ios-engine').include?("--gn-args='flutter_prebuilt_dart_sdk=false'") && + run_text_by_job.fetch('ios-engine').include?('create_macos_gen_snapshots') && + run_text_by_job.fetch('ios-engine').include?('create_macos_analyze_snapshots') && run_text_by_job.fetch('ios-engine').include?('verify_ios_interpreter_route.sh') && - run_text_by_job.fetch('ios-engine').include?('test -x flutter/engine/src/out/host_release_arm64/gen_snapshot') && + run_text_by_job.fetch('ios-engine').include?('require_executable flutter/engine/src/out/host_release_arm64/gen_snapshot') && + run_text_by_job.fetch('ios-engine').include?('require_directory flutter/engine/src/out/ios_release/Flutter.xcframework') && + run_text_by_job.fetch('ios-engine').include?('require_executable flutter/engine/src/out/ios_release/analyze_snapshot_arm64') && + run_text_by_job.fetch('ios-engine').include?('missing required path: $path') && run_text_by_job.fetch('ios-engine').include?('host_release_arm64/gen_snapshot') && run_text_by_job.fetch('ios-engine').include?('ios-release/artifacts.zip') && run_text_by_job.fetch('ios-engine').include?('mirror/shorebird/flutter_infra_release/flutter/${engine_revision}/ios-release/artifacts.zip') && From 2b3cf2592e3a983b66f32fa05ad717361b557719 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 09:51:29 +0800 Subject: [PATCH 16/21] Handle downloaded mirror metadata layout --- scripts/assemble_artifact_mirror.sh | 30 +++++++++++++++++-- ...erify_artifact_mirror_workflow_assembly.sh | 6 ++-- scripts/verify_assemble_artifact_mirror.sh | 6 ++-- scripts/verify_ci_workflow.rb | 5 ++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/scripts/assemble_artifact_mirror.sh b/scripts/assemble_artifact_mirror.sh index c4f256c..1decf0b 100755 --- a/scripts/assemble_artifact_mirror.sh +++ b/scripts/assemble_artifact_mirror.sh @@ -38,18 +38,19 @@ mkdir -p "$OUTPUT_DIR" FOUND_TREES=0 -copy_shorebird_tree() { +copy_tree_to_shorebird_prefix() { local tree="$1" + local prefix="$2" local source_file rel_file target_file FOUND_TREES=$((FOUND_TREES + 1)) while IFS= read -r -d '' source_file; do rel_file="${source_file#"$tree"/}" - target_file="$OUTPUT_DIR/shorebird/$rel_file" + target_file="$OUTPUT_DIR/shorebird/$prefix$rel_file" mkdir -p "$(dirname "$target_file")" if [[ -e "$target_file" ]]; then if ! cmp -s "$source_file" "$target_file"; then - echo "conflicting mirror file: shorebird/$rel_file" >&2 + echo "conflicting mirror file: shorebird/$prefix$rel_file" >&2 echo " existing: $target_file" >&2 echo " incoming: $source_file" >&2 exit 70 @@ -60,6 +61,10 @@ copy_shorebird_tree() { done < <(find "$tree" -type f -print0) } +copy_shorebird_tree() { + copy_tree_to_shorebird_prefix "$1" "" +} + scan_for_shorebird_trees() { local search_root="$1" local tree @@ -69,7 +74,26 @@ scan_for_shorebird_trees() { done < <(find "$search_root" -type d -name shorebird -print0) } +scan_for_downloaded_metadata_trees() { + local search_root="$1" + local manifest_path metadata_dir engine_revision + + # actions/upload-artifact strips the non-wildcard prefix from + # artifacts/mirror/shorebird/**/artifacts_manifest.yaml, so the downloaded + # mirror-metadata artifact is shaped as /artifacts_manifest.yaml. + while IFS= read -r -d '' manifest_path; do + if [[ "$manifest_path" == */shorebird/* ]]; then + continue + fi + + metadata_dir="$(dirname "$manifest_path")" + engine_revision="$(basename "$metadata_dir")" + copy_tree_to_shorebird_prefix "$metadata_dir" "$engine_revision/" + done < <(find "$search_root" -type f -name artifacts_manifest.yaml -print0) +} + scan_for_shorebird_trees "$INPUT_DIR" +scan_for_downloaded_metadata_trees "$INPUT_DIR" archive_index=0 while IFS= read -r -d '' archive_path; do diff --git a/scripts/verify_artifact_mirror_workflow_assembly.sh b/scripts/verify_artifact_mirror_workflow_assembly.sh index 96d5436..d60cab4 100755 --- a/scripts/verify_artifact_mirror_workflow_assembly.sh +++ b/scripts/verify_artifact_mirror_workflow_assembly.sh @@ -63,8 +63,8 @@ for target in \ write_artifact "$DOWNLOADED/$target" "$target" done -mkdir -p "$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION" -cat > "$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" < "$DOWNLOADED/mirror-metadata/$ENGINE_REVISION/artifacts_manifest.yaml" < "$INPUT_DIR/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" < "$INPUT_DIR/mirror-metadata/$ENGINE_REVISION/artifacts_manifest.yaml" </artifacts_manifest.yaml') && assemble_artifact_mirror.include?("find \"$INPUT_DIR\" -type f \\( -name '*.tar.gz' -o -name '*.tgz' \\)") && assemble_artifact_mirror.include?('scripts/safe_extract_tar.py') && assemble_artifact_mirror.include?('conflicting mirror file') && @@ -1294,6 +1297,7 @@ assert!( verify_assemble_artifact_mirror.include?('patch-darwin-x64.zip') && verify_assemble_artifact_mirror.include?('patch-darwin-arm64.zip') && verify_assemble_artifact_mirror.include?('patch-windows-x64.zip') && + verify_assemble_artifact_mirror.include?('mirror-metadata/$ENGINE_REVISION') && verify_assemble_artifact_mirror.include?('artifacts_manifest.yaml') && verify_assemble_artifact_mirror.include?('unexpectedly allowed a conflicting mirror file') && verify_assemble_artifact_mirror.include?('unexpectedly allowed an unsafe tar member') && @@ -1321,6 +1325,7 @@ assert!( verify_artifact_mirror_workflow_assembly.include?('verify_downloaded_release_artifacts.sh') && verify_artifact_mirror_workflow_assembly.include?('validate_artifact_mirror.py') && verify_artifact_mirror_workflow_assembly.include?('safe_extract_tar.py') && + verify_artifact_mirror_workflow_assembly.include?('mirror-metadata/$ENGINE_REVISION') && verify_artifact_mirror_workflow_assembly.include?('mirror-metadata/*artifacts_manifest.yaml') && verify_artifact_mirror_workflow_assembly.include?('flutter_patched_sdk_product.zip') && verify_artifact_mirror_workflow_assembly.include?('unexpectedly accepted downloaded artifacts for the wrong github_sha') && From dc7c5afda4a8be8595aedf420c86ca7620ef8a00 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 13:50:21 +0800 Subject: [PATCH 17/21] Update Flutter submodule --- flutter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter b/flutter index 48bd78b..1316ceb 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 48bd78b4440938ce8736328af0f9854220fd96e4 +Subproject commit 1316cebf196d9a5fc227019ac5baf47a46a7477d From 3c8b5b3e9cb0b1d8a16f3b93dcbc877ddd18c469 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 13:58:35 +0800 Subject: [PATCH 18/21] Update Flutter submodule --- flutter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter b/flutter index 1316ceb..3b68d99 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 1316cebf196d9a5fc227019ac5baf47a46a7477d +Subproject commit 3b68d99270da8bf77ceeaa77e73faf00c78d6b27 From a1a99fd5ec9c38a0f57524fd41ebfe51a7850022 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 14:03:30 +0800 Subject: [PATCH 19/21] Update Flutter submodule for CI shard fix --- flutter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter b/flutter index 3b68d99..609ac58 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 3b68d99270da8bf77ceeaa77e73faf00c78d6b27 +Subproject commit 609ac584fb0dd0e2add7e0a14a028ab5505aa509 From e17c8627e0816edc906d2e570012e988b7222d09 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 14:22:21 +0800 Subject: [PATCH 20/21] Update Flutter submodule to merged fork --- flutter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter b/flutter index 609ac58..9199400 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 609ac584fb0dd0e2add7e0a14a028ab5505aa509 +Subproject commit 9199400e83f4320fa67b6a8c0694661efbbcc50d From b93fa2573377de3f00db2059986d848b5b97453a Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 26 Jun 2026 14:24:25 +0800 Subject: [PATCH 21/21] Update Flutter submodule for default-branch CI --- flutter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter b/flutter index 9199400..0b84921 160000 --- a/flutter +++ b/flutter @@ -1 +1 @@ -Subproject commit 9199400e83f4320fa67b6a8c0694661efbbcc50d +Subproject commit 0b849218346fe188fdc18a6081431eed915b58f8