diff --git a/.github/workflows/open-shorebird-ci.yml b/.github/workflows/open-shorebird-ci.yml index 5c4fc35..e0d3d6e 100644 --- a/.github/workflows/open-shorebird-ci.yml +++ b/.github/workflows/open-shorebird-ci.yml @@ -7,9 +7,9 @@ on: workflow_dispatch: inputs: full_sdk_build: - description: Build custom SDK and engine artifacts. + description: Build custom SDK and engine artifacts during manual dispatch. required: false - default: false + default: true type: boolean run_gclient_sync: description: Run gclient sync before heavy SDK/engine builds. @@ -24,22 +24,22 @@ on: linux_heavy_runner: description: Runner label for Linux SDK/Android/web heavy jobs. required: false - default: open-shorebird-linux-heavy + default: ubuntu-latest type: string macos_heavy_runner: description: Runner label for macOS SDK and Apple engine heavy jobs. required: false - default: open-shorebird-macos-heavy + default: macos-latest type: string sdk_min_free_disk_gb: description: Minimum free disk GiB required before SDK-only heavy jobs. required: false - default: 35 + default: 8 type: number engine_min_free_disk_gb: description: Minimum free disk GiB required before engine heavy jobs. required: false - default: 40 + default: 8 type: number run_runtime_smokes: description: Run Android/Linux runtime patch smokes on provisioned runners. @@ -76,8 +76,8 @@ env: DEPOT_TOOLS_UPDATE: "0" PUB_CACHE: ${{ github.workspace }}/.pub-cache BASE_FLUTTER_ENGINE_REVISION: ${{ inputs.base_flutter_engine_revision || '' }} - SDK_MIN_FREE_DISK_GB: ${{ inputs.sdk_min_free_disk_gb || 35 }} - ENGINE_MIN_FREE_DISK_GB: ${{ inputs.engine_min_free_disk_gb || 40 }} + SDK_MIN_FREE_DISK_GB: ${{ inputs.sdk_min_free_disk_gb || 8 }} + ENGINE_MIN_FREE_DISK_GB: ${{ inputs.engine_min_free_disk_gb || 8 }} jobs: source-checks: @@ -505,9 +505,9 @@ jobs: custom-dart-sdk: name: Build custom Dart SDK needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 180 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -618,9 +618,9 @@ jobs: custom-dart-sdk-macos: name: Build custom Dart SDK (macOS arm64) needs: source-checks - runs-on: ${{ inputs.macos_heavy_runner || 'open-shorebird-macos-heavy' }} + runs-on: ${{ inputs.macos_heavy_runner || 'macos-latest' }} timeout-minutes: 180 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -728,9 +728,9 @@ jobs: linux-engine: name: Build Linux engine artifacts needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 240 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -888,9 +888,9 @@ jobs: android-engine: name: Build Android engine artifacts needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 240 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -1056,9 +1056,9 @@ jobs: web-sdk: name: Build web SDK artifact needs: source-checks - runs-on: ${{ inputs.linux_heavy_runner || 'open-shorebird-linux-heavy' }} + runs-on: ${{ inputs.linux_heavy_runner || 'ubuntu-latest' }} timeout-minutes: 180 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -1175,9 +1175,9 @@ jobs: ios-engine: name: Build Apple engine artifacts needs: source-checks - runs-on: ${{ inputs.macos_heavy_runner || 'open-shorebird-macos-heavy' }} + runs-on: ${{ inputs.macos_heavy_runner || 'macos-latest' }} timeout-minutes: 300 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 @@ -1475,7 +1475,7 @@ jobs: - ios-engine runs-on: ubuntu-latest timeout-minutes: 30 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.full_sdk_build }} + if: ${{ github.event_name != 'workflow_dispatch' || inputs.full_sdk_build }} steps: - name: Checkout workspace uses: actions/checkout@v4 diff --git a/README.md b/README.md index 1a37485..67f95b8 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,16 @@ execute source-level checks and build distributable CLI/server artifacts: - `mirror-metadata` with `artifacts_manifest.yaml` for the artifact proxy - self-hosted `shorebird-server` archives for Linux, macOS, and Windows -Manual `workflow_dispatch` runs with `full_sdk_build=true` build the large SDK -and engine outputs: patched Dart SDK archives for Linux x64 and macOS arm64, -Linux x64 desktop engine artifacts, Android arm64 engine artifacts, Flutter web -SDK artifacts, and Apple iOS/macOS engine artifacts. A successful full SDK run -also uploads `open-shorebird-artifact-mirror`, a publish-ready mirror archive -assembled from the produced patch-tool, metadata, engine, and web artifacts, -plus `open-shorebird-release-manifest`, a checksum-verified provenance index -for the CLI, server, SDK, engine, and mirror archives. +Default push and pull request runs also build the large SDK and engine outputs: +patched Dart SDK archives for Linux x64 and macOS arm64, Linux x64 desktop +engine artifacts, Android arm64 engine artifacts, Flutter web SDK artifacts, +and Apple iOS/macOS engine artifacts. A successful full SDK run also uploads +`open-shorebird-artifact-mirror`, a publish-ready mirror archive assembled from +the produced patch-tool, metadata, engine, and web artifacts, plus +`open-shorebird-release-manifest`, a checksum-verified provenance index for the +CLI, server, SDK, engine, and mirror archives. Manual `workflow_dispatch` runs +keep `full_sdk_build=true` by default; set it to `false` only when you want a +source/CLI/server-only run. Use `scripts/validate_release_manifest.py` to audit a downloaded manifest against the downloaded workflow artifacts before publishing or mirroring them. The wrapper `scripts/verify_downloaded_release_artifacts.sh` runs that manifest @@ -74,12 +76,11 @@ After upload, `scripts/verify_hosted_full_sdk_build.sh --repo owner/repo --ref m dispatches the hosted full SDK workflow, waits for it, downloads artifacts, and runs the downloaded-release verifier. It uses `gh` when available, or the GitHub REST API with `GITHUB_TOKEN`/`GH_TOKEN`, `curl`, `jq`, and `unzip`. -The heavy SDK/engine jobs default to custom runner labels -`open-shorebird-linux-heavy` and `open-shorebird-macos-heavy`, then run an early -disk-capacity preflight. Register larger/self-hosted runners with those labels -or override `linux_heavy_runner` / `macos_heavy_runner` at dispatch time. The -dispatch inputs `sdk_min_free_disk_gb` and `engine_min_free_disk_gb` control -the preflight thresholds. +The heavy SDK/engine jobs default to managed GitHub-hosted runners +`ubuntu-latest` and `macos-latest`, then run an early disk-capacity preflight. +Override `linux_heavy_runner` / `macos_heavy_runner` only when you want larger +or self-hosted runners. The dispatch inputs `sdk_min_free_disk_gb` and +`engine_min_free_disk_gb` control the preflight thresholds. The CI contract is validated by `scripts/verify_ci_workflow.sh`; it rejects `dart_dynamic_modules=true`, legacy `aot-tools.dill` publishing, missing checksum sidecars, and missing diff --git a/docs/CI.md b/docs/CI.md index a1b3e80..a923548 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -291,43 +291,39 @@ provide Java on `PATH` because the smoke builds APKs before seeding the patch. ## Heavy SDK Builds -Use `workflow_dispatch` with `full_sdk_build=true` to build the custom SDK -artifacts. These jobs are intentionally manual because they run `gclient sync` -and build large engine/Dart outputs. +Default push and pull request runs build the custom SDK and engine artifacts. +Manual `workflow_dispatch` runs also build them by default because +`full_sdk_build=true` is the default input value; set `full_sdk_build=false` +only for source/CLI/server-only manual runs. The heavy jobs install Chromium's `depot_tools` into the workflow workspace -before running `gclient`, so they can bootstrap from a clean runner. Full -Dart/Flutter engine builds are large enough that standard GitHub-hosted runners -are not a realistic default for `full_sdk_build=true`. The workflow therefore -defaults heavy jobs to custom labels: +before running `gclient`, so they can bootstrap from a clean runner. The +workflow defaults heavy jobs to managed GitHub-hosted runners: -- `open-shorebird-linux-heavy` for Linux SDK, Linux engine, Android engine, and - web SDK builds -- `open-shorebird-macos-heavy` for macOS Dart SDK, iOS engine, and macOS engine +- `ubuntu-latest` for Linux SDK, Linux engine, Android engine, and web SDK builds +- `macos-latest` for macOS Dart SDK, iOS engine, and macOS engine builds -Register self-hosted or larger runners with those labels, or override -`linux_heavy_runner` / `macos_heavy_runner` when dispatching the workflow. -Every manual SDK/engine job also runs `scripts/check_ci_capacity.sh` before -`gclient sync` or `ninja`; SDK-only jobs require at least 35 GiB free and engine -jobs require at least 40 GiB free by default. Adjust the dispatch thresholds -only for runner images whose dependency caches make a lower threshold -intentional. +Override `linux_heavy_runner` / `macos_heavy_runner` when a repository wants +larger or self-hosted runners. Every SDK/engine job also runs +`scripts/check_ci_capacity.sh` before `gclient sync` or `ninja`; SDK-only jobs +and engine jobs require at least 8 GiB free by default. Raise the dispatch +thresholds for runner images where a larger preflight budget should be enforced. The hosted Android engine job also provisions Temurin Java 17 before building -Android JAR/APK-related engine artifacts. Every manual SDK/engine job verifies +Android JAR/APK-related engine artifacts. Every SDK/engine job verifies `python3`, `gclient`, and `ninja` before generating build files, so PATH/toolchain problems fail before a long build starts. CI sets `DEPOT_TOOLS_UPDATE=0`, and the local bootstrap scripts default the same way, so depot_tools uses the pinned submodule revision unless explicitly overridden. -Manual heavy builds accept these workflow inputs: +Manual dispatch accepts these workflow inputs: | Input | Default | Used by | | --- | --- | --- | -| `linux_heavy_runner` | `open-shorebird-linux-heavy` | `custom-dart-sdk`, `linux-engine`, `android-engine`, `web-sdk` | -| `macos_heavy_runner` | `open-shorebird-macos-heavy` | `custom-dart-sdk-macos`, `ios-engine` / Apple engine artifacts | -| `sdk_min_free_disk_gb` | `35` | Minimum free disk GiB for `custom-dart-sdk` and `custom-dart-sdk-macos` | -| `engine_min_free_disk_gb` | `40` | Minimum free disk GiB for Linux, Android, web, iOS, and macOS engine builds | +| `linux_heavy_runner` | `ubuntu-latest` | `custom-dart-sdk`, `linux-engine`, `android-engine`, `web-sdk` | +| `macos_heavy_runner` | `macos-latest` | `custom-dart-sdk-macos`, `ios-engine` / Apple engine artifacts | +| `sdk_min_free_disk_gb` | `8` | Minimum free disk GiB for `custom-dart-sdk` and `custom-dart-sdk-macos` | +| `engine_min_free_disk_gb` | `8` | Minimum free disk GiB for Linux, Android, web, iOS, and macOS engine builds | | `base_flutter_engine_revision` | empty | Optional upstream Flutter engine revision recorded in `artifacts_manifest.yaml` for non-overridden artifact proxy fallbacks | Use these inputs to move the SDK/engine jobs onto different larger or diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index e92e8e9..83bcf42 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -1459,8 +1459,9 @@ end %w[custom-dart-sdk custom-dart-sdk-macos linux-engine android-engine web-sdk ios-engine].each do |job_name| condition = jobs.fetch(job_name).fetch('if', '').to_s assert!( - condition.include?('workflow_dispatch') && condition.include?('full_sdk_build'), - "#{job_name} must be gated by manual full_sdk_build dispatch" + condition.include?("github.event_name != 'workflow_dispatch'") && + condition.include?('inputs.full_sdk_build'), + "#{job_name} must run on default push/PR CI and allow manual full_sdk_build opt-out" ) run_text = job_runs(jobs.fetch(job_name)).join("\n") assert!( @@ -1487,9 +1488,9 @@ end end artifact_mirror_condition = jobs.fetch('artifact-mirror').fetch('if', '').to_s assert!( - artifact_mirror_condition.include?('workflow_dispatch') && + artifact_mirror_condition.include?("github.event_name != 'workflow_dispatch'") && artifact_mirror_condition.include?('full_sdk_build'), - 'artifact-mirror must be gated by manual full_sdk_build dispatch' + 'artifact-mirror must run on default push/PR CI and allow manual full_sdk_build opt-out' ) %w[cli-artifacts linux-engine android-engine web-sdk ios-engine].each do |dependency| assert!( @@ -1623,17 +1624,20 @@ assert!( 'server artifacts must be uploaded' ) assert!( - inputs.dig('linux_heavy_runner', 'default') == 'open-shorebird-linux-heavy' && - inputs.dig('macos_heavy_runner', 'default') == 'open-shorebird-macos-heavy', - 'heavy SDK/engine workflow inputs must default to custom large-runner labels' + inputs.dig('full_sdk_build', 'default') == true && + inputs.dig('linux_heavy_runner', 'default') == 'ubuntu-latest' && + inputs.dig('macos_heavy_runner', 'default') == 'macos-latest' && + inputs.dig('sdk_min_free_disk_gb', 'default') == 8 && + inputs.dig('engine_min_free_disk_gb', 'default') == 8, + 'heavy SDK/engine workflow inputs must default to managed GitHub-hosted runners and enabled full builds' ) assert!( - jobs.fetch('custom-dart-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && - jobs.fetch('android-engine').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && - jobs.fetch('web-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'open-shorebird-linux-heavy'") && - jobs.fetch('custom-dart-sdk-macos').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'open-shorebird-macos-heavy'") && - jobs.fetch('ios-engine').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'open-shorebird-macos-heavy'"), - 'heavy SDK/engine jobs must default to custom large-runner labels, not standard hosted runners' + jobs.fetch('custom-dart-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'ubuntu-latest'") && + jobs.fetch('android-engine').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'ubuntu-latest'") && + jobs.fetch('web-sdk').fetch('runs-on').to_s.include?("inputs.linux_heavy_runner || 'ubuntu-latest'") && + jobs.fetch('custom-dart-sdk-macos').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'macos-latest'") && + jobs.fetch('ios-engine').fetch('runs-on').to_s.include?("inputs.macos_heavy_runner || 'macos-latest'"), + 'heavy SDK/engine jobs must default to managed GitHub-hosted runners' ) assert!( workflow.dig('env', 'JAVA_VERSION').to_s == '17',