flutter/shorebird-workspace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main'

Browse Source 
# Conflicts:
#	docs/REPOSITORIES.md
This commit is contained in:
Tony
2026-08-03 00:37:03 +08:00
parent 4169dddf97 b93fa25733
commit e36cf7c3d4
52 changed files with 9060 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/open-shorebird-ci.yml
+1785
View File
File diff suppressed because it is too large Load Diff
.gitignore
+2
View File
@@ -21,6 +21,8 @@ test_bytecode.dart
# Dart/Flutter build output in root-owned test fixtures.
**/.dart_tool/
**/build/
**/__pycache__/
*.pyc
# Local editor/OS files.
.DS_Store
.gitmodules
+5 -5
View File
@@ -1,6 +1,6 @@
[submodule "dart-sdk-new"]
path = dart-sdk-new
url = https://git.tonycloud.org/dart-lang/sdk.git
[submodule "dart-sdk"]
path = dart-sdk
url = https://github.com/tony-cloud/dart-sdk.git
branch = tonycloud/dev
[submodule "depot_tools"]
path = depot_tools
@@ -8,8 +8,8 @@
branch = main
[submodule "flutter"]
path = flutter
url = https://git.tonycloud.org/flutter/flutter.git
branch = shorebird/dev
url = https://github.com/tony-cloud/flutter.git
branch = tonycloud/dev
[submodule "shorebird"]
path = shorebird
url = https://git.tonycloud.org/flutter/shorebird.git
README.md
+96 -3
View File
@@ -9,15 +9,15 @@ platform scripts in this root repository.
| Path | Role | Remote | Branch |
| --- | --- | --- | --- |
| `dart-sdk-new` | Dart SDK fork with AOT patch runtime work | `https://git.tonycloud.org/dart-lang/sdk.git` | `tonycloud/dev` |
| `flutter` | Flutter fork used by app and engine integration tests; engine sources live under `flutter/engine/src/flutter` | `https://git.tonycloud.org/flutter/flutter.git` | `shorebird/dev` |
| `dart-sdk` | Dart SDK fork with AOT patch runtime work | `https://github.com/tony-cloud/dart-sdk.git` | `tonycloud/dev` |
| `flutter` | Flutter fork used by app and engine integration tests; engine sources live under `flutter/engine/src/flutter` | `https://github.com/tony-cloud/flutter.git` | `tonycloud/dev` |
| `shorebird` | CLI, protocol/client packages, open patch tools | `https://git.tonycloud.org/flutter/shorebird.git` | `main` |
| `shorebird-server` | Self-hosted CodePush/auth/management server | `https://git.tonycloud.org/flutter/shorebird-server.git` | `main` |
| `updater` | Runtime updater and patch package tooling | `https://git.tonycloud.org/flutter/shorebird-updater.git` | `main` |
| `depot_tools` | Chromium/Dart checkout tooling | `https://chromium.googlesource.com/chromium/tools/depot_tools.git` | `main` |
The Dart SDK checkout has additional gclient-managed dependencies under
`dart-sdk-new/third_party`. They are intentionally not top-level submodules.
`dart-sdk/third_party`. They are intentionally not top-level submodules.
## First Checkout
@@ -41,6 +41,97 @@ By default the bootstrap scripts run `gclient sync`, which can download a large
toolchain/dependency set. Set `SKIP_GCLIENT_SYNC=1` when the checkout is already
synced or when you only want source-level tests.
## GitHub CI
The root workflow `.github/workflows/open-shorebird-ci.yml` is the upload-ready
CI entry point for the combined workspace. Default push and pull request runs
execute source-level checks and build distributable CLI/server artifacts:
- compiled `shorebird`, `open_aot_patch_tools`, and `artifact_proxy` archives
for Linux x64, macOS x64/arm64, and Windows x64
- public updater `patch-*.zip` mirror artifacts for the CLI cache
- `mirror-metadata` with `artifacts_manifest.yaml` for the artifact proxy
- self-hosted `shorebird-server` archives for Linux, macOS, and Windows
Default push and pull request runs also build the large SDK and engine outputs:
patched Dart SDK archives for Linux x64 and macOS arm64, Linux x64 desktop
engine artifacts, Android arm64 engine artifacts, Flutter web SDK artifacts,
and Apple iOS/macOS engine artifacts. A successful full SDK run also uploads
`open-shorebird-artifact-mirror`, a publish-ready mirror archive assembled from
the produced patch-tool, metadata, engine, and web artifacts, plus
`open-shorebird-release-manifest`, a checksum-verified provenance index for the
CLI, server, SDK, engine, and mirror archives. Manual `workflow_dispatch` runs
keep `full_sdk_build=true` by default; set it to `false` only when you want a
source/CLI/server-only run.
Use `scripts/validate_release_manifest.py` to audit a downloaded manifest
against the downloaded workflow artifacts before publishing or mirroring them.
The wrapper `scripts/verify_downloaded_release_artifacts.sh` runs that manifest
sidecar check, verifies the mirror archive sidecar, and validates the extracted
artifact mirror from a downloaded workflow run.
Before pushing the workspace and submodule forks, run
`scripts/verify_upload_readiness.sh`; it fails if any required CI support file
is untracked or if a required root/submodule checkout still has uncommitted
changes.
After upload, `scripts/verify_hosted_full_sdk_build.sh --repo owner/repo --ref main`
dispatches the hosted full SDK workflow, waits for it, downloads artifacts, and
runs the downloaded-release verifier. It uses `gh` when available, or the
GitHub REST API with `GITHUB_TOKEN`/`GH_TOKEN`, `curl`, `jq`, and `unzip`.
The heavy SDK/engine jobs default to managed GitHub-hosted runners
`ubuntu-latest` and `macos-latest`, then run an early disk-capacity preflight.
Override `linux_heavy_runner` / `macos_heavy_runner` only when you want larger
or self-hosted runners. The dispatch inputs `sdk_min_free_disk_gb` and
`engine_min_free_disk_gb` control the preflight thresholds.
The CI contract is validated by
`scripts/verify_ci_workflow.sh`; it rejects `dart_dynamic_modules=true`,
legacy `aot-tools.dill` publishing, missing checksum sidecars, and missing
required artifacts.
Manual `workflow_dispatch` runs with `run_runtime_smokes=true` run the Android
and Linux seeded runtime patch smokes on provisioned runners. These jobs are not
part of default push/PR CI because they require local engine build directories;
the Android smoke also requires an attached device or emulator.
The open CLI defaults vended Flutter installs to this workspace's open Flutter
fork and can be pointed at a future GitHub mirror with
`SHOREBIRD_FLUTTER_GIT_URL`. When no `base_url` or hosted URL override is
configured, API/auth traffic defaults to the local self-hosted server at
`http://localhost:8080` instead of Shorebird's hosted service.
The bundled updater library uses the same default when `shorebird.yaml` does not
include `base_url`.
CLI-managed patch-tool artifact downloads default to the local open mirror root
`http://localhost:8080/artifacts`; set `SHOREBIRD_ARTIFACT_BASE_URL` for a
public mirror populated with the CI-produced `patch-*.zip` files and
`artifacts_manifest.yaml`. The patch artifacts and manual engine artifacts
include publish-ready `shorebird/` mirror subtrees that can be copied to that
same mirror root. Full SDK workflow runs upload the already assembled
`open-shorebird-artifact-mirror` archive; after downloading workflow artifacts
manually, use
`scripts/assemble_artifact_mirror.sh <downloaded-artifacts-dir> <mirror-root>`
to merge those subtrees, reject conflicting artifact bytes, validate
`artifacts_manifest.yaml` overrides, verify all platform `patch-*.zip` files,
and write missing checksum sidecars. If your open engine revision differs from
the upstream Flutter engine revision used for unchanged artifacts, set the
workflow `base_flutter_engine_revision` input when generating mirror metadata.
Vended Flutter commands receive `FLUTTER_STORAGE_BASE_URL` from
`SHOREBIRD_FLUTTER_STORAGE_BASE_URL`, falling back to
`http://localhost:8080/download.flutter.io` for local self-hosted artifacts.
The Flutter fork itself uses that same open mirror as its default engine and
Android Maven artifact host when `FLUTTER_STORAGE_BASE_URL` is unset, including
its Dart SDK refresh scripts.
The iOS App Store candidate route is the encrypted Dart bytecode interpreter
path, not native AOT patch loading. Real-device testing on June 25, 2026 showed
the patched iPad app displaying `license:pro` without enabling
`DART_DYNAMIC_MODULES`. The macOS native-AOT desktop route was also verified
locally the same day by launching a saved free app with a pro Flutter snapshot
seeded as patch `1`; the patched app reported `license:pro`.
CI builds Linux, Android, and macOS engine artifacts with the native AOT patch
runtime enabled and the interpreter route disabled; iOS is the only App
Store-safe interpreter patch artifact path.
Android release APK builds pass locally with the custom Android engine; use
`scripts/android_runtime_patch_smoke.sh` on a machine with an attached Android
device or emulator to run the seeded runtime patch proof. Use
`scripts/linux_runtime_patch_smoke.sh` on a Linux desktop runner to exercise the
same saved-free-app/pro-patch flow for the Linux embedder.
## Local Test Fixture
`testapps/license_flavor_patch_test` is kept in the root repository because it
@@ -49,6 +140,8 @@ Generated build output under that app is ignored by this meta-repository.
## Documents
- `docs/CI.md` describes the GitHub Actions jobs for source checks, CLI/server
artifacts, manual custom SDK/engine builds, and opt-in runtime smokes.
- `docs/REPOSITORIES.md` records the remote URL inventory and cleanup policy.
- `docs/PLATFORM_TESTING.md` documents the Linux, macOS, and later iOS test
flow.
dart-sdk
Submodule
+1
Submodule dart-sdk added at bb9acb510b
dart-sdk-new
-1
Submodule dart-sdk-new deleted from 57b27a7b44
depot_tools
+1 -1
Submodule depot_tools updated: a8c9ac9832...226aa79e99
docs/CI.md
+457
View File
@@ -0,0 +1,457 @@
# GitHub CI
The root workflow at `.github/workflows/open-shorebird-ci.yml` validates the
combined open Shorebird workspace rather than any single submodule in isolation.
## Default CI
Push and pull request runs perform source-level checks and build distributable
binaries:
1. `source-checks` runs the bootstrap test path with `SKIP_GCLIENT_SYNC=1`,
including CLI, code push client, artifact proxy, updater library, open patch
tooling, and server tests.
This checks the Shorebird CLI focused tests, code push client open-server
default tests, open patch tools, server tests, and the root shell scripts
without downloading the full engine dependency graph. It also runs
`scripts/verify_ci_workflow.sh`, which parses the
workflow YAML, checks every workflow `run:` block with `bash -n`, and
verifies the expected open SDK/CLI/server artifact contract, including
archive creation and checksum sidecar generation. It also checks
`.gitmodules`, `docs/REPOSITORIES.md`, and generated `.gclient` files so a
fresh GitHub checkout uses open HTTPS remotes instead of local paths, SSH
remotes, or official closed Shorebird repositories. The verifier also scans
runtime/build-sensitive CLI, updater, artifact-proxy, Flutter tool, Gradle,
engine metadata, and web UI artifact-copy sources to reject hosted Shorebird
endpoints and official `shorebirdtech` GitHub dependencies. It also reads
the forked engine BUILD files and checks that the GN targets and archive
names referenced by the manual engine jobs still exist.
Before uploading the workspace, run `scripts/verify_upload_readiness.sh`.
It invokes the same workflow contract verifier in upload-ready mode, which
requires every listed CI support file to be tracked in its owning checkout
and requires the root plus required submodule checkouts to be clean.
`scripts/verify_sync_open_sources.sh` also exercises the source-link helper
in an isolated temporary workspace so clean generated engine checkouts are
replaced by the workspace Dart/updater submodules and dirty targets are not
overwritten.
`scripts/verify_open_infrastructure_defaults.sh` independently scans
build-sensitive Flutter, Shorebird CLI, artifact-proxy, updater, `.gclient`,
and submodule metadata for closed Shorebird endpoints, official
`shorebirdtech` dependency remotes, and private prebuilt buckets, while
requiring the open local server and artifact mirror defaults.
`scripts/verify_powershell_open_defaults.sh` checks the Windows launcher and
Flutter Dart SDK updater PowerShell scripts for open Flutter/artifact
defaults and parses them with `pwsh` when it is available.
`scripts/verify_write_sha256.sh` smoke-tests the portable checksum helper
used by every artifact job.
`scripts/verify_ios_interpreter_route_validator.sh` smoke-tests the iOS
App Store route gate with synthetic GN args and encrypted patch artifacts,
covering a valid interpreter artifact and rejecting dynamic-module,
non-iOS, malformed JSON, and Mach-O native patch inputs.
2. All artifact and runtime-smoke jobs declare `needs: source-checks`, so no
uploaded CLI, server, SDK, or engine artifact is produced before the
source-level open-replacement contract passes.
3. `cli-artifacts` compiles:
- `shorebird` for Linux x64, macOS x64, macOS arm64, and Windows x64
- `open_aot_patch_tools` for Linux x64, macOS x64, macOS arm64, and
Windows x64
- `artifact_proxy` for Linux x64, macOS x64, macOS arm64, and Windows x64
- public updater `patch` binaries packaged as CLI-cache mirror artifacts:
`patch-linux-x64.zip`, `patch-darwin-x64.zip`,
`patch-darwin-arm64.zip`, and `patch-windows-x64.zip`
- `mirror-metadata`, containing
`shorebird/<engine-revision>/artifacts_manifest.yaml`
The job provisions stable Rust before building the updater `patch` binary,
so those mirror artifacts do not depend on preinstalled runner state.
CI opens each generated patch mirror ZIP before upload and verifies it has
exactly the cache-facing entry name (`patch` on Unix, `patch.exe` on
Windows) with a non-empty payload.
Each CLI upload is an `open-shorebird-cli-<os>-<arch>.tar.gz` archive with
`bin/shorebird`, `bin/open_aot_patch_tools`, `bin/artifact_proxy`,
`bin/internal` version metadata, `manifest.json`, and a `.sha256` sidecar.
The manifest records the Flutter revision and engine revision used by the
bundled CLI cache metadata. The macOS CLI split is intentional: macOS x64
runs on `macos-15-intel`, and macOS arm64 runs on `macos-14`. Before upload,
CI extracts the archive and runs `shorebird --version`,
`open_aot_patch_tools --help`, and `artifact_proxy --health-check` from the
extracted layout. The `shorebird --version` smoke must report the open
`https://git.tonycloud.org/flutter/shorebird.git` fork and rejects the old
official Shorebird SSH remote.
4. `server-artifacts` runs `go test ./...` and cross-compiles
`shorebird-server` for Linux x64/arm64, macOS x64/arm64, and Windows x64
targets. Each uploaded archive contains the server binary, `web/` dashboard
assets, `.env.example`, `README.md`, `openapi.yaml`, `manifest.json`, and a
`.sha256` sidecar. The manifest records both the root workflow commit and
the server submodule commit. Keep `web/` beside the executable when
unpacking; the server looks for dashboard assets relative to its working
directory and executable path.
The Linux x64 archive is extracted and smoke-tested in CI by starting the
packaged binary with SQLite/local storage, checking `/health`, fetching the
dashboard HTML from `/`, and verifying `/openapi.yaml`.
The CLI/server artifacts are uploaded from each workflow run. CLI archives are
intended for installation or release attachments; patch mirror ZIPs are separate
because the Shorebird CLI cache expects the `patch-*.zip` names and a `patch`
or `patch.exe` entry inside each ZIP. The patch mirror ZIPs include `.sha256`
sidecars and can be copied under
`/shorebird/<engine-revision>/` on the host referenced by
`SHOREBIRD_ARTIFACT_BASE_URL`. Each `mirror-patch-*.zip` GitHub artifact also
includes a publish-ready `shorebird/<engine-revision>/patch-*.zip` copy and
matching `.sha256` sidecar, so mirror publication can copy the `shorebird/`
subtree directly. The `mirror-metadata` artifact provides the matching
`artifacts_manifest.yaml` for the artifact proxy.
Manual engine artifacts also include a `mirror/` subtree whose contents are
already laid out under `shorebird/flutter_infra_release/...`; copy that subtree
to the same artifact mirror root to satisfy the generated
`artifacts_manifest.yaml` overrides.
On `full_sdk_build=true` runs, `artifact-mirror` downloads the mirror metadata,
all platform patch ZIPs, and the Linux/Android/web/iOS/macOS engine artifacts,
then uploads `open-shorebird-artifact-mirror`. It also downloads the CLI,
server, and custom Dart SDK artifacts and uploads
`open-shorebird-release-manifest`, a JSON manifest that verifies every archived
release artifact has a matching checksum sidecar and records its SHA-256 digest,
size, relative path, downloaded artifact group, file name, and workflow commit.
Zero-byte release artifacts are rejected even when their sidecars match.
The manifest job also requires every
expected CLI, server, custom Dart SDK, engine, patch-tool, metadata, and mirror
artifact family to be present. The mirror archive is the checked publish-ready
mirror root for `SHOREBIRD_ARTIFACT_BASE_URL`; the release manifest is the
provenance index for the full SDK/CLI/server/engine output set.
The artifact-mirror job validates the generated manifest with
`scripts/validate_release_manifest.py` before upload. After downloading workflow
artifacts, the same validator can audit the manifest against the downloaded
artifact directory and will reject missing files, duplicate or unsafe paths,
checksum/size mismatches, sidecar mismatches, orphan sidecars, mismatched
artifact group or file-name provenance fields, and artifacts that are present on
disk but missing from the manifest.
Use the wrapper below after downloading a completed `full_sdk_build=true` run:
```sh
gh run download <run-id> --dir downloaded-artifacts
./scripts/verify_downloaded_release_artifacts.sh \
--github-sha <run-head-sha> \
downloaded-artifacts
```
It verifies the release manifest sidecar, validates the manifest against every
downloaded artifact, optionally requires the manifest commit to match the
expected run SHA, verifies the mirror archive sidecar, safely extracts
`open-shorebird-artifact-mirror.tar.gz` after rejecting unsafe archive members,
and validates the extracted mirror root before anything is published.
After the workspace is uploaded to GitHub, the end-to-end proof can be driven
from a local checkout with:
```sh
./scripts/verify_hosted_full_sdk_build.sh \
--repo owner/repo \
--ref main
```
That helper dispatches `open-shorebird-ci.yml` with `full_sdk_build=true`,
waits for the hosted workflow run to succeed, downloads all artifacts, and runs
`scripts/verify_downloaded_release_artifacts.sh` against the downloaded output
with the workflow run's `headSha`.
It uses GitHub CLI when `gh` is installed; otherwise it uses the GitHub REST API
with `GITHUB_TOKEN` or `GH_TOKEN` plus `curl`, `jq`, and `unzip`.
Use `--linux-heavy-runner`, `--macos-heavy-runner`, and the disk-threshold flags
when the repository uses custom larger/self-hosted runner labels.
For local release assembly or reassembly after downloading workflow artifacts,
run:
```sh
./scripts/assemble_artifact_mirror.sh downloaded-artifacts public-mirror
```
The assembler scans direct artifact contents plus `*.tar.gz` engine archives,
validates archive member paths, rejects links/devices before extraction, copies
every `shorebird/` mirror subtree into `public-mirror`, rejects conflicting
files with different bytes, validates that each
`artifacts_manifest.yaml` override resolves to a copied non-empty file, verifies
every platform `patch-*.zip` contains exactly the cache-facing `patch` or
`patch.exe` entry, and writes missing `.sha256` sidecars.
`scripts/validate_artifact_mirror.py public-mirror` can be run independently
after safely extracting `open-shorebird-artifact-mirror.tar.gz`; the
`artifact-mirror` job safe-extracts and validates the assembled archive before
upload.
`scripts/verify_assemble_artifact_mirror.sh` keeps the low-level assembler
covered in default source checks. `scripts/verify_artifact_mirror_workflow_assembly.sh`
also dry-runs the full `artifact-mirror` aggregation flow with fake downloaded
artifacts, including the mirror-only input subset, extracted mirror validation,
and release manifest requirements. This catches path-layout regressions before
the hosted `full_sdk_build=true` run.
CI writes checksum sidecars through `scripts/write_sha256.sh` so the artifact
jobs do not depend on platform-specific checksum tools.
The workflow defaults all `run` steps to Bash, including Windows matrix jobs,
because the workspace scripts and packaging commands use Bash syntax.
On June 25, 2026, the Darwin arm64 CLI artifact path was smoke-tested locally:
the workflow-equivalent commands compiled `shorebird`, `open_aot_patch_tools`,
and `artifact_proxy`; built the Rust updater `patch` binary; produced
`open-shorebird-cli-macos-arm64.tar.gz` and `patch-darwin-arm64.zip`; wrote both
checksum sidecars; extracted the CLI archive; and verified the compiled tools
could run from the extracted layout.
The patch mirror ZIP contained a single `patch` entry, matching the CLI cache
contract.
The Darwin arm64 server artifact path was smoke-tested locally the same day:
the workflow-equivalent packaging command produced
`shorebird-server-darwin-arm64.tar.gz`, wrote a `.sha256` sidecar, extracted
the archive, started the packaged server with SQLite and local storage, and
verified `/health` returned `status=ok` while `/` served the packaged dashboard
HTML from the sibling `web/` directory. The packaged server also serves the
checked OpenAPI contract at `/openapi.yaml`.
## Open Artifact Hosts
The CLI can be used with a self-hosted server and open artifact mirror without
changing source code. When no project `base_url`, user config, or environment
override is present, the open CLI defaults API/auth traffic to the local
self-hosted server at `http://localhost:8080` instead of Shorebird's hosted
service:
```sh
export SHOREBIRD_HOSTED_URL=https://updates.example.com
export SHOREBIRD_ARTIFACT_BASE_URL=https://artifacts.example.com/open
export SHOREBIRD_FLUTTER_STORAGE_BASE_URL=https://artifacts.example.com/flutter
export SHOREBIRD_FLUTTER_GIT_URL=https://github.com/example/open-flutter.git
```
`SHOREBIRD_HOSTED_URL` points API calls at the self-hosted server and is also
written into new `shorebird.yaml` files as `base_url`. The bundled updater
library also defaults to `http://localhost:8080` when a legacy app omits
`base_url`. The self-hosted server publishes its checked OpenAPI contract at
`http://localhost:8080/openapi.yaml`. The artifact mirror keeps the same path
layout as Shorebird's default storage bucket:
```text
$SHOREBIRD_ARTIFACT_BASE_URL/shorebird/<engine-revision>/patch-linux-x64.zip
$SHOREBIRD_ARTIFACT_BASE_URL/shorebird/<engine-revision>/patch-darwin-x64.zip
$SHOREBIRD_ARTIFACT_BASE_URL/shorebird/<engine-revision>/patch-darwin-arm64.zip
$SHOREBIRD_ARTIFACT_BASE_URL/shorebird/<engine-revision>/patch-windows-x64.zip
$SHOREBIRD_ARTIFACT_BASE_URL/shorebird/<engine-revision>/artifacts_manifest.yaml
```
When `SHOREBIRD_ARTIFACT_BASE_URL` is not set, the open CLI defaults CLI-managed
artifact downloads to `http://localhost:8080/artifacts`. Use that local mirror
root for development, or set `SHOREBIRD_ARTIFACT_BASE_URL` to the public mirror
where the CI-produced `patch-*.zip` artifacts are hosted.
If the mirror is a bucket-style host, `SHOREBIRD_STORAGE_BASE_URL` and
`SHOREBIRD_STORAGE_BUCKET` can be used instead of
`SHOREBIRD_ARTIFACT_BASE_URL`. `SHOREBIRD_FLUTTER_STORAGE_BASE_URL` controls the
`FLUTTER_STORAGE_BASE_URL` passed to vended Flutter commands; when unset it
defaults to `http://localhost:8080/download.flutter.io`.
`SHOREBIRD_FLUTTER_GIT_URL` controls where the CLI clones vended Flutter
revisions during cache installation. When unset, the open CLI defaults to this
workspace's open Flutter fork instead of `github.com/shorebirdtech/flutter.git`.
The Flutter fork also defaults its own engine downloads, Dart SDK refresh
scripts, doctor network check, Android Gradle Maven host, Android host-app
integration fixtures, docs artifact scripts, and Shorebird integration tests to
`http://localhost:8080/download.flutter.io` when `FLUTTER_STORAGE_BASE_URL` is
unset, so direct Flutter use does not fall back to Shorebird's closed artifact
host.
The open CI currently does not publish `aot-tools.dill`. That file is a
different Shorebird linker artifact used by the legacy native-AOT iOS linker
path. The open App Store-safe iOS path uses encrypted Dart bytecode interpreter
artifacts instead, and Android/Linux/macOS/Windows patch creation uses the
public updater `patch` binaries above.
The CLI cache does not download `aot-tools.dill` by default. Set
`SHOREBIRD_ENABLE_LEGACY_AOT_TOOLS=1` only when deliberately validating the
development-only legacy native-AOT linker path.
Runtime smoke scripts that need external targets are kept out of default CI:
`scripts/android_runtime_patch_smoke.sh` requires an attached Android
device/emulator with app-private storage access, and
`scripts/linux_runtime_patch_smoke.sh` requires a Linux desktop runtime or
`xvfb-run`. Use them on suitable self-hosted runners when extending runtime
coverage beyond the artifact build matrix.
Use `workflow_dispatch` with `run_runtime_smokes=true` to run those external
runtime smokes from CI. The runtime jobs are intentionally separate from
`full_sdk_build=true`: they are for provisioned runners that already have the
matching local engine build outputs in the checkout workspace. Runtime checkout
uses `clean: false` so self-hosted runners can preserve those `out/` directories
between preparation and smoke runs.
| Input | Default | Used by |
| --- | --- | --- |
| `run_runtime_smokes` | `false` | Enables `linux-runtime-smoke` and `android-runtime-smoke` |
| `linux_runtime_runner` | `self-hosted` | Runner label for `scripts/linux_runtime_patch_smoke.sh` |
| `android_runtime_runner` | `self-hosted` | Runner label for `scripts/android_runtime_patch_smoke.sh` |
| `android_serial` | empty | Optional `adb` serial passed as `ANDROID_SERIAL` |
The Linux runtime runner must provide
`flutter/engine/src/out/linux_release_x64`. The Android runtime runner must provide
`flutter/engine/src/out/android_release_arm64` and
`flutter/engine/src/out/host_release_arm64`, plus an Android target where either
`adb shell run-as` works for the test package or `adb root` works. It must also
provide Java on `PATH` because the smoke builds APKs before seeding the patch.
## Heavy SDK Builds
Default push and pull request runs build the custom SDK and engine artifacts.
Manual `workflow_dispatch` runs also build them by default because
`full_sdk_build=true` is the default input value; set `full_sdk_build=false`
only for source/CLI/server-only manual runs.
The heavy jobs install Chromium's `depot_tools` into the workflow workspace
before running `gclient`, so they can bootstrap from a clean runner. The
workflow defaults heavy jobs to managed GitHub-hosted runners:
- `ubuntu-latest` for Linux SDK, Linux engine, Android engine, and web SDK
builds
- `macos-latest` for macOS Dart SDK, iOS engine, and macOS engine builds
Override `linux_heavy_runner` / `macos_heavy_runner` when a repository wants
larger or self-hosted runners. Every SDK/engine job also runs
`scripts/check_ci_capacity.sh` before `gclient sync` or `ninja`; SDK-only jobs
and engine jobs require at least 8 GiB free by default. Raise the dispatch
thresholds for runner images where a larger preflight budget should be enforced.
The hosted Android engine job also provisions Temurin Java 17 before building
Android JAR/APK-related engine artifacts. Every SDK/engine job verifies
`python3`, `gclient`, and `ninja` before generating build files, so
PATH/toolchain problems fail before a long build starts. CI sets
`DEPOT_TOOLS_UPDATE=0`, and the local bootstrap scripts default the same way,
so depot_tools uses the pinned submodule revision unless explicitly overridden.
Manual dispatch accepts these workflow inputs:
| Input | Default | Used by |
| --- | --- | --- |
| `linux_heavy_runner` | `ubuntu-latest` | `custom-dart-sdk`, `linux-engine`, `android-engine`, `web-sdk` |
| `macos_heavy_runner` | `macos-latest` | `custom-dart-sdk-macos`, `ios-engine` / Apple engine artifacts |
| `sdk_min_free_disk_gb` | `8` | Minimum free disk GiB for `custom-dart-sdk` and `custom-dart-sdk-macos` |
| `engine_min_free_disk_gb` | `8` | Minimum free disk GiB for Linux, Android, web, iOS, and macOS engine builds |
| `base_flutter_engine_revision` | empty | Optional upstream Flutter engine revision recorded in `artifacts_manifest.yaml` for non-overridden artifact proxy fallbacks |
Use these inputs to move the SDK/engine jobs onto different larger or
self-hosted runners without editing the workflow file. The Linux heavy jobs also run
`scripts/free_ci_disk_linux.sh`, which removes unrelated preinstalled toolchain
caches only when `GITHUB_ACTIONS=true` and `RUNNER_ENVIRONMENT=github-hosted`.
Self-hosted runners skip that cleanup by default. Set `CI_FREE_DISK_SPACE=0` to
skip cleanup anywhere, or `CI_FREE_DISK_SPACE_FORCE=1` to opt in on a
self-hosted runner.
When `base_flutter_engine_revision` is empty, `mirror-metadata` records the
same revision as `flutter/bin/internal/engine.version`. Set it when the open
Shorebird engine revision is a custom fork revision and unchanged artifacts
should be proxied back to a different upstream Flutter engine revision.
The engine jobs perform two syncs when `run_gclient_sync=true`: the root
workspace sync for the Dart SDK checkout, then a second sync from `flutter/` for
Flutter engine dependencies written by `scripts/write_gclient.sh`. Linux engine
jobs use `INCLUDE_ENGINE_DEPS=1` to include Flutter engine dependencies,
Android dependencies, and emsdk; the Apple engine job uses the same flag to
include iOS, Android, and emsdk
dependencies on macOS.
Before building, `scripts/sync_open_sources.sh` links
`flutter/engine/src/flutter/third_party/dart` to the `dart-sdk` submodule
and `flutter/engine/src/flutter/third_party/updater` to the `updater`
submodule. It does not clone the official `shorebirdtech/updater` repository by
default; set `UPDATER_URL` explicitly only when testing a different updater
fork. The helper rejects known unpatched upstream Dart SDK remotes and official
Shorebird updater remotes so CI does not accidentally build from the wrong
component checkout.
The Flutter fork's `DEPS` file also points its Dart SDK and updater dependency
URLs at the open mirrors and does not reference Shorebird's private
`shorebird-dart-sdk-prebuilt` bucket; custom SDK jobs build and archive the
open Dart SDK fork directly.
`custom-dart-sdk` and `custom-dart-sdk-macos` build the Dart SDK fork with:
```gn
dart_dynamic_modules = false
dart_enable_aot_patching = true
dart_enable_shorebird_interpreter = true
```
They run `scripts/verify_dart_sdk_args.sh`, then the focused VM patch API
tests. The Linux job uploads `custom-dart-sdk-linux-x64`; the macOS job uploads
`custom-dart-sdk-macos-arm64`. Each archive contains the Dart SDK,
`gen_snapshot`, `dartaotruntime`, `args.gn`, and a `manifest.json` that records
the root workflow commit, Dart SDK source commit, and patch-related build
flags. The workflow also uploads `.sha256` sidecars for the SDK archives.
Before writing the checksum, each SDK job extracts the archive, verifies
`manifest.json`, `args.gn`, `gen_snapshot`, `dartaotruntime`, and
`dart-sdk/bin/dart`, then runs the extracted `dart --version`.
`ios-engine` builds:
- `host_release_arm64` with `--shorebird-interpreter`,
`dart_dynamic_modules=false`, `dart_enable_aot_patching=true`, and
`dart_enable_shorebird_interpreter=true`
- `ios_release` with `--shorebird-interpreter`
- `macos_release_arm64` with `dart_dynamic_modules=false`,
`dart_enable_aot_patching=true`, `shorebird_enable_aot_patching=true`, and
`shorebird_use_interpreter=false`
It runs `scripts/verify_ios_interpreter_route.sh` before uploading the
`ios-interpreter-engine` artifact, so the archived build proves
`DART_DYNAMIC_MODULES` is off and the iOS route uses the bytecode interpreter
instead of native AOT patch loading. The archive includes `Flutter.framework`,
`Flutter.xcframework`, a mirror-ready `ios-release/artifacts.zip` containing
the xcframework plus `gen_snapshot_arm64`, `analyze_snapshot_arm64`, and the
code-sign configuration files Flutter expects, the
`host_release_arm64/gen_snapshot` binary built with `--shorebird-interpreter`,
the iOS and host `args.gn` files, and a `manifest.json` that records
`dart_dynamic_modules=false`,
`dart_enable_aot_patching=true`, `dart_enable_shorebird_interpreter=true`,
`shorebird_enable_aot_patching=false`, `shorebird_use_interpreter=true`, and
the Flutter engine revision; the workflow also uploads a `.sha256` sidecar for
the engine archive.
Before checksum upload, CI extracts the iOS engine archive and verifies
`Flutter.framework`, `Flutter.xcframework`, `ios-release/artifacts.zip`, the
mirror copy under `mirror/shorebird/flutter_infra_release/flutter/<engine>`,
both args files, `manifest.json`, and the executable host `gen_snapshot`.
When an app or IPA is supplied to the gate, strict mode also rejects bundled
patch payloads, executable-memory entitlements, and raw `aot_patch_key_hex`
material in `shorebird.yaml`.
The same Apple job uploads a separate `macos-engine-arm64` artifact containing
`FlutterMacOS.framework.zip`, `flutter_patched_sdk_product.zip`, the macOS
`args.gn`, a mirror-ready copy of the macOS framework override, a manifest with
the Flutter engine revision, and a `.sha256` sidecar. The shared
`flutter_patched_sdk_product.zip` mirror override is published only by the Linux
engine job to avoid duplicate producers for the same mirror path. The macOS
build is checked with
`scripts/verify_engine_args.sh` so CI fails if `dart_dynamic_modules=true`
appears or if the native AOT patch runtime flags are missing from the generated
args. CI extracts the macOS engine archive before upload and verifies the
framework zip, patched SDK zip, args file, manifest, and framework mirror
subtree.
`linux-engine` builds `linux_release_x64` on Ubuntu with
`dart_dynamic_modules=false`, `dart_enable_aot_patching=true`,
`shorebird_enable_aot_patching=true`, and `shorebird_use_interpreter=false`. It
uploads `linux-x64-flutter-gtk.zip`, `flutter_patched_sdk_product.zip`,
`artifacts.zip`, mirror-ready copies of those engine override files, `args.gn`,
a manifest with the Flutter engine revision, and a `.sha256` sidecar. CI
extracts the archive before upload and verifies the GTK zip, patched SDK zip,
artifacts zip, args file, manifest, and mirror subtree.
`android-engine` builds `android_release_arm64` on Ubuntu with
`dart_dynamic_modules=false`, `dart_enable_aot_patching=true`,
`shorebird_enable_aot_patching=true`, and `shorebird_use_interpreter=false`. It
uploads `artifacts.zip`, `symbols.zip`, `flutter.jar`, `libflutter.so`, host
`gen_snapshot_arm64`, `analyze_snapshot_arm64`, mirror-ready copies of the
Android engine override files, `args.gn`, a manifest with the Flutter engine
revision, and a `.sha256` sidecar. CI extracts the archive before upload and
verifies the Android artifacts/symbols zips, `flutter.jar`, `libflutter.so`,
host snapshot/analyzer tools, args file, manifest, and mirror subtree.
`web-sdk` builds the Flutter web SDK archive from `wasm_release` with
`dart_dynamic_modules=false` and `flutter_prebuilt_dart_sdk=true`, then uploads
`flutter-web-sdk.zip`, a mirror-ready copy of that SDK archive, `args.gn`, a
manifest with the Flutter engine revision, and a `.sha256` sidecar. The web job
does not download Flutter's private prebuilt Dart SDK; it links
`dart-sdk/tools/sdks/dart-sdk` into Flutter's expected prebuilt path before
running GN. Native engine jobs use `--no-prebuilt-dart-sdk` because the open
workspace builds those SDK artifacts from the linked Dart checkout instead. Web
is still not a Shorebird CodePush release platform in this CLI/protocol; this
job exists to keep the open Flutter SDK/web artifacts buildable from the
workspace. CI extracts the web SDK archive before upload and verifies the SDK
zip, args file, manifest, and mirror subtree.
Set `run_gclient_sync=false` only for debugging a runner image that already has
all gclient-managed dependencies restored.
docs/PLATFORM_TESTING.md
+254 -22
View File
@@ -1,4 +1,4 @@
# Linux, macOS, and iOS Platform Testing
# Platform Testing
The first complete runtime proof was built on Windows. The next platform pass
should verify the same source layout on Linux and macOS, then use macOS to
@@ -14,6 +14,9 @@ Linux:
- Dart or Flutter on `PATH`
- Go 1.23+
- Python and standard build dependencies required by Dart/Flutter
- Flutter Linux desktop build dependencies (`clang`, `cmake`, `ninja`,
`pkg-config`, GTK development headers)
- A graphical session or `xvfb-run` for headless runtime smoke tests
macOS:
@@ -24,6 +27,16 @@ macOS:
- Xcode command line tools
- Full Xcode for iOS simulator/device follow-up
Android runtime validation:
- Android SDK platform tools (`adb`)
- Java/JDK on `PATH` for APK builds
- A connected Android device or emulator
- Either `adb shell run-as` access for the test package or a rooted
emulator/device. The seeded runtime smoke writes directly to the test app's
private updater directory; production installs normally receive patches from
the updater download flow.
Both platforms need network access for a fresh `git submodule update` and
`gclient sync`.
@@ -59,6 +72,11 @@ Useful environment variables:
| `IOS_IPA=/path/to/App.ipa` | Optionally verify the `Payload/*.app` inside an IPA instead of a raw app bundle. |
| `IOS_PATCH_ARTIFACT=/path/to/dlc.vmcode` | Optionally verify that the patch artifact is an encrypted iOS arm64 interpreter full-snapshot artifact. |
| `APP_STORE_STRICT=1` | Fail the iOS route check when App Store-inappropriate signing state, such as `get-task-allow=true`, is present. |
| `ANDROID_SERIAL=<serial>` | Select the Android device used by `scripts/android_runtime_patch_smoke.sh` when more than one device is attached. |
| `SKIP_ANDROID_BUILDS=1` | Reuse APKs for the Android runtime smoke instead of rebuilding; set `ANDROID_FREE_APK` and `ANDROID_PRO_APK`. |
| `KEEP_ANDROID_RUNTIME_SMOKE_ARTIFACTS=1` | Keep the temporary APK/libapp extraction directory after the Android runtime smoke. |
| `LINUX_RUNTIME_SMOKE_XVFB=0` | Disable automatic `xvfb-run` wrapping in `scripts/linux_runtime_patch_smoke.sh`. |
| `KEEP_LINUX_RUNTIME_SMOKE_ARTIFACTS=1` | Keep the temporary Linux app copy and free/pro bundles after the Linux runtime smoke. |
## Expected Checks
@@ -73,15 +91,196 @@ The bootstrap scripts run:
7. self-hosted server Go tests
8. license/flavor AOT verification when an AOT patch SDK build already exists.
The verifier checks `AOT_PATCH_BUILD_DIR` first, then common local outputs
such as `dart-sdk-new/xcodebuild/ReleaseARM64`,
`dart-sdk-new/out/ReleaseARM64AotPatch`, and
`dart-sdk-new/out/ReleaseX64AotPatch`.
such as `dart-sdk/xcodebuild/ReleaseARM64`,
`dart-sdk/out/ReleaseARM64AotPatch`, and
`dart-sdk/out/ReleaseX64AotPatch`.
9. iOS interpreter route verification when local `ios_release` and
`host_release_arm64` engine args exist. This checks that
`DART_DYNAMIC_MODULES` is off, the no-DDM interpreter route is on for both
the device engine and host snapshotter, and the native AOT patch path is off
for iOS.
## Current Coverage Status
| Platform | Open patch route | Current status |
| --- | --- | --- |
| iOS | Encrypted Dart bytecode interpreter artifact, `DART_DYNAMIC_MODULES=false` | Verified on a real iPad on June 25, 2026; the patched app displayed `license:pro`. |
| Android | Public updater binary diff `patch-*.zip` artifact with native AOT patch runtime enabled | CI builds mirror-ready patch binaries and a manual `android-engine-arm64` artifact with `dart_enable_aot_patching=true`, `shorebird_enable_aot_patching=true`, and interpreter mode off; local arm64 release APK build passed. Runtime smoke script is available, but this macOS host currently has no Android device or AVD attached. |
| macOS | Public updater binary diff `patch-*.zip` artifact with native AOT patch runtime enabled | Verified locally on June 25, 2026 with a native-AOT Flutter snapshot patch; the patched app displayed `license:pro`. CI builds macOS x64/arm64 patch binaries and a manual `macos-engine-arm64` artifact with native AOT patch runtime args verified. |
| Linux | Public updater binary diff `patch-*.zip` artifact with native AOT patch runtime enabled | CI builds the Linux x64 patch binary and a manual `linux-engine-x64` artifact with native AOT patch runtime args verified. The runtime smoke defaults to `linux_release_x64`; local runtime validation requires a Linux runner. |
| Windows | Public updater binary diff `patch-*.zip` artifact | Previously tested; CI still builds the Windows patch binary. |
| Web | Not a Shorebird CodePush release platform in this CLI/protocol | Manual CI builds `flutter-web-sdk`; local release web build passed with the matching local engine, but it is not counted as a Shorebird runtime patch test. |
The open CI currently does not publish `aot-tools.dill`. That artifact belongs
to the legacy native-AOT iOS linker path. The App Store-safe iOS route proven
above does not use it; Android/Linux/macOS/Windows patch creation uses the
public updater `patch` binary instead.
## macOS Host Smoke Results
On June 25, 2026, the default source-level bootstrap path passed locally with
`SKIP_GCLIENT_SYNC=1`, the workspace Flutter/Dart SDK, and network access for
pub packages:
```sh
SKIP_GCLIENT_SYNC=1 \
DART_BIN=/Users/tonylu/git/shorebird-workspace/flutter/bin/dart \
FLUTTER_BIN=/Users/tonylu/git/shorebird-workspace/flutter/bin/flutter \
./scripts/bootstrap_linux.sh
```
This ran the focused Shorebird CLI tests, open patch tool tests, self-hosted
server Go tests, and license/flavor AOT verifier. The AOT verifier reported:
```text
AOT patch applied to license_flavor_patch_test successfully.
base: license:free / pro-feature:off
patch: license:pro / pro-feature:enabled
```
The same app also built successfully for macOS release with the local engine:
```sh
flutter build macos --release \
--local-engine-src-path=/Users/tonylu/git/shorebird-workspace/flutter/engine/src \
--local-engine=host_release_arm64 \
--local-engine-host=host_release_arm64 \
--dart-define=LICENSE_TYPE=free
```
Output:
```text
build/macos/Build/Products/Release/license_flavor_patch_test.app
```
On June 25, 2026, the macOS runtime patch path was also verified locally. The
test saved a free baseline app, rebuilt the same Flutter app with
`--dart-define=LICENSE_TYPE=pro`, copied the resolved pro
`App.framework/Versions/A/App` Mach-O into the sandboxed updater container as:
```text
Library/Application Support/shorebird/shorebird_updater/
license-flavor-patch-test/patches/1/dlc.vmcode
```
and seeded `pointers.json` plus `patches/1/state.json` for patch `1`.
Launching the saved free app selected that patch path and wrote:
```text
license:pro
pro-feature:enabled
```
The seeded payload must be a Flutter app snapshot for this smoke. A standalone
Dart verifier snapshot is loadable by the updater handoff, but it is not a
valid Flutter application isolate and fails later with `dart:ui` missing.
When copying from `App.framework/App`, resolve the framework symlink first
(`Versions/A/App` or `cp -L`) so the updater validates the real Mach-O size
rather than the symlink size.
Android arm64 release also built successfully with the local Android engine:
```sh
flutter build apk --release \
--target-platform android-arm64 \
--local-engine-src-path=/Users/tonylu/git/shorebird-workspace/flutter/engine/src \
--local-engine=android_release_arm64 \
--local-engine-host=host_release_arm64 \
--dart-define=LICENSE_TYPE=free
```
Output:
```text
build/app/outputs/flutter-apk/app-release.apk
```
Android runtime validation is scripted but was not run to completion on this
macOS host because `adb devices -l` returned no Android devices and
`emulator -list-avds` returned no AVDs. When a target is available, run:
```sh
ANDROID_SERIAL=<device-or-emulator-serial> \
./scripts/android_runtime_patch_smoke.sh
```
The same script can be run from GitHub Actions with `workflow_dispatch` and
`run_runtime_smokes=true` on a provisioned Android runner. Use
`android_runtime_runner` to choose the runner label and `android_serial` to
select the target device when needed.
The script builds free and pro arm64 release APKs with the local Android
engine, installs the free APK, verifies the visible `license:free` and
`pro-feature:off` UI with `uiautomator`, extracts the pro `libapp.so`, seeds it
as patch `1` under the app-private `files/shorebird_updater` directory,
relaunches the free app, and verifies `license:pro` plus
`pro-feature:enabled`.
The seed step needs access to app-private storage. Use an emulator/device where
`adb shell run-as com.example.licenseflavorpatchtest.license_flavor_patch_test`
works, or a rooted emulator/device where `adb root` works. This seeded smoke is
for local runtime proof; production Android patch installation should exercise
the updater download path through the self-hosted server.
A normal web release build succeeds when pointed at the matching local engine:
```sh
flutter build web --release \
--local-engine-src-path=/Users/tonylu/git/shorebird-workspace/flutter/engine/src \
--local-engine=host_release_arm64 \
--local-engine-host=host_release_arm64 \
--dart-define=LICENSE_TYPE=free
```
Output:
```text
build/web/main.dart.js
```
Running the same web build without `--local-engine` currently fails because the
prebuilt `const_finder.dart.snapshot` in `flutter/bin/cache` expects a different
kernel binary format than this workspace Dart SDK emits. Use the local engine
when validating this workspace's web build artifacts.
The self-hosted server bundle packaging was also smoke-tested locally for
`darwin/arm64`: the archive was extracted, the server started with SQLite and
local storage, `/health` returned `status=ok`, and `/` served the packaged
dashboard HTML from the sibling `web/` directory. The server serves the checked
OpenAPI contract at `/openapi.yaml`, and CI verifies that endpoint from the
packaged Linux x64 archive. CI server artifacts therefore package `web/` and
`openapi.yaml` next to the binary instead of uploading a bare executable.
## Linux Runtime Smoke
Linux runtime validation is scripted for a Linux host:
```sh
./scripts/linux_runtime_patch_smoke.sh
```
The script copies `testapps/license_flavor_patch_test` into a temporary
directory, generates the missing Linux platform scaffold there with
`flutter create --platforms=linux`, builds free and pro release bundles with the
local Linux engine, launches the saved free bundle, verifies
`license:free`/`pro-feature:off`, seeds the pro bundle's `lib/libapp.so` as
patch `1` under:
```text
~/.shorebird_cache/shorebird_updater/license-flavor-patch-test/
```
then relaunches the saved free bundle and verifies `license:pro` plus
`pro-feature:enabled`. If no `DISPLAY` is set and `xvfb-run` exists, the script
uses it automatically.
This macOS host cannot execute Linux desktop bundles, so this check remains a
Linux-runner task. The same check can be triggered in GitHub Actions with
`workflow_dispatch` and `run_runtime_smokes=true` on a provisioned Linux
runner. Use `linux_runtime_runner` to choose the runner label; the runner must
preserve the matching `flutter/engine/src/out/host_release` local engine output.
## iOS Preparation Notes
The macOS script writes this minimal root `.gclient` by default:
@@ -127,8 +326,14 @@ request JIT or unsigned-executable-memory entitlements, and should not have
`get-task-allow=true` in strict mode. When `IOS_PATCH_ARTIFACT` is set, the gate
also rejects Mach-O and ELF patch files and requires the encrypted open artifact
metadata to declare `runtime_mode=dart-bytecode-interpreter`, `target_os=ios`,
`target_arch=arm64`, and `payload_kind=full-snapshot`. Real patches should
arrive through the updater after the reviewed app is installed.
`target_arch=arm64`, and `payload_kind=full-snapshot`. It parses the artifact
as JSON and also requires the AES-256-GCM wrapper fields, base64 ciphertext,
nonce/tag, key id, payload digest, and AAD digest to be present and well
formed. In strict mode, the gate also rejects bundled `aot_patch_key_hex`
values; production apps should provide patch keys with
`FlutterDartProject.shorebirdAotPatchKeyProvider` or an equivalent app-owned
key source. Real patches should arrive through the updater after the reviewed
app is installed.
## iOS Device Results
@@ -177,12 +382,14 @@ path when it is used to produce iOS AOT snapshots:
```sh
flutter/tools/gn --runtime-mode=release --mac-cpu=arm64 \
--target-dir=host_release_arm64 --shorebird-interpreter
--target-dir=host_release_arm64 --shorebird-interpreter \
--gn-args='dart_dynamic_modules=false dart_enable_aot_patching=true dart_enable_shorebird_interpreter=true shorebird_use_interpreter=true'
ninja -C out/host_release_arm64 gen_snapshot
```
Without the host-side interpreter flag, the generated app snapshot can bind AOT
static calls directly to base `Code` objects and omit the static-call metadata
Without the host-side interpreter and AOT patch flags, the generated app
snapshot can bind AOT static calls directly to base `Code` objects and omit the
static-call metadata
that the iOS interpreter runtime uses to enter patched bytecode.
The current open-source interpreter path wires the no-DDM artifact handoff and
@@ -190,8 +397,10 @@ an initial product-AOT function replacement mapper. Interpreter-mode artifacts
now force and validate `payload_kind=full-snapshot`; compact interpreter diffs
are rejected until runtime reconstruction exists. The engine can hand an
encrypted open patch artifact to `Dart_InstallAotPatch`, decrypt it with an
explicit `shorebird.yaml` AES key, verify the decrypted payload is Dart
bytecode, and install it through `Dart_ReloadBytecodePatch`.
app-owned AES key provider, verify the decrypted payload is Dart bytecode, and
install it through `Dart_ReloadBytecodePatch`. A development-only
`aot_patch_key_hex` field in `shorebird.yaml` is still supported when no app key
provider is configured, but strict App Store-route checks reject it.
For product iOS AOT, `Dart_ReloadBytecodePatch` no longer uses stock VM reload.
It reads patch bytecode as data, maps declarations to already-loaded
@@ -255,9 +464,9 @@ IOS_PATCH_ARTIFACT=/private/tmp/shorebird-seed-main/shorebird_updater/patches/1/
This verified the generated iOS and host engine args, found no bundled
`shorebird_updater` or `dlc.vmcode` payload in the app bundle, found no JIT or
unsigned-executable-memory entitlement, confirmed `get-task-allow` is not true,
and verified that the seeded patch artifact is an encrypted interpreter
full-snapshot for iOS arm64. Passing a Mach-O binary as `IOS_PATCH_ARTIFACT`
correctly fails the route gate.
confirmed no `aot_patch_key_hex` is bundled, and verified that the seeded patch
artifact is an encrypted interpreter full-snapshot for iOS arm64. Passing a
Mach-O binary as `IOS_PATCH_ARTIFACT` correctly fails the route gate.
The remaining App Store/TestFlight proof is a distribution test, not a local
build test:
@@ -284,12 +493,13 @@ must emit an interpreter payload in that compatible shape, and compact
reconstruction still needs to move into the SDK before compact bytecode patches
can be accepted.
For development-device encrypted interpreter tests, `shorebird.yaml` may include
these top-level fields:
For encrypted interpreter tests, `shorebird.yaml` may include these top-level
fields:
```yaml
aot_patch_runtime_mode: dart-bytecode-interpreter
aot_patch_key_id: test-key
aot_patch_key_hex: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
aot_patch_app_build_id: 1.2.3+4
aot_patch_base_flavor_id: free
aot_patch_base_license_type: free
aot_patch_flavor_id: pro
@@ -298,8 +508,30 @@ aot_patch_sdk_hash: <flutter-engine-or-dart-sdk-hash>
aot_patch_base_snapshot_hash: <sha256-of-base-bytecode-snapshot>
```
The YAML key path is only the first open bridge for local verification. A
production App Store implementation still needs an app-owned key callback or
equivalent secure key source, a verified compatible interpreter payload
compiler, compact interpreter reconstruction, and TestFlight/App Store review
validation.
`aot_patch_bytecode_path` is optional and should normally be omitted. When it
is omitted, `shorebird patch ios` compiles the patch target with the Dart SDK's
`dart2bytecode` snapshot and writes `build/ios_interpreter_patch.bytecode`.
The generated bytecode compile forwards user `--dart-define` values, Flutter's
`FLUTTER_APP_FLAVOR` define, and Flutter's standard version/revision/Dart SDK
defines from `bin/cache/flutter.version.json`. It also mirrors Flutter's
runtime feature flag define (`FLUTTER_ENABLED_FEATURE_FLAGS`) for enabled
runtime-id features.
Set `aot_patch_bytecode_path` or `SHOREBIRD_IOS_INTERPRETER_PATCH_PATH` only to
override that generated payload during local experiments.
Development-only local tests may also include `aot_patch_key_hex`, but reviewed
iOS app bundles should not. The `license_flavor_patch_test` iOS app now provides
the test AES key through `FlutterDartProject.shorebirdAotPatchKeyProvider`
instead.
`shorebird patch ios` now treats the interpreter route as the default iOS patch
artifact path. It packages the generated or overridden Dart bytecode payload as
an encrypted `open-aot-vmcode-encrypted-v1` full-snapshot artifact with
`runtime_mode=dart-bytecode-interpreter`, `target_os=ios`, and
`target_arch=arm64`. For production builds, provide the encryption key to the
CLI with `SHOREBIRD_AOT_PATCH_KEY_HEX` and keep `aot_patch_key_hex` out of the
bundled app. The legacy native-AOT iOS patch path is development-only and now
requires `SHOREBIRD_IOS_NATIVE_AOT_PATCH=1`.
A production App Store implementation still needs compact interpreter
reconstruction and TestFlight/App Store review validation.
docs/REPOSITORIES.md
+47 -47
View File
@@ -1,17 +1,17 @@
# Repository Inventory
Last verified: 2026-06-24.
Last verified: 2026-06-26.
## Active Top-Level Repositories
| Path | Remote URL | Branch | Pinned commit |
| --- | --- | --- | --- |
| `dart-sdk-new` | `https://git.tonycloud.org/dart-lang/sdk.git` | `tonycloud/dev` | `08139af589d0d32fbdec64b127c075ea3427cde8` |
| `depot_tools` | `https://chromium.googlesource.com/chromium/tools/depot_tools.git` | `main` | `90f5d4d14de66a5aae4a55b554ad4565111169af` |
| `flutter` | `https://git.tonycloud.org/flutter/flutter.git` | `shorebird/dev` | `9754ce03d76fb06c23a72ddbd8b7137bfa85446c` |
| `shorebird` | `https://git.tonycloud.org/flutter/shorebird.git` | `main` | `c824005c2dc65f56b0fa078984fadb649876fce7` |
| `shorebird-server` | `https://git.tonycloud.org/flutter/shorebird-server.git` | `main` | `774954fce70181846995ebcad842e9c54e57f1a2` |
| `updater` | `https://git.tonycloud.org/flutter/shorebird-updater.git` | `main` | `3ac748ff28eb4ebb53630c66230b9b1be5df73c2` |
| `dart-sdk` | `https://github.com/tony-cloud/dart-sdk.git` | `tonycloud/dev` | `57b27a7b44a9112c7cb3a1cb3c73636fc149bd16` |
| `depot_tools` | `https://chromium.googlesource.com/chromium/tools/depot_tools.git` | `main` | `226aa79e9947adc1e9e0c79f96b58562516535d9` |
| `flutter` | `https://github.com/tony-cloud/flutter.git` | `tonycloud/dev` | `5b96fd59be2f00061cccaaa23a02e15e720c1161` |
| `shorebird` | `https://git.tonycloud.org/flutter/shorebird.git` | `main` | `07a654e7717a20d685c3eadf4b55c2e6731e96fa` |
| `shorebird-server` | `https://git.tonycloud.org/flutter/shorebird-server.git` | `main` | `d6e5a39546905b85e2345b8058c4350ed236184c` |
| `updater` | `https://git.tonycloud.org/flutter/shorebird-updater.git` | `main` | `5d4e9c339636fc2f67cbe9892026d9e10b0888bc` |
## Submodule Policy
@@ -34,43 +34,43 @@ file.
| Path | Remote URL |
| --- | --- |
| `dart-sdk-new/buildtools/clang_format/script` | `https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git` |
| `dart-sdk-new/tests/co19/src` | `https://dart.googlesource.com/co19` |
| `dart-sdk-new/third_party/binaryen/src` | `https://chromium.googlesource.com/external/github.com/WebAssembly/binaryen.git` |
| `dart-sdk-new/third_party/boringssl/src` | `https://boringssl.googlesource.com/boringssl.git` |
| `dart-sdk-new/third_party/cpu_features/src` | `https://chromium.googlesource.com/external/github.com/google/cpu_features.git` |
| `dart-sdk-new/third_party/crashpad/crashpad` | `https://chromium.googlesource.com/crashpad/crashpad.git` |
| `dart-sdk-new/third_party/cygwin` | `https://chromium.googlesource.com/chromium/deps/cygwin.git` |
| `dart-sdk-new/third_party/emsdk` | `https://dart.googlesource.com/external/github.com/emscripten-core/emsdk.git` |
| `dart-sdk-new/third_party/googletest` | `https://fuchsia.googlesource.com/third_party/googletest` |
| `dart-sdk-new/third_party/icu` | `https://chromium.googlesource.com/chromium/deps/icu.git` |
| `dart-sdk-new/third_party/jinja2` | `https://chromium.googlesource.com/chromium/src/third_party/jinja2.git` |
| `dart-sdk-new/third_party/libc` | `https://llvm.googlesource.com/llvm-project/libc` |
| `dart-sdk-new/third_party/libcxx` | `https://llvm.googlesource.com/llvm-project/libcxx` |
| `dart-sdk-new/third_party/libcxxabi` | `https://llvm.googlesource.com/llvm-project/libcxxabi` |
| `dart-sdk-new/third_party/markupsafe` | `https://chromium.googlesource.com/chromium/src/third_party/markupsafe.git` |
| `dart-sdk-new/third_party/mini_chromium/mini_chromium` | `https://chromium.googlesource.com/chromium/mini_chromium` |
| `dart-sdk-new/third_party/perfetto/src` | `https://chromium.googlesource.com/external/github.com/google/perfetto` |
| `dart-sdk-new/third_party/pkg/core` | `https://dart.googlesource.com/core.git` |
| `dart-sdk-new/third_party/pkg/dart_style` | `https://dart.googlesource.com/dart_style.git` |
| `dart-sdk-new/third_party/pkg/dartdoc` | `https://dart.googlesource.com/dartdoc.git` |
| `dart-sdk-new/third_party/pkg/ecosystem` | `https://dart.googlesource.com/ecosystem.git` |
| `dart-sdk-new/third_party/pkg/http` | `https://dart.googlesource.com/http.git` |
| `dart-sdk-new/third_party/pkg/i18n` | `https://dart.googlesource.com/i18n.git` |
| `dart-sdk-new/third_party/pkg/leak_tracker` | `https://dart.googlesource.com/leak_tracker.git` |
| `dart-sdk-new/third_party/pkg/native` | `https://dart.googlesource.com/native.git` |
| `dart-sdk-new/third_party/pkg/protobuf` | `https://dart.googlesource.com/protobuf.git` |
| `dart-sdk-new/third_party/pkg/pub` | `https://dart.googlesource.com/pub.git` |
| `dart-sdk-new/third_party/pkg/shelf` | `https://dart.googlesource.com/shelf.git` |
| `dart-sdk-new/third_party/pkg/sync_http` | `https://dart.googlesource.com/sync_http.git` |
| `dart-sdk-new/third_party/pkg/tar` | `https://dart.googlesource.com/external/github.com/simolus3/tar.git` |
| `dart-sdk-new/third_party/pkg/test` | `https://dart.googlesource.com/test.git` |
| `dart-sdk-new/third_party/pkg/tools` | `https://dart.googlesource.com/tools.git` |
| `dart-sdk-new/third_party/pkg/vector_math` | `https://dart.googlesource.com/external/github.com/google/vector_math.dart.git` |
| `dart-sdk-new/third_party/pkg/web` | `https://dart.googlesource.com/web.git` |
| `dart-sdk-new/third_party/pkg/webdev` | `https://dart.googlesource.com/webdev.git` |
| `dart-sdk-new/third_party/pkg/webdriver` | `https://dart.googlesource.com/external/github.com/google/webdriver.dart.git` |
| `dart-sdk-new/third_party/pkg/webkit_inspection_protocol` | `https://dart.googlesource.com/external/github.com/google/webkit_inspection_protocol.dart.git` |
| `dart-sdk-new/third_party/ply` | `https://chromium.googlesource.com/chromium/src/third_party/ply.git` |
| `dart-sdk-new/third_party/WebCore` | `https://dart.googlesource.com/webcore.git` |
| `dart-sdk-new/third_party/zlib` | `https://chromium.googlesource.com/chromium/src/third_party/zlib.git` |
| `dart-sdk/buildtools/clang_format/script` | `https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git` |
| `dart-sdk/tests/co19/src` | `https://dart.googlesource.com/co19` |
| `dart-sdk/third_party/binaryen/src` | `https://chromium.googlesource.com/external/github.com/WebAssembly/binaryen.git` |
| `dart-sdk/third_party/boringssl/src` | `https://boringssl.googlesource.com/boringssl.git` |
| `dart-sdk/third_party/cpu_features/src` | `https://chromium.googlesource.com/external/github.com/google/cpu_features.git` |
| `dart-sdk/third_party/crashpad/crashpad` | `https://chromium.googlesource.com/crashpad/crashpad.git` |
| `dart-sdk/third_party/cygwin` | `https://chromium.googlesource.com/chromium/deps/cygwin.git` |
| `dart-sdk/third_party/emsdk` | `https://dart.googlesource.com/external/github.com/emscripten-core/emsdk.git` |
| `dart-sdk/third_party/googletest` | `https://fuchsia.googlesource.com/third_party/googletest` |
| `dart-sdk/third_party/icu` | `https://chromium.googlesource.com/chromium/deps/icu.git` |
| `dart-sdk/third_party/jinja2` | `https://chromium.googlesource.com/chromium/src/third_party/jinja2.git` |
| `dart-sdk/third_party/libc` | `https://llvm.googlesource.com/llvm-project/libc` |
| `dart-sdk/third_party/libcxx` | `https://llvm.googlesource.com/llvm-project/libcxx` |
| `dart-sdk/third_party/libcxxabi` | `https://llvm.googlesource.com/llvm-project/libcxxabi` |
| `dart-sdk/third_party/markupsafe` | `https://chromium.googlesource.com/chromium/src/third_party/markupsafe.git` |
| `dart-sdk/third_party/mini_chromium/mini_chromium` | `https://chromium.googlesource.com/chromium/mini_chromium` |
| `dart-sdk/third_party/perfetto/src` | `https://chromium.googlesource.com/external/github.com/google/perfetto` |
| `dart-sdk/third_party/pkg/core` | `https://dart.googlesource.com/core.git` |
| `dart-sdk/third_party/pkg/dart_style` | `https://dart.googlesource.com/dart_style.git` |
| `dart-sdk/third_party/pkg/dartdoc` | `https://dart.googlesource.com/dartdoc.git` |
| `dart-sdk/third_party/pkg/ecosystem` | `https://dart.googlesource.com/ecosystem.git` |
| `dart-sdk/third_party/pkg/http` | `https://dart.googlesource.com/http.git` |
| `dart-sdk/third_party/pkg/i18n` | `https://dart.googlesource.com/i18n.git` |
| `dart-sdk/third_party/pkg/leak_tracker` | `https://dart.googlesource.com/leak_tracker.git` |
| `dart-sdk/third_party/pkg/native` | `https://dart.googlesource.com/native.git` |
| `dart-sdk/third_party/pkg/protobuf` | `https://dart.googlesource.com/protobuf.git` |
| `dart-sdk/third_party/pkg/pub` | `https://dart.googlesource.com/pub.git` |
| `dart-sdk/third_party/pkg/shelf` | `https://dart.googlesource.com/shelf.git` |
| `dart-sdk/third_party/pkg/sync_http` | `https://dart.googlesource.com/sync_http.git` |
| `dart-sdk/third_party/pkg/tar` | `https://dart.googlesource.com/external/github.com/simolus3/tar.git` |
| `dart-sdk/third_party/pkg/test` | `https://dart.googlesource.com/test.git` |
| `dart-sdk/third_party/pkg/tools` | `https://dart.googlesource.com/tools.git` |
| `dart-sdk/third_party/pkg/vector_math` | `https://dart.googlesource.com/external/github.com/google/vector_math.dart.git` |
| `dart-sdk/third_party/pkg/web` | `https://dart.googlesource.com/web.git` |
| `dart-sdk/third_party/pkg/webdev` | `https://dart.googlesource.com/webdev.git` |
| `dart-sdk/third_party/pkg/webdriver` | `https://dart.googlesource.com/external/github.com/google/webdriver.dart.git` |
| `dart-sdk/third_party/pkg/webkit_inspection_protocol` | `https://dart.googlesource.com/external/github.com/google/webkit_inspection_protocol.dart.git` |
| `dart-sdk/third_party/ply` | `https://chromium.googlesource.com/chromium/src/third_party/ply.git` |
| `dart-sdk/third_party/WebCore` | `https://dart.googlesource.com/webcore.git` |
| `dart-sdk/third_party/zlib` | `https://chromium.googlesource.com/chromium/src/third_party/zlib.git` |
flutter
+1 -1
Submodule flutter updated: 542e6e89c8...0b84921834
scripts/android_runtime_patch_smoke.sh
Executable
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
APP_DIR="${APP_DIR:-$ROOT/testapps/license_flavor_patch_test}"
FLUTTER_BIN="${FLUTTER_BIN:-$ROOT/flutter/bin/flutter}"
ADB_BIN="${ADB_BIN:-adb}"
PACKAGE="${ANDROID_PACKAGE:-com.example.licenseflavorpatchtest.license_flavor_patch_test}"
ACTIVITY="${ANDROID_ACTIVITY:-com.example.licenseflavorpatchtest.license_flavor_patch_test.MainActivity}"
RELEASE_VERSION="${ANDROID_RELEASE_VERSION:-1.0+1}"
LOCAL_ENGINE_SRC_PATH="${LOCAL_ENGINE_SRC_PATH:-$ROOT/flutter/engine/src}"
LOCAL_ENGINE="${LOCAL_ENGINE:-android_release_arm64}"
LOCAL_ENGINE_HOST="${LOCAL_ENGINE_HOST:-host_release_arm64}"
WORK_DIR="${ANDROID_RUNTIME_SMOKE_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-android-runtime.XXXXXX")}"
SEED_REMOTE_DIR="${ANDROID_SEED_REMOTE_DIR:-/data/local/tmp/open-shorebird-android-runtime-seed}"
TARGET_PLATFORM="${ANDROID_TARGET_PLATFORM:-android-arm64}"
if [[ "${KEEP_ANDROID_RUNTIME_SMOKE_ARTIFACTS:-0}" != "1" ]]; then
trap 'rm -rf "$WORK_DIR"' EXIT
fi
ADB=("$ADB_BIN")
if [[ -n "${ANDROID_SERIAL:-}" ]]; then
ADB+=("-s" "$ANDROID_SERIAL")
fi
adb_cmd() {
"${ADB[@]}" "$@"
}
adb_shell() {
adb_cmd shell "$@"
}
python_bin() {
if command -v python3 >/dev/null 2>&1; then
printf '%s\n' python3
else
printf '%s\n' python
fi
}
require_tool() {
local tool="$1"
if ! command -v "$tool" >/dev/null 2>&1; then
echo "$tool is required" >&2
exit 127
fi
}
build_apk() {
local license="$1"
local output="$2"
if [[ "${SKIP_ANDROID_BUILDS:-0}" == "1" ]]; then
local env_name
case "$license" in
free) env_name=ANDROID_FREE_APK ;;
pro) env_name=ANDROID_PRO_APK ;;
*)
echo "Unsupported license variant: $license" >&2
exit 64
;;
esac
local existing="${!env_name:-}"
if [[ -z "$existing" || ! -f "$existing" ]]; then
echo "SKIP_ANDROID_BUILDS=1 requires $env_name to point at an APK" >&2
exit 66
fi
cp "$existing" "$output"
return
fi
(
cd "$APP_DIR"
"$FLUTTER_BIN" build apk --release \
--target-platform "$TARGET_PLATFORM" \
--local-engine-src-path="$LOCAL_ENGINE_SRC_PATH" \
--local-engine="$LOCAL_ENGINE" \
--local-engine-host="$LOCAL_ENGINE_HOST" \
--dart-define="LICENSE_TYPE=$license"
)
cp "$APP_DIR/build/app/outputs/flutter-apk/app-release.apk" "$output"
}
extract_libapp() {
local apk="$1"
local output="$2"
local py
py="$(python_bin)"
"$py" - "$apk" "$output" <<'PY'
import pathlib
import sys
import zipfile
apk_path = pathlib.Path(sys.argv[1])
output_path = pathlib.Path(sys.argv[2])
with zipfile.ZipFile(apk_path) as archive:
candidates = [
name for name in archive.namelist()
if name.endswith("/libapp.so") and "arm64-v8a/" in name
]
if not candidates:
raise SystemExit(f"missing arm64 libapp.so in {apk_path}")
output_path.write_bytes(archive.read(candidates[0]))
PY
}
wait_for_text() {
local expected="$1"
local dump="$WORK_DIR/window.xml"
for _ in $(seq 1 "${ANDROID_UI_WAIT_ATTEMPTS:-80}"); do
adb_shell uiautomator dump /sdcard/open_shorebird_window.xml >/dev/null 2>&1 || true
adb_cmd exec-out cat /sdcard/open_shorebird_window.xml >"$dump" 2>/dev/null || true
if grep -q "$expected" "$dump"; then
return 0
fi
sleep 0.25
done
echo "Timed out waiting for Android UI text: $expected" >&2
echo "Last uiautomator dump:" >&2
sed -n '1,120p' "$dump" >&2 || true
return 70
}
start_app() {
adb_shell am force-stop "$PACKAGE" >/dev/null 2>&1 || true
adb_shell am start -W -n "$PACKAGE/$ACTIVITY" >/dev/null
}
can_seed_with_run_as() {
adb_shell run-as "$PACKAGE" sh -c 'test -d files' >/dev/null 2>&1
}
can_seed_with_root() {
if adb_shell sh -c 'test "$(id -u)" = "0"' >/dev/null 2>&1; then
return 0
fi
adb_cmd root >/dev/null 2>&1 || true
adb_cmd wait-for-device >/dev/null 2>&1 || true
adb_shell sh -c 'test "$(id -u)" = "0"' >/dev/null 2>&1
}
seed_with_run_as() {
adb_shell run-as "$PACKAGE" sh -c \
"mkdir -p files && rm -rf files/shorebird_updater && cp -R '$SEED_REMOTE_DIR/shorebird_updater' files/"
}
seed_with_root() {
local target="/data/data/$PACKAGE/files"
local owner
owner="$(adb_shell sh -c "stat -c '%u:%g' '$target'" | tr -d '\r')"
adb_shell sh -c \
"rm -rf '$target/shorebird_updater' && cp -R '$SEED_REMOTE_DIR/shorebird_updater' '$target/' && chown -R '$owner' '$target/shorebird_updater'"
}
prepare_seed() {
local libapp="$1"
local seed_root="$WORK_DIR/seed/shorebird_updater"
local size
size="$(wc -c <"$libapp" | tr -d ' ')"
rm -rf "$WORK_DIR/seed"
mkdir -p "$seed_root/patches/1"
cp "$libapp" "$seed_root/patches/1/dlc.vmcode"
cat >"$seed_root/state.json" <<EOF
{
"client_id": "android-runtime-smoke",
"release_version": "$RELEASE_VERSION",
"queued_events": []
}
EOF
cat >"$seed_root/pointers.json" <<'EOF'
{
"next_boot_patch": 1,
"last_booted_patch": null,
"currently_booting_patch": null,
"boot_started_at": null
}
EOF
cat >"$seed_root/patches/1/state.json" <<EOF
{
"kind": "Installed",
"signature": null,
"size": $size
}
EOF
adb_shell rm -rf "$SEED_REMOTE_DIR" >/dev/null 2>&1 || true
adb_shell mkdir -p "$SEED_REMOTE_DIR" >/dev/null
adb_cmd push "$seed_root" "$SEED_REMOTE_DIR/" >/dev/null
echo "seeded_patch_size=$size"
}
require_tool "$ADB_BIN"
require_tool "$FLUTTER_BIN"
require_tool java
adb_cmd start-server >/dev/null
device_count="$(adb_cmd devices | awk 'NR > 1 && $2 == "device" { count++ } END { print count + 0 }')"
if [[ "$device_count" -lt 1 ]]; then
echo "No Android device or emulator is connected." >&2
echo "Start an emulator/device, then rerun this script. Use ANDROID_SERIAL to pick a device." >&2
exit 69
fi
if [[ "$device_count" -gt 1 && -z "${ANDROID_SERIAL:-}" ]]; then
echo "Multiple Android devices are connected; set ANDROID_SERIAL." >&2
adb_cmd devices -l >&2
exit 64
fi
free_apk="$WORK_DIR/free.apk"
pro_apk="$WORK_DIR/pro.apk"
pro_libapp="$WORK_DIR/pro-libapp.so"
build_apk free "$free_apk"
build_apk pro "$pro_apk"
extract_libapp "$pro_apk" "$pro_libapp"
adb_cmd uninstall "$PACKAGE" >/dev/null 2>&1 || true
adb_cmd install -r "$free_apk" >/dev/null
start_app
wait_for_text 'license:free'
wait_for_text 'pro-feature:off'
echo "android_base_status=license:free/pro-feature:off"
prepare_seed "$pro_libapp"
if can_seed_with_run_as; then
seed_with_run_as
echo "android_seed_mode=run-as"
elif can_seed_with_root; then
seed_with_root
echo "android_seed_mode=root"
else
cat >&2 <<EOF
Unable to seed the Android app-private updater directory.
Use a debuggable build/device where 'adb shell run-as $PACKAGE' works, or use
a rooted emulator/device where 'adb root' works. The smoke seed target is:
/data/data/$PACKAGE/files/shorebird_updater
EOF
exit 77
fi
start_app
wait_for_text 'license:pro'
wait_for_text 'pro-feature:enabled'
echo "android_patch_status=license:pro/pro-feature:enabled"
adb_shell am force-stop "$PACKAGE" >/dev/null 2>&1 || true
echo "android_runtime_patch_smoke=passed"
scripts/assemble_artifact_mirror.sh
Executable
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat >&2 <<'EOF'
usage: assemble_artifact_mirror.sh <downloaded-workflow-artifacts-dir> <output-mirror-root>
Copies every publish-ready shorebird/ mirror subtree from downloaded GitHub
Actions artifacts into <output-mirror-root>. Engine archives are extracted and
scanned for nested mirror/shorebird trees. Existing files may be reused only
when their bytes match.
EOF
}
if [[ "$#" -ne 2 ]]; then
usage
exit 64
fi
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
INPUT_DIR="$1"
OUTPUT_DIR="$2"
if [[ ! -d "$INPUT_DIR" ]]; then
echo "input artifact directory does not exist: $INPUT_DIR" >&2
exit 66
fi
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-mirror.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
mkdir -p "$OUTPUT_DIR"
FOUND_TREES=0
copy_tree_to_shorebird_prefix() {
local tree="$1"
local prefix="$2"
local source_file rel_file target_file
FOUND_TREES=$((FOUND_TREES + 1))
while IFS= read -r -d '' source_file; do
rel_file="${source_file#"$tree"/}"
target_file="$OUTPUT_DIR/shorebird/$prefix$rel_file"
mkdir -p "$(dirname "$target_file")"
if [[ -e "$target_file" ]]; then
if ! cmp -s "$source_file" "$target_file"; then
echo "conflicting mirror file: shorebird/$prefix$rel_file" >&2
echo " existing: $target_file" >&2
echo " incoming: $source_file" >&2
exit 70
fi
continue
fi
cp -p "$source_file" "$target_file"
done < <(find "$tree" -type f -print0)
}
copy_shorebird_tree() {
copy_tree_to_shorebird_prefix "$1" ""
}
scan_for_shorebird_trees() {
local search_root="$1"
local tree
while IFS= read -r -d '' tree; do
copy_shorebird_tree "$tree"
done < <(find "$search_root" -type d -name shorebird -print0)
}
scan_for_downloaded_metadata_trees() {
local search_root="$1"
local manifest_path metadata_dir engine_revision
# actions/upload-artifact strips the non-wildcard prefix from
# artifacts/mirror/shorebird/**/artifacts_manifest.yaml, so the downloaded
# mirror-metadata artifact is shaped as <engine>/artifacts_manifest.yaml.
while IFS= read -r -d '' manifest_path; do
if [[ "$manifest_path" == */shorebird/* ]]; then
continue
fi
metadata_dir="$(dirname "$manifest_path")"
engine_revision="$(basename "$metadata_dir")"
copy_tree_to_shorebird_prefix "$metadata_dir" "$engine_revision/"
done < <(find "$search_root" -type f -name artifacts_manifest.yaml -print0)
}
scan_for_shorebird_trees "$INPUT_DIR"
scan_for_downloaded_metadata_trees "$INPUT_DIR"
archive_index=0
while IFS= read -r -d '' archive_path; do
archive_index=$((archive_index + 1))
extract_dir="$TMP_DIR/archive-$archive_index"
mkdir -p "$extract_dir"
"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" "$archive_path" "$extract_dir"
scan_for_shorebird_trees "$extract_dir"
done < <(find "$INPUT_DIR" -type f \( -name '*.tar.gz' -o -name '*.tgz' \) -print0)
if [[ "$FOUND_TREES" -eq 0 ]]; then
echo "no shorebird/ mirror subtrees found under $INPUT_DIR" >&2
exit 65
fi
while IFS= read -r -d '' mirror_file; do
if [[ "$mirror_file" == *.sha256 ]]; then
continue
fi
sidecar="$mirror_file.sha256"
if [[ ! -f "$sidecar" ]]; then
"$ROOT/scripts/write_sha256.sh" "$mirror_file" "$sidecar"
fi
done < <(find "$OUTPUT_DIR/shorebird" -type f -print0)
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$OUTPUT_DIR"
echo "assembled artifact mirror at $OUTPUT_DIR"
scripts/check_ci_capacity.sh
Executable
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -euo pipefail
MIN_FREE_DISK_GB="${CI_MIN_FREE_DISK_GB:-0}"
CHECK_PATH="${CI_CAPACITY_PATH:-$PWD}"
AVAILABLE_DISK_KB_OVERRIDE="${CI_AVAILABLE_DISK_KB_OVERRIDE:-}"
fail() {
echo "error: $*" >&2
exit 70
}
case "$MIN_FREE_DISK_GB" in
''|*[!0-9]*)
fail "CI_MIN_FREE_DISK_GB must be a non-negative integer, got '$MIN_FREE_DISK_GB'"
;;
esac
if [[ "$MIN_FREE_DISK_GB" == "0" ]]; then
echo "CI capacity check skipped because CI_MIN_FREE_DISK_GB=0"
exit 0
fi
if [[ ! -e "$CHECK_PATH" ]]; then
fail "capacity check path does not exist: $CHECK_PATH"
fi
available_disk_kb() {
if [[ -n "$AVAILABLE_DISK_KB_OVERRIDE" ]]; then
case "$AVAILABLE_DISK_KB_OVERRIDE" in
*[!0-9]*)
fail "CI_AVAILABLE_DISK_KB_OVERRIDE must be an integer, got '$AVAILABLE_DISK_KB_OVERRIDE'"
;;
esac
printf '%s\n' "$AVAILABLE_DISK_KB_OVERRIDE"
return
fi
df -Pk "$CHECK_PATH" | awk 'NR == 2 { print $4 }'
}
available_kb="$(available_disk_kb)"
case "$available_kb" in
''|*[!0-9]*)
fail "could not determine available disk space for $CHECK_PATH"
;;
esac
required_kb=$((MIN_FREE_DISK_GB * 1024 * 1024))
available_gb=$((available_kb / 1024 / 1024))
echo "CI capacity: ${available_gb} GiB free at $CHECK_PATH; required: ${MIN_FREE_DISK_GB} GiB"
if (( available_kb < required_kb )); then
fail "runner has insufficient free disk for the heavy SDK/engine build. Use a larger or self-hosted runner label, or free disk before this step."
fi
scripts/free_ci_disk_linux.sh
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${GITHUB_ACTIONS:-}" != "true" ]]; then
echo "Refusing to free disk outside GitHub Actions." >&2
exit 64
fi
if [[ "$(uname -s)" != "Linux" ]]; then
echo "Skipping Linux disk cleanup on $(uname -s)."
exit 0
fi
if [[ "${CI_FREE_DISK_SPACE:-1}" == "0" ]]; then
echo "Skipping disk cleanup because CI_FREE_DISK_SPACE=0."
df -h
exit 0
fi
if [[ "${RUNNER_ENVIRONMENT:-github-hosted}" != "github-hosted" &&
"${CI_FREE_DISK_SPACE_FORCE:-0}" != "1" ]]; then
echo "Skipping disk cleanup on ${RUNNER_ENVIRONMENT} runner."
echo "Set CI_FREE_DISK_SPACE_FORCE=1 to opt in on non-hosted runners."
df -h
exit 0
fi
echo "Disk before cleanup:"
df -h
# GitHub-hosted Ubuntu images include large toolchains that are unrelated to
# Dart SDK, Flutter engine, and updater builds. Remove only well-known cache
# directories on ephemeral GitHub Actions runners.
for path in \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/usr/local/.ghcup \
/usr/local/lib/android/sdk \
/usr/local/share/boost \
/usr/share/dotnet; do
if [[ -e "$path" ]]; then
echo "Removing $path"
sudo rm -rf "$path"
fi
done
echo "Disk after cleanup:"
df -h
scripts/linux_runtime_patch_smoke.sh
Executable
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SOURCE_APP_DIR="${SOURCE_APP_DIR:-$ROOT/testapps/license_flavor_patch_test}"
FLUTTER_BIN="${FLUTTER_BIN:-$ROOT/flutter/bin/flutter}"
APP_ID="${SHOREBIRD_APP_ID:-license-flavor-patch-test}"
LOCAL_ENGINE_SRC_PATH="${LOCAL_ENGINE_SRC_PATH:-$ROOT/flutter/engine/src}"
DEFAULT_LOCAL_ENGINE="linux_release_x64"
if [[ ! -d "$LOCAL_ENGINE_SRC_PATH/out/$DEFAULT_LOCAL_ENGINE" &&
-d "$LOCAL_ENGINE_SRC_PATH/out/host_release" ]]; then
DEFAULT_LOCAL_ENGINE="host_release"
fi
LOCAL_ENGINE="${LOCAL_ENGINE:-$DEFAULT_LOCAL_ENGINE}"
LOCAL_ENGINE_HOST="${LOCAL_ENGINE_HOST:-$LOCAL_ENGINE}"
WORK_DIR="${LINUX_RUNTIME_SMOKE_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-linux-runtime.XXXXXX")}"
APP_COPY="$WORK_DIR/app"
HOME_DIR="$WORK_DIR/home"
if [[ "${KEEP_LINUX_RUNTIME_SMOKE_ARTIFACTS:-0}" != "1" ]]; then
trap 'rm -rf "$WORK_DIR"' EXIT
fi
require_tool() {
local tool="$1"
if ! command -v "$tool" >/dev/null 2>&1; then
echo "$tool is required" >&2
exit 127
fi
}
python_bin() {
if command -v python3 >/dev/null 2>&1; then
printf '%s\n' python3
else
printf '%s\n' python
fi
}
copy_app_fixture() {
mkdir -p "$APP_COPY"
(
cd "$SOURCE_APP_DIR"
tar \
--exclude='./build' \
--exclude='./.dart_tool' \
--exclude='./android/.gradle' \
--exclude='./ios/Pods' \
--exclude='./macos/Flutter/ephemeral' \
-cf - .
) | tar -C "$APP_COPY" -xf -
}
ensure_linux_platform() {
if [[ -d "$APP_COPY/linux" ]]; then
return
fi
(
cd "$APP_COPY"
"$FLUTTER_BIN" create --platforms=linux --project-name=license_flavor_patch_test .
)
}
build_linux_bundle() {
local license="$1"
local output="$2"
(
cd "$APP_COPY"
"$FLUTTER_BIN" build linux --release \
--local-engine-src-path="$LOCAL_ENGINE_SRC_PATH" \
--local-engine="$LOCAL_ENGINE" \
--local-engine-host="$LOCAL_ENGINE_HOST" \
--dart-define="LICENSE_TYPE=$license"
)
local bundle
bundle="$(find "$APP_COPY/build/linux" -path '*/release/bundle' -type d -print -quit)"
if [[ -z "$bundle" || ! -x "$bundle/license_flavor_patch_test" ]]; then
echo "Failed to find Linux release bundle under $APP_COPY/build/linux" >&2
exit 66
fi
rm -rf "$output"
cp -a "$bundle" "$output"
}
release_version_for_bundle() {
local bundle="$1"
local version_json="$bundle/data/flutter_assets/version.json"
local py
py="$(python_bin)"
"$py" - "$version_json" <<'PY'
import json
import pathlib
import sys
version = json.loads(pathlib.Path(sys.argv[1]).read_text())
build_name = str(version.get("version", ""))
build_number = str(version.get("build_number", ""))
if build_number:
print(f"{build_name}+{build_number}")
else:
print(build_name)
PY
}
run_saved_app() {
local label="$1"
local bundle="$2"
local stdout="$WORK_DIR/$label.stdout"
local stderr="$WORK_DIR/$label.stderr"
local tmp_status="$WORK_DIR/tmp/license_flavor_patch_status.txt"
local home_status="$HOME_DIR/Library/Application Support/license_flavor_patch_status.txt"
rm -f "$tmp_status" "$home_status"
mkdir -p "$WORK_DIR/tmp" "$HOME_DIR"
local -a app_command=(env HOME="$HOME_DIR" TMPDIR="$WORK_DIR/tmp" "$bundle/license_flavor_patch_test")
if [[ "${LINUX_RUNTIME_SMOKE_XVFB:-auto}" != "0" && -z "${DISPLAY:-}" && "$(command -v xvfb-run || true)" != "" ]]; then
app_command=(xvfb-run -a "${app_command[@]}")
fi
"${app_command[@]}" >"$stdout" 2>"$stderr" &
local pid=$!
for _ in $(seq 1 "${LINUX_RUNTIME_WAIT_ATTEMPTS:-120}"); do
if [[ -f "$tmp_status" || -f "$home_status" ]]; then
break
fi
if ! kill -0 "$pid" 2>/dev/null; then
break
fi
sleep 0.25
done
if kill -0 "$pid" 2>/dev/null; then
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
else
wait "$pid" 2>/dev/null || true
fi
echo "${label}_stdout=$stdout"
echo "${label}_stderr=$stderr"
if [[ -f "$tmp_status" ]]; then
echo "${label}_status_file=$tmp_status"
cat "$tmp_status"
elif [[ -f "$home_status" ]]; then
echo "${label}_status_file=$home_status"
cat "$home_status"
else
echo "${label}_status_file_missing" >&2
echo "--- $label stderr ---" >&2
sed -n '1,180p' "$stderr" >&2 || true
return 70
fi
}
read_launch_status() {
local tmp_status="$WORK_DIR/tmp/license_flavor_patch_status.txt"
local home_status="$HOME_DIR/Library/Application Support/license_flavor_patch_status.txt"
if [[ -f "$tmp_status" ]]; then
cat "$tmp_status"
elif [[ -f "$home_status" ]]; then
cat "$home_status"
fi
}
require_status() {
local label="$1"
local expected_license="$2"
local expected_feature="$3"
local status
status="$(read_launch_status)"
if ! grep -q "license:$expected_license" <<<"$status" ||
! grep -q "pro-feature:$expected_feature" <<<"$status"; then
echo "$label Linux app did not report expected status." >&2
echo "Expected: license:$expected_license / pro-feature:$expected_feature" >&2
echo "Actual:" >&2
printf '%s\n' "$status" >&2
exit 70
fi
}
seed_patch() {
local pro_bundle="$1"
local release_version="$2"
local patch_file="$pro_bundle/lib/libapp.so"
local state_root="$HOME_DIR/.shorebird_cache/shorebird_updater/$APP_ID"
local size
size="$(wc -c <"$patch_file" | tr -d ' ')"
rm -rf "$state_root"
mkdir -p "$state_root/patches/1"
cp "$patch_file" "$state_root/patches/1/dlc.vmcode"
cat >"$state_root/state.json" <<EOF
{
"client_id": "linux-runtime-smoke",
"release_version": "$release_version",
"queued_events": []
}
EOF
cat >"$state_root/pointers.json" <<'EOF'
{
"next_boot_patch": 1,
"last_booted_patch": null,
"currently_booting_patch": null,
"boot_started_at": null
}
EOF
cat >"$state_root/patches/1/state.json" <<EOF
{
"kind": "Installed",
"signature": null,
"size": $size
}
EOF
echo "linux_seeded_patch=$state_root/patches/1/dlc.vmcode"
echo "linux_seeded_patch_size=$size"
}
require_tool "$FLUTTER_BIN"
require_tool tar
copy_app_fixture
ensure_linux_platform
free_bundle="$WORK_DIR/free-bundle"
pro_bundle="$WORK_DIR/pro-bundle"
build_linux_bundle free "$free_bundle"
build_linux_bundle pro "$pro_bundle"
release_version="$(release_version_for_bundle "$free_bundle")"
run_saved_app base "$free_bundle"
require_status base free off
seed_patch "$pro_bundle" "$release_version"
run_saved_app patch "$free_bundle"
require_status patch pro enabled
echo "linux_runtime_patch_smoke=passed"
scripts/platform_test_common.sh
+14 -4
View File
@@ -6,6 +6,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DART_BIN="${DART_BIN:-dart}"
FLUTTER_BIN="${FLUTTER_BIN:-flutter}"
GO_BIN="${GO_BIN:-go}"
CARGO_BIN="${CARGO_BIN:-cargo}"
run() {
echo
@@ -23,6 +24,7 @@ require_command() {
require_command git
require_command "$DART_BIN"
require_command "$GO_BIN"
require_command "$CARGO_BIN"
if [[ "$PLATFORM" == "macos" ]]; then
if command -v xcodebuild >/dev/null 2>&1; then
@@ -37,9 +39,13 @@ run "$ROOT/scripts/write_gclient.sh" "$PLATFORM"
run "$ROOT/scripts/sync_open_sources.sh"
export PATH="$ROOT/depot_tools:$PATH"
export DEPOT_TOOLS_UPDATE="${DEPOT_TOOLS_UPDATE:-0}"
if [[ "${SKIP_GCLIENT_SYNC:-0}" != "1" ]]; then
require_command gclient
run gclient sync --no-history
if [[ "${INCLUDE_ENGINE_DEPS:-0}" == "1" && -f "$ROOT/flutter/.gclient" ]]; then
run bash -lc "cd '$ROOT/flutter' && gclient sync --no-history"
fi
else
echo
echo "==> skipping gclient sync because SKIP_GCLIENT_SYNC=1"
@@ -51,16 +57,20 @@ if [[ "${SKIP_TESTS:-0}" == "1" ]]; then
exit 0
fi
run bash -lc "cd '$ROOT/shorebird/packages/shorebird_cli' && '$DART_BIN' pub get && '$DART_BIN' test test/src/user_config_test.dart test/src/shorebird_env_test.dart test/src/shorebird_cli_command_runner_test.dart test/src/commands/init_command_test.dart"
run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' pub get"
run bash -lc "cd '$ROOT/shorebird/packages/shorebird_cli' && '$DART_BIN' pub get && '$DART_BIN' test test/src/user_config_test.dart test/src/shorebird_env_test.dart test/src/shorebird_cli_command_runner_test.dart test/src/commands/doctor_command_test.dart test/src/commands/init_command_test.dart test/src/cache_test.dart test/src/shorebird_process_test.dart test/src/network_checker_test.dart test/src/shorebird_web_console_test.dart test/src/auth/auth_test.dart test/src/commands/login_command_test.dart test/src/commands/login_ci_command_test.dart test/src/commands/release/aar_releaser_test.dart test/src/shorebird_validator_test.dart test/src/shorebird_flutter_test.dart test/src/shorebird_artifacts_test.dart test/src/artifact_builder/artifact_builder_test.dart test/src/config/shorebird_yaml_test.dart test/src/commands/patch/ios_patcher_test.dart"
run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' test packages/shorebird_code_push_client/test/src/code_push_client_test.dart"
run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' test packages/artifact_proxy/test/artifact_proxy_test.dart packages/artifact_proxy/test/server_bin_test.dart packages/artifact_proxy/test/src/artifact_manifest_client_test.dart"
run bash -lc "cd '$ROOT/shorebird/packages/open_aot_patch_tools' && '$DART_BIN' pub get && '$DART_BIN' test"
run "$CARGO_BIN" test --manifest-path "$ROOT/updater/library/Cargo.toml"
run bash -lc "cd '$ROOT/shorebird-server' && '$GO_BIN' test ./..."
AOT_PATCH_BUILD_DIR="${AOT_PATCH_BUILD_DIR:-}"
if [[ -z "$AOT_PATCH_BUILD_DIR" ]]; then
for candidate in \
"$ROOT/dart-sdk-new/xcodebuild/ReleaseARM64" \
"$ROOT/dart-sdk-new/out/ReleaseARM64AotPatch" \
"$ROOT/dart-sdk-new/out/ReleaseX64AotPatch"; do
"$ROOT/dart-sdk/xcodebuild/ReleaseARM64" \
"$ROOT/dart-sdk/out/ReleaseARM64AotPatch" \
"$ROOT/dart-sdk/out/ReleaseX64AotPatch"; do
if [[ -f "$candidate/args.gn" ]]; then
AOT_PATCH_BUILD_DIR="$candidate"
break
scripts/safe_extract_tar.py
Executable
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
from pathlib import Path, PurePosixPath
import shutil
import sys
import tarfile
def fail(message: str) -> None:
print(f"error: {message}", file=sys.stderr)
raise SystemExit(70)
def safe_member_name(member: tarfile.TarInfo) -> str | None:
name = member.name
if member.isdir():
name = name.rstrip("/")
if not name or name == ".":
return None
if "\\" in name or "\x00" in name:
return None
if any(ord(character) < 32 for character in name):
return None
candidate = PurePosixPath(name)
if candidate.is_absolute():
return None
if any(part in ("", ".", "..") for part in candidate.parts):
return None
if candidate.parts and ":" in candidate.parts[0]:
return None
return candidate.as_posix()
def validate_members(archive_path: Path, members: list[tarfile.TarInfo]) -> None:
member_types: dict[str, str] = {}
for member in members:
safe_name = safe_member_name(member)
if safe_name is None:
fail(f"{archive_path}: unsafe archive member path {member.name!r}")
if not (member.isdir() or member.isfile()):
fail(f"{archive_path}: unsupported archive member type {member.name!r}")
member_type = "dir" if member.isdir() else "file"
previous_type = member_types.get(safe_name)
if previous_type is None:
member_types[safe_name] = member_type
continue
if previous_type != "dir" or member_type != "dir":
fail(f"{archive_path}: duplicate archive member path {member.name!r}")
def ensure_within_root(archive_path: Path, extract_root: Path, target: Path, name: str) -> None:
try:
target.relative_to(extract_root)
except ValueError:
fail(f"{archive_path}: archive member escapes extraction root {name!r}")
def extract_safe_tar_archive(archive_path: Path, extract_dir: Path) -> None:
extract_root = extract_dir.resolve()
extracted_files: set[Path] = set()
try:
with tarfile.open(archive_path, "r:*") as archive:
members = archive.getmembers()
validate_members(archive_path, members)
for member in members:
safe_name = safe_member_name(member)
assert safe_name is not None
target = (extract_root / safe_name).resolve()
ensure_within_root(archive_path, extract_root, target, member.name)
if member.isdir():
if target in extracted_files:
fail(f"{archive_path}: directory collides with file {member.name!r}")
target.mkdir(parents=True, exist_ok=True)
continue
for parent in target.parents:
if parent == extract_root:
break
if parent in extracted_files:
fail(f"{archive_path}: file parent collides with file {member.name!r}")
if target.exists() and not target.is_file():
fail(f"{archive_path}: file collides with directory {member.name!r}")
target.parent.mkdir(parents=True, exist_ok=True)
source = archive.extractfile(member)
if source is None:
fail(f"{archive_path}: unable to read archive member {member.name!r}")
with source, target.open("wb") as output:
shutil.copyfileobj(source, output)
target.chmod(member.mode & 0o777)
extracted_files.add(target)
except tarfile.TarError as error:
fail(f"{archive_path}: invalid tar archive: {error}")
def main() -> int:
parser = argparse.ArgumentParser(
description="Safely extract a tar archive containing only files and directories.",
)
parser.add_argument("archive", type=Path)
parser.add_argument("extract_dir", type=Path)
args = parser.parse_args()
args.extract_dir.mkdir(parents=True, exist_ok=True)
extract_safe_tar_archive(args.archive, args.extract_dir)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/sync_flutter_prebuilt_dart_sdk.sh
Executable
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
HOST_CONFIG="${1:-linux-x64}"
DART_SDK_SOURCE="${DART_SDK_SOURCE:-$ROOT/dart-sdk/tools/sdks/dart-sdk}"
TARGET="$ROOT/flutter/engine/src/flutter/prebuilts/$HOST_CONFIG/dart-sdk"
relative_path() {
python3 - "$1" "$2" <<'PY'
import os
import sys
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
PY
}
real_path() {
python3 - "$1" <<'PY'
import os
import sys
print(os.path.realpath(sys.argv[1]))
PY
}
case "$HOST_CONFIG" in
linux-x64|macos-x64|macos-arm64)
;;
*)
echo "unsupported Flutter prebuilt Dart SDK host config: $HOST_CONFIG" >&2
exit 64
;;
esac
if [[ ! -d "$DART_SDK_SOURCE" ]]; then
echo "missing Dart tool SDK source: $DART_SDK_SOURCE" >&2
exit 66
fi
for required in \
bin/dart \
bin/dartaotruntime \
bin/snapshots/dartdevc_aot.dart.snapshot \
bin/snapshots/kernel_worker_aot.dart.snapshot
do
if [[ ! -e "$DART_SDK_SOURCE/$required" ]]; then
echo "Dart tool SDK is missing $required: $DART_SDK_SOURCE" >&2
exit 66
fi
done
mkdir -p "$(dirname "$TARGET")"
if [[ -L "$TARGET" ]]; then
source_real="$(real_path "$DART_SDK_SOURCE")"
target_real="$(real_path "$TARGET")"
if [[ "$source_real" != "$target_real" ]]; then
echo "Flutter prebuilt Dart SDK link points at $target_real, expected $source_real" >&2
exit 70
fi
elif [[ -e "$TARGET" ]]; then
echo "Flutter prebuilt Dart SDK target exists but is not the workspace Dart tool SDK symlink: $TARGET" >&2
exit 70
else
ln -s "$(relative_path "$TARGET" "$DART_SDK_SOURCE")" "$TARGET"
fi
echo "[open-source-sync] Flutter $HOST_CONFIG prebuilt Dart SDK uses $DART_SDK_SOURCE"
scripts/sync_open_sources.sh
+120 -40
View File
@@ -2,16 +2,102 @@
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DART_SRC="${DART_SRC:-$ROOT/dart-sdk-new}"
DART_SRC="${DART_SRC:-$ROOT/dart-sdk}"
DART_TARGET="$ROOT/flutter/engine/src/flutter/third_party/dart"
UPDATER_SRC="${UPDATER_SRC:-$ROOT/updater}"
UPDATER_URL="${UPDATER_URL:-https://github.com/shorebirdtech/updater.git}"
UPDATER_URL="${UPDATER_URL:-}"
TARGET="$ROOT/flutter/engine/src/flutter/third_party/updater"
is_git_checkout() {
git -C "$1" rev-parse --git-dir >/dev/null 2>&1
}
relative_path() {
python3 - "$1" "$2" <<'PY'
import os
import sys
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
PY
}
real_path() {
python3 - "$1" <<'PY'
import os
import sys
print(os.path.realpath(sys.argv[1]))
PY
}
link_checkout() {
local target="$1"
local source="$2"
local label="$3"
local rel_target
rel_target="$(relative_path "$target" "$source")"
ln -s "$rel_target" "$target"
echo "[open-source-sync] linked $label checkout into Flutter engine."
}
is_clean_git_checkout() {
[[ -z "$(git -C "$1" status --porcelain)" ]]
}
reject_forbidden_remotes() {
local source="$1"
local label="$2"
shift 2
local remotes
remotes="$(git -C "$source" remote -v 2>/dev/null || true)"
if [[ -z "$remotes" ]]; then
return
fi
local forbidden
for forbidden in "$@"; do
if grep -Fq "$forbidden" <<<"$remotes"; then
echo "$label source checkout uses forbidden remote fragment '$forbidden': $source" >&2
echo "$remotes" >&2
exit 1
fi
done
}
ensure_source_link() {
local target="$1"
local source="$2"
local label="$3"
if [[ ! -d "$source" ]] || ! is_git_checkout "$source"; then
echo "$label source checkout is missing: $source" >&2
exit 1
fi
if [[ -L "$target" ]]; then
local target_real
local source_real
target_real="$(real_path "$target")"
source_real="$(real_path "$source")"
if [[ "$target_real" != "$source_real" ]]; then
echo "$label target symlink points at $target_real, expected $source_real" >&2
exit 1
fi
echo "[open-source-sync] $label target already links to the workspace checkout."
elif is_git_checkout "$target"; then
if ! is_clean_git_checkout "$target"; then
echo "$label target is a dirty git checkout and cannot be replaced: $target" >&2
exit 1
fi
rm -rf "$target"
link_checkout "$target" "$source" "$label"
elif [[ -e "$target" ]]; then
echo "$label target exists but is not a symlink or git checkout: $target" >&2
exit 1
else
link_checkout "$target" "$source" "$label"
fi
}
echo "[open-source-sync] dart source: $DART_SRC"
echo "[open-source-sync] dart target: $DART_TARGET"
echo "[open-source-sync] updater source: $UPDATER_SRC"
@@ -20,60 +106,54 @@ echo "[open-source-sync] target: $TARGET"
mkdir -p "$(dirname "$DART_TARGET")"
mkdir -p "$(dirname "$TARGET")"
if [[ -L "$DART_TARGET" ]]; then
echo "[open-source-sync] Dart target is already a symlink."
elif is_git_checkout "$DART_TARGET"; then
echo "[open-source-sync] Dart target is already a git checkout."
elif [[ -e "$DART_TARGET" ]]; then
echo "Dart target exists but is not a symlink or git checkout: $DART_TARGET" >&2
exit 1
elif is_git_checkout "$DART_SRC"; then
rel_target="$(python3 - "$DART_TARGET" "$DART_SRC" <<'PY'
import os
import sys
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
PY
)"
ln -s "$rel_target" "$DART_TARGET"
echo "[open-source-sync] linked Dart SDK checkout into Flutter engine."
else
echo "Dart source checkout is missing: $DART_SRC" >&2
exit 1
fi
ensure_source_link "$DART_TARGET" "$DART_SRC" "Dart SDK"
reject_forbidden_remotes \
"$DART_SRC" \
"Dart SDK" \
"github.com/dart-lang/sdk" \
"dart.googlesource.com/sdk"
if [[ ! -f "$DART_TARGET/runtime/vm/dart_api_impl.h" ]]; then
echo "Dart checkout is missing runtime/vm/dart_api_impl.h" >&2
exit 1
fi
if [[ -L "$TARGET" ]]; then
echo "[open-source-sync] updater target is already a symlink."
exit 0
fi
if is_git_checkout "$TARGET"; then
if is_git_checkout "$UPDATER_SRC"; then
ensure_source_link "$TARGET" "$UPDATER_SRC" "updater submodule"
reject_forbidden_remotes \
"$UPDATER_SRC" \
"updater submodule" \
"github.com/shorebirdtech/updater" \
"github.com/shorebirdtech/shorebird-updater"
elif [[ -n "$UPDATER_URL" ]]; then
if [[ "$UPDATER_URL" == *github.com/shorebirdtech/updater* ||
"$UPDATER_URL" == *github.com/shorebirdtech/shorebird-updater* ]]; then
echo "UPDATER_URL points at a forbidden official Shorebird updater remote: $UPDATER_URL" >&2
exit 1
fi
if [[ -L "$TARGET" ]]; then
rm "$TARGET"
fi
if is_git_checkout "$TARGET"; then
echo "[open-source-sync] updating existing updater checkout."
git -C "$TARGET" remote set-url origin "$UPDATER_URL"
git -C "$TARGET" fetch --tags origin
git -C "$TARGET" checkout "${UPDATER_REVISION:-main}"
if [[ "${UPDATER_REVISION:-main}" == "main" ]]; then
git -C "$TARGET" pull --ff-only
fi
elif [[ -e "$TARGET" ]]; then
elif [[ -e "$TARGET" ]]; then
echo "target exists but is not a symlink or git checkout: $TARGET" >&2
exit 1
elif is_git_checkout "$UPDATER_SRC"; then
rel_target="$(python3 - "$TARGET" "$UPDATER_SRC" <<'PY'
import os
import sys
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
PY
)"
ln -s "$rel_target" "$TARGET"
echo "[open-source-sync] linked updater submodule into Flutter engine."
else
echo "[open-source-sync] cloning public updater checkout."
else
echo "[open-source-sync] cloning updater checkout from explicit UPDATER_URL."
git clone "$UPDATER_URL" "$TARGET"
git -C "$TARGET" checkout "${UPDATER_REVISION:-main}"
fi
else
echo "updater source checkout is missing: $UPDATER_SRC" >&2
echo "Set UPDATER_SRC to a local fork or set UPDATER_URL explicitly." >&2
exit 1
fi
if [[ ! -f "$TARGET/library/include/updater_engine.h" ]]; then
scripts/validate_artifact_mirror.py
Executable
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Validate an assembled open Shorebird artifact mirror."""
from __future__ import annotations
import argparse
import hashlib
from pathlib import Path, PurePosixPath
import re
import sys
import zipfile
REQUIRED_PATCH_ZIPS = {
"patch-linux-x64.zip": "patch",
"patch-darwin-x64.zip": "patch",
"patch-darwin-arm64.zip": "patch",
"patch-windows-x64.zip": "patch.exe",
}
def digest_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as file:
for chunk in iter(lambda: file.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def is_plain_file(path: Path) -> bool:
return path.is_file() and not path.is_symlink()
def parse_sidecar(path: Path) -> tuple[str, str]:
text = path.read_text(encoding="utf-8").strip()
parts = text.split()
if len(parts) != 2:
raise ValueError(f"expected '<sha256> <filename>', got {text!r}")
digest, filename = parts
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
raise ValueError(f"invalid sha256 digest {digest!r}")
return digest, filename
def is_safe_relative_path(path: str) -> bool:
if not path or path == ".":
return False
if "\\" in path or "\x00" in path or path.endswith("/"):
return False
if any(ord(character) < 32 for character in path):
return False
candidate = PurePosixPath(path)
if candidate.is_absolute():
return False
if any(part in ("", ".", "..") for part in candidate.parts):
return False
if candidate.parts and ":" in candidate.parts[0]:
return False
return True
def validate_sidecars(shorebird_root: Path) -> list[str]:
errors: list[str] = []
for artifact_path in sorted(shorebird_root.rglob("*")):
if artifact_path.is_symlink():
errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: symlink entries are not allowed")
continue
if not artifact_path.is_file():
continue
if artifact_path.suffix == ".sha256":
artifact_path_without_suffix = Path(str(artifact_path)[: -len(".sha256")])
if not is_plain_file(artifact_path_without_suffix):
errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: orphan sidecar")
continue
sidecar_path = Path(f"{artifact_path}.sha256")
if not is_plain_file(sidecar_path):
errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: missing sidecar")
continue
try:
sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path)
except ValueError as error:
errors.append(f"{sidecar_path.relative_to(shorebird_root.parent)}: {error}")
continue
actual_digest = digest_file(artifact_path)
if sidecar_digest != actual_digest:
errors.append(
f"{sidecar_path.relative_to(shorebird_root.parent)}: digest mismatch "
f"{sidecar_digest} != {actual_digest}"
)
if sidecar_filename != artifact_path.name:
errors.append(
f"{sidecar_path.relative_to(shorebird_root.parent)}: filename mismatch "
f"{sidecar_filename!r} != {artifact_path.name!r}"
)
return errors
def validate_manifest_overrides(mirror_root: Path, manifest_paths: list[Path]) -> list[str]:
override_pattern = re.compile(r"^\s*-\s*'?(?P<path>[^'#\n]+?)'?\s*(?:#.*)?$")
shorebird_root = mirror_root / "shorebird"
errors: list[str] = []
for manifest_path in manifest_paths:
engine_revision = manifest_path.parent.name
for line in manifest_path.read_text(encoding="utf-8").splitlines():
match = override_pattern.match(line)
if not match:
continue
artifact_path = match.group("path").replace("$engine", engine_revision)
if not is_safe_relative_path(artifact_path):
errors.append(
f"{manifest_path.relative_to(mirror_root)} -> "
f"unsafe artifact override path: {artifact_path}"
)
continue
resolved_artifact_path = shorebird_root / artifact_path
if not is_plain_file(resolved_artifact_path):
errors.append(
f"{manifest_path.relative_to(mirror_root)} -> shorebird/{artifact_path}"
)
continue
if resolved_artifact_path.stat().st_size <= 0:
errors.append(
f"{manifest_path.relative_to(mirror_root)} -> "
f"shorebird/{artifact_path}: artifact override is empty"
)
return errors
def validate_patch_zips(mirror_root: Path, manifest_paths: list[Path]) -> list[str]:
errors: list[str] = []
for manifest_path in manifest_paths:
engine_dir = manifest_path.parent
for zip_name, expected_entry in REQUIRED_PATCH_ZIPS.items():
zip_path = engine_dir / zip_name
display_path = zip_path.relative_to(mirror_root)
if not is_plain_file(zip_path):
errors.append(f"{display_path}: missing")
continue
try:
with zipfile.ZipFile(zip_path) as archive:
names = archive.namelist()
if names != [expected_entry]:
errors.append(
f"{display_path}: expected only {expected_entry!r}, got {names!r}"
)
continue
if archive.getinfo(expected_entry).file_size <= 0:
errors.append(f"{display_path}: {expected_entry} is empty")
except zipfile.BadZipFile:
errors.append(f"{display_path}: invalid zip")
return errors
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("mirror_root", type=Path)
return parser.parse_args()
def main() -> int:
args = parse_args()
mirror_root = args.mirror_root
shorebird_root = mirror_root / "shorebird"
if not mirror_root.is_dir():
print(f"missing mirror root: {mirror_root}", file=sys.stderr)
return 66
if not shorebird_root.is_dir():
print(f"mirror root is missing shorebird/: {mirror_root}", file=sys.stderr)
return 70
manifest_paths = sorted(shorebird_root.glob("*/artifacts_manifest.yaml"))
if not manifest_paths:
print(
"artifact mirror is missing shorebird/<engine>/artifacts_manifest.yaml",
file=sys.stderr,
)
return 70
errors: list[str] = []
sidecar_errors = validate_sidecars(shorebird_root)
if sidecar_errors:
errors.append("invalid checksum sidecars:\n" + "\n".join(f" {e}" for e in sidecar_errors))
override_errors = validate_manifest_overrides(mirror_root, manifest_paths)
if override_errors:
errors.append(
"invalid files referenced by artifacts_manifest.yaml:\n"
+ "\n".join(f" {path}" for path in override_errors)
)
patch_errors = validate_patch_zips(mirror_root, manifest_paths)
if patch_errors:
errors.append(
"invalid CLI patch-tool artifacts:\n"
+ "\n".join(f" {error}" for error in patch_errors)
)
if errors:
print("artifact mirror validation failed:\n" + "\n".join(errors), file=sys.stderr)
return 70
print(f"artifact mirror validated: {mirror_root}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/validate_release_manifest.py
Executable
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python3
"""Validate a JSON release manifest against downloaded CI artifacts."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path, PurePosixPath
import sys
def digest_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as file:
for chunk in iter(lambda: file.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def is_plain_file(path: Path) -> bool:
return path.is_file() and not path.is_symlink()
def parse_sidecar(path: Path) -> tuple[str, str]:
text = path.read_text(encoding="utf-8").strip()
parts = text.split()
if len(parts) != 2:
raise ValueError(f"expected '<sha256> <filename>', got {text!r}")
digest, filename = parts
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
raise ValueError(f"invalid sha256 digest {digest!r}")
return digest, filename
def is_safe_relative_path(path: str) -> bool:
if not path or path == ".":
return False
if "\\" in path or "\x00" in path or path.endswith("/"):
return False
if any(ord(character) < 32 for character in path):
return False
candidate = PurePosixPath(path)
if candidate.is_absolute():
return False
if any(part in ("", ".", "..") for part in candidate.parts):
return False
if candidate.parts and ":" in candidate.parts[0]:
return False
return True
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--github-sha",
default="",
help="Require the manifest github_sha field to match this commit SHA.",
)
parser.add_argument("input_dir", type=Path)
parser.add_argument("manifest", type=Path)
return parser.parse_args()
def main() -> int:
args = parse_args()
input_dir = args.input_dir
manifest_path = args.manifest
if not input_dir.is_dir():
print(f"missing input directory: {input_dir}", file=sys.stderr)
return 66
if not manifest_path.is_file():
print(f"missing release manifest: {manifest_path}", file=sys.stderr)
return 66
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as error:
print(f"invalid release manifest JSON: {error}", file=sys.stderr)
return 70
errors: list[str] = []
if manifest.get("format_version") != 1:
errors.append(f"format_version is {manifest.get('format_version')!r}; expected 1")
if not isinstance(manifest.get("github_sha", ""), str):
errors.append("github_sha must be a string")
elif args.github_sha and manifest.get("github_sha") != args.github_sha:
errors.append(
f"github_sha is {manifest.get('github_sha')!r}; expected {args.github_sha!r}"
)
artifacts = manifest.get("artifacts")
if not isinstance(artifacts, list):
errors.append("artifacts must be a list")
artifacts = []
expected_count = manifest.get("artifact_count")
if expected_count != len(artifacts):
errors.append(
f"artifact_count is {expected_count!r}; expected {len(artifacts)}"
)
seen_paths: set[str] = set()
seen_sidecars: set[str] = set()
for index, artifact in enumerate(artifacts):
if not isinstance(artifact, dict):
errors.append(f"artifacts[{index}] must be an object")
continue
artifact_path_text = artifact.get("path")
artifact_group = artifact.get("artifact_group")
filename = artifact.get("filename")
sidecar_path_text = artifact.get("sidecar")
expected_digest = artifact.get("sha256")
expected_size = artifact.get("size")
if not isinstance(artifact_path_text, str):
errors.append(f"artifacts[{index}].path must be a string")
continue
if artifact_path_text in seen_paths:
errors.append(f"{artifact_path_text}: duplicate artifact path")
seen_paths.add(artifact_path_text)
if not is_safe_relative_path(artifact_path_text):
errors.append(f"{artifact_path_text}: unsafe artifact path")
continue
artifact_relative = PurePosixPath(artifact_path_text)
if not isinstance(artifact_group, str):
errors.append(f"{artifact_path_text}: artifact_group must be a string")
elif artifact_group != artifact_relative.parts[0]:
errors.append(
f"{artifact_path_text}: artifact_group {artifact_group!r} "
f"does not match path group {artifact_relative.parts[0]!r}"
)
if not isinstance(filename, str):
errors.append(f"{artifact_path_text}: filename must be a string")
elif filename != artifact_relative.name:
errors.append(
f"{artifact_path_text}: filename {filename!r} "
f"does not match path filename {artifact_relative.name!r}"
)
if not isinstance(sidecar_path_text, str):
errors.append(f"{artifact_path_text}: sidecar must be a string")
continue
if sidecar_path_text in seen_sidecars:
errors.append(f"{sidecar_path_text}: duplicate sidecar path")
seen_sidecars.add(sidecar_path_text)
if not is_safe_relative_path(sidecar_path_text):
errors.append(f"{sidecar_path_text}: unsafe sidecar path")
continue
artifact_path = input_dir / artifact_path_text
sidecar_path = input_dir / sidecar_path_text
if not is_plain_file(artifact_path):
errors.append(f"{artifact_path_text}: missing artifact file")
continue
if not is_plain_file(sidecar_path):
errors.append(f"{sidecar_path_text}: missing sidecar file")
continue
actual_size = artifact_path.stat().st_size
if actual_size <= 0:
errors.append(f"{artifact_path_text}: empty artifacts are not allowed")
if sidecar_path != Path(f"{artifact_path}.sha256"):
errors.append(
f"{artifact_path_text}: sidecar path {sidecar_path_text!r} "
f"does not match sibling {artifact_path.name}.sha256"
)
actual_digest = digest_file(artifact_path)
if expected_digest != actual_digest:
errors.append(
f"{artifact_path_text}: digest mismatch "
f"{expected_digest!r} != {actual_digest}"
)
if expected_size != actual_size:
errors.append(
f"{artifact_path_text}: size mismatch "
f"{expected_size!r} != {actual_size}"
)
try:
sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path)
except ValueError as error:
errors.append(f"{sidecar_path_text}: {error}")
continue
if sidecar_digest != actual_digest:
errors.append(
f"{sidecar_path_text}: sidecar digest mismatch "
f"{sidecar_digest} != {actual_digest}"
)
if sidecar_filename != artifact_path.name:
errors.append(
f"{sidecar_path_text}: sidecar filename mismatch "
f"{sidecar_filename!r} != {artifact_path.name!r}"
)
manifest_resolved = manifest_path.resolve()
manifest_sidecar_resolved = Path(f"{manifest_path}.sha256").resolve()
actual_artifacts: set[str] = set()
actual_sidecars: set[str] = set()
for path in sorted(input_dir.rglob("*")):
if not path.is_file():
continue
if path.is_symlink():
relative = path.relative_to(input_dir).as_posix()
if path.suffix == ".sha256":
actual_sidecars.add(relative)
else:
actual_artifacts.add(relative)
continue
resolved = path.resolve()
if resolved == manifest_resolved or resolved == manifest_sidecar_resolved:
continue
relative = path.relative_to(input_dir).as_posix()
if path.suffix == ".sha256":
actual_sidecars.add(relative)
else:
actual_artifacts.add(relative)
missing_from_manifest = sorted(actual_artifacts - seen_paths)
if missing_from_manifest:
errors.append(
"artifacts missing from release manifest: "
+ ", ".join(missing_from_manifest)
)
orphan_sidecars = sorted(actual_sidecars - seen_sidecars)
if orphan_sidecars:
errors.append(
"sidecars missing from release manifest: " + ", ".join(orphan_sidecars)
)
if errors:
print(
"release manifest validation failed:\n"
+ "\n".join(f" {error}" for error in errors),
file=sys.stderr,
)
return 70
print(f"release manifest validated: {manifest_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/verify_artifact_mirror_validator.sh
Executable
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-mirror-validator.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
"$ROOT/scripts/verify_assemble_artifact_mirror.sh" >/dev/null
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
ENGINE_REVISION=engine123
MIRROR_ROOT="$TMP_DIR/mirror"
mkdir -p "$MIRROR_ROOT/shorebird/$ENGINE_REVISION"
cat > "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
flutter_engine_revision: 'base-engine'
storage_bucket: 'shorebird'
artifact_overrides:
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/artifacts.zip'
EOF
"$PYTHON_BIN" - "$MIRROR_ROOT/shorebird/$ENGINE_REVISION" <<'PY'
from pathlib import Path
import sys
import zipfile
root = Path(sys.argv[1])
patch_zips = {
"patch-linux-x64.zip": "patch",
"patch-darwin-x64.zip": "patch",
"patch-darwin-arm64.zip": "patch",
"patch-windows-x64.zip": "patch.exe",
}
for zip_name, entry_name in patch_zips.items():
with zipfile.ZipFile(root / zip_name, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr(entry_name, f"{zip_name}:{entry_name}\n")
PY
mkdir -p "$MIRROR_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release"
printf 'linux-engine-artifacts\n' \
> "$MIRROR_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip"
while IFS= read -r -d '' mirror_file; do
if [[ "$mirror_file" == *.sha256 ]]; then
continue
fi
"$ROOT/scripts/write_sha256.sh" "$mirror_file"
done < <(find "$MIRROR_ROOT/shorebird" -type f -print0)
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$MIRROR_ROOT" >/dev/null
UNSAFE_MIRROR_ROOT="$TMP_DIR/unsafe-mirror"
cp -R "$MIRROR_ROOT" "$UNSAFE_MIRROR_ROOT"
"$PYTHON_BIN" - "$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<'PY'
from pathlib import Path
import sys
manifest_path = Path(sys.argv[1])
manifest_path.write_text(
"\n".join(
[
"flutter_engine_revision: 'base-engine'",
"storage_bucket: 'shorebird'",
"artifact_overrides:",
" - '../outside/artifacts.zip'",
]
)
+ "\n",
encoding="utf-8",
)
PY
"$ROOT/scripts/write_sha256.sh" \
"$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" \
"$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml.sha256"
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
"$UNSAFE_MIRROR_ROOT" >"$TMP_DIR/unsafe-mirror.log" 2>&1; then
echo "validate_artifact_mirror.py unexpectedly accepted an unsafe manifest override" >&2
exit 70
fi
grep -q "unsafe artifact override path" "$TMP_DIR/unsafe-mirror.log"
SYMLINK_MIRROR_ROOT="$TMP_DIR/symlink-mirror"
cp -R "$MIRROR_ROOT" "$SYMLINK_MIRROR_ROOT"
rm "$SYMLINK_MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
ln -s "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" \
"$SYMLINK_MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
"$SYMLINK_MIRROR_ROOT" >"$TMP_DIR/symlink-mirror.log" 2>&1; then
echo "validate_artifact_mirror.py unexpectedly accepted a symlink artifact" >&2
exit 70
fi
grep -q "symlink entries are not allowed" "$TMP_DIR/symlink-mirror.log"
EMPTY_OVERRIDE_ROOT="$TMP_DIR/empty-override-mirror"
cp -R "$MIRROR_ROOT" "$EMPTY_OVERRIDE_ROOT"
"$PYTHON_BIN" - "$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" <<'PY'
from pathlib import Path
import sys
Path(sys.argv[1]).write_bytes(b"")
PY
"$ROOT/scripts/write_sha256.sh" \
"$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" \
"$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip.sha256"
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
"$EMPTY_OVERRIDE_ROOT" >"$TMP_DIR/empty-override-mirror.log" 2>&1; then
echo "validate_artifact_mirror.py unexpectedly accepted an empty manifest override artifact" >&2
exit 70
fi
grep -q "artifact override is empty" "$TMP_DIR/empty-override-mirror.log"
printf 'tampered\n' >> "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$MIRROR_ROOT" >/dev/null 2>&1; then
echo "validate_artifact_mirror.py unexpectedly accepted a stale sidecar" >&2
exit 70
fi
echo "validate_artifact_mirror.py smoke test passed"
scripts/verify_artifact_mirror_workflow_assembly.sh
Executable
+287
View File
@@ -0,0 +1,287 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-artifact-job.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
ENGINE_REVISION=engine123
DOWNLOADED="$TMP_DIR/downloaded-artifacts"
mkdir -p "$DOWNLOADED"
write_artifact() {
local path="$1"
local content="$2"
mkdir -p "$(dirname "$path")"
printf '%s\n' "$content" > "$path"
"$ROOT/scripts/write_sha256.sh" "$path"
}
write_zip() {
local zip_path="$1"
local entry_name="$2"
local content="$3"
mkdir -p "$(dirname "$zip_path")"
"$PYTHON_BIN" - "$zip_path" "$entry_name" "$content" <<'PY'
from pathlib import Path
import sys
import zipfile
zip_path, entry_name, content = sys.argv[1:]
with zipfile.ZipFile(Path(zip_path), "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr(entry_name, content + "\n")
PY
"$ROOT/scripts/write_sha256.sh" "$zip_path"
}
write_tgz() {
local archive_path="$1"
local staging_dir="$2"
local root_entry="$3"
mkdir -p "$(dirname "$archive_path")"
tar -C "$staging_dir" -czf "$archive_path" "$root_entry"
"$ROOT/scripts/write_sha256.sh" "$archive_path"
}
for target in \
cli-linux-x64/open-shorebird-cli-linux-x64.tar.gz \
cli-macos-x64/open-shorebird-cli-macos-x64.tar.gz \
cli-macos-arm64/open-shorebird-cli-macos-arm64.tar.gz \
cli-windows-x64/open-shorebird-cli-windows-x64.tar.gz \
shorebird-server-linux-amd64/shorebird-server-linux-amd64.tar.gz \
shorebird-server-linux-arm64/shorebird-server-linux-arm64.tar.gz \
shorebird-server-darwin-amd64/shorebird-server-darwin-amd64.tar.gz \
shorebird-server-darwin-arm64/shorebird-server-darwin-arm64.tar.gz \
shorebird-server-windows-amd64/shorebird-server-windows-amd64.tar.gz \
custom-dart-sdk-linux-x64/custom-dart-sdk-linux-x64.tar.gz \
custom-dart-sdk-macos-arm64/custom-dart-sdk-macos-arm64.tar.gz; do
write_artifact "$DOWNLOADED/$target" "$target"
done
mkdir -p "$DOWNLOADED/mirror-metadata/$ENGINE_REVISION"
cat > "$DOWNLOADED/mirror-metadata/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
flutter_engine_revision: 'base-engine'
storage_bucket: 'shorebird'
artifact_overrides:
- 'flutter_infra_release/flutter/\$engine/android-arm64-release/artifacts.zip'
- 'flutter_infra_release/flutter/\$engine/android-arm64-release/symbols.zip'
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/artifacts.zip'
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/linux-x64-flutter-gtk.zip'
- 'flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip'
- 'flutter_infra_release/flutter/\$engine/flutter_patched_sdk_product.zip'
- 'flutter_infra_release/flutter/\$engine/flutter-web-sdk.zip'
- 'flutter_infra_release/flutter/\$engine/darwin-arm64-release/FlutterMacOS.framework.zip'
EOF
"$ROOT/scripts/write_sha256.sh" \
"$DOWNLOADED/mirror-metadata/$ENGINE_REVISION/artifacts_manifest.yaml"
write_patch_artifact() {
local artifact_name="$1"
local zip_name="$2"
local entry_name="$3"
write_zip "$DOWNLOADED/$artifact_name/artifacts/mirror/$zip_name" "$entry_name" "$zip_name"
write_zip "$DOWNLOADED/$artifact_name/artifacts/mirror/shorebird/$ENGINE_REVISION/$zip_name" "$entry_name" "$zip_name"
}
write_patch_artifact mirror-patch-linux-x64.zip patch-linux-x64.zip patch
write_patch_artifact mirror-patch-darwin-x64.zip patch-darwin-x64.zip patch
write_patch_artifact mirror-patch-darwin-arm64.zip patch-darwin-arm64.zip patch
write_patch_artifact mirror-patch-windows-x64.zip patch-windows-x64.zip patch.exe
engine_stage="$TMP_DIR/engine-stage"
mkdir -p "$engine_stage"
make_engine_archive() {
local artifact_dir="$1"
local archive_name="$2"
local root_name="$3"
local mirror_subdir="$4"
shift 4
local stage="$engine_stage/$root_name"
rm -rf "$stage"
mkdir -p "$stage/$root_name/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/$mirror_subdir"
while [[ "$#" -gt 0 ]]; do
local file_name="$1"
local content="$2"
shift 2
write_artifact \
"$stage/$root_name/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/$mirror_subdir/$file_name" \
"$content"
done
write_tgz "$DOWNLOADED/$artifact_dir/$archive_name" "$stage" "$root_name"
}
make_engine_archive \
linux-engine-x64 linux-engine-x64.tar.gz linux-engine linux-x64-release \
artifacts.zip linux-artifacts \
linux-x64-flutter-gtk.zip linux-gtk
mkdir -p "$engine_stage/linux-engine/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION"
write_artifact \
"$engine_stage/linux-engine/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/flutter_patched_sdk_product.zip" \
linux-patched-sdk
write_tgz "$DOWNLOADED/linux-engine-x64/linux-engine-x64.tar.gz" \
"$engine_stage/linux-engine" \
linux-engine
make_engine_archive \
android-engine-arm64 android-engine-arm64.tar.gz android-engine android-arm64-release \
artifacts.zip android-artifacts \
symbols.zip android-symbols
make_engine_archive \
flutter-web-sdk flutter-web-sdk.tar.gz web-sdk . \
flutter-web-sdk.zip web-sdk
make_engine_archive \
ios-interpreter-engine ios-interpreter-engine.tar.gz ios-engine ios-release \
artifacts.zip ios-artifacts
make_engine_archive \
macos-engine-arm64 macos-engine-arm64.tar.gz macos-engine darwin-arm64-release \
FlutterMacOS.framework.zip macos-framework
mirror_input="$TMP_DIR/mirror-input"
mkdir -p "$mirror_input"
cp -R "$DOWNLOADED"/mirror-* "$mirror_input/"
cp -R "$DOWNLOADED/linux-engine-x64" "$mirror_input/"
cp -R "$DOWNLOADED/android-engine-arm64" "$mirror_input/"
cp -R "$DOWNLOADED/flutter-web-sdk" "$mirror_input/"
cp -R "$DOWNLOADED/ios-interpreter-engine" "$mirror_input/"
cp -R "$DOWNLOADED/macos-engine-arm64" "$mirror_input/"
assembled="$TMP_DIR/artifacts/open-shorebird-artifact-mirror"
mkdir -p "$TMP_DIR/artifacts"
"$ROOT/scripts/assemble_artifact_mirror.sh" "$mirror_input" "$assembled" >/dev/null
tar -C "$TMP_DIR/artifacts" -czf "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" open-shorebird-artifact-mirror
mirror_extract_dir="$TMP_DIR/mirror-extract"
mkdir -p "$mirror_extract_dir"
"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" \
"$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" \
"$mirror_extract_dir"
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
"$mirror_extract_dir/open-shorebird-artifact-mirror" >/dev/null
"$ROOT/scripts/write_sha256.sh" "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz"
manifest_input="$TMP_DIR/manifest-input"
mkdir -p "$manifest_input"
cp -R "$DOWNLOADED"/. "$manifest_input/"
mkdir -p "$manifest_input/open-shorebird-artifact-mirror"
cp "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" "$manifest_input/open-shorebird-artifact-mirror/"
cp "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz.sha256" "$manifest_input/open-shorebird-artifact-mirror/"
"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$manifest_input" \
--github-sha test-sha \
--require 'cli-linux-x64/*open-shorebird-cli-linux-x64.tar.gz' \
--require 'cli-macos-x64/*open-shorebird-cli-macos-x64.tar.gz' \
--require 'cli-macos-arm64/*open-shorebird-cli-macos-arm64.tar.gz' \
--require 'cli-windows-x64/*open-shorebird-cli-windows-x64.tar.gz' \
--require 'shorebird-server-linux-amd64/*shorebird-server-linux-amd64.tar.gz' \
--require 'shorebird-server-linux-arm64/*shorebird-server-linux-arm64.tar.gz' \
--require 'shorebird-server-darwin-amd64/*shorebird-server-darwin-amd64.tar.gz' \
--require 'shorebird-server-darwin-arm64/*shorebird-server-darwin-arm64.tar.gz' \
--require 'shorebird-server-windows-amd64/*shorebird-server-windows-amd64.tar.gz' \
--require 'custom-dart-sdk-linux-x64/*custom-dart-sdk-linux-x64.tar.gz' \
--require 'custom-dart-sdk-macos-arm64/*custom-dart-sdk-macos-arm64.tar.gz' \
--require 'linux-engine-x64/*linux-engine-x64.tar.gz' \
--require 'android-engine-arm64/*android-engine-arm64.tar.gz' \
--require 'flutter-web-sdk/*flutter-web-sdk.tar.gz' \
--require 'ios-interpreter-engine/*ios-interpreter-engine.tar.gz' \
--require 'macos-engine-arm64/*macos-engine-arm64.tar.gz' \
--require 'mirror-patch-linux-x64.zip/*patch-linux-x64.zip' \
--require 'mirror-patch-darwin-x64.zip/*patch-darwin-x64.zip' \
--require 'mirror-patch-darwin-arm64.zip/*patch-darwin-arm64.zip' \
--require 'mirror-patch-windows-x64.zip/*patch-windows-x64.zip' \
--require 'mirror-metadata/*artifacts_manifest.yaml' \
--require 'open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz' \
--output "$TMP_DIR/open-shorebird-release-manifest.json"
"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$manifest_input" \
"$TMP_DIR/open-shorebird-release-manifest.json" >/dev/null
mkdir -p "$manifest_input/open-shorebird-release-manifest"
cp "$TMP_DIR/open-shorebird-release-manifest.json" \
"$manifest_input/open-shorebird-release-manifest/"
"$ROOT/scripts/write_sha256.sh" \
"$manifest_input/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha test-sha \
"$manifest_input" >/dev/null
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha wrong-sha \
"$manifest_input" >"$TMP_DIR/wrong-download-sha.log" 2>&1; then
echo "unexpectedly accepted downloaded artifacts for the wrong github_sha" >&2
exit 1
fi
grep -q "github_sha is" "$TMP_DIR/wrong-download-sha.log"
release_manifest_path="$manifest_input/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
mirror_archive_path="$manifest_input/open-shorebird-artifact-mirror/open-shorebird-artifact-mirror.tar.gz"
printf '%064d open-shorebird-release-manifest.json\n' 0 > "$release_manifest_path.sha256"
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha test-sha \
"$manifest_input" >/dev/null 2>&1; then
echo "unexpectedly accepted a stale downloaded release manifest sidecar" >&2
exit 1
fi
"$ROOT/scripts/write_sha256.sh" "$release_manifest_path"
printf '%064d open-shorebird-artifact-mirror.tar.gz\n' 0 > "$mirror_archive_path.sha256"
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha test-sha \
"$manifest_input" >/dev/null 2>&1; then
echo "unexpectedly accepted a stale downloaded mirror archive sidecar" >&2
exit 1
fi
"$ROOT/scripts/write_sha256.sh" "$mirror_archive_path"
"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha test-sha \
"$manifest_input" >/dev/null
unsafe_download="$TMP_DIR/unsafe-download"
cp -R "$manifest_input" "$unsafe_download"
unsafe_mirror_archive="$unsafe_download/open-shorebird-artifact-mirror/open-shorebird-artifact-mirror.tar.gz"
"$PYTHON_BIN" - "$unsafe_mirror_archive" <<'PY'
import io
import sys
import tarfile
archive_path = sys.argv[1]
with tarfile.open(archive_path, "w:gz") as archive:
content = b"unsafe\n"
member = tarfile.TarInfo("../outside.txt")
member.size = len(content)
archive.addfile(member, io.BytesIO(content))
PY
"$ROOT/scripts/write_sha256.sh" "$unsafe_mirror_archive"
"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$unsafe_download" \
--github-sha test-sha \
--require 'open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz' \
--output "$unsafe_download/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
"$ROOT/scripts/write_sha256.sh" \
"$unsafe_download/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha test-sha \
"$unsafe_download" >"$TMP_DIR/unsafe-download.log" 2>&1; then
echo "unexpectedly accepted an unsafe downloaded mirror archive" >&2
exit 1
fi
grep -q "unsafe archive member path" "$TMP_DIR/unsafe-download.log"
"$PYTHON_BIN" - "$TMP_DIR/open-shorebird-release-manifest.json" <<'PY'
import json
import sys
manifest = json.load(open(sys.argv[1], encoding="utf-8"))
paths = {artifact["path"] for artifact in manifest["artifacts"]}
assert any(path.endswith("open-shorebird-artifact-mirror.tar.gz") for path in paths)
assert any(path.endswith("linux-engine-x64.tar.gz") for path in paths)
assert any(path.endswith("patch-windows-x64.zip") for path in paths)
assert any(path.endswith("artifacts_manifest.yaml") for path in paths)
PY
echo "artifact-mirror workflow assembly smoke test passed"
scripts/verify_assemble_artifact_mirror.sh
Executable
+180
View File
@@ -0,0 +1,180 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-assemble.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
ENGINE_REVISION=engine123
INPUT_DIR="$TMP_DIR/downloaded-artifacts"
OUTPUT_DIR="$TMP_DIR/mirror"
mkdir -p "$INPUT_DIR"
mkdir -p "$INPUT_DIR/mirror-metadata/$ENGINE_REVISION"
cat > "$INPUT_DIR/mirror-metadata/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
flutter_engine_revision: 'base-engine'
storage_bucket: 'shorebird'
artifact_overrides:
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/artifacts.zip'
EOF
"$ROOT/scripts/write_sha256.sh" \
"$INPUT_DIR/mirror-metadata/$ENGINE_REVISION/artifacts_manifest.yaml"
mkdir -p "$INPUT_DIR/mirror-patch/artifacts/mirror/shorebird/$ENGINE_REVISION"
"$PYTHON_BIN" - "$INPUT_DIR/mirror-patch/artifacts/mirror/shorebird/$ENGINE_REVISION" <<'PY'
from pathlib import Path
import sys
import zipfile
root = Path(sys.argv[1])
patch_zips = {
"patch-linux-x64.zip": "patch",
"patch-darwin-x64.zip": "patch",
"patch-darwin-arm64.zip": "patch",
"patch-windows-x64.zip": "patch.exe",
}
for zip_name, entry_name in patch_zips.items():
with zipfile.ZipFile(root / zip_name, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr(entry_name, f"{zip_name}:{entry_name}\n")
PY
engine_staging="$TMP_DIR/linux-engine"
mkdir -p "$engine_staging/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release"
printf 'linux-engine-artifacts\n' \
> "$engine_staging/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip"
tar -C "$engine_staging" -czf "$INPUT_DIR/linux-engine-x64.tar.gz" linux-engine
"$ROOT/scripts/assemble_artifact_mirror.sh" "$INPUT_DIR" "$OUTPUT_DIR"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml.sha256"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-linux-x64.zip.sha256"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-x64.zip"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-x64.zip.sha256"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-arm64.zip"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-arm64.zip.sha256"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-windows-x64.zip"
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-windows-x64.zip.sha256"
test -f "$OUTPUT_DIR/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip"
test -f "$OUTPUT_DIR/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip.sha256"
CONFLICT_INPUT="$TMP_DIR/conflicting-artifacts"
mkdir -p "$CONFLICT_INPUT/conflict/artifacts/mirror/shorebird/$ENGINE_REVISION"
"$PYTHON_BIN" - "$CONFLICT_INPUT/conflict/artifacts/mirror/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" <<'PY'
from pathlib import Path
import sys
import zipfile
with zipfile.ZipFile(Path(sys.argv[1]), "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr("patch", "different-patch\n")
PY
if "$ROOT/scripts/assemble_artifact_mirror.sh" "$CONFLICT_INPUT" "$OUTPUT_DIR" >/dev/null 2>&1; then
echo "assemble_artifact_mirror.sh unexpectedly allowed a conflicting mirror file" >&2
exit 70
fi
UNSAFE_TAR_INPUT="$TMP_DIR/unsafe-tar"
UNSAFE_TAR_OUTPUT="$TMP_DIR/unsafe-tar-output"
mkdir -p "$UNSAFE_TAR_INPUT"
"$PYTHON_BIN" - "$UNSAFE_TAR_INPUT/unsafe-engine.tar.gz" <<'PY'
import io
import sys
import tarfile
archive_path = sys.argv[1]
with tarfile.open(archive_path, "w:gz") as archive:
content = b"unsafe\n"
member = tarfile.TarInfo("../outside.txt")
member.size = len(content)
archive.addfile(member, io.BytesIO(content))
PY
if "$ROOT/scripts/assemble_artifact_mirror.sh" \
"$UNSAFE_TAR_INPUT" \
"$UNSAFE_TAR_OUTPUT" >"$TMP_DIR/unsafe-tar.log" 2>&1; then
echo "assemble_artifact_mirror.sh unexpectedly allowed an unsafe tar member" >&2
exit 70
fi
grep -q "unsafe archive member path" "$TMP_DIR/unsafe-tar.log"
SYMLINK_TAR_INPUT="$TMP_DIR/symlink-tar"
SYMLINK_TAR_OUTPUT="$TMP_DIR/symlink-tar-output"
mkdir -p "$SYMLINK_TAR_INPUT"
"$PYTHON_BIN" - "$SYMLINK_TAR_INPUT/symlink-engine.tar.gz" <<'PY'
import sys
import tarfile
archive_path = sys.argv[1]
with tarfile.open(archive_path, "w:gz") as archive:
member = tarfile.TarInfo("engine/mirror/shorebird/link")
member.type = tarfile.SYMTYPE
member.linkname = "/tmp/outside"
archive.addfile(member)
PY
if "$ROOT/scripts/assemble_artifact_mirror.sh" \
"$SYMLINK_TAR_INPUT" \
"$SYMLINK_TAR_OUTPUT" >"$TMP_DIR/symlink-tar.log" 2>&1; then
echo "assemble_artifact_mirror.sh unexpectedly allowed a tar symlink member" >&2
exit 70
fi
grep -q "unsupported archive member type" "$TMP_DIR/symlink-tar.log"
DUPLICATE_TAR_INPUT="$TMP_DIR/duplicate-tar"
DUPLICATE_TAR_OUTPUT="$TMP_DIR/duplicate-tar-output"
mkdir -p "$DUPLICATE_TAR_INPUT"
"$PYTHON_BIN" - "$DUPLICATE_TAR_INPUT/duplicate-engine.tar.gz" <<'PY'
import io
import sys
import tarfile
archive_path = sys.argv[1]
with tarfile.open(archive_path, "w:gz") as archive:
for content in (b"first\n", b"second\n"):
member = tarfile.TarInfo("engine/mirror/shorebird/duplicate.txt")
member.size = len(content)
archive.addfile(member, io.BytesIO(content))
PY
if "$ROOT/scripts/assemble_artifact_mirror.sh" \
"$DUPLICATE_TAR_INPUT" \
"$DUPLICATE_TAR_OUTPUT" >"$TMP_DIR/duplicate-tar.log" 2>&1; then
echo "assemble_artifact_mirror.sh unexpectedly allowed a duplicate tar member" >&2
exit 70
fi
grep -q "duplicate archive member path" "$TMP_DIR/duplicate-tar.log"
BAD_ZIP_INPUT="$TMP_DIR/bad-zip"
BAD_ZIP_OUTPUT="$TMP_DIR/bad-zip-output"
mkdir -p "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION"
cat > "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
flutter_engine_revision: 'base-engine'
storage_bucket: 'shorebird'
artifact_overrides: []
EOF
printf 'not a zip\n' > "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
if "$ROOT/scripts/assemble_artifact_mirror.sh" "$BAD_ZIP_INPUT" "$BAD_ZIP_OUTPUT" >/dev/null 2>&1; then
echo "assemble_artifact_mirror.sh unexpectedly allowed an invalid patch zip" >&2
exit 70
fi
MISSING_INPUT="$TMP_DIR/missing-override"
MISSING_OUTPUT="$TMP_DIR/missing-output"
mkdir -p "$MISSING_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION"
cat > "$MISSING_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
flutter_engine_revision: 'base-engine'
storage_bucket: 'shorebird'
artifact_overrides:
- 'flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip'
EOF
if "$ROOT/scripts/assemble_artifact_mirror.sh" "$MISSING_INPUT" "$MISSING_OUTPUT" >/dev/null 2>&1; then
echo "assemble_artifact_mirror.sh unexpectedly allowed a missing manifest override" >&2
exit 70
fi
echo "assemble_artifact_mirror.sh smoke test passed"
scripts/verify_ci_capacity.sh
Executable
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CI_MIN_FREE_DISK_GB=2 \
CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \
"$ROOT/scripts/check_ci_capacity.sh" >/dev/null
if CI_MIN_FREE_DISK_GB=4 \
CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \
"$ROOT/scripts/check_ci_capacity.sh" >/dev/null 2>&1; then
echo "check_ci_capacity.sh unexpectedly accepted insufficient disk" >&2
exit 70
fi
if CI_MIN_FREE_DISK_GB=not-a-number \
CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \
"$ROOT/scripts/check_ci_capacity.sh" >/dev/null 2>&1; then
echo "check_ci_capacity.sh unexpectedly accepted an invalid minimum" >&2
exit 70
fi
CI_MIN_FREE_DISK_GB=0 "$ROOT/scripts/check_ci_capacity.sh" >/dev/null
echo "check_ci_capacity.sh smoke test passed"
scripts/verify_ci_workflow.rb
Executable
+2251
View File
File diff suppressed because it is too large Load Diff
scripts/verify_ci_workflow.sh
Executable
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
WORKFLOW="$ROOT/.github/workflows/open-shorebird-ci.yml"
RUBY_ARGS=()
usage() {
cat >&2 <<'EOF'
usage: verify_ci_workflow.sh [--require-tracked] [--require-clean] [--require-upload-ready] [workflow.yml]
Validates the Open Shorebird GitHub Actions workflow contract. The upload-ready
mode additionally requires every required CI support file to be tracked in its
own git checkout and every required owning checkout to be clean.
EOF
}
while [[ "$#" -gt 0 ]]; do
case "$1" in
--require-tracked|--require-clean|--require-upload-ready)
RUBY_ARGS+=("$1")
shift
;;
-h|--help)
usage
exit 0
;;
-*)
echo "unknown argument: $1" >&2
usage
exit 64
;;
*)
WORKFLOW="$1"
shift
if [[ "$#" -gt 0 ]]; then
echo "unexpected extra argument: $1" >&2
usage
exit 64
fi
;;
esac
done
if [[ ! -f "$WORKFLOW" ]]; then
echo "missing workflow: $WORKFLOW" >&2
exit 66
fi
command -v ruby >/dev/null 2>&1 || {
echo "ruby is required to validate GitHub workflow YAML" >&2
exit 127
}
if [[ "${#RUBY_ARGS[@]}" -eq 0 ]]; then
ruby "$ROOT/scripts/verify_ci_workflow.rb" "$WORKFLOW"
else
ruby "$ROOT/scripts/verify_ci_workflow.rb" "${RUBY_ARGS[@]}" "$WORKFLOW"
fi
scripts/verify_dart_sdk_args.sh
Executable
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" -eq 0 ]]; then
echo "usage: $0 <args.gn>..." >&2
exit 64
fi
require_gn_value() {
local args_file="$1"
local key="$2"
local value="$3"
local actual
if ! actual="$(read_gn_value "$args_file" "$key")"; then
echo "expected $key = $value in $args_file, but $key is missing" >&2
exit 70
fi
if [[ "$actual" != "$value" ]]; then
echo "expected $key = $value in $args_file, found $actual" >&2
exit 70
fi
}
read_gn_value() {
local file="$1"
local key="$2"
awk -v key="$key" '
$1 == key && $2 == "=" {
value = $0
sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value)
sub("[[:space:]]*$", "", value)
found = 1
}
END {
if (!found) {
exit 1
}
print value
}
' "$file"
}
for args_file in "$@"; do
if [[ ! -f "$args_file" ]]; then
echo "missing args.gn: $args_file" >&2
exit 66
fi
if [[ "$(read_gn_value "$args_file" dart_dynamic_modules)" == "true" ]]; then
echo "DART_DYNAMIC_MODULES must not be enabled: $args_file" >&2
exit 70
fi
require_gn_value "$args_file" dart_dynamic_modules false
require_gn_value "$args_file" dart_enable_aot_patching true
require_gn_value "$args_file" dart_enable_shorebird_interpreter true
echo "Verified $args_file: patched Dart SDK flags are enabled without DDM"
done
scripts/verify_dart_tool_sdk.sh
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DART_SRC="${DART_SRC:-$ROOT/dart-sdk}"
TOOL_SDK="$DART_SRC/tools/sdks/dart-sdk"
ENGINE_DART="$ROOT/flutter/engine/src/flutter/third_party/dart"
required_constraint="$(
ruby -ryaml -e 'puts YAML.load_file(ARGV[0]).fetch("environment").fetch("sdk")' \
"$DART_SRC/pkg/front_end/pubspec.yaml"
)"
required_major_minor="$(
ruby -e 'ARGV[0] =~ /([0-9]+)\.([0-9]+)/ or abort "unable to parse SDK constraint"; puts "#{$1}.#{$2}"' \
"$required_constraint"
)"
if [[ ! -x "$TOOL_SDK/bin/dart" ]]; then
echo "missing executable Dart tool SDK: $TOOL_SDK/bin/dart" >&2
exit 66
fi
version_output="$("$TOOL_SDK/bin/dart" --version 2>&1)"
actual_major_minor="$(
ruby -e 'ARGV[0] =~ /Dart SDK version: ([0-9]+)\.([0-9]+)/ or abort "unable to parse Dart version"; puts "#{$1}.#{$2}"' \
"$version_output"
)"
if [[ "$actual_major_minor" != "$required_major_minor" ]]; then
echo "Dart tool SDK version does not satisfy front_end SDK constraint." >&2
echo " required major.minor: $required_major_minor from $required_constraint" >&2
echo " actual: $version_output" >&2
exit 70
fi
if [[ -e "$ENGINE_DART" ]]; then
engine_real="$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$ENGINE_DART")"
dart_real="$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$DART_SRC")"
if [[ "$engine_real" != "$dart_real" ]]; then
echo "Flutter engine Dart checkout points at $engine_real, expected $dart_real" >&2
exit 70
fi
fi
echo "[open-source-sync] Dart tool SDK is compatible: $version_output"
scripts/verify_downloaded_release_artifacts.sh
Executable
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
usage() {
cat >&2 <<'EOF'
usage: verify_downloaded_release_artifacts.sh [--github-sha sha] downloaded-artifacts
Verifies a downloaded full SDK build artifact set. The release manifest and
artifact mirror archive must have valid checksum sidecars, the manifest must
cover every downloaded artifact, and the mirror archive must safely extract to a
valid open Shorebird artifact mirror.
EOF
}
DOWNLOAD_DIR=""
EXPECTED_GITHUB_SHA=""
while [[ "$#" -gt 0 ]]; do
case "$1" in
--github-sha)
if [[ "$#" -lt 2 || -z "${2:-}" ]]; then
echo "--github-sha value is required" >&2
usage
exit 64
fi
EXPECTED_GITHUB_SHA="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
-*)
echo "unknown argument: $1" >&2
usage
exit 64
;;
*)
if [[ -n "$DOWNLOAD_DIR" ]]; then
echo "unexpected extra argument: $1" >&2
usage
exit 64
fi
DOWNLOAD_DIR="$1"
shift
;;
esac
done
if [[ -z "$DOWNLOAD_DIR" ]]; then
usage
exit 64
fi
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-downloaded-release.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
fail() {
echo "error: $*" >&2
exit 70
}
verify_sha256_sidecar() {
local artifact_path="$1"
local sidecar_path="$2"
[[ -f "$sidecar_path" ]] || fail "missing checksum sidecar: $sidecar_path"
"$PYTHON_BIN" - "$artifact_path" "$sidecar_path" <<'PY'
import hashlib
from pathlib import Path
import sys
artifact_path = Path(sys.argv[1])
sidecar_path = Path(sys.argv[2])
try:
text = sidecar_path.read_text(encoding="utf-8").strip()
except UnicodeDecodeError as error:
print(f"error: {sidecar_path}: invalid UTF-8: {error}", file=sys.stderr)
sys.exit(70)
parts = text.split()
if len(parts) != 2:
print(
f"error: {sidecar_path}: expected '<sha256> <filename>', got {text!r}",
file=sys.stderr,
)
sys.exit(70)
expected_digest, expected_filename = parts
if len(expected_digest) != 64 or any(
char not in "0123456789abcdef" for char in expected_digest
):
print(f"error: {sidecar_path}: invalid sha256 digest {expected_digest!r}", file=sys.stderr)
sys.exit(70)
if expected_filename != artifact_path.name:
print(
f"error: {sidecar_path}: filename mismatch "
f"{expected_filename!r} != {artifact_path.name!r}",
file=sys.stderr,
)
sys.exit(70)
digest = hashlib.sha256()
with artifact_path.open("rb") as artifact:
for chunk in iter(lambda: artifact.read(1024 * 1024), b""):
digest.update(chunk)
actual_digest = digest.hexdigest()
if expected_digest != actual_digest:
print(
f"error: {sidecar_path}: digest mismatch "
f"{expected_digest} != {actual_digest}",
file=sys.stderr,
)
sys.exit(70)
PY
}
[[ -d "$DOWNLOAD_DIR" ]] || fail "missing downloaded artifacts directory: $DOWNLOAD_DIR"
manifest_paths=()
while IFS= read -r path; do
manifest_paths+=("$path")
done < <(find "$DOWNLOAD_DIR" -type f -name open-shorebird-release-manifest.json | sort)
if [[ "${#manifest_paths[@]}" -ne 1 ]]; then
fail "expected exactly one open-shorebird-release-manifest.json, found ${#manifest_paths[@]}"
fi
manifest_path="${manifest_paths[0]}"
manifest_sidecar="$manifest_path.sha256"
verify_sha256_sidecar "$manifest_path" "$manifest_sidecar"
validate_manifest_args=()
if [[ -n "$EXPECTED_GITHUB_SHA" ]]; then
validate_manifest_args+=(--github-sha "$EXPECTED_GITHUB_SHA")
fi
"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"${validate_manifest_args[@]}" \
"$DOWNLOAD_DIR" \
"$manifest_path"
mirror_archives=()
while IFS= read -r path; do
mirror_archives+=("$path")
done < <(find "$DOWNLOAD_DIR" -type f -name open-shorebird-artifact-mirror.tar.gz | sort)
if [[ "${#mirror_archives[@]}" -ne 1 ]]; then
fail "expected exactly one open-shorebird-artifact-mirror.tar.gz, found ${#mirror_archives[@]}"
fi
mirror_archive="${mirror_archives[0]}"
mirror_sidecar="$mirror_archive.sha256"
verify_sha256_sidecar "$mirror_archive" "$mirror_sidecar"
"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" "$mirror_archive" "$TMP_DIR"
mirror_root="$TMP_DIR/open-shorebird-artifact-mirror"
[[ -d "$mirror_root" ]] || fail "mirror archive did not contain open-shorebird-artifact-mirror/"
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$mirror_root"
echo "downloaded release artifacts verified: $DOWNLOAD_DIR"
scripts/verify_engine_args.sh
Executable
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" -eq 0 ]]; then
echo "usage: $0 <args.gn>... | <args.gn> key=value ..." >&2
exit 64
fi
verify_no_ddm() {
local args_file="$1"
if [[ ! -f "$args_file" ]]; then
echo "missing args.gn: $args_file" >&2
exit 66
fi
local actual
if ! actual="$(read_gn_value "$args_file" dart_dynamic_modules)"; then
return 0
fi
if [[ "$actual" == "true" ]]; then
echo "DART_DYNAMIC_MODULES must not be enabled: $args_file" >&2
exit 70
fi
}
read_gn_value() {
local file="$1"
local key="$2"
awk -v key="$key" '
$1 == key && $2 == "=" {
value = $0
sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value)
sub("[[:space:]]*$", "", value)
found = 1
}
END {
if (!found) {
exit 1
}
print value
}
' "$file"
}
has_expectations=0
for arg in "$@"; do
if [[ "$arg" == *=* ]]; then
has_expectations=1
break
fi
done
if [[ "$has_expectations" == "0" ]]; then
for args_file in "$@"; do
verify_no_ddm "$args_file"
echo "Verified $args_file: dart_dynamic_modules is not true"
done
exit 0
fi
args_file="$1"
shift
verify_no_ddm "$args_file"
for expectation in "$@"; do
if [[ "$expectation" != *=* ]]; then
echo "invalid expectation: $expectation; expected key=value" >&2
exit 64
fi
key="${expectation%%=*}"
value="${expectation#*=}"
if ! actual="$(read_gn_value "$args_file" "$key")"; then
echo "expected $key = $value in $args_file, but $key is missing" >&2
exit 70
fi
if [[ "$actual" != "$value" ]]; then
echo "expected $key = $value in $args_file, found $actual" >&2
exit 70
fi
done
echo "Verified $args_file: dart_dynamic_modules is not true and expected flags are present"
scripts/verify_hosted_full_sdk_build.sh
Executable
+381
View File
@@ -0,0 +1,381 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat >&2 <<'EOF'
usage: verify_hosted_full_sdk_build.sh --repo owner/name [options]
Dispatches the Open Shorebird full SDK build on GitHub Actions, waits for the
workflow run to finish, downloads all artifacts, and verifies the release
manifest plus assembled artifact mirror.
Uses GitHub CLI when `gh` is available. Otherwise uses the GitHub REST API with
`GITHUB_TOKEN` or `GH_TOKEN`; that fallback also requires `curl`, `jq`, and
`unzip`.
Options:
--repo owner/name GitHub repository to run against.
--ref branch-or-sha Ref to dispatch. Defaults to current branch.
--workflow file Workflow file. Defaults to open-shorebird-ci.yml.
--download-dir path Artifact download directory. Defaults to hosted-full-sdk-artifacts.
--timeout-minutes minutes Maximum wait time. Defaults to 720.
--poll-seconds seconds Poll interval. Defaults to 30.
--linux-heavy-runner label Override linux_heavy_runner.
--macos-heavy-runner label Override macos_heavy_runner.
--sdk-min-free-disk-gb value Override sdk_min_free_disk_gb.
--engine-min-free-disk-gb value Override engine_min_free_disk_gb.
--base-flutter-engine-revision v Override base_flutter_engine_revision.
--skip-gclient-sync Dispatch with run_gclient_sync=false.
EOF
}
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO=""
REF=""
WORKFLOW="open-shorebird-ci.yml"
DOWNLOAD_DIR="hosted-full-sdk-artifacts"
TIMEOUT_MINUTES=720
POLL_SECONDS=30
LINUX_HEAVY_RUNNER=""
MACOS_HEAVY_RUNNER=""
SDK_MIN_FREE_DISK_GB=""
ENGINE_MIN_FREE_DISK_GB=""
BASE_FLUTTER_ENGINE_REVISION=""
RUN_GCLIENT_SYNC=true
GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}"
while [[ "$#" -gt 0 ]]; do
case "$1" in
--repo)
REPO="${2:-}"
shift 2
;;
--ref)
REF="${2:-}"
shift 2
;;
--workflow)
WORKFLOW="${2:-}"
shift 2
;;
--download-dir)
DOWNLOAD_DIR="${2:-}"
shift 2
;;
--timeout-minutes)
TIMEOUT_MINUTES="${2:-}"
shift 2
;;
--poll-seconds)
POLL_SECONDS="${2:-}"
shift 2
;;
--linux-heavy-runner)
LINUX_HEAVY_RUNNER="${2:-}"
shift 2
;;
--macos-heavy-runner)
MACOS_HEAVY_RUNNER="${2:-}"
shift 2
;;
--sdk-min-free-disk-gb)
SDK_MIN_FREE_DISK_GB="${2:-}"
shift 2
;;
--engine-min-free-disk-gb)
ENGINE_MIN_FREE_DISK_GB="${2:-}"
shift 2
;;
--base-flutter-engine-revision)
BASE_FLUTTER_ENGINE_REVISION="${2:-}"
shift 2
;;
--skip-gclient-sync)
RUN_GCLIENT_SYNC=false
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage
exit 64
;;
esac
done
if [[ -z "$REPO" ]]; then
echo "--repo owner/name is required" >&2
usage
exit 64
fi
if [[ -z "$REF" ]]; then
REF="$(git -C "$ROOT" branch --show-current 2>/dev/null || true)"
fi
if [[ -z "$REF" ]]; then
echo "--ref is required when the current checkout is detached" >&2
exit 64
fi
run_fields=(
-f full_sdk_build=true
-f run_gclient_sync="$RUN_GCLIENT_SYNC"
-f run_runtime_smokes=false
)
[[ -z "$LINUX_HEAVY_RUNNER" ]] || run_fields+=(-f linux_heavy_runner="$LINUX_HEAVY_RUNNER")
[[ -z "$MACOS_HEAVY_RUNNER" ]] || run_fields+=(-f macos_heavy_runner="$MACOS_HEAVY_RUNNER")
[[ -z "$SDK_MIN_FREE_DISK_GB" ]] || run_fields+=(-f sdk_min_free_disk_gb="$SDK_MIN_FREE_DISK_GB")
[[ -z "$ENGINE_MIN_FREE_DISK_GB" ]] || run_fields+=(-f engine_min_free_disk_gb="$ENGINE_MIN_FREE_DISK_GB")
[[ -z "$BASE_FLUTTER_ENGINE_REVISION" ]] || run_fields+=(-f base_flutter_engine_revision="$BASE_FLUTTER_ENGINE_REVISION")
api_token() {
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
printf '%s' "$GITHUB_TOKEN"
elif [[ -n "${GH_TOKEN:-}" ]]; then
printf '%s' "$GH_TOKEN"
else
echo "GITHUB_TOKEN or GH_TOKEN is required when gh is not installed" >&2
exit 69
fi
}
api_require_tool() {
local tool="$1"
if ! command -v "$tool" >/dev/null 2>&1; then
echo "$tool is required when gh is not installed" >&2
exit 69
fi
}
api_request() {
local method="$1"
local path="$2"
local data="${3:-}"
local token
token="$(api_token)"
local curl_args=(
-fsSL
-X "$method"
-H "Accept: application/vnd.github+json"
-H "Authorization: Bearer $token"
-H "X-GitHub-Api-Version: 2022-11-28"
)
if [[ -n "$data" ]]; then
curl_args+=(-H "Content-Type: application/json" -d "$data")
fi
curl "${curl_args[@]}" "$GITHUB_API_URL/repos/$REPO$path"
}
api_dispatch_payload() {
local inputs_filter
inputs_filter='{full_sdk_build: "true", run_gclient_sync: $run_gclient_sync, run_runtime_smokes: "false"}'
local jq_args=(
--arg ref "$REF"
--arg run_gclient_sync "$RUN_GCLIENT_SYNC"
)
if [[ -n "$LINUX_HEAVY_RUNNER" ]]; then
inputs_filter="$inputs_filter + {linux_heavy_runner: \$linux_heavy_runner}"
jq_args+=(--arg linux_heavy_runner "$LINUX_HEAVY_RUNNER")
fi
if [[ -n "$MACOS_HEAVY_RUNNER" ]]; then
inputs_filter="$inputs_filter + {macos_heavy_runner: \$macos_heavy_runner}"
jq_args+=(--arg macos_heavy_runner "$MACOS_HEAVY_RUNNER")
fi
if [[ -n "$SDK_MIN_FREE_DISK_GB" ]]; then
inputs_filter="$inputs_filter + {sdk_min_free_disk_gb: \$sdk_min_free_disk_gb}"
jq_args+=(--arg sdk_min_free_disk_gb "$SDK_MIN_FREE_DISK_GB")
fi
if [[ -n "$ENGINE_MIN_FREE_DISK_GB" ]]; then
inputs_filter="$inputs_filter + {engine_min_free_disk_gb: \$engine_min_free_disk_gb}"
jq_args+=(--arg engine_min_free_disk_gb "$ENGINE_MIN_FREE_DISK_GB")
fi
if [[ -n "$BASE_FLUTTER_ENGINE_REVISION" ]]; then
inputs_filter="$inputs_filter + {base_flutter_engine_revision: \$base_flutter_engine_revision}"
jq_args+=(--arg base_flutter_engine_revision "$BASE_FLUTTER_ENGINE_REVISION")
fi
jq -n "${jq_args[@]}" "{ref: \$ref, inputs: ($inputs_filter)}"
}
api_download_artifacts() {
rm -rf "$DOWNLOAD_DIR"
mkdir -p "$DOWNLOAD_DIR"
local page=1
local downloaded_count=0
while true; do
local response
response="$(api_request GET "/actions/runs/$run_id/artifacts?per_page=100&page=$page")"
local artifact_count
artifact_count="$(jq '.artifacts | length' <<<"$response")"
[[ "$artifact_count" == "0" ]] && break
while IFS= read -r artifact; do
local name
local url
name="$(jq -r '.name' <<<"$artifact")"
url="$(jq -r '.archive_download_url' <<<"$artifact")"
case "$name" in
""|*/*|*..*)
echo "unsafe GitHub artifact name: $name" >&2
exit 70
;;
esac
if [[ -z "$url" || "$url" == "null" ]]; then
echo "missing archive_download_url for artifact: $name" >&2
exit 70
fi
local artifact_dir
local zip_path
artifact_dir="$DOWNLOAD_DIR/$name"
zip_path="$(mktemp "${TMPDIR:-/tmp}/github-artifact.XXXXXX")"
curl \
-fsSL \
-L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $(api_token)" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-o "$zip_path" \
"$url"
rm -rf "$artifact_dir"
mkdir -p "$artifact_dir"
unzip -q "$zip_path" -d "$artifact_dir"
rm -f "$zip_path"
downloaded_count=$((downloaded_count + 1))
done < <(jq -c '.artifacts[] | select(.expired | not)' <<<"$response")
page=$((page + 1))
done
if [[ "$downloaded_count" -eq 0 ]]; then
echo "no non-expired artifacts were available for workflow run $run_id" >&2
exit 70
fi
}
use_gh=0
if command -v gh >/dev/null 2>&1; then
use_gh=1
else
api_require_tool curl
api_require_tool jq
api_require_tool unzip
api_token >/dev/null
fi
echo "Dispatching $WORKFLOW on $REPO@$REF with full_sdk_build=true"
start_epoch="$(date +%s)"
start_iso="$(date -u -r "$start_epoch" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "@$start_epoch" +"%Y-%m-%dT%H:%M:%SZ")"
if [[ "$use_gh" == "1" ]]; then
gh workflow run "$WORKFLOW" \
--repo "$REPO" \
--ref "$REF" \
"${run_fields[@]}"
else
api_request POST "/actions/workflows/$WORKFLOW/dispatches" "$(api_dispatch_payload)" >/dev/null
fi
run_id=""
for _ in {1..40}; do
if [[ "$use_gh" == "1" ]]; then
run_list_args=(
--repo "$REPO"
--workflow "$WORKFLOW"
--event workflow_dispatch
--json databaseId,createdAt
--limit 20
)
if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then
run_list_args+=(--commit "$REF")
else
run_list_args+=(--branch "$REF")
fi
run_id="$(
gh run list \
"${run_list_args[@]}" \
--jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \
|
head -n 1
)"
else
runs_response="$(api_request GET "/actions/workflows/$WORKFLOW/runs?event=workflow_dispatch&per_page=20")"
run_id="$(
jq -r \
--arg ref "$REF" \
--arg start_iso "$start_iso" \
'
.workflow_runs
| map(select(.created_at >= $start_iso))
| map(select(.head_branch == $ref or .head_sha == $ref))
| sort_by(.created_at)
| reverse
| .[0].id // ""
' \
<<<"$runs_response"
)"
fi
[[ -z "$run_id" ]] || break
sleep 3
done
if [[ -z "$run_id" ]]; then
echo "unable to find dispatched workflow run for $WORKFLOW on $REF" >&2
exit 70
fi
echo "Waiting for hosted full SDK run: $run_id"
deadline=$((start_epoch + TIMEOUT_MINUTES * 60))
run_head_sha=""
while true; do
if [[ "$use_gh" == "1" ]]; then
IFS=$'\t' read -r status conclusion url run_head_sha < <(
gh run view "$run_id" \
--repo "$REPO" \
--json status,conclusion,url,headSha \
--jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv'
)
else
run_response="$(api_request GET "/actions/runs/$run_id")"
IFS=$'\t' read -r status conclusion url run_head_sha < <(
jq -r '[.status, (.conclusion // ""), .html_url, (.head_sha // "")] | @tsv' \
<<<"$run_response"
)
fi
echo "run $run_id status=$status conclusion=${conclusion:-null} url=$url"
if [[ "$status" == "completed" ]]; then
if [[ "$conclusion" != "success" ]]; then
echo "hosted full SDK build failed: conclusion=$conclusion" >&2
exit 70
fi
break
fi
if [[ "$(date +%s)" -ge "$deadline" ]]; then
echo "timed out waiting for hosted full SDK build after $TIMEOUT_MINUTES minutes" >&2
exit 70
fi
sleep "$POLL_SECONDS"
done
if [[ "$use_gh" == "1" ]]; then
rm -rf "$DOWNLOAD_DIR"
mkdir -p "$DOWNLOAD_DIR"
gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR"
else
api_download_artifacts
fi
if [[ -z "$run_head_sha" ]]; then
echo "unable to read headSha for workflow run $run_id" >&2
exit 70
fi
"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
--github-sha "$run_head_sha" \
"$DOWNLOAD_DIR"
echo "hosted full SDK build verified: $run_id"
scripts/verify_ios_interpreter_route.sh
+127 -26
View File
@@ -14,6 +14,7 @@ CHECKED_IPA=""
CHECKED_PATCH_ARTIFACT=""
ENTITLEMENTS_CHECKED=0
APP_STORE_STRICT_CHECKED=0
NO_BUNDLED_KEY_CHECKED=0
CLEANUP_DIR=""
cleanup() {
@@ -35,14 +36,14 @@ read_gn_value() {
$1 == key && $2 == "=" {
value = $0
sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value)
print value
sub("[[:space:]]*$", "", value)
found = 1
exit
}
END {
if (!found) {
exit 1
}
print value
}
' "$file"
}
@@ -66,9 +67,11 @@ verify_ios_engine_args() {
require_gn_value "$args_file" target_os '"ios"'
require_gn_value "$args_file" dart_dynamic_modules false
require_gn_value "$args_file" dart_enable_aot_patching true
require_gn_value "$args_file" dart_enable_shorebird_interpreter true
require_gn_value "$args_file" shorebird_use_interpreter true
require_gn_value "$args_file" shorebird_enable_aot_patching false
require_gn_value "$args_file" flutter_prebuilt_dart_sdk false
}
verify_host_engine_args() {
@@ -77,8 +80,10 @@ verify_host_engine_args() {
require_gn_value "$args_file" target_os '"mac"'
require_gn_value "$args_file" dart_dynamic_modules false
require_gn_value "$args_file" dart_enable_aot_patching true
require_gn_value "$args_file" dart_enable_shorebird_interpreter true
require_gn_value "$args_file" shorebird_use_interpreter true
require_gn_value "$args_file" flutter_prebuilt_dart_sdk false
}
app_bundle_from_ipa() {
@@ -143,6 +148,24 @@ verify_entitlements() {
fi
}
verify_no_bundled_patch_key() {
local app_bundle="$1"
[[ "$APP_STORE_STRICT" == "1" ]] || return 0
local shorebird_yaml
shorebird_yaml="$(
find "$app_bundle" -name "shorebird.yaml" -type f -print -quit
)"
if [[ -z "$shorebird_yaml" ]]; then
return 0
fi
if grep -Eq '^[[:space:]]*aot_patch_key_hex[[:space:]]*:' "$shorebird_yaml"; then
fail "APP_STORE_STRICT=1 rejects bundled aot_patch_key_hex in $shorebird_yaml"
fi
NO_BUNDLED_KEY_CHECKED=1
}
verify_app_bundle() {
if [[ -n "$IOS_APP_BUNDLE" && -n "$IOS_IPA" ]]; then
fail "set only one of IOS_APP_BUNDLE or IOS_IPA"
@@ -165,6 +188,7 @@ verify_app_bundle() {
fi
verify_entitlements "$IOS_APP_BUNDLE"
verify_no_bundled_patch_key "$IOS_APP_BUNDLE"
}
file_magic_hex() {
@@ -186,31 +210,105 @@ verify_patch_artifact() {
;;
esac
local compact_json
compact_json="$(LC_ALL=C tr -d '[:space:]' < "$IOS_PATCH_ARTIFACT")"
local python_bin
python_bin=python3
if ! command -v "$python_bin" >/dev/null 2>&1; then
python_bin=python
fi
command -v "$python_bin" >/dev/null 2>&1 ||
fail "python3 or python is required to inspect IOS_PATCH_ARTIFACT"
if [[ "$compact_json" != \{* ]]; then
fail "iOS patch artifact must be the encrypted open JSON wrapper, not a raw native/code payload"
fi
if ! grep -Fq '"format":"open-aot-vmcode-encrypted-v1"' <<<"$compact_json"; then
fail "iOS patch artifact is not an open encrypted VM code artifact"
fi
if grep -Fq '"runtime_mode":"dart-dynamic-modules"' <<<"$compact_json" ||
grep -Fq '"runtime_mode":"dynamic-modules"' <<<"$compact_json"; then
fail "iOS patch artifact uses DART_DYNAMIC_MODULES runtime mode"
fi
if ! grep -Fq '"runtime_mode":"dart-bytecode-interpreter"' <<<"$compact_json"; then
fail "iOS patch artifact must declare runtime_mode dart-bytecode-interpreter"
fi
if ! grep -Fq '"target_os":"ios"' <<<"$compact_json"; then
fail "iOS patch artifact must target iOS"
fi
if ! grep -Fq '"target_arch":"arm64"' <<<"$compact_json"; then
fail "iOS patch artifact must target arm64"
fi
if ! grep -Fq '"payload_kind":"full-snapshot"' <<<"$compact_json"; then
fail "current iOS interpreter mapper requires payload_kind full-snapshot"
fi
"$python_bin" - "$IOS_PATCH_ARTIFACT" <<'PY'
import base64
import binascii
import json
import re
import sys
path = sys.argv[1]
try:
with open(path, encoding="utf-8") as file:
artifact = json.load(file)
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
raise SystemExit(
"iOS patch artifact must be the encrypted open JSON wrapper, "
f"not a raw native/code payload: {error}"
)
if not isinstance(artifact, dict):
raise SystemExit("iOS patch artifact JSON must be an object")
metadata = artifact.get("metadata")
if not isinstance(metadata, dict):
raise SystemExit("iOS patch artifact must contain metadata object")
def require(mapping, key, expected, scope):
actual = mapping.get(key)
if actual != expected:
raise SystemExit(
f"iOS patch artifact {scope}.{key} is {actual!r}; "
f"expected {expected!r}"
)
require(artifact, "format", "open-aot-vmcode-encrypted-v1", "artifact")
runtime_mode = metadata.get("runtime_mode")
if runtime_mode in {"dart-dynamic-modules", "dynamic-modules"}:
raise SystemExit("iOS patch artifact uses DART_DYNAMIC_MODULES runtime mode")
require(metadata, "runtime_mode", "dart-bytecode-interpreter", "metadata")
require(metadata, "target_os", "ios", "metadata")
require(metadata, "target_arch", "arm64", "metadata")
require(artifact, "payload_kind", "full-snapshot", "artifact")
encryption = artifact.get("encryption")
if not isinstance(encryption, dict):
raise SystemExit("iOS patch artifact must contain encryption object")
require(encryption, "algorithm", "AES-256-GCM", "encryption")
def require_base64(mapping, key, scope):
value = mapping.get(key)
if not isinstance(value, str) or not value:
raise SystemExit(f"iOS patch artifact {scope}.{key} must be non-empty")
try:
decoded = base64.b64decode(value, validate=True)
except (binascii.Error, ValueError) as error:
raise SystemExit(
f"iOS patch artifact {scope}.{key} is not valid base64: {error}"
)
if not decoded:
raise SystemExit(f"iOS patch artifact {scope}.{key} decodes to empty bytes")
return decoded
require_base64(artifact, "encrypted_payload_base64", "artifact")
require_base64(encryption, "nonce_base64", "encryption")
require_base64(encryption, "tag_base64", "encryption")
key_id = encryption.get("key_id")
if not isinstance(key_id, str) or not key_id:
raise SystemExit("iOS patch artifact encryption.key_id must be non-empty")
hex_pattern = re.compile(r"^[0-9a-f]{64}$")
for scope, mapping, key in (
("artifact", artifact, "payload_sha256"),
("encryption", encryption, "aad_sha256"),
):
value = mapping.get(key)
if not isinstance(value, str) or not hex_pattern.fullmatch(value):
raise SystemExit(
f"iOS patch artifact {scope}.{key} must be a lowercase SHA-256 hex digest"
)
reconstructed_size = artifact.get("reconstructed_size")
if reconstructed_size is not None:
if not isinstance(reconstructed_size, int) or reconstructed_size <= 0:
raise SystemExit(
"iOS patch artifact reconstructed_size must be a positive integer"
)
PY
CHECKED_PATCH_ARTIFACT="$IOS_PATCH_ARTIFACT"
}
@@ -240,6 +338,9 @@ fi
if [[ "$APP_STORE_STRICT_CHECKED" == "1" ]]; then
echo " App Store strict: get-task-allow is not true"
fi
if [[ "$NO_BUNDLED_KEY_CHECKED" == "1" ]]; then
echo " key material: no bundled aot_patch_key_hex"
fi
if [[ -n "$CHECKED_PATCH_ARTIFACT" ]]; then
echo " patch artifact: encrypted interpreter full-snapshot for ios/arm64"
fi
scripts/verify_ios_interpreter_route_validator.sh
Executable
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/ios-route-validator.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
IOS_ENGINE_DIR="$TMP_DIR/ios_release"
HOST_ENGINE_DIR="$TMP_DIR/host_release_arm64"
mkdir -p "$IOS_ENGINE_DIR" "$HOST_ENGINE_DIR"
cat > "$IOS_ENGINE_DIR/args.gn" <<'EOF'
target_os = "android"
target_os = "ios"
dart_dynamic_modules = false
dart_enable_aot_patching = false
dart_enable_aot_patching = true
dart_enable_shorebird_interpreter = false
dart_enable_shorebird_interpreter = true
shorebird_use_interpreter = false
shorebird_use_interpreter = true
shorebird_enable_aot_patching = true
shorebird_enable_aot_patching = false
flutter_prebuilt_dart_sdk = false
EOF
cat > "$HOST_ENGINE_DIR/args.gn" <<'EOF'
target_os = "linux"
target_os = "mac"
dart_dynamic_modules = false
dart_enable_aot_patching = false
dart_enable_aot_patching = true
dart_enable_shorebird_interpreter = false
dart_enable_shorebird_interpreter = true
shorebird_use_interpreter = false
shorebird_use_interpreter = true
flutter_prebuilt_dart_sdk = false
EOF
write_artifact() {
local path="$1"
local runtime_mode="$2"
local target_os="${3:-ios}"
cat > "$path" <<EOF
{
"format": "open-aot-vmcode-encrypted-v1",
"metadata": {
"app_id": "app.test",
"app_build_id": "1",
"flavor_id": "pro",
"license_type": "pro",
"sdk_hash": "sdk",
"base_snapshot_hash": "0000000000000000000000000000000000000000000000000000000000000000",
"patch_snapshot_hash": "1111111111111111111111111111111111111111111111111111111111111111",
"target_os": "$target_os",
"target_arch": "arm64",
"runtime_mode": "$runtime_mode"
},
"payload_kind": "full-snapshot",
"reconstructed_size": 4,
"payload_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
"encrypted_payload_base64": "AQIDBA==",
"encryption": {
"algorithm": "AES-256-GCM",
"key_id": "test-key",
"nonce_base64": "AQIDBAUGBwgJCgsM",
"tag_base64": "AQIDBAUGBwgJCgsMDQ4PEA==",
"aad_sha256": "3333333333333333333333333333333333333333333333333333333333333333"
}
}
EOF
}
valid_artifact="$TMP_DIR/valid.vmcode"
write_artifact "$valid_artifact" "dart-bytecode-interpreter"
IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
IOS_PATCH_ARTIFACT="$valid_artifact" \
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null
bad_runtime="$TMP_DIR/bad-runtime.vmcode"
write_artifact "$bad_runtime" "dart-dynamic-modules"
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
IOS_PATCH_ARTIFACT="$bad_runtime" \
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
echo "iOS route validator unexpectedly accepted DART_DYNAMIC_MODULES metadata" >&2
exit 70
fi
bad_target="$TMP_DIR/bad-target.vmcode"
write_artifact "$bad_target" "dart-bytecode-interpreter" "android"
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
IOS_PATCH_ARTIFACT="$bad_target" \
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
echo "iOS route validator unexpectedly accepted a non-iOS patch artifact" >&2
exit 70
fi
bad_json="$TMP_DIR/bad-json.vmcode"
printf '{"format":"open-aot-vmcode-encrypted-v1"\n' > "$bad_json"
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
IOS_PATCH_ARTIFACT="$bad_json" \
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
echo "iOS route validator unexpectedly accepted malformed JSON" >&2
exit 70
fi
bad_native="$TMP_DIR/bad-native.vmcode"
printf '\xcf\xfa\xed\xfe' > "$bad_native"
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
IOS_PATCH_ARTIFACT="$bad_native" \
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
echo "iOS route validator unexpectedly accepted a Mach-O patch artifact" >&2
exit 70
fi
echo "iOS interpreter route validator smoke test passed"
scripts/verify_open_infrastructure_defaults.sh
Executable
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
fail() {
echo "error: $*" >&2
exit 70
}
require_contains() {
local path="$1"
local needle="$2"
grep -Fq "$needle" "$path" || fail "$path is missing required text: $needle"
}
reject_contains() {
local path="$1"
local needle="$2"
if grep -Fq "$needle" "$path"; then
fail "$path contains forbidden text: $needle"
fi
}
check_forbidden_in_file() {
local path="$1"
local pattern
[[ -f "$path" ]] || fail "missing open-infrastructure check input: $path"
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
reject_contains "$path" "$pattern"
done
}
check_forbidden_in_tree() {
local tree="$1"
local extension="$2"
local path
[[ -d "$tree" ]] || fail "missing open-infrastructure check tree: $tree"
while IFS= read -r -d '' path; do
check_forbidden_in_file "$path"
done < <(find "$tree" -type f -name "*.$extension" -print0)
}
FORBIDDEN_PATTERNS=(
"https://download.shorebird.dev"
"download.shorebird.dev"
"api.shorebird.dev"
"auth.shorebird.dev"
"console.shorebird.dev"
"cdn.shorebird.cloud"
"git@github.com:shorebirdtech/dart-sdk.git"
"github.com/shorebirdtech/updater.git"
"github.com/shorebirdtech/flutter.git"
"shorebird-dart-sdk-prebuilt"
"shorebirdtech/_build_engine"
)
BUILD_SENSITIVE_FILES=(
"$ROOT/.gitmodules"
"$ROOT/flutter/DEPS"
"$ROOT/flutter/bin/internal/update_dart_sdk.ps1"
"$ROOT/flutter/bin/internal/update_dart_sdk.sh"
"$ROOT/flutter/dev/bots/post_process_docs.dart"
"$ROOT/flutter/dev/bots/unpublish_package.dart"
"$ROOT/flutter/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle"
"$ROOT/flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts"
"$ROOT/flutter/dev/tools/create_api_docs.dart"
"$ROOT/flutter/engine/src/flutter/build/zip_bundle.gni"
"$ROOT/flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart"
"$ROOT/flutter/packages/flutter_tools/gradle/aar_init_script.gradle"
"$ROOT/flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt"
"$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart"
"$ROOT/flutter/packages/flutter_tools/lib/src/http_host_validator.dart"
"$ROOT/flutter/packages/flutter_tools/pubspec.yaml"
"$ROOT/flutter/packages/shorebird_tests/test/shorebird_tests.dart"
"$ROOT/scripts/write_gclient.sh"
"$ROOT/shorebird/bin/shorebird.ps1"
"$ROOT/shorebird/third_party/flutter/bin/internal/shared.sh"
"$ROOT/updater/library/src/config.rs"
)
for path in "${BUILD_SENSITIVE_FILES[@]}"; do
check_forbidden_in_file "$path"
done
check_forbidden_in_tree "$ROOT/shorebird/packages/artifact_proxy/lib" dart
check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_cli/lib" dart
check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_code_push_client/lib" dart
require_contains "$ROOT/.gitmodules" "https://github.com/tony-cloud/dart-sdk.git"
require_contains "$ROOT/.gitmodules" "https://github.com/tony-cloud/flutter.git"
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird.git"
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-server.git"
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-updater.git"
require_contains "$ROOT/flutter/DEPS" '"dart_sdk_git": "https://github.com/tony-cloud/dart-sdk.git"'
require_contains "$ROOT/flutter/DEPS" '"updater_git": "https://git.tonycloud.org/flutter/shorebird-updater.git"'
require_contains "$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart" \
"kOpenFlutterStorageUrl = 'http://localhost:8080/download.flutter.io'"
require_contains "$ROOT/flutter/bin/internal/update_dart_sdk.sh" \
"http://localhost:8080/download.flutter.io"
require_contains "$ROOT/flutter/bin/internal/update_dart_sdk.ps1" \
"http://localhost:8080/download.flutter.io"
require_contains "$ROOT/flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt" \
'DEFAULT_MAVEN_HOST = "http://localhost:8080/download.flutter.io"'
require_contains "$ROOT/flutter/dev/tools/create_api_docs.dart" \
"Platform.environment['FLUTTER_STORAGE_BASE_URL']"
require_contains "$ROOT/flutter/packages/shorebird_tests/test/shorebird_tests.dart" \
"'FLUTTER_STORAGE_BASE_URL': 'http://localhost:8080/download.flutter.io'"
require_contains "$ROOT/shorebird/packages/shorebird_cli/lib/src/shorebird_env.dart" \
"defaultHostedUrl = 'http://localhost:8080'"
require_contains "$ROOT/shorebird/packages/shorebird_cli/lib/src/cache.dart" \
"defaultArtifactBaseUrl = 'http://localhost:8080/artifacts'"
require_contains "$ROOT/updater/library/src/config.rs" \
'const DEFAULT_BASE_URL: &str = "http://localhost:8080";'
echo "open infrastructure defaults check passed"
scripts/verify_powershell_open_defaults.sh
Executable
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
fail() {
echo "error: $*" >&2
exit 70
}
require_contains() {
local path="$1"
local needle="$2"
grep -Fq "$needle" "$path" || fail "$path is missing required text: $needle"
}
reject_contains() {
local path="$1"
local needle="$2"
if grep -Fq "$needle" "$path"; then
fail "$path contains forbidden text: $needle"
fi
}
shorebird_launcher="$ROOT/shorebird/bin/shorebird.ps1"
flutter_dart_updater="$ROOT/flutter/bin/internal/update_dart_sdk.ps1"
[[ -f "$shorebird_launcher" ]] || fail "missing PowerShell launcher: $shorebird_launcher"
[[ -f "$flutter_dart_updater" ]] || fail "missing Flutter Dart SDK updater: $flutter_dart_updater"
require_contains "$shorebird_launcher" \
'$defaultFlutterGitUrl = "https://github.com/tony-cloud/flutter.git"'
require_contains "$shorebird_launcher" \
'$defaultFlutterStorageBaseUrl = "http://localhost:8080/download.flutter.io"'
require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_GIT_URL'
require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_STORAGE_BASE_URL'
require_contains "$shorebird_launcher" 'FLUTTER_STORAGE_BASE_URL'
require_contains "$flutter_dart_updater" '$Env:FLUTTER_STORAGE_BASE_URL'
require_contains "$flutter_dart_updater" \
'$dartSdkBaseUrl = "http://localhost:8080/download.flutter.io"'
for path in "$shorebird_launcher" "$flutter_dart_updater"; do
reject_contains "$path" 'download.shorebird.dev'
reject_contains "$path" 'api.shorebird.dev'
reject_contains "$path" 'auth.shorebird.dev'
reject_contains "$path" 'console.shorebird.dev'
reject_contains "$path" 'docs.shorebird.dev'
reject_contains "$path" 'github.com/shorebirdtech/flutter.git'
reject_contains "$path" 'github.com/shorebirdtech/shorebird'
reject_contains "$path" 'git@github.com:shorebirdtech'
done
if command -v pwsh >/dev/null 2>&1; then
SHOREBIRD_POWERSHELL_LAUNCHER="$shorebird_launcher" \
FLUTTER_DART_SDK_POWERSHELL_UPDATER="$flutter_dart_updater" \
pwsh -NoProfile -NonInteractive -Command '
$paths = @(
$env:SHOREBIRD_POWERSHELL_LAUNCHER,
$env:FLUTTER_DART_SDK_POWERSHELL_UPDATER
)
$failed = $false
foreach ($path in $paths) {
$tokens = $null
$errors = $null
[System.Management.Automation.Language.Parser]::ParseFile(
$path,
[ref]$tokens,
[ref]$errors
) | Out-Null
if ($errors.Count -gt 0) {
Write-Error "$path has PowerShell parse errors:"
foreach ($errorRecord in $errors) {
Write-Error " $($errorRecord.Message)"
}
$failed = $true
}
}
if ($failed) {
exit 70
}
'
else
echo "warning: pwsh not found; skipped PowerShell parse check" >&2
fi
echo "PowerShell open-default checks passed"
scripts/verify_release_manifest.sh
Executable
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-release-manifest.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
INPUT_DIR="$TMP_DIR/artifacts"
mkdir -p "$INPUT_DIR/cli-linux" "$INPUT_DIR/server-linux" "$INPUT_DIR/mirror-patch/artifacts/mirror"
printf 'cli archive\n' > "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
printf 'server archive\n' > "$INPUT_DIR/server-linux/shorebird-server-linux-amd64.tar.gz"
printf 'patch archive\n' > "$INPUT_DIR/mirror-patch/artifacts/mirror/patch-linux-x64.zip"
"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/server-linux/shorebird-server-linux-amd64.tar.gz"
"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/mirror-patch/artifacts/mirror/patch-linux-x64.zip"
"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$INPUT_DIR" \
--github-sha test-sha \
--require 'cli-linux/*open-shorebird-cli-linux-x64.tar.gz' \
--require 'server-linux/*shorebird-server-linux-amd64.tar.gz' \
--require 'mirror-patch/*patch-linux-x64.zip' \
--output "$TMP_DIR/release-manifest.json"
"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
--github-sha test-sha \
"$INPUT_DIR" \
"$TMP_DIR/release-manifest.json"
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" <<'PY'
import json
import sys
manifest = json.load(open(sys.argv[1], encoding="utf-8"))
assert manifest["format_version"] == 1
assert manifest["github_sha"] == "test-sha"
assert manifest["artifact_count"] == 3
paths = {artifact["path"] for artifact in manifest["artifacts"]}
assert "cli-linux/open-shorebird-cli-linux-x64.tar.gz" in paths
assert "server-linux/shorebird-server-linux-amd64.tar.gz" in paths
assert "mirror-patch/artifacts/mirror/patch-linux-x64.zip" in paths
for artifact in manifest["artifacts"]:
assert artifact["artifact_group"] in {
"cli-linux",
"server-linux",
"mirror-patch",
}
assert artifact["filename"] == artifact["path"].split("/")[-1]
assert len(artifact["sha256"]) == 64
assert artifact["size"] > 0
assert artifact["sidecar"].endswith(".sha256")
PY
TAMPERED_MANIFEST="$TMP_DIR/tampered-release-manifest.json"
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$TAMPERED_MANIFEST" <<'PY'
import json
import sys
source, output = sys.argv[1:]
manifest = json.load(open(source, encoding="utf-8"))
manifest["artifacts"][0]["sha256"] = "0" * 64
json.dump(manifest, open(output, "w", encoding="utf-8"))
PY
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$INPUT_DIR" \
"$TAMPERED_MANIFEST" >/dev/null 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted a tampered digest" >&2
exit 70
fi
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
--github-sha wrong-sha \
"$INPUT_DIR" \
"$TMP_DIR/release-manifest.json" >"$TMP_DIR/wrong-sha.log" 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted the wrong github_sha" >&2
exit 70
fi
grep -q "github_sha is" "$TMP_DIR/wrong-sha.log"
UNLISTED_ARTIFACTS="$TMP_DIR/unlisted-artifacts"
cp -R "$INPUT_DIR" "$UNLISTED_ARTIFACTS"
printf 'extra artifact\n' > "$UNLISTED_ARTIFACTS/extra.tar.gz"
"$ROOT/scripts/write_sha256.sh" "$UNLISTED_ARTIFACTS/extra.tar.gz"
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$UNLISTED_ARTIFACTS" \
"$TMP_DIR/release-manifest.json" >/dev/null 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted an unlisted artifact" >&2
exit 70
fi
UNSAFE_MANIFEST="$TMP_DIR/unsafe-release-manifest.json"
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$UNSAFE_MANIFEST" <<'PY'
import json
import sys
source, output = sys.argv[1:]
manifest = json.load(open(source, encoding="utf-8"))
manifest["artifacts"][0]["path"] = r"cli-linux\open-shorebird-cli-linux-x64.tar.gz"
manifest["artifacts"][0]["sidecar"] = (
r"cli-linux\open-shorebird-cli-linux-x64.tar.gz.sha256"
)
json.dump(manifest, open(output, "w", encoding="utf-8"))
PY
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$INPUT_DIR" \
"$UNSAFE_MANIFEST" >/dev/null 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted an unsafe artifact path" >&2
exit 70
fi
BAD_PROVENANCE_MANIFEST="$TMP_DIR/bad-provenance-release-manifest.json"
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$BAD_PROVENANCE_MANIFEST" <<'PY'
import json
import sys
source, output = sys.argv[1:]
manifest = json.load(open(source, encoding="utf-8"))
manifest["artifacts"][0]["artifact_group"] = "wrong-group"
manifest["artifacts"][1]["filename"] = "wrong-name.tar.gz"
json.dump(manifest, open(output, "w", encoding="utf-8"))
PY
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$INPUT_DIR" \
"$BAD_PROVENANCE_MANIFEST" >/dev/null 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted bad provenance fields" >&2
exit 70
fi
SYMLINK_LISTED_ARTIFACT="$TMP_DIR/symlink-listed-artifact"
cp -R "$INPUT_DIR" "$SYMLINK_LISTED_ARTIFACT"
rm "$SYMLINK_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
ln -s "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz" \
"$SYMLINK_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$SYMLINK_LISTED_ARTIFACT" \
"$TMP_DIR/release-manifest.json" >/dev/null 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted a symlink artifact" >&2
exit 70
fi
EMPTY_LISTED_ARTIFACT="$TMP_DIR/empty-listed-artifact"
cp -R "$INPUT_DIR" "$EMPTY_LISTED_ARTIFACT"
printf '' > "$EMPTY_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
"$ROOT/scripts/write_sha256.sh" \
"$EMPTY_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
EMPTY_LISTED_MANIFEST="$TMP_DIR/empty-listed-release-manifest.json"
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$EMPTY_LISTED_MANIFEST" <<'PY'
import hashlib
import json
import sys
source, output = sys.argv[1:]
manifest = json.load(open(source, encoding="utf-8"))
empty_digest = hashlib.sha256(b"").hexdigest()
for artifact in manifest["artifacts"]:
if artifact["path"] == "cli-linux/open-shorebird-cli-linux-x64.tar.gz":
artifact["sha256"] = empty_digest
artifact["size"] = 0
json.dump(manifest, open(output, "w", encoding="utf-8"))
PY
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
"$EMPTY_LISTED_ARTIFACT" \
"$EMPTY_LISTED_MANIFEST" >"$TMP_DIR/empty-listed.log" 2>&1; then
echo "validate_release_manifest.py unexpectedly accepted an empty artifact" >&2
exit 70
fi
grep -q "empty artifacts are not allowed" "$TMP_DIR/empty-listed.log"
SYMLINK_INPUT="$TMP_DIR/symlink-input"
mkdir -p "$SYMLINK_INPUT"
printf 'symlink target\n' > "$SYMLINK_INPUT/target.tar.gz"
"$ROOT/scripts/write_sha256.sh" "$SYMLINK_INPUT/target.tar.gz"
ln -s target.tar.gz "$SYMLINK_INPUT/link.tar.gz"
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$SYMLINK_INPUT" \
--output "$TMP_DIR/symlink-input.json" >/dev/null 2>&1; then
echo "write_release_manifest.py unexpectedly accepted a symlink artifact" >&2
exit 70
fi
MISSING_SIDECAR="$TMP_DIR/missing-sidecar"
mkdir -p "$MISSING_SIDECAR"
printf 'missing sidecar\n' > "$MISSING_SIDECAR/artifact.tar.gz"
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$MISSING_SIDECAR" \
--output "$TMP_DIR/missing.json" >/dev/null 2>&1; then
echo "write_release_manifest.py unexpectedly accepted a missing sidecar" >&2
exit 70
fi
BAD_SIDECAR="$TMP_DIR/bad-sidecar"
mkdir -p "$BAD_SIDECAR"
printf 'bad sidecar\n' > "$BAD_SIDECAR/artifact.tar.gz"
printf '%064d artifact.tar.gz\n' 0 > "$BAD_SIDECAR/artifact.tar.gz.sha256"
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$BAD_SIDECAR" \
--output "$TMP_DIR/bad.json" >/dev/null 2>&1; then
echo "write_release_manifest.py unexpectedly accepted a bad sidecar" >&2
exit 70
fi
EMPTY_ARTIFACT="$TMP_DIR/empty-artifact"
mkdir -p "$EMPTY_ARTIFACT"
printf '' > "$EMPTY_ARTIFACT/artifact.tar.gz"
"$ROOT/scripts/write_sha256.sh" "$EMPTY_ARTIFACT/artifact.tar.gz"
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$EMPTY_ARTIFACT" \
--output "$TMP_DIR/empty.json" >"$TMP_DIR/empty-artifact.log" 2>&1; then
echo "write_release_manifest.py unexpectedly accepted an empty artifact" >&2
exit 70
fi
grep -q "empty artifacts are not allowed" "$TMP_DIR/empty-artifact.log"
ORPHAN_SIDECAR="$TMP_DIR/orphan-sidecar"
mkdir -p "$ORPHAN_SIDECAR"
printf '%064d missing.tar.gz\n' 0 > "$ORPHAN_SIDECAR/missing.tar.gz.sha256"
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$ORPHAN_SIDECAR" \
--output "$TMP_DIR/orphan.json" >/dev/null 2>&1; then
echo "write_release_manifest.py unexpectedly accepted an orphan sidecar" >&2
exit 70
fi
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
"$INPUT_DIR" \
--require 'missing-artifact/*.tar.gz' \
--output "$TMP_DIR/missing-required.json" >/dev/null 2>&1; then
echo "write_release_manifest.py unexpectedly accepted a missing required artifact" >&2
exit 70
fi
echo "write_release_manifest.py smoke test passed"
scripts/verify_sync_open_sources.sh
Executable
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-source-sync.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
init_git_checkout() {
local path="$1"
mkdir -p "$path"
git -C "$path" init -q
}
real_path() {
python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$1"
}
make_workspace() {
local workspace="$1"
mkdir -p "$workspace/scripts"
cp "$ROOT/scripts/sync_open_sources.sh" "$workspace/scripts/sync_open_sources.sh"
chmod +x "$workspace/scripts/sync_open_sources.sh"
init_git_checkout "$workspace/dart-sdk"
mkdir -p "$workspace/dart-sdk/runtime/vm"
: > "$workspace/dart-sdk/runtime/vm/dart_api_impl.h"
init_git_checkout "$workspace/updater"
mkdir -p "$workspace/updater/library/include"
: > "$workspace/updater/library/include/updater_engine.h"
mkdir -p "$workspace/flutter/engine/src/flutter/third_party"
}
run_sync() {
local workspace="$1"
DART_SRC="$workspace/dart-sdk" \
UPDATER_SRC="$workspace/updater" \
"$workspace/scripts/sync_open_sources.sh"
}
assert_links_to_workspace_sources() {
local workspace="$1"
local dart_target="$workspace/flutter/engine/src/flutter/third_party/dart"
local updater_target="$workspace/flutter/engine/src/flutter/third_party/updater"
test -L "$dart_target"
test -L "$updater_target"
[[ "$(real_path "$dart_target")" == "$(real_path "$workspace/dart-sdk")" ]]
[[ "$(real_path "$updater_target")" == "$(real_path "$workspace/updater")" ]]
test -f "$dart_target/runtime/vm/dart_api_impl.h"
test -f "$updater_target/library/include/updater_engine.h"
}
clean_checkout_workspace="$TMP_DIR/clean-checkouts"
make_workspace "$clean_checkout_workspace"
init_git_checkout "$clean_checkout_workspace/flutter/engine/src/flutter/third_party/dart"
init_git_checkout "$clean_checkout_workspace/flutter/engine/src/flutter/third_party/updater"
run_sync "$clean_checkout_workspace"
assert_links_to_workspace_sources "$clean_checkout_workspace"
run_sync "$clean_checkout_workspace"
assert_links_to_workspace_sources "$clean_checkout_workspace"
stale_link_workspace="$TMP_DIR/stale-link"
make_workspace "$stale_link_workspace"
mkdir -p "$stale_link_workspace/other-dart"
ln -s "../../../../../other-dart" \
"$stale_link_workspace/flutter/engine/src/flutter/third_party/dart"
if run_sync "$stale_link_workspace" >"$TMP_DIR/stale-link.log" 2>&1; then
echo "expected stale Dart symlink to fail" >&2
exit 1
fi
grep -q "Dart SDK target symlink points at" "$TMP_DIR/stale-link.log"
dirty_checkout_workspace="$TMP_DIR/dirty-checkout"
make_workspace "$dirty_checkout_workspace"
dirty_dart="$dirty_checkout_workspace/flutter/engine/src/flutter/third_party/dart"
init_git_checkout "$dirty_dart"
: > "$dirty_dart/untracked.txt"
if run_sync "$dirty_checkout_workspace" >"$TMP_DIR/dirty-checkout.log" 2>&1; then
echo "expected dirty Dart checkout to fail" >&2
exit 1
fi
grep -q "Dart SDK target is a dirty git checkout" "$TMP_DIR/dirty-checkout.log"
test -d "$dirty_dart/.git"
test -f "$dirty_dart/untracked.txt"
forbidden_dart_remote_workspace="$TMP_DIR/forbidden-dart-remote"
make_workspace "$forbidden_dart_remote_workspace"
git -C "$forbidden_dart_remote_workspace/dart-sdk" remote add origin \
https://github.com/dart-lang/sdk.git
if run_sync "$forbidden_dart_remote_workspace" >"$TMP_DIR/forbidden-dart-remote.log" 2>&1; then
echo "expected forbidden Dart SDK remote to fail" >&2
exit 1
fi
grep -q "Dart SDK source checkout uses forbidden remote fragment" \
"$TMP_DIR/forbidden-dart-remote.log"
forbidden_updater_remote_workspace="$TMP_DIR/forbidden-updater-remote"
make_workspace "$forbidden_updater_remote_workspace"
git -C "$forbidden_updater_remote_workspace/updater" remote add origin \
https://github.com/shorebirdtech/updater.git
if run_sync "$forbidden_updater_remote_workspace" >"$TMP_DIR/forbidden-updater-remote.log" 2>&1; then
echo "expected forbidden updater source remote to fail" >&2
exit 1
fi
grep -q "updater submodule source checkout uses forbidden remote fragment" \
"$TMP_DIR/forbidden-updater-remote.log"
forbidden_updater_url_workspace="$TMP_DIR/forbidden-updater-url"
make_workspace "$forbidden_updater_url_workspace"
rm -rf "$forbidden_updater_url_workspace/updater"
if DART_SRC="$forbidden_updater_url_workspace/dart-sdk" \
UPDATER_SRC="$forbidden_updater_url_workspace/missing-updater" \
UPDATER_URL=https://github.com/shorebirdtech/updater.git \
"$forbidden_updater_url_workspace/scripts/sync_open_sources.sh" \
>"$TMP_DIR/forbidden-updater-url.log" 2>&1; then
echo "expected forbidden explicit UPDATER_URL to fail" >&2
exit 1
fi
grep -q "UPDATER_URL points at a forbidden official Shorebird updater remote" \
"$TMP_DIR/forbidden-updater-url.log"
echo "sync_open_sources.sh smoke test passed"
scripts/verify_upload_readiness.sh
Executable
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
"$ROOT/scripts/verify_ci_workflow.sh" \
--require-upload-ready \
"$ROOT/.github/workflows/open-shorebird-ci.yml"
echo "upload readiness check passed"
scripts/verify_write_sha256.sh
Executable
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-sha256.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
ARTIFACT="$TMP_DIR/artifact.txt"
SIDECAR="$TMP_DIR/artifact.txt.sha256"
CUSTOM_SIDECAR="$TMP_DIR/custom.sha256"
printf 'open-shorebird\n' > "$ARTIFACT"
"$ROOT/scripts/write_sha256.sh" "$ARTIFACT"
EXPECTED_HASH="$("$PYTHON_BIN" - "$ARTIFACT" <<'PY'
import hashlib
import pathlib
import sys
print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())
PY
)"
EXPECTED_LINE="$EXPECTED_HASH artifact.txt"
ACTUAL_LINE="$(cat "$SIDECAR")"
if [[ "$ACTUAL_LINE" != "$EXPECTED_LINE" ]]; then
echo "unexpected sha256 sidecar: $ACTUAL_LINE" >&2
echo "expected: $EXPECTED_LINE" >&2
exit 70
fi
"$ROOT/scripts/write_sha256.sh" "$ARTIFACT" "$CUSTOM_SIDECAR"
CUSTOM_LINE="$(cat "$CUSTOM_SIDECAR")"
if [[ "$CUSTOM_LINE" != "$EXPECTED_LINE" ]]; then
echo "unexpected custom sha256 sidecar: $CUSTOM_LINE" >&2
echo "expected: $EXPECTED_LINE" >&2
exit 70
fi
if "$ROOT/scripts/write_sha256.sh" "$TMP_DIR/missing.txt" >/dev/null 2>&1; then
echo "write_sha256.sh unexpectedly succeeded for a missing artifact" >&2
exit 70
fi
echo "write_sha256.sh smoke test passed"
scripts/write_artifact_manifest.py
Executable
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Write an open Shorebird artifact proxy manifest.
The artifact proxy expects this file at:
/shorebird/<shorebird-engine-revision>/artifacts_manifest.yaml
It maps a custom Shorebird engine revision back to the upstream Flutter engine
revision for unchanged artifacts, and lists the artifact paths that should be
served from the open Shorebird mirror.
"""
from __future__ import annotations
import argparse
from pathlib import Path
import sys
DEFAULT_ARTIFACT_OVERRIDES = (
"flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip",
"flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip",
"flutter_infra_release/flutter/$engine/linux-x64-release/artifacts.zip",
"flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip",
"flutter_infra_release/flutter/$engine/ios-release/artifacts.zip",
"flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip",
"flutter_infra_release/flutter/$engine/flutter-web-sdk.zip",
"flutter_infra_release/flutter/$engine/darwin-arm64-release/FlutterMacOS.framework.zip",
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--flutter-engine-revision",
required=True,
help="Upstream Flutter engine revision used for non-overridden artifacts.",
)
parser.add_argument(
"--storage-bucket",
default="shorebird",
help="Bucket/path prefix under the Shorebird artifact mirror.",
)
parser.add_argument(
"--output",
type=Path,
help="Output path. Writes to stdout when omitted.",
)
return parser.parse_args()
def yaml_quote(value: str) -> str:
return "'" + value.replace("'", "''") + "'"
def build_manifest(flutter_engine_revision: str, storage_bucket: str) -> str:
lines = [
f"flutter_engine_revision: {yaml_quote(flutter_engine_revision)}",
f"storage_bucket: {yaml_quote(storage_bucket)}",
"artifact_overrides:",
]
lines.extend(
f" - {yaml_quote(override)}" for override in DEFAULT_ARTIFACT_OVERRIDES
)
return "\n".join(lines) + "\n"
def main() -> int:
args = parse_args()
manifest = build_manifest(
flutter_engine_revision=args.flutter_engine_revision,
storage_bucket=args.storage_bucket,
)
if args.output is None:
sys.stdout.write(manifest)
return 0
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(manifest, encoding="utf-8")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/write_gclient.sh
+9 -5
View File
@@ -7,8 +7,13 @@ INCLUDE_ENGINE_DEPS="${INCLUDE_ENGINE_DEPS:-0}"
case "$PLATFORM" in
linux)
if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then
TARGET_OS='["linux", "android"]'
DART_DOWNLOAD_ANDROID_DEPS="True"
else
TARGET_OS='["linux"]'
DART_DOWNLOAD_ANDROID_DEPS="False"
fi
;;
macos)
if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then
@@ -28,13 +33,13 @@ esac
cat > "$ROOT/.gclient" <<EOF
solutions = [
{
"name": "dart-sdk-new",
"url": "https://git.tonycloud.org/dart-lang/sdk.git",
"name": "dart-sdk",
"url": "https://github.com/tony-cloud/dart-sdk.git",
"deps_file": "DEPS",
"managed": False,
"custom_deps": {},
"custom_vars": {
"dart_root": "dart-sdk-new",
"dart_root": "dart-sdk",
"download_android_deps": $DART_DOWNLOAD_ANDROID_DEPS,
"checkout_javascript_engines": False,
"checkout_benchmarks_internal": False,
@@ -52,7 +57,7 @@ if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then
solutions = [
{
"name": ".",
"url": "https://git.tonycloud.org/flutter/flutter.git",
"url": "https://github.com/tony-cloud/flutter.git",
"deps_file": "DEPS",
"managed": False,
"custom_deps": {
@@ -77,7 +82,6 @@ solutions = [
"engine/src/flutter/third_party/dart/third_party/pkg/tools": None,
"engine/src/flutter/third_party/dart/third_party/pkg/vector_math": None,
"engine/src/flutter/third_party/dart/third_party/pkg/web": None,
"engine/src/flutter/third_party/dart/third_party/pkg/webdev": None,
"engine/src/flutter/third_party/dart/third_party/pkg/webdriver": None,
"engine/src/flutter/third_party/dart/third_party/pkg/webkit_inspection_protocol": None,
"engine/src/flutter/third_party/dart/tools/sdks/dart-sdk": None,
scripts/write_release_manifest.py
Executable
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Write a JSON manifest for CI release artifacts.
Every non-sidecar file under the input directory must have a sibling
`<file>.sha256` sidecar in the format written by scripts/write_sha256.py:
<hex sha256> <basename>
"""
from __future__ import annotations
import argparse
import fnmatch
import hashlib
import json
from pathlib import Path, PurePosixPath
import sys
def digest_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as file:
for chunk in iter(lambda: file.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def is_plain_file(path: Path) -> bool:
return path.is_file() and not path.is_symlink()
def parse_sidecar(path: Path) -> tuple[str, str]:
text = path.read_text(encoding="utf-8").strip()
parts = text.split()
if len(parts) != 2:
raise ValueError(f"expected '<sha256> <filename>', got {text!r}")
digest, filename = parts
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
raise ValueError(f"invalid sha256 digest {digest!r}")
return digest, filename
def is_safe_relative_path(path: str) -> bool:
if not path or path == ".":
return False
if "\\" in path or "\x00" in path or path.endswith("/"):
return False
if any(ord(character) < 32 for character in path):
return False
candidate = PurePosixPath(path)
if candidate.is_absolute():
return False
if any(part in ("", ".", "..") for part in candidate.parts):
return False
if candidate.parts and ":" in candidate.parts[0]:
return False
return True
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("input_dir", type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--github-sha", default="")
parser.add_argument(
"--require",
action="append",
default=[],
metavar="GLOB",
help="Require at least one artifact path matching this glob. May be repeated.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
input_dir = args.input_dir
output_path = args.output
if not input_dir.is_dir():
print(f"missing input directory: {input_dir}", file=sys.stderr)
return 66
artifacts = []
errors = []
for artifact_path in sorted(path for path in input_dir.rglob("*") if path.is_file()):
if artifact_path.is_symlink():
errors.append(f"{artifact_path.relative_to(input_dir).as_posix()}: symlink artifacts are not allowed")
continue
if artifact_path.suffix == ".sha256":
continue
if artifact_path.resolve() == output_path.resolve():
continue
artifact_relative_path = artifact_path.relative_to(input_dir).as_posix()
if not is_safe_relative_path(artifact_relative_path):
errors.append(f"{artifact_relative_path}: unsafe artifact path")
continue
if artifact_path.stat().st_size <= 0:
errors.append(f"{artifact_relative_path}: empty artifacts are not allowed")
continue
artifact_relative = PurePosixPath(artifact_relative_path)
sidecar_path = Path(f"{artifact_path}.sha256")
if not is_plain_file(sidecar_path):
errors.append(f"{artifact_relative_path}: missing .sha256 sidecar")
continue
actual_digest = digest_file(artifact_path)
try:
sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path)
except ValueError as error:
errors.append(f"{sidecar_path.relative_to(input_dir).as_posix()}: {error}")
continue
if sidecar_digest != actual_digest:
errors.append(
f"{sidecar_path.relative_to(input_dir).as_posix()}: digest mismatch "
f"{sidecar_digest} != {actual_digest}"
)
if sidecar_filename != artifact_path.name:
errors.append(
f"{sidecar_path.relative_to(input_dir).as_posix()}: filename mismatch "
f"{sidecar_filename!r} != {artifact_path.name!r}"
)
artifacts.append(
{
"path": artifact_relative_path,
"artifact_group": artifact_relative.parts[0],
"filename": artifact_path.name,
"sha256": actual_digest,
"size": artifact_path.stat().st_size,
"sidecar": sidecar_path.relative_to(input_dir).as_posix(),
}
)
orphan_sidecars = []
for sidecar_path in sorted(input_dir.rglob("*.sha256")):
artifact_path = Path(str(sidecar_path)[: -len(".sha256")])
if sidecar_path.is_symlink():
orphan_sidecars.append(sidecar_path.relative_to(input_dir).as_posix())
elif not is_plain_file(artifact_path):
orphan_sidecars.append(sidecar_path.relative_to(input_dir).as_posix())
if orphan_sidecars:
errors.append(f"orphan .sha256 sidecars: {', '.join(orphan_sidecars)}")
artifact_paths = [artifact["path"] for artifact in artifacts]
for required_glob in args.require:
if not any(fnmatch.fnmatchcase(path, required_glob) for path in artifact_paths):
errors.append(f"missing required artifact matching {required_glob!r}")
if errors:
print(
"release manifest validation failed:\n"
+ "\n".join(f" {error}" for error in errors),
file=sys.stderr,
)
return 70
manifest = {
"format_version": 1,
"github_sha": args.github_sha,
"artifact_count": len(artifacts),
"artifacts": artifacts,
}
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/write_sha256.py
Executable
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""Write a sha256 sidecar in the common `digest filename` format."""
from __future__ import annotations
import hashlib
import pathlib
import sys
def digest_file(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as file:
for chunk in iter(lambda: file.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def main() -> int:
if len(sys.argv) not in (2, 3):
print("usage: write_sha256.py <artifact> [output]", file=sys.stderr)
return 64
artifact_path = pathlib.Path(sys.argv[1])
output_path = (
pathlib.Path(sys.argv[2])
if len(sys.argv) == 3
else pathlib.Path(f"{artifact_path}.sha256")
)
if not artifact_path.is_file():
print(f"missing artifact: {artifact_path}", file=sys.stderr)
return 66
output_path.write_text(
f"{digest_file(artifact_path)} {artifact_path.name}\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/write_sha256.sh
Executable
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" -ne 1 && "$#" -ne 2 ]]; then
echo "usage: $0 <artifact> [output]" >&2
exit 64
fi
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PYTHON_BIN=python3
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
PYTHON_BIN=python
fi
"$PYTHON_BIN" "$ROOT/scripts/write_sha256.py" "$@"
shorebird
+1 -1
Submodule shorebird updated: 08b8f07bfb...cdd8cefb27
shorebird-server
+1 -1
Submodule shorebird-server updated: 774954fce7...d6e5a39546
testapps/license_flavor_patch_test/ios/Runner/SceneDelegate.swift
+34
View File
@@ -2,5 +2,39 @@ import Flutter
import UIKit
class SceneDelegate: FlutterSceneDelegate {
override func scene(
_ scene: UIScene,
willConnectTo session: UISceneSession,
options connectionOptions: UIScene.ConnectionOptions
) {
guard let windowScene = scene as? UIWindowScene else {
return
}
let project = FlutterDartProject()
project.shorebirdAotPatchKeyProvider = { keyIdentifier in
guard keyIdentifier == "test-key" else {
return nil
}
let bytes: [UInt8] = [
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
]
return Data(bytes)
}
let flutterViewController = FlutterViewController(
project: project,
nibName: nil,
bundle: nil
)
GeneratedPluginRegistrant.register(with: flutterViewController.engine)
let window = UIWindow(windowScene: windowScene)
window.rootViewController = flutterViewController
self.window = window
window.makeKeyAndVisible()
}
}
testapps/license_flavor_patch_test/shorebird.yaml
-1
View File
@@ -1,6 +1,5 @@
app_id: license-flavor-patch-test
aot_patch_key_id: test-key
aot_patch_key_hex: "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
aot_patch_base_flavor_id: free
aot_patch_base_license_type: free
aot_patch_flavor_id: pro
testapps/license_flavor_patch_test/tool/verify_aot_patch.dart
+1 -1
View File
@@ -4,7 +4,7 @@ import 'dart:io';
Future<void> main() async {
final appDir = File.fromUri(Platform.script).parent.parent;
final workspaceRoot = appDir.parent.parent;
final dartRoot = Directory('${workspaceRoot.path}/dart-sdk-new');
final dartRoot = Directory('${workspaceRoot.path}/dart-sdk');
final patchBuildDir = _resolvePatchBuildDir(dartRoot);
final toolWorkspace = Directory('${workspaceRoot.path}/shorebird');
final workDir = Directory('${appDir.path}/build/open_aot_patch_verify');
updater
+1 -1
Submodule updater updated: 3ac748ff28...6e9aab2ce7