Add open Shorebird CI and release tooling
This commit is contained in:
Tony
Executable
+249
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
APP_DIR="${APP_DIR:-$ROOT/testapps/license_flavor_patch_test}"
|
||||
FLUTTER_BIN="${FLUTTER_BIN:-$ROOT/flutter/bin/flutter}"
|
||||
ADB_BIN="${ADB_BIN:-adb}"
|
||||
PACKAGE="${ANDROID_PACKAGE:-com.example.licenseflavorpatchtest.license_flavor_patch_test}"
|
||||
ACTIVITY="${ANDROID_ACTIVITY:-com.example.licenseflavorpatchtest.license_flavor_patch_test.MainActivity}"
|
||||
RELEASE_VERSION="${ANDROID_RELEASE_VERSION:-1.0+1}"
|
||||
LOCAL_ENGINE_SRC_PATH="${LOCAL_ENGINE_SRC_PATH:-$ROOT/flutter/engine/src}"
|
||||
LOCAL_ENGINE="${LOCAL_ENGINE:-android_release_arm64}"
|
||||
LOCAL_ENGINE_HOST="${LOCAL_ENGINE_HOST:-host_release_arm64}"
|
||||
WORK_DIR="${ANDROID_RUNTIME_SMOKE_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-android-runtime.XXXXXX")}"
|
||||
SEED_REMOTE_DIR="${ANDROID_SEED_REMOTE_DIR:-/data/local/tmp/open-shorebird-android-runtime-seed}"
|
||||
TARGET_PLATFORM="${ANDROID_TARGET_PLATFORM:-android-arm64}"
|
||||
|
||||
if [[ "${KEEP_ANDROID_RUNTIME_SMOKE_ARTIFACTS:-0}" != "1" ]]; then
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
fi
|
||||
|
||||
ADB=("$ADB_BIN")
|
||||
if [[ -n "${ANDROID_SERIAL:-}" ]]; then
|
||||
ADB+=("-s" "$ANDROID_SERIAL")
|
||||
fi
|
||||
|
||||
adb_cmd() {
|
||||
"${ADB[@]}" "$@"
|
||||
}
|
||||
|
||||
adb_shell() {
|
||||
adb_cmd shell "$@"
|
||||
}
|
||||
|
||||
python_bin() {
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
printf '%s\n' python3
|
||||
else
|
||||
printf '%s\n' python
|
||||
fi
|
||||
}
|
||||
|
||||
require_tool() {
|
||||
local tool="$1"
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
echo "$tool is required" >&2
|
||||
exit 127
|
||||
fi
|
||||
}
|
||||
|
||||
build_apk() {
|
||||
local license="$1"
|
||||
local output="$2"
|
||||
if [[ "${SKIP_ANDROID_BUILDS:-0}" == "1" ]]; then
|
||||
local env_name
|
||||
case "$license" in
|
||||
free) env_name=ANDROID_FREE_APK ;;
|
||||
pro) env_name=ANDROID_PRO_APK ;;
|
||||
*)
|
||||
echo "Unsupported license variant: $license" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
local existing="${!env_name:-}"
|
||||
if [[ -z "$existing" || ! -f "$existing" ]]; then
|
||||
echo "SKIP_ANDROID_BUILDS=1 requires $env_name to point at an APK" >&2
|
||||
exit 66
|
||||
fi
|
||||
cp "$existing" "$output"
|
||||
return
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$APP_DIR"
|
||||
"$FLUTTER_BIN" build apk --release \
|
||||
--target-platform "$TARGET_PLATFORM" \
|
||||
--local-engine-src-path="$LOCAL_ENGINE_SRC_PATH" \
|
||||
--local-engine="$LOCAL_ENGINE" \
|
||||
--local-engine-host="$LOCAL_ENGINE_HOST" \
|
||||
--dart-define="LICENSE_TYPE=$license"
|
||||
)
|
||||
cp "$APP_DIR/build/app/outputs/flutter-apk/app-release.apk" "$output"
|
||||
}
|
||||
|
||||
extract_libapp() {
|
||||
local apk="$1"
|
||||
local output="$2"
|
||||
local py
|
||||
py="$(python_bin)"
|
||||
"$py" - "$apk" "$output" <<'PY'
|
||||
import pathlib
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
apk_path = pathlib.Path(sys.argv[1])
|
||||
output_path = pathlib.Path(sys.argv[2])
|
||||
with zipfile.ZipFile(apk_path) as archive:
|
||||
candidates = [
|
||||
name for name in archive.namelist()
|
||||
if name.endswith("/libapp.so") and "arm64-v8a/" in name
|
||||
]
|
||||
if not candidates:
|
||||
raise SystemExit(f"missing arm64 libapp.so in {apk_path}")
|
||||
output_path.write_bytes(archive.read(candidates[0]))
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_text() {
|
||||
local expected="$1"
|
||||
local dump="$WORK_DIR/window.xml"
|
||||
for _ in $(seq 1 "${ANDROID_UI_WAIT_ATTEMPTS:-80}"); do
|
||||
adb_shell uiautomator dump /sdcard/open_shorebird_window.xml >/dev/null 2>&1 || true
|
||||
adb_cmd exec-out cat /sdcard/open_shorebird_window.xml >"$dump" 2>/dev/null || true
|
||||
if grep -q "$expected" "$dump"; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
echo "Timed out waiting for Android UI text: $expected" >&2
|
||||
echo "Last uiautomator dump:" >&2
|
||||
sed -n '1,120p' "$dump" >&2 || true
|
||||
return 70
|
||||
}
|
||||
|
||||
start_app() {
|
||||
adb_shell am force-stop "$PACKAGE" >/dev/null 2>&1 || true
|
||||
adb_shell am start -W -n "$PACKAGE/$ACTIVITY" >/dev/null
|
||||
}
|
||||
|
||||
can_seed_with_run_as() {
|
||||
adb_shell run-as "$PACKAGE" sh -c 'test -d files' >/dev/null 2>&1
|
||||
}
|
||||
|
||||
can_seed_with_root() {
|
||||
if adb_shell sh -c 'test "$(id -u)" = "0"' >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
adb_cmd root >/dev/null 2>&1 || true
|
||||
adb_cmd wait-for-device >/dev/null 2>&1 || true
|
||||
adb_shell sh -c 'test "$(id -u)" = "0"' >/dev/null 2>&1
|
||||
}
|
||||
|
||||
seed_with_run_as() {
|
||||
adb_shell run-as "$PACKAGE" sh -c \
|
||||
"mkdir -p files && rm -rf files/shorebird_updater && cp -R '$SEED_REMOTE_DIR/shorebird_updater' files/"
|
||||
}
|
||||
|
||||
seed_with_root() {
|
||||
local target="/data/data/$PACKAGE/files"
|
||||
local owner
|
||||
owner="$(adb_shell sh -c "stat -c '%u:%g' '$target'" | tr -d '\r')"
|
||||
adb_shell sh -c \
|
||||
"rm -rf '$target/shorebird_updater' && cp -R '$SEED_REMOTE_DIR/shorebird_updater' '$target/' && chown -R '$owner' '$target/shorebird_updater'"
|
||||
}
|
||||
|
||||
prepare_seed() {
|
||||
local libapp="$1"
|
||||
local seed_root="$WORK_DIR/seed/shorebird_updater"
|
||||
local size
|
||||
size="$(wc -c <"$libapp" | tr -d ' ')"
|
||||
rm -rf "$WORK_DIR/seed"
|
||||
mkdir -p "$seed_root/patches/1"
|
||||
cp "$libapp" "$seed_root/patches/1/dlc.vmcode"
|
||||
cat >"$seed_root/state.json" <<EOF
|
||||
{
|
||||
"client_id": "android-runtime-smoke",
|
||||
"release_version": "$RELEASE_VERSION",
|
||||
"queued_events": []
|
||||
}
|
||||
EOF
|
||||
cat >"$seed_root/pointers.json" <<'EOF'
|
||||
{
|
||||
"next_boot_patch": 1,
|
||||
"last_booted_patch": null,
|
||||
"currently_booting_patch": null,
|
||||
"boot_started_at": null
|
||||
}
|
||||
EOF
|
||||
cat >"$seed_root/patches/1/state.json" <<EOF
|
||||
{
|
||||
"kind": "Installed",
|
||||
"signature": null,
|
||||
"size": $size
|
||||
}
|
||||
EOF
|
||||
adb_shell rm -rf "$SEED_REMOTE_DIR" >/dev/null 2>&1 || true
|
||||
adb_shell mkdir -p "$SEED_REMOTE_DIR" >/dev/null
|
||||
adb_cmd push "$seed_root" "$SEED_REMOTE_DIR/" >/dev/null
|
||||
echo "seeded_patch_size=$size"
|
||||
}
|
||||
|
||||
require_tool "$ADB_BIN"
|
||||
require_tool "$FLUTTER_BIN"
|
||||
require_tool java
|
||||
|
||||
adb_cmd start-server >/dev/null
|
||||
device_count="$(adb_cmd devices | awk 'NR > 1 && $2 == "device" { count++ } END { print count + 0 }')"
|
||||
if [[ "$device_count" -lt 1 ]]; then
|
||||
echo "No Android device or emulator is connected." >&2
|
||||
echo "Start an emulator/device, then rerun this script. Use ANDROID_SERIAL to pick a device." >&2
|
||||
exit 69
|
||||
fi
|
||||
if [[ "$device_count" -gt 1 && -z "${ANDROID_SERIAL:-}" ]]; then
|
||||
echo "Multiple Android devices are connected; set ANDROID_SERIAL." >&2
|
||||
adb_cmd devices -l >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
free_apk="$WORK_DIR/free.apk"
|
||||
pro_apk="$WORK_DIR/pro.apk"
|
||||
pro_libapp="$WORK_DIR/pro-libapp.so"
|
||||
|
||||
build_apk free "$free_apk"
|
||||
build_apk pro "$pro_apk"
|
||||
extract_libapp "$pro_apk" "$pro_libapp"
|
||||
|
||||
adb_cmd uninstall "$PACKAGE" >/dev/null 2>&1 || true
|
||||
adb_cmd install -r "$free_apk" >/dev/null
|
||||
|
||||
start_app
|
||||
wait_for_text 'license:free'
|
||||
wait_for_text 'pro-feature:off'
|
||||
echo "android_base_status=license:free/pro-feature:off"
|
||||
|
||||
prepare_seed "$pro_libapp"
|
||||
if can_seed_with_run_as; then
|
||||
seed_with_run_as
|
||||
echo "android_seed_mode=run-as"
|
||||
elif can_seed_with_root; then
|
||||
seed_with_root
|
||||
echo "android_seed_mode=root"
|
||||
else
|
||||
cat >&2 <<EOF
|
||||
Unable to seed the Android app-private updater directory.
|
||||
|
||||
Use a debuggable build/device where 'adb shell run-as $PACKAGE' works, or use
|
||||
a rooted emulator/device where 'adb root' works. The smoke seed target is:
|
||||
/data/data/$PACKAGE/files/shorebird_updater
|
||||
EOF
|
||||
exit 77
|
||||
fi
|
||||
|
||||
start_app
|
||||
wait_for_text 'license:pro'
|
||||
wait_for_text 'pro-feature:enabled'
|
||||
echo "android_patch_status=license:pro/pro-feature:enabled"
|
||||
|
||||
adb_shell am force-stop "$PACKAGE" >/dev/null 2>&1 || true
|
||||
echo "android_runtime_patch_smoke=passed"
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: assemble_artifact_mirror.sh <downloaded-workflow-artifacts-dir> <output-mirror-root>
|
||||
|
||||
Copies every publish-ready shorebird/ mirror subtree from downloaded GitHub
|
||||
Actions artifacts into <output-mirror-root>. Engine archives are extracted and
|
||||
scanned for nested mirror/shorebird trees. Existing files may be reused only
|
||||
when their bytes match.
|
||||
EOF
|
||||
}
|
||||
|
||||
if [[ "$#" -ne 2 ]]; then
|
||||
usage
|
||||
exit 64
|
||||
fi
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
INPUT_DIR="$1"
|
||||
OUTPUT_DIR="$2"
|
||||
|
||||
if [[ ! -d "$INPUT_DIR" ]]; then
|
||||
echo "input artifact directory does not exist: $INPUT_DIR" >&2
|
||||
exit 66
|
||||
fi
|
||||
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-mirror.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
FOUND_TREES=0
|
||||
|
||||
copy_shorebird_tree() {
|
||||
local tree="$1"
|
||||
local source_file rel_file target_file
|
||||
|
||||
FOUND_TREES=$((FOUND_TREES + 1))
|
||||
while IFS= read -r -d '' source_file; do
|
||||
rel_file="${source_file#"$tree"/}"
|
||||
target_file="$OUTPUT_DIR/shorebird/$rel_file"
|
||||
mkdir -p "$(dirname "$target_file")"
|
||||
if [[ -e "$target_file" ]]; then
|
||||
if ! cmp -s "$source_file" "$target_file"; then
|
||||
echo "conflicting mirror file: shorebird/$rel_file" >&2
|
||||
echo " existing: $target_file" >&2
|
||||
echo " incoming: $source_file" >&2
|
||||
exit 70
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
cp -p "$source_file" "$target_file"
|
||||
done < <(find "$tree" -type f -print0)
|
||||
}
|
||||
|
||||
scan_for_shorebird_trees() {
|
||||
local search_root="$1"
|
||||
local tree
|
||||
|
||||
while IFS= read -r -d '' tree; do
|
||||
copy_shorebird_tree "$tree"
|
||||
done < <(find "$search_root" -type d -name shorebird -print0)
|
||||
}
|
||||
|
||||
scan_for_shorebird_trees "$INPUT_DIR"
|
||||
|
||||
archive_index=0
|
||||
while IFS= read -r -d '' archive_path; do
|
||||
archive_index=$((archive_index + 1))
|
||||
extract_dir="$TMP_DIR/archive-$archive_index"
|
||||
mkdir -p "$extract_dir"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" "$archive_path" "$extract_dir"
|
||||
scan_for_shorebird_trees "$extract_dir"
|
||||
done < <(find "$INPUT_DIR" -type f \( -name '*.tar.gz' -o -name '*.tgz' \) -print0)
|
||||
|
||||
if [[ "$FOUND_TREES" -eq 0 ]]; then
|
||||
echo "no shorebird/ mirror subtrees found under $INPUT_DIR" >&2
|
||||
exit 65
|
||||
fi
|
||||
|
||||
while IFS= read -r -d '' mirror_file; do
|
||||
if [[ "$mirror_file" == *.sha256 ]]; then
|
||||
continue
|
||||
fi
|
||||
sidecar="$mirror_file.sha256"
|
||||
if [[ ! -f "$sidecar" ]]; then
|
||||
"$ROOT/scripts/write_sha256.sh" "$mirror_file" "$sidecar"
|
||||
fi
|
||||
done < <(find "$OUTPUT_DIR/shorebird" -type f -print0)
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$OUTPUT_DIR"
|
||||
|
||||
echo "assembled artifact mirror at $OUTPUT_DIR"
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MIN_FREE_DISK_GB="${CI_MIN_FREE_DISK_GB:-0}"
|
||||
CHECK_PATH="${CI_CAPACITY_PATH:-$PWD}"
|
||||
AVAILABLE_DISK_KB_OVERRIDE="${CI_AVAILABLE_DISK_KB_OVERRIDE:-}"
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 70
|
||||
}
|
||||
|
||||
case "$MIN_FREE_DISK_GB" in
|
||||
''|*[!0-9]*)
|
||||
fail "CI_MIN_FREE_DISK_GB must be a non-negative integer, got '$MIN_FREE_DISK_GB'"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$MIN_FREE_DISK_GB" == "0" ]]; then
|
||||
echo "CI capacity check skipped because CI_MIN_FREE_DISK_GB=0"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! -e "$CHECK_PATH" ]]; then
|
||||
fail "capacity check path does not exist: $CHECK_PATH"
|
||||
fi
|
||||
|
||||
available_disk_kb() {
|
||||
if [[ -n "$AVAILABLE_DISK_KB_OVERRIDE" ]]; then
|
||||
case "$AVAILABLE_DISK_KB_OVERRIDE" in
|
||||
*[!0-9]*)
|
||||
fail "CI_AVAILABLE_DISK_KB_OVERRIDE must be an integer, got '$AVAILABLE_DISK_KB_OVERRIDE'"
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' "$AVAILABLE_DISK_KB_OVERRIDE"
|
||||
return
|
||||
fi
|
||||
df -Pk "$CHECK_PATH" | awk 'NR == 2 { print $4 }'
|
||||
}
|
||||
|
||||
available_kb="$(available_disk_kb)"
|
||||
case "$available_kb" in
|
||||
''|*[!0-9]*)
|
||||
fail "could not determine available disk space for $CHECK_PATH"
|
||||
;;
|
||||
esac
|
||||
|
||||
required_kb=$((MIN_FREE_DISK_GB * 1024 * 1024))
|
||||
available_gb=$((available_kb / 1024 / 1024))
|
||||
|
||||
echo "CI capacity: ${available_gb} GiB free at $CHECK_PATH; required: ${MIN_FREE_DISK_GB} GiB"
|
||||
|
||||
if (( available_kb < required_kb )); then
|
||||
fail "runner has insufficient free disk for the heavy SDK/engine build. Use a larger or self-hosted runner label, or free disk before this step."
|
||||
fi
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${GITHUB_ACTIONS:-}" != "true" ]]; then
|
||||
echo "Refusing to free disk outside GitHub Actions." >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
if [[ "$(uname -s)" != "Linux" ]]; then
|
||||
echo "Skipping Linux disk cleanup on $(uname -s)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${CI_FREE_DISK_SPACE:-1}" == "0" ]]; then
|
||||
echo "Skipping disk cleanup because CI_FREE_DISK_SPACE=0."
|
||||
df -h
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_ENVIRONMENT:-github-hosted}" != "github-hosted" &&
|
||||
"${CI_FREE_DISK_SPACE_FORCE:-0}" != "1" ]]; then
|
||||
echo "Skipping disk cleanup on ${RUNNER_ENVIRONMENT} runner."
|
||||
echo "Set CI_FREE_DISK_SPACE_FORCE=1 to opt in on non-hosted runners."
|
||||
df -h
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Disk before cleanup:"
|
||||
df -h
|
||||
|
||||
# GitHub-hosted Ubuntu images include large toolchains that are unrelated to
|
||||
# Dart SDK, Flutter engine, and updater builds. Remove only well-known cache
|
||||
# directories on ephemeral GitHub Actions runners.
|
||||
for path in \
|
||||
/opt/ghc \
|
||||
/opt/hostedtoolcache/CodeQL \
|
||||
/usr/local/.ghcup \
|
||||
/usr/local/lib/android/sdk \
|
||||
/usr/local/share/boost \
|
||||
/usr/share/dotnet; do
|
||||
if [[ -e "$path" ]]; then
|
||||
echo "Removing $path"
|
||||
sudo rm -rf "$path"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Disk after cleanup:"
|
||||
df -h
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SOURCE_APP_DIR="${SOURCE_APP_DIR:-$ROOT/testapps/license_flavor_patch_test}"
|
||||
FLUTTER_BIN="${FLUTTER_BIN:-$ROOT/flutter/bin/flutter}"
|
||||
APP_ID="${SHOREBIRD_APP_ID:-license-flavor-patch-test}"
|
||||
LOCAL_ENGINE_SRC_PATH="${LOCAL_ENGINE_SRC_PATH:-$ROOT/flutter/engine/src}"
|
||||
DEFAULT_LOCAL_ENGINE="linux_release_x64"
|
||||
if [[ ! -d "$LOCAL_ENGINE_SRC_PATH/out/$DEFAULT_LOCAL_ENGINE" &&
|
||||
-d "$LOCAL_ENGINE_SRC_PATH/out/host_release" ]]; then
|
||||
DEFAULT_LOCAL_ENGINE="host_release"
|
||||
fi
|
||||
LOCAL_ENGINE="${LOCAL_ENGINE:-$DEFAULT_LOCAL_ENGINE}"
|
||||
LOCAL_ENGINE_HOST="${LOCAL_ENGINE_HOST:-$LOCAL_ENGINE}"
|
||||
WORK_DIR="${LINUX_RUNTIME_SMOKE_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-linux-runtime.XXXXXX")}"
|
||||
APP_COPY="$WORK_DIR/app"
|
||||
HOME_DIR="$WORK_DIR/home"
|
||||
|
||||
if [[ "${KEEP_LINUX_RUNTIME_SMOKE_ARTIFACTS:-0}" != "1" ]]; then
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
fi
|
||||
|
||||
require_tool() {
|
||||
local tool="$1"
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
echo "$tool is required" >&2
|
||||
exit 127
|
||||
fi
|
||||
}
|
||||
|
||||
python_bin() {
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
printf '%s\n' python3
|
||||
else
|
||||
printf '%s\n' python
|
||||
fi
|
||||
}
|
||||
|
||||
copy_app_fixture() {
|
||||
mkdir -p "$APP_COPY"
|
||||
(
|
||||
cd "$SOURCE_APP_DIR"
|
||||
tar \
|
||||
--exclude='./build' \
|
||||
--exclude='./.dart_tool' \
|
||||
--exclude='./android/.gradle' \
|
||||
--exclude='./ios/Pods' \
|
||||
--exclude='./macos/Flutter/ephemeral' \
|
||||
-cf - .
|
||||
) | tar -C "$APP_COPY" -xf -
|
||||
}
|
||||
|
||||
ensure_linux_platform() {
|
||||
if [[ -d "$APP_COPY/linux" ]]; then
|
||||
return
|
||||
fi
|
||||
(
|
||||
cd "$APP_COPY"
|
||||
"$FLUTTER_BIN" create --platforms=linux --project-name=license_flavor_patch_test .
|
||||
)
|
||||
}
|
||||
|
||||
build_linux_bundle() {
|
||||
local license="$1"
|
||||
local output="$2"
|
||||
(
|
||||
cd "$APP_COPY"
|
||||
"$FLUTTER_BIN" build linux --release \
|
||||
--local-engine-src-path="$LOCAL_ENGINE_SRC_PATH" \
|
||||
--local-engine="$LOCAL_ENGINE" \
|
||||
--local-engine-host="$LOCAL_ENGINE_HOST" \
|
||||
--dart-define="LICENSE_TYPE=$license"
|
||||
)
|
||||
local bundle
|
||||
bundle="$(find "$APP_COPY/build/linux" -path '*/release/bundle' -type d -print -quit)"
|
||||
if [[ -z "$bundle" || ! -x "$bundle/license_flavor_patch_test" ]]; then
|
||||
echo "Failed to find Linux release bundle under $APP_COPY/build/linux" >&2
|
||||
exit 66
|
||||
fi
|
||||
rm -rf "$output"
|
||||
cp -a "$bundle" "$output"
|
||||
}
|
||||
|
||||
release_version_for_bundle() {
|
||||
local bundle="$1"
|
||||
local version_json="$bundle/data/flutter_assets/version.json"
|
||||
local py
|
||||
py="$(python_bin)"
|
||||
"$py" - "$version_json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
version = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
build_name = str(version.get("version", ""))
|
||||
build_number = str(version.get("build_number", ""))
|
||||
if build_number:
|
||||
print(f"{build_name}+{build_number}")
|
||||
else:
|
||||
print(build_name)
|
||||
PY
|
||||
}
|
||||
|
||||
run_saved_app() {
|
||||
local label="$1"
|
||||
local bundle="$2"
|
||||
local stdout="$WORK_DIR/$label.stdout"
|
||||
local stderr="$WORK_DIR/$label.stderr"
|
||||
local tmp_status="$WORK_DIR/tmp/license_flavor_patch_status.txt"
|
||||
local home_status="$HOME_DIR/Library/Application Support/license_flavor_patch_status.txt"
|
||||
|
||||
rm -f "$tmp_status" "$home_status"
|
||||
mkdir -p "$WORK_DIR/tmp" "$HOME_DIR"
|
||||
|
||||
local -a app_command=(env HOME="$HOME_DIR" TMPDIR="$WORK_DIR/tmp" "$bundle/license_flavor_patch_test")
|
||||
if [[ "${LINUX_RUNTIME_SMOKE_XVFB:-auto}" != "0" && -z "${DISPLAY:-}" && "$(command -v xvfb-run || true)" != "" ]]; then
|
||||
app_command=(xvfb-run -a "${app_command[@]}")
|
||||
fi
|
||||
|
||||
"${app_command[@]}" >"$stdout" 2>"$stderr" &
|
||||
local pid=$!
|
||||
for _ in $(seq 1 "${LINUX_RUNTIME_WAIT_ATTEMPTS:-120}"); do
|
||||
if [[ -f "$tmp_status" || -f "$home_status" ]]; then
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
else
|
||||
wait "$pid" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "${label}_stdout=$stdout"
|
||||
echo "${label}_stderr=$stderr"
|
||||
if [[ -f "$tmp_status" ]]; then
|
||||
echo "${label}_status_file=$tmp_status"
|
||||
cat "$tmp_status"
|
||||
elif [[ -f "$home_status" ]]; then
|
||||
echo "${label}_status_file=$home_status"
|
||||
cat "$home_status"
|
||||
else
|
||||
echo "${label}_status_file_missing" >&2
|
||||
echo "--- $label stderr ---" >&2
|
||||
sed -n '1,180p' "$stderr" >&2 || true
|
||||
return 70
|
||||
fi
|
||||
}
|
||||
|
||||
read_launch_status() {
|
||||
local tmp_status="$WORK_DIR/tmp/license_flavor_patch_status.txt"
|
||||
local home_status="$HOME_DIR/Library/Application Support/license_flavor_patch_status.txt"
|
||||
if [[ -f "$tmp_status" ]]; then
|
||||
cat "$tmp_status"
|
||||
elif [[ -f "$home_status" ]]; then
|
||||
cat "$home_status"
|
||||
fi
|
||||
}
|
||||
|
||||
require_status() {
|
||||
local label="$1"
|
||||
local expected_license="$2"
|
||||
local expected_feature="$3"
|
||||
local status
|
||||
status="$(read_launch_status)"
|
||||
if ! grep -q "license:$expected_license" <<<"$status" ||
|
||||
! grep -q "pro-feature:$expected_feature" <<<"$status"; then
|
||||
echo "$label Linux app did not report expected status." >&2
|
||||
echo "Expected: license:$expected_license / pro-feature:$expected_feature" >&2
|
||||
echo "Actual:" >&2
|
||||
printf '%s\n' "$status" >&2
|
||||
exit 70
|
||||
fi
|
||||
}
|
||||
|
||||
seed_patch() {
|
||||
local pro_bundle="$1"
|
||||
local release_version="$2"
|
||||
local patch_file="$pro_bundle/lib/libapp.so"
|
||||
local state_root="$HOME_DIR/.shorebird_cache/shorebird_updater/$APP_ID"
|
||||
local size
|
||||
size="$(wc -c <"$patch_file" | tr -d ' ')"
|
||||
|
||||
rm -rf "$state_root"
|
||||
mkdir -p "$state_root/patches/1"
|
||||
cp "$patch_file" "$state_root/patches/1/dlc.vmcode"
|
||||
cat >"$state_root/state.json" <<EOF
|
||||
{
|
||||
"client_id": "linux-runtime-smoke",
|
||||
"release_version": "$release_version",
|
||||
"queued_events": []
|
||||
}
|
||||
EOF
|
||||
cat >"$state_root/pointers.json" <<'EOF'
|
||||
{
|
||||
"next_boot_patch": 1,
|
||||
"last_booted_patch": null,
|
||||
"currently_booting_patch": null,
|
||||
"boot_started_at": null
|
||||
}
|
||||
EOF
|
||||
cat >"$state_root/patches/1/state.json" <<EOF
|
||||
{
|
||||
"kind": "Installed",
|
||||
"signature": null,
|
||||
"size": $size
|
||||
}
|
||||
EOF
|
||||
echo "linux_seeded_patch=$state_root/patches/1/dlc.vmcode"
|
||||
echo "linux_seeded_patch_size=$size"
|
||||
}
|
||||
|
||||
require_tool "$FLUTTER_BIN"
|
||||
require_tool tar
|
||||
|
||||
copy_app_fixture
|
||||
ensure_linux_platform
|
||||
|
||||
free_bundle="$WORK_DIR/free-bundle"
|
||||
pro_bundle="$WORK_DIR/pro-bundle"
|
||||
|
||||
build_linux_bundle free "$free_bundle"
|
||||
build_linux_bundle pro "$pro_bundle"
|
||||
|
||||
release_version="$(release_version_for_bundle "$free_bundle")"
|
||||
|
||||
run_saved_app base "$free_bundle"
|
||||
require_status base free off
|
||||
seed_patch "$pro_bundle" "$release_version"
|
||||
run_saved_app patch "$free_bundle"
|
||||
require_status patch pro enabled
|
||||
|
||||
echo "linux_runtime_patch_smoke=passed"
|
||||
@@ -6,6 +6,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DART_BIN="${DART_BIN:-dart}"
|
||||
FLUTTER_BIN="${FLUTTER_BIN:-flutter}"
|
||||
GO_BIN="${GO_BIN:-go}"
|
||||
CARGO_BIN="${CARGO_BIN:-cargo}"
|
||||
|
||||
run() {
|
||||
echo
|
||||
@@ -23,6 +24,7 @@ require_command() {
|
||||
require_command git
|
||||
require_command "$DART_BIN"
|
||||
require_command "$GO_BIN"
|
||||
require_command "$CARGO_BIN"
|
||||
|
||||
if [[ "$PLATFORM" == "macos" ]]; then
|
||||
if command -v xcodebuild >/dev/null 2>&1; then
|
||||
@@ -37,9 +39,13 @@ run "$ROOT/scripts/write_gclient.sh" "$PLATFORM"
|
||||
run "$ROOT/scripts/sync_open_sources.sh"
|
||||
|
||||
export PATH="$ROOT/depot_tools:$PATH"
|
||||
export DEPOT_TOOLS_UPDATE="${DEPOT_TOOLS_UPDATE:-0}"
|
||||
if [[ "${SKIP_GCLIENT_SYNC:-0}" != "1" ]]; then
|
||||
require_command gclient
|
||||
run gclient sync --no-history
|
||||
if [[ "${INCLUDE_ENGINE_DEPS:-0}" == "1" && -f "$ROOT/flutter/.gclient" ]]; then
|
||||
run bash -lc "cd '$ROOT/flutter' && gclient sync --no-history"
|
||||
fi
|
||||
else
|
||||
echo
|
||||
echo "==> skipping gclient sync because SKIP_GCLIENT_SYNC=1"
|
||||
@@ -51,8 +57,12 @@ if [[ "${SKIP_TESTS:-0}" == "1" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
run bash -lc "cd '$ROOT/shorebird/packages/shorebird_cli' && '$DART_BIN' pub get && '$DART_BIN' test test/src/user_config_test.dart test/src/shorebird_env_test.dart test/src/shorebird_cli_command_runner_test.dart test/src/commands/init_command_test.dart"
|
||||
run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' pub get"
|
||||
run bash -lc "cd '$ROOT/shorebird/packages/shorebird_cli' && '$DART_BIN' pub get && '$DART_BIN' test test/src/user_config_test.dart test/src/shorebird_env_test.dart test/src/shorebird_cli_command_runner_test.dart test/src/commands/doctor_command_test.dart test/src/commands/init_command_test.dart test/src/cache_test.dart test/src/shorebird_process_test.dart test/src/network_checker_test.dart test/src/shorebird_web_console_test.dart test/src/auth/auth_test.dart test/src/commands/login_command_test.dart test/src/commands/login_ci_command_test.dart test/src/commands/release/aar_releaser_test.dart test/src/shorebird_validator_test.dart test/src/shorebird_flutter_test.dart test/src/shorebird_artifacts_test.dart test/src/artifact_builder/artifact_builder_test.dart test/src/config/shorebird_yaml_test.dart test/src/commands/patch/ios_patcher_test.dart"
|
||||
run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' test packages/shorebird_code_push_client/test/src/code_push_client_test.dart"
|
||||
run bash -lc "cd '$ROOT/shorebird' && '$DART_BIN' test packages/artifact_proxy/test/artifact_proxy_test.dart packages/artifact_proxy/test/server_bin_test.dart packages/artifact_proxy/test/src/artifact_manifest_client_test.dart"
|
||||
run bash -lc "cd '$ROOT/shorebird/packages/open_aot_patch_tools' && '$DART_BIN' pub get && '$DART_BIN' test"
|
||||
run "$CARGO_BIN" test --manifest-path "$ROOT/updater/library/Cargo.toml"
|
||||
run bash -lc "cd '$ROOT/shorebird-server' && '$GO_BIN' test ./..."
|
||||
|
||||
AOT_PATCH_BUILD_DIR="${AOT_PATCH_BUILD_DIR:-}"
|
||||
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path, PurePosixPath
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
print(f"error: {message}", file=sys.stderr)
|
||||
raise SystemExit(70)
|
||||
|
||||
|
||||
def safe_member_name(member: tarfile.TarInfo) -> str | None:
|
||||
name = member.name
|
||||
if member.isdir():
|
||||
name = name.rstrip("/")
|
||||
if not name or name == ".":
|
||||
return None
|
||||
if "\\" in name or "\x00" in name:
|
||||
return None
|
||||
if any(ord(character) < 32 for character in name):
|
||||
return None
|
||||
|
||||
candidate = PurePosixPath(name)
|
||||
if candidate.is_absolute():
|
||||
return None
|
||||
if any(part in ("", ".", "..") for part in candidate.parts):
|
||||
return None
|
||||
if candidate.parts and ":" in candidate.parts[0]:
|
||||
return None
|
||||
return candidate.as_posix()
|
||||
|
||||
|
||||
def validate_members(archive_path: Path, members: list[tarfile.TarInfo]) -> None:
|
||||
member_types: dict[str, str] = {}
|
||||
for member in members:
|
||||
safe_name = safe_member_name(member)
|
||||
if safe_name is None:
|
||||
fail(f"{archive_path}: unsafe archive member path {member.name!r}")
|
||||
if not (member.isdir() or member.isfile()):
|
||||
fail(f"{archive_path}: unsupported archive member type {member.name!r}")
|
||||
|
||||
member_type = "dir" if member.isdir() else "file"
|
||||
previous_type = member_types.get(safe_name)
|
||||
if previous_type is None:
|
||||
member_types[safe_name] = member_type
|
||||
continue
|
||||
if previous_type != "dir" or member_type != "dir":
|
||||
fail(f"{archive_path}: duplicate archive member path {member.name!r}")
|
||||
|
||||
|
||||
def ensure_within_root(archive_path: Path, extract_root: Path, target: Path, name: str) -> None:
|
||||
try:
|
||||
target.relative_to(extract_root)
|
||||
except ValueError:
|
||||
fail(f"{archive_path}: archive member escapes extraction root {name!r}")
|
||||
|
||||
|
||||
def extract_safe_tar_archive(archive_path: Path, extract_dir: Path) -> None:
|
||||
extract_root = extract_dir.resolve()
|
||||
extracted_files: set[Path] = set()
|
||||
|
||||
try:
|
||||
with tarfile.open(archive_path, "r:*") as archive:
|
||||
members = archive.getmembers()
|
||||
validate_members(archive_path, members)
|
||||
|
||||
for member in members:
|
||||
safe_name = safe_member_name(member)
|
||||
assert safe_name is not None
|
||||
target = (extract_root / safe_name).resolve()
|
||||
ensure_within_root(archive_path, extract_root, target, member.name)
|
||||
|
||||
if member.isdir():
|
||||
if target in extracted_files:
|
||||
fail(f"{archive_path}: directory collides with file {member.name!r}")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
|
||||
for parent in target.parents:
|
||||
if parent == extract_root:
|
||||
break
|
||||
if parent in extracted_files:
|
||||
fail(f"{archive_path}: file parent collides with file {member.name!r}")
|
||||
|
||||
if target.exists() and not target.is_file():
|
||||
fail(f"{archive_path}: file collides with directory {member.name!r}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = archive.extractfile(member)
|
||||
if source is None:
|
||||
fail(f"{archive_path}: unable to read archive member {member.name!r}")
|
||||
with source, target.open("wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
target.chmod(member.mode & 0o777)
|
||||
extracted_files.add(target)
|
||||
except tarfile.TarError as error:
|
||||
fail(f"{archive_path}: invalid tar archive: {error}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Safely extract a tar archive containing only files and directories.",
|
||||
)
|
||||
parser.add_argument("archive", type=Path)
|
||||
parser.add_argument("extract_dir", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
args.extract_dir.mkdir(parents=True, exist_ok=True)
|
||||
extract_safe_tar_archive(args.archive, args.extract_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+127
-47
@@ -5,13 +5,99 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DART_SRC="${DART_SRC:-$ROOT/dart-sdk-new}"
|
||||
DART_TARGET="$ROOT/flutter/engine/src/flutter/third_party/dart"
|
||||
UPDATER_SRC="${UPDATER_SRC:-$ROOT/updater}"
|
||||
UPDATER_URL="${UPDATER_URL:-https://github.com/shorebirdtech/updater.git}"
|
||||
UPDATER_URL="${UPDATER_URL:-}"
|
||||
TARGET="$ROOT/flutter/engine/src/flutter/third_party/updater"
|
||||
|
||||
is_git_checkout() {
|
||||
git -C "$1" rev-parse --git-dir >/dev/null 2>&1
|
||||
}
|
||||
|
||||
relative_path() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
|
||||
PY
|
||||
}
|
||||
|
||||
real_path() {
|
||||
python3 - "$1" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
print(os.path.realpath(sys.argv[1]))
|
||||
PY
|
||||
}
|
||||
|
||||
link_checkout() {
|
||||
local target="$1"
|
||||
local source="$2"
|
||||
local label="$3"
|
||||
local rel_target
|
||||
rel_target="$(relative_path "$target" "$source")"
|
||||
ln -s "$rel_target" "$target"
|
||||
echo "[open-source-sync] linked $label checkout into Flutter engine."
|
||||
}
|
||||
|
||||
is_clean_git_checkout() {
|
||||
[[ -z "$(git -C "$1" status --porcelain)" ]]
|
||||
}
|
||||
|
||||
reject_forbidden_remotes() {
|
||||
local source="$1"
|
||||
local label="$2"
|
||||
shift 2
|
||||
|
||||
local remotes
|
||||
remotes="$(git -C "$source" remote -v 2>/dev/null || true)"
|
||||
if [[ -z "$remotes" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local forbidden
|
||||
for forbidden in "$@"; do
|
||||
if grep -Fq "$forbidden" <<<"$remotes"; then
|
||||
echo "$label source checkout uses forbidden remote fragment '$forbidden': $source" >&2
|
||||
echo "$remotes" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
ensure_source_link() {
|
||||
local target="$1"
|
||||
local source="$2"
|
||||
local label="$3"
|
||||
|
||||
if [[ ! -d "$source" ]] || ! is_git_checkout "$source"; then
|
||||
echo "$label source checkout is missing: $source" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -L "$target" ]]; then
|
||||
local target_real
|
||||
local source_real
|
||||
target_real="$(real_path "$target")"
|
||||
source_real="$(real_path "$source")"
|
||||
if [[ "$target_real" != "$source_real" ]]; then
|
||||
echo "$label target symlink points at $target_real, expected $source_real" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[open-source-sync] $label target already links to the workspace checkout."
|
||||
elif is_git_checkout "$target"; then
|
||||
if ! is_clean_git_checkout "$target"; then
|
||||
echo "$label target is a dirty git checkout and cannot be replaced: $target" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -rf "$target"
|
||||
link_checkout "$target" "$source" "$label"
|
||||
elif [[ -e "$target" ]]; then
|
||||
echo "$label target exists but is not a symlink or git checkout: $target" >&2
|
||||
exit 1
|
||||
else
|
||||
link_checkout "$target" "$source" "$label"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "[open-source-sync] dart source: $DART_SRC"
|
||||
echo "[open-source-sync] dart target: $DART_TARGET"
|
||||
echo "[open-source-sync] updater source: $UPDATER_SRC"
|
||||
@@ -20,60 +106,54 @@ echo "[open-source-sync] target: $TARGET"
|
||||
mkdir -p "$(dirname "$DART_TARGET")"
|
||||
mkdir -p "$(dirname "$TARGET")"
|
||||
|
||||
if [[ -L "$DART_TARGET" ]]; then
|
||||
echo "[open-source-sync] Dart target is already a symlink."
|
||||
elif is_git_checkout "$DART_TARGET"; then
|
||||
echo "[open-source-sync] Dart target is already a git checkout."
|
||||
elif [[ -e "$DART_TARGET" ]]; then
|
||||
echo "Dart target exists but is not a symlink or git checkout: $DART_TARGET" >&2
|
||||
exit 1
|
||||
elif is_git_checkout "$DART_SRC"; then
|
||||
rel_target="$(python3 - "$DART_TARGET" "$DART_SRC" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
|
||||
PY
|
||||
)"
|
||||
ln -s "$rel_target" "$DART_TARGET"
|
||||
echo "[open-source-sync] linked Dart SDK checkout into Flutter engine."
|
||||
else
|
||||
echo "Dart source checkout is missing: $DART_SRC" >&2
|
||||
exit 1
|
||||
fi
|
||||
ensure_source_link "$DART_TARGET" "$DART_SRC" "Dart SDK"
|
||||
reject_forbidden_remotes \
|
||||
"$DART_SRC" \
|
||||
"Dart SDK" \
|
||||
"github.com/dart-lang/sdk" \
|
||||
"dart.googlesource.com/sdk"
|
||||
|
||||
if [[ ! -f "$DART_TARGET/runtime/vm/dart_api_impl.h" ]]; then
|
||||
echo "Dart checkout is missing runtime/vm/dart_api_impl.h" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -L "$TARGET" ]]; then
|
||||
echo "[open-source-sync] updater target is already a symlink."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if is_git_checkout "$TARGET"; then
|
||||
echo "[open-source-sync] updating existing updater checkout."
|
||||
git -C "$TARGET" fetch --tags origin
|
||||
git -C "$TARGET" checkout "${UPDATER_REVISION:-main}"
|
||||
if [[ "${UPDATER_REVISION:-main}" == "main" ]]; then
|
||||
git -C "$TARGET" pull --ff-only
|
||||
if is_git_checkout "$UPDATER_SRC"; then
|
||||
ensure_source_link "$TARGET" "$UPDATER_SRC" "updater submodule"
|
||||
reject_forbidden_remotes \
|
||||
"$UPDATER_SRC" \
|
||||
"updater submodule" \
|
||||
"github.com/shorebirdtech/updater" \
|
||||
"github.com/shorebirdtech/shorebird-updater"
|
||||
elif [[ -n "$UPDATER_URL" ]]; then
|
||||
if [[ "$UPDATER_URL" == *github.com/shorebirdtech/updater* ||
|
||||
"$UPDATER_URL" == *github.com/shorebirdtech/shorebird-updater* ]]; then
|
||||
echo "UPDATER_URL points at a forbidden official Shorebird updater remote: $UPDATER_URL" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -L "$TARGET" ]]; then
|
||||
rm "$TARGET"
|
||||
fi
|
||||
if is_git_checkout "$TARGET"; then
|
||||
echo "[open-source-sync] updating existing updater checkout."
|
||||
git -C "$TARGET" remote set-url origin "$UPDATER_URL"
|
||||
git -C "$TARGET" fetch --tags origin
|
||||
git -C "$TARGET" checkout "${UPDATER_REVISION:-main}"
|
||||
if [[ "${UPDATER_REVISION:-main}" == "main" ]]; then
|
||||
git -C "$TARGET" pull --ff-only
|
||||
fi
|
||||
elif [[ -e "$TARGET" ]]; then
|
||||
echo "target exists but is not a symlink or git checkout: $TARGET" >&2
|
||||
exit 1
|
||||
else
|
||||
echo "[open-source-sync] cloning updater checkout from explicit UPDATER_URL."
|
||||
git clone "$UPDATER_URL" "$TARGET"
|
||||
git -C "$TARGET" checkout "${UPDATER_REVISION:-main}"
|
||||
fi
|
||||
elif [[ -e "$TARGET" ]]; then
|
||||
echo "target exists but is not a symlink or git checkout: $TARGET" >&2
|
||||
exit 1
|
||||
elif is_git_checkout "$UPDATER_SRC"; then
|
||||
rel_target="$(python3 - "$TARGET" "$UPDATER_SRC" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
print(os.path.relpath(sys.argv[2], os.path.dirname(sys.argv[1])))
|
||||
PY
|
||||
)"
|
||||
ln -s "$rel_target" "$TARGET"
|
||||
echo "[open-source-sync] linked updater submodule into Flutter engine."
|
||||
else
|
||||
echo "[open-source-sync] cloning public updater checkout."
|
||||
git clone "$UPDATER_URL" "$TARGET"
|
||||
git -C "$TARGET" checkout "${UPDATER_REVISION:-main}"
|
||||
echo "updater source checkout is missing: $UPDATER_SRC" >&2
|
||||
echo "Set UPDATER_SRC to a local fork or set UPDATER_URL explicitly." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$TARGET/library/include/updater_engine.h" ]]; then
|
||||
|
||||
Executable
+210
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate an assembled open Shorebird artifact mirror."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
REQUIRED_PATCH_ZIPS = {
|
||||
"patch-linux-x64.zip": "patch",
|
||||
"patch-darwin-x64.zip": "patch",
|
||||
"patch-darwin-arm64.zip": "patch",
|
||||
"patch-windows-x64.zip": "patch.exe",
|
||||
}
|
||||
|
||||
|
||||
def digest_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(lambda: file.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_plain_file(path: Path) -> bool:
|
||||
return path.is_file() and not path.is_symlink()
|
||||
|
||||
|
||||
def parse_sidecar(path: Path) -> tuple[str, str]:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
parts = text.split()
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"expected '<sha256> <filename>', got {text!r}")
|
||||
digest, filename = parts
|
||||
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
|
||||
raise ValueError(f"invalid sha256 digest {digest!r}")
|
||||
return digest, filename
|
||||
|
||||
|
||||
def is_safe_relative_path(path: str) -> bool:
|
||||
if not path or path == ".":
|
||||
return False
|
||||
if "\\" in path or "\x00" in path or path.endswith("/"):
|
||||
return False
|
||||
if any(ord(character) < 32 for character in path):
|
||||
return False
|
||||
|
||||
candidate = PurePosixPath(path)
|
||||
if candidate.is_absolute():
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in candidate.parts):
|
||||
return False
|
||||
if candidate.parts and ":" in candidate.parts[0]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def validate_sidecars(shorebird_root: Path) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for artifact_path in sorted(shorebird_root.rglob("*")):
|
||||
if artifact_path.is_symlink():
|
||||
errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: symlink entries are not allowed")
|
||||
continue
|
||||
if not artifact_path.is_file():
|
||||
continue
|
||||
if artifact_path.suffix == ".sha256":
|
||||
artifact_path_without_suffix = Path(str(artifact_path)[: -len(".sha256")])
|
||||
if not is_plain_file(artifact_path_without_suffix):
|
||||
errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: orphan sidecar")
|
||||
continue
|
||||
|
||||
sidecar_path = Path(f"{artifact_path}.sha256")
|
||||
if not is_plain_file(sidecar_path):
|
||||
errors.append(f"{artifact_path.relative_to(shorebird_root.parent)}: missing sidecar")
|
||||
continue
|
||||
try:
|
||||
sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path)
|
||||
except ValueError as error:
|
||||
errors.append(f"{sidecar_path.relative_to(shorebird_root.parent)}: {error}")
|
||||
continue
|
||||
actual_digest = digest_file(artifact_path)
|
||||
if sidecar_digest != actual_digest:
|
||||
errors.append(
|
||||
f"{sidecar_path.relative_to(shorebird_root.parent)}: digest mismatch "
|
||||
f"{sidecar_digest} != {actual_digest}"
|
||||
)
|
||||
if sidecar_filename != artifact_path.name:
|
||||
errors.append(
|
||||
f"{sidecar_path.relative_to(shorebird_root.parent)}: filename mismatch "
|
||||
f"{sidecar_filename!r} != {artifact_path.name!r}"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def validate_manifest_overrides(mirror_root: Path, manifest_paths: list[Path]) -> list[str]:
|
||||
override_pattern = re.compile(r"^\s*-\s*'?(?P<path>[^'#\n]+?)'?\s*(?:#.*)?$")
|
||||
shorebird_root = mirror_root / "shorebird"
|
||||
errors: list[str] = []
|
||||
for manifest_path in manifest_paths:
|
||||
engine_revision = manifest_path.parent.name
|
||||
for line in manifest_path.read_text(encoding="utf-8").splitlines():
|
||||
match = override_pattern.match(line)
|
||||
if not match:
|
||||
continue
|
||||
artifact_path = match.group("path").replace("$engine", engine_revision)
|
||||
if not is_safe_relative_path(artifact_path):
|
||||
errors.append(
|
||||
f"{manifest_path.relative_to(mirror_root)} -> "
|
||||
f"unsafe artifact override path: {artifact_path}"
|
||||
)
|
||||
continue
|
||||
resolved_artifact_path = shorebird_root / artifact_path
|
||||
if not is_plain_file(resolved_artifact_path):
|
||||
errors.append(
|
||||
f"{manifest_path.relative_to(mirror_root)} -> shorebird/{artifact_path}"
|
||||
)
|
||||
continue
|
||||
if resolved_artifact_path.stat().st_size <= 0:
|
||||
errors.append(
|
||||
f"{manifest_path.relative_to(mirror_root)} -> "
|
||||
f"shorebird/{artifact_path}: artifact override is empty"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def validate_patch_zips(mirror_root: Path, manifest_paths: list[Path]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for manifest_path in manifest_paths:
|
||||
engine_dir = manifest_path.parent
|
||||
for zip_name, expected_entry in REQUIRED_PATCH_ZIPS.items():
|
||||
zip_path = engine_dir / zip_name
|
||||
display_path = zip_path.relative_to(mirror_root)
|
||||
if not is_plain_file(zip_path):
|
||||
errors.append(f"{display_path}: missing")
|
||||
continue
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path) as archive:
|
||||
names = archive.namelist()
|
||||
if names != [expected_entry]:
|
||||
errors.append(
|
||||
f"{display_path}: expected only {expected_entry!r}, got {names!r}"
|
||||
)
|
||||
continue
|
||||
if archive.getinfo(expected_entry).file_size <= 0:
|
||||
errors.append(f"{display_path}: {expected_entry} is empty")
|
||||
except zipfile.BadZipFile:
|
||||
errors.append(f"{display_path}: invalid zip")
|
||||
return errors
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("mirror_root", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
mirror_root = args.mirror_root
|
||||
shorebird_root = mirror_root / "shorebird"
|
||||
|
||||
if not mirror_root.is_dir():
|
||||
print(f"missing mirror root: {mirror_root}", file=sys.stderr)
|
||||
return 66
|
||||
if not shorebird_root.is_dir():
|
||||
print(f"mirror root is missing shorebird/: {mirror_root}", file=sys.stderr)
|
||||
return 70
|
||||
|
||||
manifest_paths = sorted(shorebird_root.glob("*/artifacts_manifest.yaml"))
|
||||
if not manifest_paths:
|
||||
print(
|
||||
"artifact mirror is missing shorebird/<engine>/artifacts_manifest.yaml",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 70
|
||||
|
||||
errors: list[str] = []
|
||||
sidecar_errors = validate_sidecars(shorebird_root)
|
||||
if sidecar_errors:
|
||||
errors.append("invalid checksum sidecars:\n" + "\n".join(f" {e}" for e in sidecar_errors))
|
||||
|
||||
override_errors = validate_manifest_overrides(mirror_root, manifest_paths)
|
||||
if override_errors:
|
||||
errors.append(
|
||||
"invalid files referenced by artifacts_manifest.yaml:\n"
|
||||
+ "\n".join(f" {path}" for path in override_errors)
|
||||
)
|
||||
|
||||
patch_errors = validate_patch_zips(mirror_root, manifest_paths)
|
||||
if patch_errors:
|
||||
errors.append(
|
||||
"invalid CLI patch-tool artifacts:\n"
|
||||
+ "\n".join(f" {error}" for error in patch_errors)
|
||||
)
|
||||
|
||||
if errors:
|
||||
print("artifact mirror validation failed:\n" + "\n".join(errors), file=sys.stderr)
|
||||
return 70
|
||||
|
||||
print(f"artifact mirror validated: {mirror_root}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+249
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a JSON release manifest against downloaded CI artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import sys
|
||||
|
||||
|
||||
def digest_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(lambda: file.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_plain_file(path: Path) -> bool:
|
||||
return path.is_file() and not path.is_symlink()
|
||||
|
||||
|
||||
def parse_sidecar(path: Path) -> tuple[str, str]:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
parts = text.split()
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"expected '<sha256> <filename>', got {text!r}")
|
||||
digest, filename = parts
|
||||
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
|
||||
raise ValueError(f"invalid sha256 digest {digest!r}")
|
||||
return digest, filename
|
||||
|
||||
|
||||
def is_safe_relative_path(path: str) -> bool:
|
||||
if not path or path == ".":
|
||||
return False
|
||||
if "\\" in path or "\x00" in path or path.endswith("/"):
|
||||
return False
|
||||
if any(ord(character) < 32 for character in path):
|
||||
return False
|
||||
|
||||
candidate = PurePosixPath(path)
|
||||
if candidate.is_absolute():
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in candidate.parts):
|
||||
return False
|
||||
if candidate.parts and ":" in candidate.parts[0]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--github-sha",
|
||||
default="",
|
||||
help="Require the manifest github_sha field to match this commit SHA.",
|
||||
)
|
||||
parser.add_argument("input_dir", type=Path)
|
||||
parser.add_argument("manifest", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
input_dir = args.input_dir
|
||||
manifest_path = args.manifest
|
||||
|
||||
if not input_dir.is_dir():
|
||||
print(f"missing input directory: {input_dir}", file=sys.stderr)
|
||||
return 66
|
||||
if not manifest_path.is_file():
|
||||
print(f"missing release manifest: {manifest_path}", file=sys.stderr)
|
||||
return 66
|
||||
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as error:
|
||||
print(f"invalid release manifest JSON: {error}", file=sys.stderr)
|
||||
return 70
|
||||
|
||||
errors: list[str] = []
|
||||
if manifest.get("format_version") != 1:
|
||||
errors.append(f"format_version is {manifest.get('format_version')!r}; expected 1")
|
||||
if not isinstance(manifest.get("github_sha", ""), str):
|
||||
errors.append("github_sha must be a string")
|
||||
elif args.github_sha and manifest.get("github_sha") != args.github_sha:
|
||||
errors.append(
|
||||
f"github_sha is {manifest.get('github_sha')!r}; expected {args.github_sha!r}"
|
||||
)
|
||||
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list):
|
||||
errors.append("artifacts must be a list")
|
||||
artifacts = []
|
||||
|
||||
expected_count = manifest.get("artifact_count")
|
||||
if expected_count != len(artifacts):
|
||||
errors.append(
|
||||
f"artifact_count is {expected_count!r}; expected {len(artifacts)}"
|
||||
)
|
||||
|
||||
seen_paths: set[str] = set()
|
||||
seen_sidecars: set[str] = set()
|
||||
for index, artifact in enumerate(artifacts):
|
||||
if not isinstance(artifact, dict):
|
||||
errors.append(f"artifacts[{index}] must be an object")
|
||||
continue
|
||||
|
||||
artifact_path_text = artifact.get("path")
|
||||
artifact_group = artifact.get("artifact_group")
|
||||
filename = artifact.get("filename")
|
||||
sidecar_path_text = artifact.get("sidecar")
|
||||
expected_digest = artifact.get("sha256")
|
||||
expected_size = artifact.get("size")
|
||||
if not isinstance(artifact_path_text, str):
|
||||
errors.append(f"artifacts[{index}].path must be a string")
|
||||
continue
|
||||
if artifact_path_text in seen_paths:
|
||||
errors.append(f"{artifact_path_text}: duplicate artifact path")
|
||||
seen_paths.add(artifact_path_text)
|
||||
if not is_safe_relative_path(artifact_path_text):
|
||||
errors.append(f"{artifact_path_text}: unsafe artifact path")
|
||||
continue
|
||||
artifact_relative = PurePosixPath(artifact_path_text)
|
||||
|
||||
if not isinstance(artifact_group, str):
|
||||
errors.append(f"{artifact_path_text}: artifact_group must be a string")
|
||||
elif artifact_group != artifact_relative.parts[0]:
|
||||
errors.append(
|
||||
f"{artifact_path_text}: artifact_group {artifact_group!r} "
|
||||
f"does not match path group {artifact_relative.parts[0]!r}"
|
||||
)
|
||||
|
||||
if not isinstance(filename, str):
|
||||
errors.append(f"{artifact_path_text}: filename must be a string")
|
||||
elif filename != artifact_relative.name:
|
||||
errors.append(
|
||||
f"{artifact_path_text}: filename {filename!r} "
|
||||
f"does not match path filename {artifact_relative.name!r}"
|
||||
)
|
||||
|
||||
if not isinstance(sidecar_path_text, str):
|
||||
errors.append(f"{artifact_path_text}: sidecar must be a string")
|
||||
continue
|
||||
if sidecar_path_text in seen_sidecars:
|
||||
errors.append(f"{sidecar_path_text}: duplicate sidecar path")
|
||||
seen_sidecars.add(sidecar_path_text)
|
||||
if not is_safe_relative_path(sidecar_path_text):
|
||||
errors.append(f"{sidecar_path_text}: unsafe sidecar path")
|
||||
continue
|
||||
|
||||
artifact_path = input_dir / artifact_path_text
|
||||
sidecar_path = input_dir / sidecar_path_text
|
||||
if not is_plain_file(artifact_path):
|
||||
errors.append(f"{artifact_path_text}: missing artifact file")
|
||||
continue
|
||||
if not is_plain_file(sidecar_path):
|
||||
errors.append(f"{sidecar_path_text}: missing sidecar file")
|
||||
continue
|
||||
actual_size = artifact_path.stat().st_size
|
||||
if actual_size <= 0:
|
||||
errors.append(f"{artifact_path_text}: empty artifacts are not allowed")
|
||||
if sidecar_path != Path(f"{artifact_path}.sha256"):
|
||||
errors.append(
|
||||
f"{artifact_path_text}: sidecar path {sidecar_path_text!r} "
|
||||
f"does not match sibling {artifact_path.name}.sha256"
|
||||
)
|
||||
|
||||
actual_digest = digest_file(artifact_path)
|
||||
if expected_digest != actual_digest:
|
||||
errors.append(
|
||||
f"{artifact_path_text}: digest mismatch "
|
||||
f"{expected_digest!r} != {actual_digest}"
|
||||
)
|
||||
if expected_size != actual_size:
|
||||
errors.append(
|
||||
f"{artifact_path_text}: size mismatch "
|
||||
f"{expected_size!r} != {actual_size}"
|
||||
)
|
||||
|
||||
try:
|
||||
sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path)
|
||||
except ValueError as error:
|
||||
errors.append(f"{sidecar_path_text}: {error}")
|
||||
continue
|
||||
if sidecar_digest != actual_digest:
|
||||
errors.append(
|
||||
f"{sidecar_path_text}: sidecar digest mismatch "
|
||||
f"{sidecar_digest} != {actual_digest}"
|
||||
)
|
||||
if sidecar_filename != artifact_path.name:
|
||||
errors.append(
|
||||
f"{sidecar_path_text}: sidecar filename mismatch "
|
||||
f"{sidecar_filename!r} != {artifact_path.name!r}"
|
||||
)
|
||||
|
||||
manifest_resolved = manifest_path.resolve()
|
||||
manifest_sidecar_resolved = Path(f"{manifest_path}.sha256").resolve()
|
||||
actual_artifacts: set[str] = set()
|
||||
actual_sidecars: set[str] = set()
|
||||
for path in sorted(input_dir.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
if path.is_symlink():
|
||||
relative = path.relative_to(input_dir).as_posix()
|
||||
if path.suffix == ".sha256":
|
||||
actual_sidecars.add(relative)
|
||||
else:
|
||||
actual_artifacts.add(relative)
|
||||
continue
|
||||
resolved = path.resolve()
|
||||
if resolved == manifest_resolved or resolved == manifest_sidecar_resolved:
|
||||
continue
|
||||
relative = path.relative_to(input_dir).as_posix()
|
||||
if path.suffix == ".sha256":
|
||||
actual_sidecars.add(relative)
|
||||
else:
|
||||
actual_artifacts.add(relative)
|
||||
|
||||
missing_from_manifest = sorted(actual_artifacts - seen_paths)
|
||||
if missing_from_manifest:
|
||||
errors.append(
|
||||
"artifacts missing from release manifest: "
|
||||
+ ", ".join(missing_from_manifest)
|
||||
)
|
||||
|
||||
orphan_sidecars = sorted(actual_sidecars - seen_sidecars)
|
||||
if orphan_sidecars:
|
||||
errors.append(
|
||||
"sidecars missing from release manifest: " + ", ".join(orphan_sidecars)
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(
|
||||
"release manifest validation failed:\n"
|
||||
+ "\n".join(f" {error}" for error in errors),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 70
|
||||
|
||||
print(f"release manifest validated: {manifest_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-mirror-validator.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
"$ROOT/scripts/verify_assemble_artifact_mirror.sh" >/dev/null
|
||||
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
ENGINE_REVISION=engine123
|
||||
MIRROR_ROOT="$TMP_DIR/mirror"
|
||||
mkdir -p "$MIRROR_ROOT/shorebird/$ENGINE_REVISION"
|
||||
cat > "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
|
||||
flutter_engine_revision: 'base-engine'
|
||||
storage_bucket: 'shorebird'
|
||||
artifact_overrides:
|
||||
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/artifacts.zip'
|
||||
EOF
|
||||
|
||||
"$PYTHON_BIN" - "$MIRROR_ROOT/shorebird/$ENGINE_REVISION" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
patch_zips = {
|
||||
"patch-linux-x64.zip": "patch",
|
||||
"patch-darwin-x64.zip": "patch",
|
||||
"patch-darwin-arm64.zip": "patch",
|
||||
"patch-windows-x64.zip": "patch.exe",
|
||||
}
|
||||
for zip_name, entry_name in patch_zips.items():
|
||||
with zipfile.ZipFile(root / zip_name, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(entry_name, f"{zip_name}:{entry_name}\n")
|
||||
PY
|
||||
|
||||
mkdir -p "$MIRROR_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release"
|
||||
printf 'linux-engine-artifacts\n' \
|
||||
> "$MIRROR_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip"
|
||||
|
||||
while IFS= read -r -d '' mirror_file; do
|
||||
if [[ "$mirror_file" == *.sha256 ]]; then
|
||||
continue
|
||||
fi
|
||||
"$ROOT/scripts/write_sha256.sh" "$mirror_file"
|
||||
done < <(find "$MIRROR_ROOT/shorebird" -type f -print0)
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$MIRROR_ROOT" >/dev/null
|
||||
|
||||
UNSAFE_MIRROR_ROOT="$TMP_DIR/unsafe-mirror"
|
||||
cp -R "$MIRROR_ROOT" "$UNSAFE_MIRROR_ROOT"
|
||||
"$PYTHON_BIN" - "$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
manifest_path = Path(sys.argv[1])
|
||||
manifest_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"flutter_engine_revision: 'base-engine'",
|
||||
"storage_bucket: 'shorebird'",
|
||||
"artifact_overrides:",
|
||||
" - '../outside/artifacts.zip'",
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
"$ROOT/scripts/write_sha256.sh" \
|
||||
"$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" \
|
||||
"$UNSAFE_MIRROR_ROOT/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml.sha256"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
|
||||
"$UNSAFE_MIRROR_ROOT" >"$TMP_DIR/unsafe-mirror.log" 2>&1; then
|
||||
echo "validate_artifact_mirror.py unexpectedly accepted an unsafe manifest override" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "unsafe artifact override path" "$TMP_DIR/unsafe-mirror.log"
|
||||
|
||||
SYMLINK_MIRROR_ROOT="$TMP_DIR/symlink-mirror"
|
||||
cp -R "$MIRROR_ROOT" "$SYMLINK_MIRROR_ROOT"
|
||||
rm "$SYMLINK_MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
|
||||
ln -s "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" \
|
||||
"$SYMLINK_MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
|
||||
"$SYMLINK_MIRROR_ROOT" >"$TMP_DIR/symlink-mirror.log" 2>&1; then
|
||||
echo "validate_artifact_mirror.py unexpectedly accepted a symlink artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "symlink entries are not allowed" "$TMP_DIR/symlink-mirror.log"
|
||||
|
||||
EMPTY_OVERRIDE_ROOT="$TMP_DIR/empty-override-mirror"
|
||||
cp -R "$MIRROR_ROOT" "$EMPTY_OVERRIDE_ROOT"
|
||||
"$PYTHON_BIN" - "$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
Path(sys.argv[1]).write_bytes(b"")
|
||||
PY
|
||||
"$ROOT/scripts/write_sha256.sh" \
|
||||
"$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip" \
|
||||
"$EMPTY_OVERRIDE_ROOT/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip.sha256"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
|
||||
"$EMPTY_OVERRIDE_ROOT" >"$TMP_DIR/empty-override-mirror.log" 2>&1; then
|
||||
echo "validate_artifact_mirror.py unexpectedly accepted an empty manifest override artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "artifact override is empty" "$TMP_DIR/empty-override-mirror.log"
|
||||
|
||||
printf 'tampered\n' >> "$MIRROR_ROOT/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$MIRROR_ROOT" >/dev/null 2>&1; then
|
||||
echo "validate_artifact_mirror.py unexpectedly accepted a stale sidecar" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
echo "validate_artifact_mirror.py smoke test passed"
|
||||
+287
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-artifact-job.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
ENGINE_REVISION=engine123
|
||||
DOWNLOADED="$TMP_DIR/downloaded-artifacts"
|
||||
mkdir -p "$DOWNLOADED"
|
||||
|
||||
write_artifact() {
|
||||
local path="$1"
|
||||
local content="$2"
|
||||
mkdir -p "$(dirname "$path")"
|
||||
printf '%s\n' "$content" > "$path"
|
||||
"$ROOT/scripts/write_sha256.sh" "$path"
|
||||
}
|
||||
|
||||
write_zip() {
|
||||
local zip_path="$1"
|
||||
local entry_name="$2"
|
||||
local content="$3"
|
||||
mkdir -p "$(dirname "$zip_path")"
|
||||
"$PYTHON_BIN" - "$zip_path" "$entry_name" "$content" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
zip_path, entry_name, content = sys.argv[1:]
|
||||
with zipfile.ZipFile(Path(zip_path), "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(entry_name, content + "\n")
|
||||
PY
|
||||
"$ROOT/scripts/write_sha256.sh" "$zip_path"
|
||||
}
|
||||
|
||||
write_tgz() {
|
||||
local archive_path="$1"
|
||||
local staging_dir="$2"
|
||||
local root_entry="$3"
|
||||
mkdir -p "$(dirname "$archive_path")"
|
||||
tar -C "$staging_dir" -czf "$archive_path" "$root_entry"
|
||||
"$ROOT/scripts/write_sha256.sh" "$archive_path"
|
||||
}
|
||||
|
||||
for target in \
|
||||
cli-linux-x64/open-shorebird-cli-linux-x64.tar.gz \
|
||||
cli-macos-x64/open-shorebird-cli-macos-x64.tar.gz \
|
||||
cli-macos-arm64/open-shorebird-cli-macos-arm64.tar.gz \
|
||||
cli-windows-x64/open-shorebird-cli-windows-x64.tar.gz \
|
||||
shorebird-server-linux-amd64/shorebird-server-linux-amd64.tar.gz \
|
||||
shorebird-server-linux-arm64/shorebird-server-linux-arm64.tar.gz \
|
||||
shorebird-server-darwin-amd64/shorebird-server-darwin-amd64.tar.gz \
|
||||
shorebird-server-darwin-arm64/shorebird-server-darwin-arm64.tar.gz \
|
||||
shorebird-server-windows-amd64/shorebird-server-windows-amd64.tar.gz \
|
||||
custom-dart-sdk-linux-x64/custom-dart-sdk-linux-x64.tar.gz \
|
||||
custom-dart-sdk-macos-arm64/custom-dart-sdk-macos-arm64.tar.gz; do
|
||||
write_artifact "$DOWNLOADED/$target" "$target"
|
||||
done
|
||||
|
||||
mkdir -p "$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION"
|
||||
cat > "$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
|
||||
flutter_engine_revision: 'base-engine'
|
||||
storage_bucket: 'shorebird'
|
||||
artifact_overrides:
|
||||
- 'flutter_infra_release/flutter/\$engine/android-arm64-release/artifacts.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/android-arm64-release/symbols.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/artifacts.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/linux-x64-flutter-gtk.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/flutter_patched_sdk_product.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/flutter-web-sdk.zip'
|
||||
- 'flutter_infra_release/flutter/\$engine/darwin-arm64-release/FlutterMacOS.framework.zip'
|
||||
EOF
|
||||
"$ROOT/scripts/write_sha256.sh" \
|
||||
"$DOWNLOADED/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml"
|
||||
|
||||
write_patch_artifact() {
|
||||
local artifact_name="$1"
|
||||
local zip_name="$2"
|
||||
local entry_name="$3"
|
||||
write_zip "$DOWNLOADED/$artifact_name/artifacts/mirror/$zip_name" "$entry_name" "$zip_name"
|
||||
write_zip "$DOWNLOADED/$artifact_name/artifacts/mirror/shorebird/$ENGINE_REVISION/$zip_name" "$entry_name" "$zip_name"
|
||||
}
|
||||
write_patch_artifact mirror-patch-linux-x64.zip patch-linux-x64.zip patch
|
||||
write_patch_artifact mirror-patch-darwin-x64.zip patch-darwin-x64.zip patch
|
||||
write_patch_artifact mirror-patch-darwin-arm64.zip patch-darwin-arm64.zip patch
|
||||
write_patch_artifact mirror-patch-windows-x64.zip patch-windows-x64.zip patch.exe
|
||||
|
||||
engine_stage="$TMP_DIR/engine-stage"
|
||||
mkdir -p "$engine_stage"
|
||||
|
||||
make_engine_archive() {
|
||||
local artifact_dir="$1"
|
||||
local archive_name="$2"
|
||||
local root_name="$3"
|
||||
local mirror_subdir="$4"
|
||||
shift 4
|
||||
|
||||
local stage="$engine_stage/$root_name"
|
||||
rm -rf "$stage"
|
||||
mkdir -p "$stage/$root_name/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/$mirror_subdir"
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
local file_name="$1"
|
||||
local content="$2"
|
||||
shift 2
|
||||
write_artifact \
|
||||
"$stage/$root_name/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/$mirror_subdir/$file_name" \
|
||||
"$content"
|
||||
done
|
||||
write_tgz "$DOWNLOADED/$artifact_dir/$archive_name" "$stage" "$root_name"
|
||||
}
|
||||
|
||||
make_engine_archive \
|
||||
linux-engine-x64 linux-engine-x64.tar.gz linux-engine linux-x64-release \
|
||||
artifacts.zip linux-artifacts \
|
||||
linux-x64-flutter-gtk.zip linux-gtk
|
||||
mkdir -p "$engine_stage/linux-engine/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION"
|
||||
write_artifact \
|
||||
"$engine_stage/linux-engine/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/flutter_patched_sdk_product.zip" \
|
||||
linux-patched-sdk
|
||||
write_tgz "$DOWNLOADED/linux-engine-x64/linux-engine-x64.tar.gz" \
|
||||
"$engine_stage/linux-engine" \
|
||||
linux-engine
|
||||
|
||||
make_engine_archive \
|
||||
android-engine-arm64 android-engine-arm64.tar.gz android-engine android-arm64-release \
|
||||
artifacts.zip android-artifacts \
|
||||
symbols.zip android-symbols
|
||||
make_engine_archive \
|
||||
flutter-web-sdk flutter-web-sdk.tar.gz web-sdk . \
|
||||
flutter-web-sdk.zip web-sdk
|
||||
make_engine_archive \
|
||||
ios-interpreter-engine ios-interpreter-engine.tar.gz ios-engine ios-release \
|
||||
artifacts.zip ios-artifacts
|
||||
make_engine_archive \
|
||||
macos-engine-arm64 macos-engine-arm64.tar.gz macos-engine darwin-arm64-release \
|
||||
FlutterMacOS.framework.zip macos-framework
|
||||
|
||||
mirror_input="$TMP_DIR/mirror-input"
|
||||
mkdir -p "$mirror_input"
|
||||
cp -R "$DOWNLOADED"/mirror-* "$mirror_input/"
|
||||
cp -R "$DOWNLOADED/linux-engine-x64" "$mirror_input/"
|
||||
cp -R "$DOWNLOADED/android-engine-arm64" "$mirror_input/"
|
||||
cp -R "$DOWNLOADED/flutter-web-sdk" "$mirror_input/"
|
||||
cp -R "$DOWNLOADED/ios-interpreter-engine" "$mirror_input/"
|
||||
cp -R "$DOWNLOADED/macos-engine-arm64" "$mirror_input/"
|
||||
|
||||
assembled="$TMP_DIR/artifacts/open-shorebird-artifact-mirror"
|
||||
mkdir -p "$TMP_DIR/artifacts"
|
||||
"$ROOT/scripts/assemble_artifact_mirror.sh" "$mirror_input" "$assembled" >/dev/null
|
||||
tar -C "$TMP_DIR/artifacts" -czf "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" open-shorebird-artifact-mirror
|
||||
mirror_extract_dir="$TMP_DIR/mirror-extract"
|
||||
mkdir -p "$mirror_extract_dir"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" \
|
||||
"$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" \
|
||||
"$mirror_extract_dir"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" \
|
||||
"$mirror_extract_dir/open-shorebird-artifact-mirror" >/dev/null
|
||||
"$ROOT/scripts/write_sha256.sh" "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz"
|
||||
|
||||
manifest_input="$TMP_DIR/manifest-input"
|
||||
mkdir -p "$manifest_input"
|
||||
cp -R "$DOWNLOADED"/. "$manifest_input/"
|
||||
mkdir -p "$manifest_input/open-shorebird-artifact-mirror"
|
||||
cp "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz" "$manifest_input/open-shorebird-artifact-mirror/"
|
||||
cp "$TMP_DIR/open-shorebird-artifact-mirror.tar.gz.sha256" "$manifest_input/open-shorebird-artifact-mirror/"
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$manifest_input" \
|
||||
--github-sha test-sha \
|
||||
--require 'cli-linux-x64/*open-shorebird-cli-linux-x64.tar.gz' \
|
||||
--require 'cli-macos-x64/*open-shorebird-cli-macos-x64.tar.gz' \
|
||||
--require 'cli-macos-arm64/*open-shorebird-cli-macos-arm64.tar.gz' \
|
||||
--require 'cli-windows-x64/*open-shorebird-cli-windows-x64.tar.gz' \
|
||||
--require 'shorebird-server-linux-amd64/*shorebird-server-linux-amd64.tar.gz' \
|
||||
--require 'shorebird-server-linux-arm64/*shorebird-server-linux-arm64.tar.gz' \
|
||||
--require 'shorebird-server-darwin-amd64/*shorebird-server-darwin-amd64.tar.gz' \
|
||||
--require 'shorebird-server-darwin-arm64/*shorebird-server-darwin-arm64.tar.gz' \
|
||||
--require 'shorebird-server-windows-amd64/*shorebird-server-windows-amd64.tar.gz' \
|
||||
--require 'custom-dart-sdk-linux-x64/*custom-dart-sdk-linux-x64.tar.gz' \
|
||||
--require 'custom-dart-sdk-macos-arm64/*custom-dart-sdk-macos-arm64.tar.gz' \
|
||||
--require 'linux-engine-x64/*linux-engine-x64.tar.gz' \
|
||||
--require 'android-engine-arm64/*android-engine-arm64.tar.gz' \
|
||||
--require 'flutter-web-sdk/*flutter-web-sdk.tar.gz' \
|
||||
--require 'ios-interpreter-engine/*ios-interpreter-engine.tar.gz' \
|
||||
--require 'macos-engine-arm64/*macos-engine-arm64.tar.gz' \
|
||||
--require 'mirror-patch-linux-x64.zip/*patch-linux-x64.zip' \
|
||||
--require 'mirror-patch-darwin-x64.zip/*patch-darwin-x64.zip' \
|
||||
--require 'mirror-patch-darwin-arm64.zip/*patch-darwin-arm64.zip' \
|
||||
--require 'mirror-patch-windows-x64.zip/*patch-windows-x64.zip' \
|
||||
--require 'mirror-metadata/*artifacts_manifest.yaml' \
|
||||
--require 'open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz' \
|
||||
--output "$TMP_DIR/open-shorebird-release-manifest.json"
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$manifest_input" \
|
||||
"$TMP_DIR/open-shorebird-release-manifest.json" >/dev/null
|
||||
|
||||
mkdir -p "$manifest_input/open-shorebird-release-manifest"
|
||||
cp "$TMP_DIR/open-shorebird-release-manifest.json" \
|
||||
"$manifest_input/open-shorebird-release-manifest/"
|
||||
"$ROOT/scripts/write_sha256.sh" \
|
||||
"$manifest_input/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
|
||||
"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha test-sha \
|
||||
"$manifest_input" >/dev/null
|
||||
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha wrong-sha \
|
||||
"$manifest_input" >"$TMP_DIR/wrong-download-sha.log" 2>&1; then
|
||||
echo "unexpectedly accepted downloaded artifacts for the wrong github_sha" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "github_sha is" "$TMP_DIR/wrong-download-sha.log"
|
||||
|
||||
release_manifest_path="$manifest_input/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
|
||||
mirror_archive_path="$manifest_input/open-shorebird-artifact-mirror/open-shorebird-artifact-mirror.tar.gz"
|
||||
printf '%064d open-shorebird-release-manifest.json\n' 0 > "$release_manifest_path.sha256"
|
||||
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha test-sha \
|
||||
"$manifest_input" >/dev/null 2>&1; then
|
||||
echo "unexpectedly accepted a stale downloaded release manifest sidecar" >&2
|
||||
exit 1
|
||||
fi
|
||||
"$ROOT/scripts/write_sha256.sh" "$release_manifest_path"
|
||||
|
||||
printf '%064d open-shorebird-artifact-mirror.tar.gz\n' 0 > "$mirror_archive_path.sha256"
|
||||
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha test-sha \
|
||||
"$manifest_input" >/dev/null 2>&1; then
|
||||
echo "unexpectedly accepted a stale downloaded mirror archive sidecar" >&2
|
||||
exit 1
|
||||
fi
|
||||
"$ROOT/scripts/write_sha256.sh" "$mirror_archive_path"
|
||||
"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha test-sha \
|
||||
"$manifest_input" >/dev/null
|
||||
|
||||
unsafe_download="$TMP_DIR/unsafe-download"
|
||||
cp -R "$manifest_input" "$unsafe_download"
|
||||
unsafe_mirror_archive="$unsafe_download/open-shorebird-artifact-mirror/open-shorebird-artifact-mirror.tar.gz"
|
||||
"$PYTHON_BIN" - "$unsafe_mirror_archive" <<'PY'
|
||||
import io
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive_path = sys.argv[1]
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
content = b"unsafe\n"
|
||||
member = tarfile.TarInfo("../outside.txt")
|
||||
member.size = len(content)
|
||||
archive.addfile(member, io.BytesIO(content))
|
||||
PY
|
||||
"$ROOT/scripts/write_sha256.sh" "$unsafe_mirror_archive"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$unsafe_download" \
|
||||
--github-sha test-sha \
|
||||
--require 'open-shorebird-artifact-mirror/*open-shorebird-artifact-mirror.tar.gz' \
|
||||
--output "$unsafe_download/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
|
||||
"$ROOT/scripts/write_sha256.sh" \
|
||||
"$unsafe_download/open-shorebird-release-manifest/open-shorebird-release-manifest.json"
|
||||
if "$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha test-sha \
|
||||
"$unsafe_download" >"$TMP_DIR/unsafe-download.log" 2>&1; then
|
||||
echo "unexpectedly accepted an unsafe downloaded mirror archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "unsafe archive member path" "$TMP_DIR/unsafe-download.log"
|
||||
|
||||
"$PYTHON_BIN" - "$TMP_DIR/open-shorebird-release-manifest.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
manifest = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
paths = {artifact["path"] for artifact in manifest["artifacts"]}
|
||||
assert any(path.endswith("open-shorebird-artifact-mirror.tar.gz") for path in paths)
|
||||
assert any(path.endswith("linux-engine-x64.tar.gz") for path in paths)
|
||||
assert any(path.endswith("patch-windows-x64.zip") for path in paths)
|
||||
assert any(path.endswith("artifacts_manifest.yaml") for path in paths)
|
||||
PY
|
||||
|
||||
echo "artifact-mirror workflow assembly smoke test passed"
|
||||
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-assemble.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
ENGINE_REVISION=engine123
|
||||
INPUT_DIR="$TMP_DIR/downloaded-artifacts"
|
||||
OUTPUT_DIR="$TMP_DIR/mirror"
|
||||
mkdir -p "$INPUT_DIR"
|
||||
|
||||
mkdir -p "$INPUT_DIR/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION"
|
||||
cat > "$INPUT_DIR/mirror-metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
|
||||
flutter_engine_revision: 'base-engine'
|
||||
storage_bucket: 'shorebird'
|
||||
artifact_overrides:
|
||||
- 'flutter_infra_release/flutter/\$engine/linux-x64-release/artifacts.zip'
|
||||
EOF
|
||||
|
||||
mkdir -p "$INPUT_DIR/mirror-patch/artifacts/mirror/shorebird/$ENGINE_REVISION"
|
||||
"$PYTHON_BIN" - "$INPUT_DIR/mirror-patch/artifacts/mirror/shorebird/$ENGINE_REVISION" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
patch_zips = {
|
||||
"patch-linux-x64.zip": "patch",
|
||||
"patch-darwin-x64.zip": "patch",
|
||||
"patch-darwin-arm64.zip": "patch",
|
||||
"patch-windows-x64.zip": "patch.exe",
|
||||
}
|
||||
for zip_name, entry_name in patch_zips.items():
|
||||
with zipfile.ZipFile(root / zip_name, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(entry_name, f"{zip_name}:{entry_name}\n")
|
||||
PY
|
||||
|
||||
engine_staging="$TMP_DIR/linux-engine"
|
||||
mkdir -p "$engine_staging/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release"
|
||||
printf 'linux-engine-artifacts\n' \
|
||||
> "$engine_staging/linux-engine/mirror/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip"
|
||||
tar -C "$engine_staging" -czf "$INPUT_DIR/linux-engine-x64.tar.gz" linux-engine
|
||||
|
||||
"$ROOT/scripts/assemble_artifact_mirror.sh" "$INPUT_DIR" "$OUTPUT_DIR"
|
||||
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml.sha256"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-linux-x64.zip.sha256"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-x64.zip"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-x64.zip.sha256"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-arm64.zip"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-darwin-arm64.zip.sha256"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-windows-x64.zip"
|
||||
test -f "$OUTPUT_DIR/shorebird/$ENGINE_REVISION/patch-windows-x64.zip.sha256"
|
||||
test -f "$OUTPUT_DIR/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip"
|
||||
test -f "$OUTPUT_DIR/shorebird/flutter_infra_release/flutter/$ENGINE_REVISION/linux-x64-release/artifacts.zip.sha256"
|
||||
|
||||
CONFLICT_INPUT="$TMP_DIR/conflicting-artifacts"
|
||||
mkdir -p "$CONFLICT_INPUT/conflict/artifacts/mirror/shorebird/$ENGINE_REVISION"
|
||||
"$PYTHON_BIN" - "$CONFLICT_INPUT/conflict/artifacts/mirror/shorebird/$ENGINE_REVISION/patch-linux-x64.zip" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(Path(sys.argv[1]), "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("patch", "different-patch\n")
|
||||
PY
|
||||
|
||||
if "$ROOT/scripts/assemble_artifact_mirror.sh" "$CONFLICT_INPUT" "$OUTPUT_DIR" >/dev/null 2>&1; then
|
||||
echo "assemble_artifact_mirror.sh unexpectedly allowed a conflicting mirror file" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
UNSAFE_TAR_INPUT="$TMP_DIR/unsafe-tar"
|
||||
UNSAFE_TAR_OUTPUT="$TMP_DIR/unsafe-tar-output"
|
||||
mkdir -p "$UNSAFE_TAR_INPUT"
|
||||
"$PYTHON_BIN" - "$UNSAFE_TAR_INPUT/unsafe-engine.tar.gz" <<'PY'
|
||||
import io
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive_path = sys.argv[1]
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
content = b"unsafe\n"
|
||||
member = tarfile.TarInfo("../outside.txt")
|
||||
member.size = len(content)
|
||||
archive.addfile(member, io.BytesIO(content))
|
||||
PY
|
||||
if "$ROOT/scripts/assemble_artifact_mirror.sh" \
|
||||
"$UNSAFE_TAR_INPUT" \
|
||||
"$UNSAFE_TAR_OUTPUT" >"$TMP_DIR/unsafe-tar.log" 2>&1; then
|
||||
echo "assemble_artifact_mirror.sh unexpectedly allowed an unsafe tar member" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "unsafe archive member path" "$TMP_DIR/unsafe-tar.log"
|
||||
|
||||
SYMLINK_TAR_INPUT="$TMP_DIR/symlink-tar"
|
||||
SYMLINK_TAR_OUTPUT="$TMP_DIR/symlink-tar-output"
|
||||
mkdir -p "$SYMLINK_TAR_INPUT"
|
||||
"$PYTHON_BIN" - "$SYMLINK_TAR_INPUT/symlink-engine.tar.gz" <<'PY'
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive_path = sys.argv[1]
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
member = tarfile.TarInfo("engine/mirror/shorebird/link")
|
||||
member.type = tarfile.SYMTYPE
|
||||
member.linkname = "/tmp/outside"
|
||||
archive.addfile(member)
|
||||
PY
|
||||
if "$ROOT/scripts/assemble_artifact_mirror.sh" \
|
||||
"$SYMLINK_TAR_INPUT" \
|
||||
"$SYMLINK_TAR_OUTPUT" >"$TMP_DIR/symlink-tar.log" 2>&1; then
|
||||
echo "assemble_artifact_mirror.sh unexpectedly allowed a tar symlink member" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "unsupported archive member type" "$TMP_DIR/symlink-tar.log"
|
||||
|
||||
DUPLICATE_TAR_INPUT="$TMP_DIR/duplicate-tar"
|
||||
DUPLICATE_TAR_OUTPUT="$TMP_DIR/duplicate-tar-output"
|
||||
mkdir -p "$DUPLICATE_TAR_INPUT"
|
||||
"$PYTHON_BIN" - "$DUPLICATE_TAR_INPUT/duplicate-engine.tar.gz" <<'PY'
|
||||
import io
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive_path = sys.argv[1]
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
for content in (b"first\n", b"second\n"):
|
||||
member = tarfile.TarInfo("engine/mirror/shorebird/duplicate.txt")
|
||||
member.size = len(content)
|
||||
archive.addfile(member, io.BytesIO(content))
|
||||
PY
|
||||
if "$ROOT/scripts/assemble_artifact_mirror.sh" \
|
||||
"$DUPLICATE_TAR_INPUT" \
|
||||
"$DUPLICATE_TAR_OUTPUT" >"$TMP_DIR/duplicate-tar.log" 2>&1; then
|
||||
echo "assemble_artifact_mirror.sh unexpectedly allowed a duplicate tar member" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "duplicate archive member path" "$TMP_DIR/duplicate-tar.log"
|
||||
|
||||
BAD_ZIP_INPUT="$TMP_DIR/bad-zip"
|
||||
BAD_ZIP_OUTPUT="$TMP_DIR/bad-zip-output"
|
||||
mkdir -p "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION"
|
||||
cat > "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
|
||||
flutter_engine_revision: 'base-engine'
|
||||
storage_bucket: 'shorebird'
|
||||
artifact_overrides: []
|
||||
EOF
|
||||
printf 'not a zip\n' > "$BAD_ZIP_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/patch-linux-x64.zip"
|
||||
|
||||
if "$ROOT/scripts/assemble_artifact_mirror.sh" "$BAD_ZIP_INPUT" "$BAD_ZIP_OUTPUT" >/dev/null 2>&1; then
|
||||
echo "assemble_artifact_mirror.sh unexpectedly allowed an invalid patch zip" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
MISSING_INPUT="$TMP_DIR/missing-override"
|
||||
MISSING_OUTPUT="$TMP_DIR/missing-output"
|
||||
mkdir -p "$MISSING_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION"
|
||||
cat > "$MISSING_INPUT/metadata/artifacts/mirror/shorebird/$ENGINE_REVISION/artifacts_manifest.yaml" <<EOF
|
||||
flutter_engine_revision: 'base-engine'
|
||||
storage_bucket: 'shorebird'
|
||||
artifact_overrides:
|
||||
- 'flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip'
|
||||
EOF
|
||||
|
||||
if "$ROOT/scripts/assemble_artifact_mirror.sh" "$MISSING_INPUT" "$MISSING_OUTPUT" >/dev/null 2>&1; then
|
||||
echo "assemble_artifact_mirror.sh unexpectedly allowed a missing manifest override" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
echo "assemble_artifact_mirror.sh smoke test passed"
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
CI_MIN_FREE_DISK_GB=2 \
|
||||
CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \
|
||||
"$ROOT/scripts/check_ci_capacity.sh" >/dev/null
|
||||
|
||||
if CI_MIN_FREE_DISK_GB=4 \
|
||||
CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \
|
||||
"$ROOT/scripts/check_ci_capacity.sh" >/dev/null 2>&1; then
|
||||
echo "check_ci_capacity.sh unexpectedly accepted insufficient disk" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
if CI_MIN_FREE_DISK_GB=not-a-number \
|
||||
CI_AVAILABLE_DISK_KB_OVERRIDE=$((3 * 1024 * 1024)) \
|
||||
"$ROOT/scripts/check_ci_capacity.sh" >/dev/null 2>&1; then
|
||||
echo "check_ci_capacity.sh unexpectedly accepted an invalid minimum" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
CI_MIN_FREE_DISK_GB=0 "$ROOT/scripts/check_ci_capacity.sh" >/dev/null
|
||||
|
||||
echo "check_ci_capacity.sh smoke test passed"
|
||||
Executable
+2124
File diff suppressed because it is too large
Load Diff
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WORKFLOW="$ROOT/.github/workflows/open-shorebird-ci.yml"
|
||||
RUBY_ARGS=()
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: verify_ci_workflow.sh [--require-tracked] [--require-clean] [--require-upload-ready] [workflow.yml]
|
||||
|
||||
Validates the Open Shorebird GitHub Actions workflow contract. The upload-ready
|
||||
mode additionally requires every required CI support file to be tracked in its
|
||||
own git checkout and every required owning checkout to be clean.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--require-tracked|--require-clean|--require-upload-ready)
|
||||
RUBY_ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage
|
||||
exit 64
|
||||
;;
|
||||
*)
|
||||
WORKFLOW="$1"
|
||||
shift
|
||||
if [[ "$#" -gt 0 ]]; then
|
||||
echo "unexpected extra argument: $1" >&2
|
||||
usage
|
||||
exit 64
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ! -f "$WORKFLOW" ]]; then
|
||||
echo "missing workflow: $WORKFLOW" >&2
|
||||
exit 66
|
||||
fi
|
||||
|
||||
command -v ruby >/dev/null 2>&1 || {
|
||||
echo "ruby is required to validate GitHub workflow YAML" >&2
|
||||
exit 127
|
||||
}
|
||||
|
||||
if [[ "${#RUBY_ARGS[@]}" -eq 0 ]]; then
|
||||
ruby "$ROOT/scripts/verify_ci_workflow.rb" "$WORKFLOW"
|
||||
else
|
||||
ruby "$ROOT/scripts/verify_ci_workflow.rb" "${RUBY_ARGS[@]}" "$WORKFLOW"
|
||||
fi
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
echo "usage: $0 <args.gn>..." >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
require_gn_value() {
|
||||
local args_file="$1"
|
||||
local key="$2"
|
||||
local value="$3"
|
||||
local actual
|
||||
|
||||
if ! actual="$(read_gn_value "$args_file" "$key")"; then
|
||||
echo "expected $key = $value in $args_file, but $key is missing" >&2
|
||||
exit 70
|
||||
fi
|
||||
if [[ "$actual" != "$value" ]]; then
|
||||
echo "expected $key = $value in $args_file, found $actual" >&2
|
||||
exit 70
|
||||
fi
|
||||
}
|
||||
|
||||
read_gn_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
awk -v key="$key" '
|
||||
$1 == key && $2 == "=" {
|
||||
value = $0
|
||||
sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value)
|
||||
sub("[[:space:]]*$", "", value)
|
||||
found = 1
|
||||
}
|
||||
END {
|
||||
if (!found) {
|
||||
exit 1
|
||||
}
|
||||
print value
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
|
||||
for args_file in "$@"; do
|
||||
if [[ ! -f "$args_file" ]]; then
|
||||
echo "missing args.gn: $args_file" >&2
|
||||
exit 66
|
||||
fi
|
||||
|
||||
if [[ "$(read_gn_value "$args_file" dart_dynamic_modules)" == "true" ]]; then
|
||||
echo "DART_DYNAMIC_MODULES must not be enabled: $args_file" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
require_gn_value "$args_file" dart_dynamic_modules false
|
||||
require_gn_value "$args_file" dart_enable_aot_patching true
|
||||
require_gn_value "$args_file" dart_enable_shorebird_interpreter true
|
||||
|
||||
echo "Verified $args_file: patched Dart SDK flags are enabled without DDM"
|
||||
done
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: verify_downloaded_release_artifacts.sh [--github-sha sha] downloaded-artifacts
|
||||
|
||||
Verifies a downloaded full SDK build artifact set. The release manifest and
|
||||
artifact mirror archive must have valid checksum sidecars, the manifest must
|
||||
cover every downloaded artifact, and the mirror archive must safely extract to a
|
||||
valid open Shorebird artifact mirror.
|
||||
EOF
|
||||
}
|
||||
|
||||
DOWNLOAD_DIR=""
|
||||
EXPECTED_GITHUB_SHA=""
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--github-sha)
|
||||
if [[ "$#" -lt 2 || -z "${2:-}" ]]; then
|
||||
echo "--github-sha value is required" >&2
|
||||
usage
|
||||
exit 64
|
||||
fi
|
||||
EXPECTED_GITHUB_SHA="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage
|
||||
exit 64
|
||||
;;
|
||||
*)
|
||||
if [[ -n "$DOWNLOAD_DIR" ]]; then
|
||||
echo "unexpected extra argument: $1" >&2
|
||||
usage
|
||||
exit 64
|
||||
fi
|
||||
DOWNLOAD_DIR="$1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$DOWNLOAD_DIR" ]]; then
|
||||
usage
|
||||
exit 64
|
||||
fi
|
||||
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-downloaded-release.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 70
|
||||
}
|
||||
|
||||
verify_sha256_sidecar() {
|
||||
local artifact_path="$1"
|
||||
local sidecar_path="$2"
|
||||
|
||||
[[ -f "$sidecar_path" ]] || fail "missing checksum sidecar: $sidecar_path"
|
||||
|
||||
"$PYTHON_BIN" - "$artifact_path" "$sidecar_path" <<'PY'
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
artifact_path = Path(sys.argv[1])
|
||||
sidecar_path = Path(sys.argv[2])
|
||||
|
||||
try:
|
||||
text = sidecar_path.read_text(encoding="utf-8").strip()
|
||||
except UnicodeDecodeError as error:
|
||||
print(f"error: {sidecar_path}: invalid UTF-8: {error}", file=sys.stderr)
|
||||
sys.exit(70)
|
||||
|
||||
parts = text.split()
|
||||
if len(parts) != 2:
|
||||
print(
|
||||
f"error: {sidecar_path}: expected '<sha256> <filename>', got {text!r}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(70)
|
||||
|
||||
expected_digest, expected_filename = parts
|
||||
if len(expected_digest) != 64 or any(
|
||||
char not in "0123456789abcdef" for char in expected_digest
|
||||
):
|
||||
print(f"error: {sidecar_path}: invalid sha256 digest {expected_digest!r}", file=sys.stderr)
|
||||
sys.exit(70)
|
||||
|
||||
if expected_filename != artifact_path.name:
|
||||
print(
|
||||
f"error: {sidecar_path}: filename mismatch "
|
||||
f"{expected_filename!r} != {artifact_path.name!r}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(70)
|
||||
|
||||
digest = hashlib.sha256()
|
||||
with artifact_path.open("rb") as artifact:
|
||||
for chunk in iter(lambda: artifact.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
|
||||
actual_digest = digest.hexdigest()
|
||||
if expected_digest != actual_digest:
|
||||
print(
|
||||
f"error: {sidecar_path}: digest mismatch "
|
||||
f"{expected_digest} != {actual_digest}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(70)
|
||||
PY
|
||||
}
|
||||
|
||||
[[ -d "$DOWNLOAD_DIR" ]] || fail "missing downloaded artifacts directory: $DOWNLOAD_DIR"
|
||||
|
||||
manifest_paths=()
|
||||
while IFS= read -r path; do
|
||||
manifest_paths+=("$path")
|
||||
done < <(find "$DOWNLOAD_DIR" -type f -name open-shorebird-release-manifest.json | sort)
|
||||
if [[ "${#manifest_paths[@]}" -ne 1 ]]; then
|
||||
fail "expected exactly one open-shorebird-release-manifest.json, found ${#manifest_paths[@]}"
|
||||
fi
|
||||
manifest_path="${manifest_paths[0]}"
|
||||
manifest_sidecar="$manifest_path.sha256"
|
||||
|
||||
verify_sha256_sidecar "$manifest_path" "$manifest_sidecar"
|
||||
|
||||
validate_manifest_args=()
|
||||
if [[ -n "$EXPECTED_GITHUB_SHA" ]]; then
|
||||
validate_manifest_args+=(--github-sha "$EXPECTED_GITHUB_SHA")
|
||||
fi
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"${validate_manifest_args[@]}" \
|
||||
"$DOWNLOAD_DIR" \
|
||||
"$manifest_path"
|
||||
|
||||
mirror_archives=()
|
||||
while IFS= read -r path; do
|
||||
mirror_archives+=("$path")
|
||||
done < <(find "$DOWNLOAD_DIR" -type f -name open-shorebird-artifact-mirror.tar.gz | sort)
|
||||
if [[ "${#mirror_archives[@]}" -ne 1 ]]; then
|
||||
fail "expected exactly one open-shorebird-artifact-mirror.tar.gz, found ${#mirror_archives[@]}"
|
||||
fi
|
||||
mirror_archive="${mirror_archives[0]}"
|
||||
|
||||
mirror_sidecar="$mirror_archive.sha256"
|
||||
verify_sha256_sidecar "$mirror_archive" "$mirror_sidecar"
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/safe_extract_tar.py" "$mirror_archive" "$TMP_DIR"
|
||||
mirror_root="$TMP_DIR/open-shorebird-artifact-mirror"
|
||||
[[ -d "$mirror_root" ]] || fail "mirror archive did not contain open-shorebird-artifact-mirror/"
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_artifact_mirror.py" "$mirror_root"
|
||||
|
||||
echo "downloaded release artifacts verified: $DOWNLOAD_DIR"
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
echo "usage: $0 <args.gn>... | <args.gn> key=value ..." >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
verify_no_ddm() {
|
||||
local args_file="$1"
|
||||
|
||||
if [[ ! -f "$args_file" ]]; then
|
||||
echo "missing args.gn: $args_file" >&2
|
||||
exit 66
|
||||
fi
|
||||
|
||||
local actual
|
||||
if ! actual="$(read_gn_value "$args_file" dart_dynamic_modules)"; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "$actual" == "true" ]]; then
|
||||
echo "DART_DYNAMIC_MODULES must not be enabled: $args_file" >&2
|
||||
exit 70
|
||||
fi
|
||||
}
|
||||
|
||||
read_gn_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
awk -v key="$key" '
|
||||
$1 == key && $2 == "=" {
|
||||
value = $0
|
||||
sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value)
|
||||
sub("[[:space:]]*$", "", value)
|
||||
found = 1
|
||||
}
|
||||
END {
|
||||
if (!found) {
|
||||
exit 1
|
||||
}
|
||||
print value
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
|
||||
has_expectations=0
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == *=* ]]; then
|
||||
has_expectations=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$has_expectations" == "0" ]]; then
|
||||
for args_file in "$@"; do
|
||||
verify_no_ddm "$args_file"
|
||||
echo "Verified $args_file: dart_dynamic_modules is not true"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
args_file="$1"
|
||||
shift
|
||||
verify_no_ddm "$args_file"
|
||||
|
||||
for expectation in "$@"; do
|
||||
if [[ "$expectation" != *=* ]]; then
|
||||
echo "invalid expectation: $expectation; expected key=value" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
key="${expectation%%=*}"
|
||||
value="${expectation#*=}"
|
||||
if ! actual="$(read_gn_value "$args_file" "$key")"; then
|
||||
echo "expected $key = $value in $args_file, but $key is missing" >&2
|
||||
exit 70
|
||||
fi
|
||||
if [[ "$actual" != "$value" ]]; then
|
||||
echo "expected $key = $value in $args_file, found $actual" >&2
|
||||
exit 70
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Verified $args_file: dart_dynamic_modules is not true and expected flags are present"
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: verify_hosted_full_sdk_build.sh --repo owner/name [options]
|
||||
|
||||
Dispatches the Open Shorebird full SDK build on GitHub Actions, waits for the
|
||||
workflow run to finish, downloads all artifacts, and verifies the release
|
||||
manifest plus assembled artifact mirror.
|
||||
|
||||
Options:
|
||||
--repo owner/name GitHub repository to run against.
|
||||
--ref branch-or-sha Ref to dispatch. Defaults to current branch.
|
||||
--workflow file Workflow file. Defaults to open-shorebird-ci.yml.
|
||||
--download-dir path Artifact download directory. Defaults to hosted-full-sdk-artifacts.
|
||||
--timeout-minutes minutes Maximum wait time. Defaults to 720.
|
||||
--poll-seconds seconds Poll interval. Defaults to 30.
|
||||
--linux-heavy-runner label Override linux_heavy_runner.
|
||||
--macos-heavy-runner label Override macos_heavy_runner.
|
||||
--sdk-min-free-disk-gb value Override sdk_min_free_disk_gb.
|
||||
--engine-min-free-disk-gb value Override engine_min_free_disk_gb.
|
||||
--base-flutter-engine-revision v Override base_flutter_engine_revision.
|
||||
--skip-gclient-sync Dispatch with run_gclient_sync=false.
|
||||
EOF
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REPO=""
|
||||
REF=""
|
||||
WORKFLOW="open-shorebird-ci.yml"
|
||||
DOWNLOAD_DIR="hosted-full-sdk-artifacts"
|
||||
TIMEOUT_MINUTES=720
|
||||
POLL_SECONDS=30
|
||||
LINUX_HEAVY_RUNNER=""
|
||||
MACOS_HEAVY_RUNNER=""
|
||||
SDK_MIN_FREE_DISK_GB=""
|
||||
ENGINE_MIN_FREE_DISK_GB=""
|
||||
BASE_FLUTTER_ENGINE_REVISION=""
|
||||
RUN_GCLIENT_SYNC=true
|
||||
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--repo)
|
||||
REPO="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--ref)
|
||||
REF="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--workflow)
|
||||
WORKFLOW="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--download-dir)
|
||||
DOWNLOAD_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--timeout-minutes)
|
||||
TIMEOUT_MINUTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--poll-seconds)
|
||||
POLL_SECONDS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--linux-heavy-runner)
|
||||
LINUX_HEAVY_RUNNER="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--macos-heavy-runner)
|
||||
MACOS_HEAVY_RUNNER="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--sdk-min-free-disk-gb)
|
||||
SDK_MIN_FREE_DISK_GB="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--engine-min-free-disk-gb)
|
||||
ENGINE_MIN_FREE_DISK_GB="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--base-flutter-engine-revision)
|
||||
BASE_FLUTTER_ENGINE_REVISION="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--skip-gclient-sync)
|
||||
RUN_GCLIENT_SYNC=false
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$REPO" ]]; then
|
||||
echo "--repo owner/name is required" >&2
|
||||
usage
|
||||
exit 64
|
||||
fi
|
||||
|
||||
if ! command -v gh >/dev/null 2>&1; then
|
||||
echo "GitHub CLI 'gh' is required" >&2
|
||||
exit 69
|
||||
fi
|
||||
|
||||
if [[ -z "$REF" ]]; then
|
||||
REF="$(git -C "$ROOT" branch --show-current 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -z "$REF" ]]; then
|
||||
echo "--ref is required when the current checkout is detached" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
run_fields=(
|
||||
-f full_sdk_build=true
|
||||
-f run_gclient_sync="$RUN_GCLIENT_SYNC"
|
||||
-f run_runtime_smokes=false
|
||||
)
|
||||
[[ -z "$LINUX_HEAVY_RUNNER" ]] || run_fields+=(-f linux_heavy_runner="$LINUX_HEAVY_RUNNER")
|
||||
[[ -z "$MACOS_HEAVY_RUNNER" ]] || run_fields+=(-f macos_heavy_runner="$MACOS_HEAVY_RUNNER")
|
||||
[[ -z "$SDK_MIN_FREE_DISK_GB" ]] || run_fields+=(-f sdk_min_free_disk_gb="$SDK_MIN_FREE_DISK_GB")
|
||||
[[ -z "$ENGINE_MIN_FREE_DISK_GB" ]] || run_fields+=(-f engine_min_free_disk_gb="$ENGINE_MIN_FREE_DISK_GB")
|
||||
[[ -z "$BASE_FLUTTER_ENGINE_REVISION" ]] || run_fields+=(-f base_flutter_engine_revision="$BASE_FLUTTER_ENGINE_REVISION")
|
||||
|
||||
echo "Dispatching $WORKFLOW on $REPO@$REF with full_sdk_build=true"
|
||||
start_epoch="$(date +%s)"
|
||||
start_iso="$(date -u -r "$start_epoch" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "@$start_epoch" +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
gh workflow run "$WORKFLOW" \
|
||||
--repo "$REPO" \
|
||||
--ref "$REF" \
|
||||
"${run_fields[@]}"
|
||||
|
||||
run_id=""
|
||||
for _ in {1..40}; do
|
||||
run_list_args=(
|
||||
--repo "$REPO"
|
||||
--workflow "$WORKFLOW"
|
||||
--event workflow_dispatch
|
||||
--json databaseId,createdAt
|
||||
--limit 20
|
||||
)
|
||||
if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
run_list_args+=(--commit "$REF")
|
||||
else
|
||||
run_list_args+=(--branch "$REF")
|
||||
fi
|
||||
run_id="$(
|
||||
gh run list \
|
||||
"${run_list_args[@]}" \
|
||||
--jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \
|
||||
|
|
||||
head -n 1
|
||||
)"
|
||||
[[ -z "$run_id" ]] || break
|
||||
sleep 3
|
||||
done
|
||||
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "unable to find dispatched workflow run for $WORKFLOW on $REF" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
echo "Waiting for hosted full SDK run: $run_id"
|
||||
deadline=$((start_epoch + TIMEOUT_MINUTES * 60))
|
||||
run_head_sha=""
|
||||
while true; do
|
||||
IFS=$'\t' read -r status conclusion url run_head_sha < <(
|
||||
gh run view "$run_id" \
|
||||
--repo "$REPO" \
|
||||
--json status,conclusion,url,headSha \
|
||||
--jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv'
|
||||
)
|
||||
echo "run $run_id status=$status conclusion=${conclusion:-null} url=$url"
|
||||
|
||||
if [[ "$status" == "completed" ]]; then
|
||||
if [[ "$conclusion" != "success" ]]; then
|
||||
echo "hosted full SDK build failed: conclusion=$conclusion" >&2
|
||||
exit 70
|
||||
fi
|
||||
break
|
||||
fi
|
||||
if [[ "$(date +%s)" -ge "$deadline" ]]; then
|
||||
echo "timed out waiting for hosted full SDK build after $TIMEOUT_MINUTES minutes" >&2
|
||||
exit 70
|
||||
fi
|
||||
sleep "$POLL_SECONDS"
|
||||
done
|
||||
|
||||
rm -rf "$DOWNLOAD_DIR"
|
||||
mkdir -p "$DOWNLOAD_DIR"
|
||||
gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR"
|
||||
if [[ -z "$run_head_sha" ]]; then
|
||||
echo "unable to read headSha for workflow run $run_id" >&2
|
||||
exit 70
|
||||
fi
|
||||
"$ROOT/scripts/verify_downloaded_release_artifacts.sh" \
|
||||
--github-sha "$run_head_sha" \
|
||||
"$DOWNLOAD_DIR"
|
||||
|
||||
echo "hosted full SDK build verified: $run_id"
|
||||
@@ -14,6 +14,7 @@ CHECKED_IPA=""
|
||||
CHECKED_PATCH_ARTIFACT=""
|
||||
ENTITLEMENTS_CHECKED=0
|
||||
APP_STORE_STRICT_CHECKED=0
|
||||
NO_BUNDLED_KEY_CHECKED=0
|
||||
CLEANUP_DIR=""
|
||||
|
||||
cleanup() {
|
||||
@@ -35,14 +36,14 @@ read_gn_value() {
|
||||
$1 == key && $2 == "=" {
|
||||
value = $0
|
||||
sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*", "", value)
|
||||
print value
|
||||
sub("[[:space:]]*$", "", value)
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
END {
|
||||
if (!found) {
|
||||
exit 1
|
||||
}
|
||||
print value
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
@@ -66,6 +67,7 @@ verify_ios_engine_args() {
|
||||
|
||||
require_gn_value "$args_file" target_os '"ios"'
|
||||
require_gn_value "$args_file" dart_dynamic_modules false
|
||||
require_gn_value "$args_file" dart_enable_aot_patching true
|
||||
require_gn_value "$args_file" dart_enable_shorebird_interpreter true
|
||||
require_gn_value "$args_file" shorebird_use_interpreter true
|
||||
require_gn_value "$args_file" shorebird_enable_aot_patching false
|
||||
@@ -77,6 +79,7 @@ verify_host_engine_args() {
|
||||
|
||||
require_gn_value "$args_file" target_os '"mac"'
|
||||
require_gn_value "$args_file" dart_dynamic_modules false
|
||||
require_gn_value "$args_file" dart_enable_aot_patching true
|
||||
require_gn_value "$args_file" dart_enable_shorebird_interpreter true
|
||||
require_gn_value "$args_file" shorebird_use_interpreter true
|
||||
}
|
||||
@@ -143,6 +146,24 @@ verify_entitlements() {
|
||||
fi
|
||||
}
|
||||
|
||||
verify_no_bundled_patch_key() {
|
||||
local app_bundle="$1"
|
||||
[[ "$APP_STORE_STRICT" == "1" ]] || return 0
|
||||
|
||||
local shorebird_yaml
|
||||
shorebird_yaml="$(
|
||||
find "$app_bundle" -name "shorebird.yaml" -type f -print -quit
|
||||
)"
|
||||
if [[ -z "$shorebird_yaml" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if grep -Eq '^[[:space:]]*aot_patch_key_hex[[:space:]]*:' "$shorebird_yaml"; then
|
||||
fail "APP_STORE_STRICT=1 rejects bundled aot_patch_key_hex in $shorebird_yaml"
|
||||
fi
|
||||
NO_BUNDLED_KEY_CHECKED=1
|
||||
}
|
||||
|
||||
verify_app_bundle() {
|
||||
if [[ -n "$IOS_APP_BUNDLE" && -n "$IOS_IPA" ]]; then
|
||||
fail "set only one of IOS_APP_BUNDLE or IOS_IPA"
|
||||
@@ -165,6 +186,7 @@ verify_app_bundle() {
|
||||
fi
|
||||
|
||||
verify_entitlements "$IOS_APP_BUNDLE"
|
||||
verify_no_bundled_patch_key "$IOS_APP_BUNDLE"
|
||||
}
|
||||
|
||||
file_magic_hex() {
|
||||
@@ -186,31 +208,105 @@ verify_patch_artifact() {
|
||||
;;
|
||||
esac
|
||||
|
||||
local compact_json
|
||||
compact_json="$(LC_ALL=C tr -d '[:space:]' < "$IOS_PATCH_ARTIFACT")"
|
||||
local python_bin
|
||||
python_bin=python3
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin=python
|
||||
fi
|
||||
command -v "$python_bin" >/dev/null 2>&1 ||
|
||||
fail "python3 or python is required to inspect IOS_PATCH_ARTIFACT"
|
||||
|
||||
if [[ "$compact_json" != \{* ]]; then
|
||||
fail "iOS patch artifact must be the encrypted open JSON wrapper, not a raw native/code payload"
|
||||
fi
|
||||
if ! grep -Fq '"format":"open-aot-vmcode-encrypted-v1"' <<<"$compact_json"; then
|
||||
fail "iOS patch artifact is not an open encrypted VM code artifact"
|
||||
fi
|
||||
if grep -Fq '"runtime_mode":"dart-dynamic-modules"' <<<"$compact_json" ||
|
||||
grep -Fq '"runtime_mode":"dynamic-modules"' <<<"$compact_json"; then
|
||||
fail "iOS patch artifact uses DART_DYNAMIC_MODULES runtime mode"
|
||||
fi
|
||||
if ! grep -Fq '"runtime_mode":"dart-bytecode-interpreter"' <<<"$compact_json"; then
|
||||
fail "iOS patch artifact must declare runtime_mode dart-bytecode-interpreter"
|
||||
fi
|
||||
if ! grep -Fq '"target_os":"ios"' <<<"$compact_json"; then
|
||||
fail "iOS patch artifact must target iOS"
|
||||
fi
|
||||
if ! grep -Fq '"target_arch":"arm64"' <<<"$compact_json"; then
|
||||
fail "iOS patch artifact must target arm64"
|
||||
fi
|
||||
if ! grep -Fq '"payload_kind":"full-snapshot"' <<<"$compact_json"; then
|
||||
fail "current iOS interpreter mapper requires payload_kind full-snapshot"
|
||||
fi
|
||||
"$python_bin" - "$IOS_PATCH_ARTIFACT" <<'PY'
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
|
||||
try:
|
||||
with open(path, encoding="utf-8") as file:
|
||||
artifact = json.load(file)
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
raise SystemExit(
|
||||
"iOS patch artifact must be the encrypted open JSON wrapper, "
|
||||
f"not a raw native/code payload: {error}"
|
||||
)
|
||||
|
||||
if not isinstance(artifact, dict):
|
||||
raise SystemExit("iOS patch artifact JSON must be an object")
|
||||
|
||||
metadata = artifact.get("metadata")
|
||||
if not isinstance(metadata, dict):
|
||||
raise SystemExit("iOS patch artifact must contain metadata object")
|
||||
|
||||
|
||||
def require(mapping, key, expected, scope):
|
||||
actual = mapping.get(key)
|
||||
if actual != expected:
|
||||
raise SystemExit(
|
||||
f"iOS patch artifact {scope}.{key} is {actual!r}; "
|
||||
f"expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
require(artifact, "format", "open-aot-vmcode-encrypted-v1", "artifact")
|
||||
runtime_mode = metadata.get("runtime_mode")
|
||||
if runtime_mode in {"dart-dynamic-modules", "dynamic-modules"}:
|
||||
raise SystemExit("iOS patch artifact uses DART_DYNAMIC_MODULES runtime mode")
|
||||
require(metadata, "runtime_mode", "dart-bytecode-interpreter", "metadata")
|
||||
require(metadata, "target_os", "ios", "metadata")
|
||||
require(metadata, "target_arch", "arm64", "metadata")
|
||||
require(artifact, "payload_kind", "full-snapshot", "artifact")
|
||||
|
||||
encryption = artifact.get("encryption")
|
||||
if not isinstance(encryption, dict):
|
||||
raise SystemExit("iOS patch artifact must contain encryption object")
|
||||
require(encryption, "algorithm", "AES-256-GCM", "encryption")
|
||||
|
||||
|
||||
def require_base64(mapping, key, scope):
|
||||
value = mapping.get(key)
|
||||
if not isinstance(value, str) or not value:
|
||||
raise SystemExit(f"iOS patch artifact {scope}.{key} must be non-empty")
|
||||
try:
|
||||
decoded = base64.b64decode(value, validate=True)
|
||||
except (binascii.Error, ValueError) as error:
|
||||
raise SystemExit(
|
||||
f"iOS patch artifact {scope}.{key} is not valid base64: {error}"
|
||||
)
|
||||
if not decoded:
|
||||
raise SystemExit(f"iOS patch artifact {scope}.{key} decodes to empty bytes")
|
||||
return decoded
|
||||
|
||||
|
||||
require_base64(artifact, "encrypted_payload_base64", "artifact")
|
||||
require_base64(encryption, "nonce_base64", "encryption")
|
||||
require_base64(encryption, "tag_base64", "encryption")
|
||||
|
||||
key_id = encryption.get("key_id")
|
||||
if not isinstance(key_id, str) or not key_id:
|
||||
raise SystemExit("iOS patch artifact encryption.key_id must be non-empty")
|
||||
|
||||
hex_pattern = re.compile(r"^[0-9a-f]{64}$")
|
||||
for scope, mapping, key in (
|
||||
("artifact", artifact, "payload_sha256"),
|
||||
("encryption", encryption, "aad_sha256"),
|
||||
):
|
||||
value = mapping.get(key)
|
||||
if not isinstance(value, str) or not hex_pattern.fullmatch(value):
|
||||
raise SystemExit(
|
||||
f"iOS patch artifact {scope}.{key} must be a lowercase SHA-256 hex digest"
|
||||
)
|
||||
|
||||
reconstructed_size = artifact.get("reconstructed_size")
|
||||
if reconstructed_size is not None:
|
||||
if not isinstance(reconstructed_size, int) or reconstructed_size <= 0:
|
||||
raise SystemExit(
|
||||
"iOS patch artifact reconstructed_size must be a positive integer"
|
||||
)
|
||||
PY
|
||||
|
||||
CHECKED_PATCH_ARTIFACT="$IOS_PATCH_ARTIFACT"
|
||||
}
|
||||
@@ -240,6 +336,9 @@ fi
|
||||
if [[ "$APP_STORE_STRICT_CHECKED" == "1" ]]; then
|
||||
echo " App Store strict: get-task-allow is not true"
|
||||
fi
|
||||
if [[ "$NO_BUNDLED_KEY_CHECKED" == "1" ]]; then
|
||||
echo " key material: no bundled aot_patch_key_hex"
|
||||
fi
|
||||
if [[ -n "$CHECKED_PATCH_ARTIFACT" ]]; then
|
||||
echo " patch artifact: encrypted interpreter full-snapshot for ios/arm64"
|
||||
fi
|
||||
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/ios-route-validator.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
IOS_ENGINE_DIR="$TMP_DIR/ios_release"
|
||||
HOST_ENGINE_DIR="$TMP_DIR/host_release_arm64"
|
||||
mkdir -p "$IOS_ENGINE_DIR" "$HOST_ENGINE_DIR"
|
||||
|
||||
cat > "$IOS_ENGINE_DIR/args.gn" <<'EOF'
|
||||
target_os = "android"
|
||||
target_os = "ios"
|
||||
dart_dynamic_modules = false
|
||||
dart_enable_aot_patching = false
|
||||
dart_enable_aot_patching = true
|
||||
dart_enable_shorebird_interpreter = false
|
||||
dart_enable_shorebird_interpreter = true
|
||||
shorebird_use_interpreter = false
|
||||
shorebird_use_interpreter = true
|
||||
shorebird_enable_aot_patching = true
|
||||
shorebird_enable_aot_patching = false
|
||||
EOF
|
||||
|
||||
cat > "$HOST_ENGINE_DIR/args.gn" <<'EOF'
|
||||
target_os = "linux"
|
||||
target_os = "mac"
|
||||
dart_dynamic_modules = false
|
||||
dart_enable_aot_patching = false
|
||||
dart_enable_aot_patching = true
|
||||
dart_enable_shorebird_interpreter = false
|
||||
dart_enable_shorebird_interpreter = true
|
||||
shorebird_use_interpreter = false
|
||||
shorebird_use_interpreter = true
|
||||
EOF
|
||||
|
||||
write_artifact() {
|
||||
local path="$1"
|
||||
local runtime_mode="$2"
|
||||
local target_os="${3:-ios}"
|
||||
cat > "$path" <<EOF
|
||||
{
|
||||
"format": "open-aot-vmcode-encrypted-v1",
|
||||
"metadata": {
|
||||
"app_id": "app.test",
|
||||
"app_build_id": "1",
|
||||
"flavor_id": "pro",
|
||||
"license_type": "pro",
|
||||
"sdk_hash": "sdk",
|
||||
"base_snapshot_hash": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"patch_snapshot_hash": "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"target_os": "$target_os",
|
||||
"target_arch": "arm64",
|
||||
"runtime_mode": "$runtime_mode"
|
||||
},
|
||||
"payload_kind": "full-snapshot",
|
||||
"reconstructed_size": 4,
|
||||
"payload_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
|
||||
"encrypted_payload_base64": "AQIDBA==",
|
||||
"encryption": {
|
||||
"algorithm": "AES-256-GCM",
|
||||
"key_id": "test-key",
|
||||
"nonce_base64": "AQIDBAUGBwgJCgsM",
|
||||
"tag_base64": "AQIDBAUGBwgJCgsMDQ4PEA==",
|
||||
"aad_sha256": "3333333333333333333333333333333333333333333333333333333333333333"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
valid_artifact="$TMP_DIR/valid.vmcode"
|
||||
write_artifact "$valid_artifact" "dart-bytecode-interpreter"
|
||||
IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
|
||||
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
|
||||
IOS_PATCH_ARTIFACT="$valid_artifact" \
|
||||
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null
|
||||
|
||||
bad_runtime="$TMP_DIR/bad-runtime.vmcode"
|
||||
write_artifact "$bad_runtime" "dart-dynamic-modules"
|
||||
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
|
||||
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
|
||||
IOS_PATCH_ARTIFACT="$bad_runtime" \
|
||||
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
|
||||
echo "iOS route validator unexpectedly accepted DART_DYNAMIC_MODULES metadata" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
bad_target="$TMP_DIR/bad-target.vmcode"
|
||||
write_artifact "$bad_target" "dart-bytecode-interpreter" "android"
|
||||
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
|
||||
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
|
||||
IOS_PATCH_ARTIFACT="$bad_target" \
|
||||
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
|
||||
echo "iOS route validator unexpectedly accepted a non-iOS patch artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
bad_json="$TMP_DIR/bad-json.vmcode"
|
||||
printf '{"format":"open-aot-vmcode-encrypted-v1"\n' > "$bad_json"
|
||||
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
|
||||
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
|
||||
IOS_PATCH_ARTIFACT="$bad_json" \
|
||||
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
|
||||
echo "iOS route validator unexpectedly accepted malformed JSON" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
bad_native="$TMP_DIR/bad-native.vmcode"
|
||||
printf '\xcf\xfa\xed\xfe' > "$bad_native"
|
||||
if IOS_ENGINE_DIR="$IOS_ENGINE_DIR" \
|
||||
HOST_ENGINE_DIR="$HOST_ENGINE_DIR" \
|
||||
IOS_PATCH_ARTIFACT="$bad_native" \
|
||||
"$ROOT/scripts/verify_ios_interpreter_route.sh" >/dev/null 2>&1; then
|
||||
echo "iOS route validator unexpectedly accepted a Mach-O patch artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
echo "iOS interpreter route validator smoke test passed"
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 70
|
||||
}
|
||||
|
||||
require_contains() {
|
||||
local path="$1"
|
||||
local needle="$2"
|
||||
grep -Fq "$needle" "$path" || fail "$path is missing required text: $needle"
|
||||
}
|
||||
|
||||
reject_contains() {
|
||||
local path="$1"
|
||||
local needle="$2"
|
||||
if grep -Fq "$needle" "$path"; then
|
||||
fail "$path contains forbidden text: $needle"
|
||||
fi
|
||||
}
|
||||
|
||||
check_forbidden_in_file() {
|
||||
local path="$1"
|
||||
local pattern
|
||||
|
||||
[[ -f "$path" ]] || fail "missing open-infrastructure check input: $path"
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
reject_contains "$path" "$pattern"
|
||||
done
|
||||
}
|
||||
|
||||
check_forbidden_in_tree() {
|
||||
local tree="$1"
|
||||
local extension="$2"
|
||||
local path
|
||||
|
||||
[[ -d "$tree" ]] || fail "missing open-infrastructure check tree: $tree"
|
||||
while IFS= read -r -d '' path; do
|
||||
check_forbidden_in_file "$path"
|
||||
done < <(find "$tree" -type f -name "*.$extension" -print0)
|
||||
}
|
||||
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"https://download.shorebird.dev"
|
||||
"download.shorebird.dev"
|
||||
"api.shorebird.dev"
|
||||
"auth.shorebird.dev"
|
||||
"console.shorebird.dev"
|
||||
"cdn.shorebird.cloud"
|
||||
"git@github.com:shorebirdtech/dart-sdk.git"
|
||||
"github.com/shorebirdtech/updater.git"
|
||||
"github.com/shorebirdtech/flutter.git"
|
||||
"shorebird-dart-sdk-prebuilt"
|
||||
"shorebirdtech/_build_engine"
|
||||
)
|
||||
|
||||
BUILD_SENSITIVE_FILES=(
|
||||
"$ROOT/.gitmodules"
|
||||
"$ROOT/flutter/DEPS"
|
||||
"$ROOT/flutter/bin/internal/update_dart_sdk.ps1"
|
||||
"$ROOT/flutter/bin/internal/update_dart_sdk.sh"
|
||||
"$ROOT/flutter/dev/bots/post_process_docs.dart"
|
||||
"$ROOT/flutter/dev/bots/unpublish_package.dart"
|
||||
"$ROOT/flutter/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle"
|
||||
"$ROOT/flutter/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts"
|
||||
"$ROOT/flutter/dev/tools/create_api_docs.dart"
|
||||
"$ROOT/flutter/engine/src/flutter/build/zip_bundle.gni"
|
||||
"$ROOT/flutter/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart"
|
||||
"$ROOT/flutter/packages/flutter_tools/gradle/aar_init_script.gradle"
|
||||
"$ROOT/flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt"
|
||||
"$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart"
|
||||
"$ROOT/flutter/packages/flutter_tools/lib/src/http_host_validator.dart"
|
||||
"$ROOT/flutter/packages/flutter_tools/pubspec.yaml"
|
||||
"$ROOT/flutter/packages/shorebird_tests/test/shorebird_tests.dart"
|
||||
"$ROOT/scripts/write_gclient.sh"
|
||||
"$ROOT/shorebird/bin/shorebird.ps1"
|
||||
"$ROOT/shorebird/third_party/flutter/bin/internal/shared.sh"
|
||||
"$ROOT/updater/library/src/config.rs"
|
||||
)
|
||||
|
||||
for path in "${BUILD_SENSITIVE_FILES[@]}"; do
|
||||
check_forbidden_in_file "$path"
|
||||
done
|
||||
|
||||
check_forbidden_in_tree "$ROOT/shorebird/packages/artifact_proxy/lib" dart
|
||||
check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_cli/lib" dart
|
||||
check_forbidden_in_tree "$ROOT/shorebird/packages/shorebird_code_push_client/lib" dart
|
||||
|
||||
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/dart-lang/sdk.git"
|
||||
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/flutter.git"
|
||||
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird.git"
|
||||
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-server.git"
|
||||
require_contains "$ROOT/.gitmodules" "https://git.tonycloud.org/flutter/shorebird-updater.git"
|
||||
|
||||
require_contains "$ROOT/flutter/DEPS" '"dart_sdk_git": "https://git.tonycloud.org/dart-lang/sdk.git"'
|
||||
require_contains "$ROOT/flutter/DEPS" '"updater_git": "https://git.tonycloud.org/flutter/shorebird-updater.git"'
|
||||
|
||||
require_contains "$ROOT/flutter/packages/flutter_tools/lib/src/cache.dart" \
|
||||
"kOpenFlutterStorageUrl = 'http://localhost:8080/download.flutter.io'"
|
||||
require_contains "$ROOT/flutter/bin/internal/update_dart_sdk.sh" \
|
||||
"http://localhost:8080/download.flutter.io"
|
||||
require_contains "$ROOT/flutter/bin/internal/update_dart_sdk.ps1" \
|
||||
"http://localhost:8080/download.flutter.io"
|
||||
require_contains "$ROOT/flutter/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt" \
|
||||
'DEFAULT_MAVEN_HOST = "http://localhost:8080/download.flutter.io"'
|
||||
require_contains "$ROOT/flutter/dev/tools/create_api_docs.dart" \
|
||||
"Platform.environment['FLUTTER_STORAGE_BASE_URL']"
|
||||
require_contains "$ROOT/flutter/packages/shorebird_tests/test/shorebird_tests.dart" \
|
||||
"'FLUTTER_STORAGE_BASE_URL': 'http://localhost:8080/download.flutter.io'"
|
||||
|
||||
require_contains "$ROOT/shorebird/packages/shorebird_cli/lib/src/shorebird_env.dart" \
|
||||
"defaultHostedUrl = 'http://localhost:8080'"
|
||||
require_contains "$ROOT/shorebird/packages/shorebird_cli/lib/src/cache.dart" \
|
||||
"defaultArtifactBaseUrl = 'http://localhost:8080/artifacts'"
|
||||
require_contains "$ROOT/updater/library/src/config.rs" \
|
||||
'const DEFAULT_BASE_URL: &str = "http://localhost:8080";'
|
||||
|
||||
echo "open infrastructure defaults check passed"
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 70
|
||||
}
|
||||
|
||||
require_contains() {
|
||||
local path="$1"
|
||||
local needle="$2"
|
||||
grep -Fq "$needle" "$path" || fail "$path is missing required text: $needle"
|
||||
}
|
||||
|
||||
reject_contains() {
|
||||
local path="$1"
|
||||
local needle="$2"
|
||||
if grep -Fq "$needle" "$path"; then
|
||||
fail "$path contains forbidden text: $needle"
|
||||
fi
|
||||
}
|
||||
|
||||
shorebird_launcher="$ROOT/shorebird/bin/shorebird.ps1"
|
||||
flutter_dart_updater="$ROOT/flutter/bin/internal/update_dart_sdk.ps1"
|
||||
|
||||
[[ -f "$shorebird_launcher" ]] || fail "missing PowerShell launcher: $shorebird_launcher"
|
||||
[[ -f "$flutter_dart_updater" ]] || fail "missing Flutter Dart SDK updater: $flutter_dart_updater"
|
||||
|
||||
require_contains "$shorebird_launcher" \
|
||||
'$defaultFlutterGitUrl = "https://git.tonycloud.org/flutter/flutter.git"'
|
||||
require_contains "$shorebird_launcher" \
|
||||
'$defaultFlutterStorageBaseUrl = "http://localhost:8080/download.flutter.io"'
|
||||
require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_GIT_URL'
|
||||
require_contains "$shorebird_launcher" 'SHOREBIRD_FLUTTER_STORAGE_BASE_URL'
|
||||
require_contains "$shorebird_launcher" 'FLUTTER_STORAGE_BASE_URL'
|
||||
|
||||
require_contains "$flutter_dart_updater" '$Env:FLUTTER_STORAGE_BASE_URL'
|
||||
require_contains "$flutter_dart_updater" \
|
||||
'$dartSdkBaseUrl = "http://localhost:8080/download.flutter.io"'
|
||||
|
||||
for path in "$shorebird_launcher" "$flutter_dart_updater"; do
|
||||
reject_contains "$path" 'download.shorebird.dev'
|
||||
reject_contains "$path" 'api.shorebird.dev'
|
||||
reject_contains "$path" 'auth.shorebird.dev'
|
||||
reject_contains "$path" 'console.shorebird.dev'
|
||||
reject_contains "$path" 'docs.shorebird.dev'
|
||||
reject_contains "$path" 'github.com/shorebirdtech/flutter.git'
|
||||
reject_contains "$path" 'github.com/shorebirdtech/shorebird'
|
||||
reject_contains "$path" 'git@github.com:shorebirdtech'
|
||||
done
|
||||
|
||||
if command -v pwsh >/dev/null 2>&1; then
|
||||
SHOREBIRD_POWERSHELL_LAUNCHER="$shorebird_launcher" \
|
||||
FLUTTER_DART_SDK_POWERSHELL_UPDATER="$flutter_dart_updater" \
|
||||
pwsh -NoProfile -NonInteractive -Command '
|
||||
$paths = @(
|
||||
$env:SHOREBIRD_POWERSHELL_LAUNCHER,
|
||||
$env:FLUTTER_DART_SDK_POWERSHELL_UPDATER
|
||||
)
|
||||
$failed = $false
|
||||
foreach ($path in $paths) {
|
||||
$tokens = $null
|
||||
$errors = $null
|
||||
[System.Management.Automation.Language.Parser]::ParseFile(
|
||||
$path,
|
||||
[ref]$tokens,
|
||||
[ref]$errors
|
||||
) | Out-Null
|
||||
if ($errors.Count -gt 0) {
|
||||
Write-Error "$path has PowerShell parse errors:"
|
||||
foreach ($errorRecord in $errors) {
|
||||
Write-Error " $($errorRecord.Message)"
|
||||
}
|
||||
$failed = $true
|
||||
}
|
||||
}
|
||||
if ($failed) {
|
||||
exit 70
|
||||
}
|
||||
'
|
||||
else
|
||||
echo "warning: pwsh not found; skipped PowerShell parse check" >&2
|
||||
fi
|
||||
|
||||
echo "PowerShell open-default checks passed"
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-release-manifest.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
INPUT_DIR="$TMP_DIR/artifacts"
|
||||
mkdir -p "$INPUT_DIR/cli-linux" "$INPUT_DIR/server-linux" "$INPUT_DIR/mirror-patch/artifacts/mirror"
|
||||
printf 'cli archive\n' > "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
|
||||
printf 'server archive\n' > "$INPUT_DIR/server-linux/shorebird-server-linux-amd64.tar.gz"
|
||||
printf 'patch archive\n' > "$INPUT_DIR/mirror-patch/artifacts/mirror/patch-linux-x64.zip"
|
||||
|
||||
"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
|
||||
"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/server-linux/shorebird-server-linux-amd64.tar.gz"
|
||||
"$ROOT/scripts/write_sha256.sh" "$INPUT_DIR/mirror-patch/artifacts/mirror/patch-linux-x64.zip"
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$INPUT_DIR" \
|
||||
--github-sha test-sha \
|
||||
--require 'cli-linux/*open-shorebird-cli-linux-x64.tar.gz' \
|
||||
--require 'server-linux/*shorebird-server-linux-amd64.tar.gz' \
|
||||
--require 'mirror-patch/*patch-linux-x64.zip' \
|
||||
--output "$TMP_DIR/release-manifest.json"
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
--github-sha test-sha \
|
||||
"$INPUT_DIR" \
|
||||
"$TMP_DIR/release-manifest.json"
|
||||
|
||||
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
manifest = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert manifest["format_version"] == 1
|
||||
assert manifest["github_sha"] == "test-sha"
|
||||
assert manifest["artifact_count"] == 3
|
||||
paths = {artifact["path"] for artifact in manifest["artifacts"]}
|
||||
assert "cli-linux/open-shorebird-cli-linux-x64.tar.gz" in paths
|
||||
assert "server-linux/shorebird-server-linux-amd64.tar.gz" in paths
|
||||
assert "mirror-patch/artifacts/mirror/patch-linux-x64.zip" in paths
|
||||
for artifact in manifest["artifacts"]:
|
||||
assert artifact["artifact_group"] in {
|
||||
"cli-linux",
|
||||
"server-linux",
|
||||
"mirror-patch",
|
||||
}
|
||||
assert artifact["filename"] == artifact["path"].split("/")[-1]
|
||||
assert len(artifact["sha256"]) == 64
|
||||
assert artifact["size"] > 0
|
||||
assert artifact["sidecar"].endswith(".sha256")
|
||||
PY
|
||||
|
||||
TAMPERED_MANIFEST="$TMP_DIR/tampered-release-manifest.json"
|
||||
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$TAMPERED_MANIFEST" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
source, output = sys.argv[1:]
|
||||
manifest = json.load(open(source, encoding="utf-8"))
|
||||
manifest["artifacts"][0]["sha256"] = "0" * 64
|
||||
json.dump(manifest, open(output, "w", encoding="utf-8"))
|
||||
PY
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$INPUT_DIR" \
|
||||
"$TAMPERED_MANIFEST" >/dev/null 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted a tampered digest" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
--github-sha wrong-sha \
|
||||
"$INPUT_DIR" \
|
||||
"$TMP_DIR/release-manifest.json" >"$TMP_DIR/wrong-sha.log" 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted the wrong github_sha" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "github_sha is" "$TMP_DIR/wrong-sha.log"
|
||||
|
||||
UNLISTED_ARTIFACTS="$TMP_DIR/unlisted-artifacts"
|
||||
cp -R "$INPUT_DIR" "$UNLISTED_ARTIFACTS"
|
||||
printf 'extra artifact\n' > "$UNLISTED_ARTIFACTS/extra.tar.gz"
|
||||
"$ROOT/scripts/write_sha256.sh" "$UNLISTED_ARTIFACTS/extra.tar.gz"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$UNLISTED_ARTIFACTS" \
|
||||
"$TMP_DIR/release-manifest.json" >/dev/null 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted an unlisted artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
UNSAFE_MANIFEST="$TMP_DIR/unsafe-release-manifest.json"
|
||||
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$UNSAFE_MANIFEST" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
source, output = sys.argv[1:]
|
||||
manifest = json.load(open(source, encoding="utf-8"))
|
||||
manifest["artifacts"][0]["path"] = r"cli-linux\open-shorebird-cli-linux-x64.tar.gz"
|
||||
manifest["artifacts"][0]["sidecar"] = (
|
||||
r"cli-linux\open-shorebird-cli-linux-x64.tar.gz.sha256"
|
||||
)
|
||||
json.dump(manifest, open(output, "w", encoding="utf-8"))
|
||||
PY
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$INPUT_DIR" \
|
||||
"$UNSAFE_MANIFEST" >/dev/null 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted an unsafe artifact path" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
BAD_PROVENANCE_MANIFEST="$TMP_DIR/bad-provenance-release-manifest.json"
|
||||
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$BAD_PROVENANCE_MANIFEST" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
source, output = sys.argv[1:]
|
||||
manifest = json.load(open(source, encoding="utf-8"))
|
||||
manifest["artifacts"][0]["artifact_group"] = "wrong-group"
|
||||
manifest["artifacts"][1]["filename"] = "wrong-name.tar.gz"
|
||||
json.dump(manifest, open(output, "w", encoding="utf-8"))
|
||||
PY
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$INPUT_DIR" \
|
||||
"$BAD_PROVENANCE_MANIFEST" >/dev/null 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted bad provenance fields" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
SYMLINK_LISTED_ARTIFACT="$TMP_DIR/symlink-listed-artifact"
|
||||
cp -R "$INPUT_DIR" "$SYMLINK_LISTED_ARTIFACT"
|
||||
rm "$SYMLINK_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
|
||||
ln -s "$INPUT_DIR/cli-linux/open-shorebird-cli-linux-x64.tar.gz" \
|
||||
"$SYMLINK_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$SYMLINK_LISTED_ARTIFACT" \
|
||||
"$TMP_DIR/release-manifest.json" >/dev/null 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted a symlink artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
EMPTY_LISTED_ARTIFACT="$TMP_DIR/empty-listed-artifact"
|
||||
cp -R "$INPUT_DIR" "$EMPTY_LISTED_ARTIFACT"
|
||||
printf '' > "$EMPTY_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
|
||||
"$ROOT/scripts/write_sha256.sh" \
|
||||
"$EMPTY_LISTED_ARTIFACT/cli-linux/open-shorebird-cli-linux-x64.tar.gz"
|
||||
EMPTY_LISTED_MANIFEST="$TMP_DIR/empty-listed-release-manifest.json"
|
||||
"$PYTHON_BIN" - "$TMP_DIR/release-manifest.json" "$EMPTY_LISTED_MANIFEST" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
source, output = sys.argv[1:]
|
||||
manifest = json.load(open(source, encoding="utf-8"))
|
||||
empty_digest = hashlib.sha256(b"").hexdigest()
|
||||
for artifact in manifest["artifacts"]:
|
||||
if artifact["path"] == "cli-linux/open-shorebird-cli-linux-x64.tar.gz":
|
||||
artifact["sha256"] = empty_digest
|
||||
artifact["size"] = 0
|
||||
json.dump(manifest, open(output, "w", encoding="utf-8"))
|
||||
PY
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/validate_release_manifest.py" \
|
||||
"$EMPTY_LISTED_ARTIFACT" \
|
||||
"$EMPTY_LISTED_MANIFEST" >"$TMP_DIR/empty-listed.log" 2>&1; then
|
||||
echo "validate_release_manifest.py unexpectedly accepted an empty artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "empty artifacts are not allowed" "$TMP_DIR/empty-listed.log"
|
||||
|
||||
SYMLINK_INPUT="$TMP_DIR/symlink-input"
|
||||
mkdir -p "$SYMLINK_INPUT"
|
||||
printf 'symlink target\n' > "$SYMLINK_INPUT/target.tar.gz"
|
||||
"$ROOT/scripts/write_sha256.sh" "$SYMLINK_INPUT/target.tar.gz"
|
||||
ln -s target.tar.gz "$SYMLINK_INPUT/link.tar.gz"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$SYMLINK_INPUT" \
|
||||
--output "$TMP_DIR/symlink-input.json" >/dev/null 2>&1; then
|
||||
echo "write_release_manifest.py unexpectedly accepted a symlink artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
MISSING_SIDECAR="$TMP_DIR/missing-sidecar"
|
||||
mkdir -p "$MISSING_SIDECAR"
|
||||
printf 'missing sidecar\n' > "$MISSING_SIDECAR/artifact.tar.gz"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$MISSING_SIDECAR" \
|
||||
--output "$TMP_DIR/missing.json" >/dev/null 2>&1; then
|
||||
echo "write_release_manifest.py unexpectedly accepted a missing sidecar" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
BAD_SIDECAR="$TMP_DIR/bad-sidecar"
|
||||
mkdir -p "$BAD_SIDECAR"
|
||||
printf 'bad sidecar\n' > "$BAD_SIDECAR/artifact.tar.gz"
|
||||
printf '%064d artifact.tar.gz\n' 0 > "$BAD_SIDECAR/artifact.tar.gz.sha256"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$BAD_SIDECAR" \
|
||||
--output "$TMP_DIR/bad.json" >/dev/null 2>&1; then
|
||||
echo "write_release_manifest.py unexpectedly accepted a bad sidecar" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
EMPTY_ARTIFACT="$TMP_DIR/empty-artifact"
|
||||
mkdir -p "$EMPTY_ARTIFACT"
|
||||
printf '' > "$EMPTY_ARTIFACT/artifact.tar.gz"
|
||||
"$ROOT/scripts/write_sha256.sh" "$EMPTY_ARTIFACT/artifact.tar.gz"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$EMPTY_ARTIFACT" \
|
||||
--output "$TMP_DIR/empty.json" >"$TMP_DIR/empty-artifact.log" 2>&1; then
|
||||
echo "write_release_manifest.py unexpectedly accepted an empty artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
grep -q "empty artifacts are not allowed" "$TMP_DIR/empty-artifact.log"
|
||||
|
||||
ORPHAN_SIDECAR="$TMP_DIR/orphan-sidecar"
|
||||
mkdir -p "$ORPHAN_SIDECAR"
|
||||
printf '%064d missing.tar.gz\n' 0 > "$ORPHAN_SIDECAR/missing.tar.gz.sha256"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$ORPHAN_SIDECAR" \
|
||||
--output "$TMP_DIR/orphan.json" >/dev/null 2>&1; then
|
||||
echo "write_release_manifest.py unexpectedly accepted an orphan sidecar" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/write_release_manifest.py" \
|
||||
"$INPUT_DIR" \
|
||||
--require 'missing-artifact/*.tar.gz' \
|
||||
--output "$TMP_DIR/missing-required.json" >/dev/null 2>&1; then
|
||||
echo "write_release_manifest.py unexpectedly accepted a missing required artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
echo "write_release_manifest.py smoke test passed"
|
||||
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-source-sync.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
init_git_checkout() {
|
||||
local path="$1"
|
||||
mkdir -p "$path"
|
||||
git -C "$path" init -q
|
||||
}
|
||||
|
||||
real_path() {
|
||||
python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$1"
|
||||
}
|
||||
|
||||
make_workspace() {
|
||||
local workspace="$1"
|
||||
|
||||
mkdir -p "$workspace/scripts"
|
||||
cp "$ROOT/scripts/sync_open_sources.sh" "$workspace/scripts/sync_open_sources.sh"
|
||||
chmod +x "$workspace/scripts/sync_open_sources.sh"
|
||||
|
||||
init_git_checkout "$workspace/dart-sdk-new"
|
||||
mkdir -p "$workspace/dart-sdk-new/runtime/vm"
|
||||
: > "$workspace/dart-sdk-new/runtime/vm/dart_api_impl.h"
|
||||
|
||||
init_git_checkout "$workspace/updater"
|
||||
mkdir -p "$workspace/updater/library/include"
|
||||
: > "$workspace/updater/library/include/updater_engine.h"
|
||||
|
||||
mkdir -p "$workspace/flutter/engine/src/flutter/third_party"
|
||||
}
|
||||
|
||||
run_sync() {
|
||||
local workspace="$1"
|
||||
DART_SRC="$workspace/dart-sdk-new" \
|
||||
UPDATER_SRC="$workspace/updater" \
|
||||
"$workspace/scripts/sync_open_sources.sh"
|
||||
}
|
||||
|
||||
assert_links_to_workspace_sources() {
|
||||
local workspace="$1"
|
||||
local dart_target="$workspace/flutter/engine/src/flutter/third_party/dart"
|
||||
local updater_target="$workspace/flutter/engine/src/flutter/third_party/updater"
|
||||
|
||||
test -L "$dart_target"
|
||||
test -L "$updater_target"
|
||||
[[ "$(real_path "$dart_target")" == "$(real_path "$workspace/dart-sdk-new")" ]]
|
||||
[[ "$(real_path "$updater_target")" == "$(real_path "$workspace/updater")" ]]
|
||||
test -f "$dart_target/runtime/vm/dart_api_impl.h"
|
||||
test -f "$updater_target/library/include/updater_engine.h"
|
||||
}
|
||||
|
||||
clean_checkout_workspace="$TMP_DIR/clean-checkouts"
|
||||
make_workspace "$clean_checkout_workspace"
|
||||
init_git_checkout "$clean_checkout_workspace/flutter/engine/src/flutter/third_party/dart"
|
||||
init_git_checkout "$clean_checkout_workspace/flutter/engine/src/flutter/third_party/updater"
|
||||
run_sync "$clean_checkout_workspace"
|
||||
assert_links_to_workspace_sources "$clean_checkout_workspace"
|
||||
run_sync "$clean_checkout_workspace"
|
||||
assert_links_to_workspace_sources "$clean_checkout_workspace"
|
||||
|
||||
stale_link_workspace="$TMP_DIR/stale-link"
|
||||
make_workspace "$stale_link_workspace"
|
||||
mkdir -p "$stale_link_workspace/other-dart"
|
||||
ln -s "../../../../../other-dart" \
|
||||
"$stale_link_workspace/flutter/engine/src/flutter/third_party/dart"
|
||||
if run_sync "$stale_link_workspace" >"$TMP_DIR/stale-link.log" 2>&1; then
|
||||
echo "expected stale Dart symlink to fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "Dart SDK target symlink points at" "$TMP_DIR/stale-link.log"
|
||||
|
||||
dirty_checkout_workspace="$TMP_DIR/dirty-checkout"
|
||||
make_workspace "$dirty_checkout_workspace"
|
||||
dirty_dart="$dirty_checkout_workspace/flutter/engine/src/flutter/third_party/dart"
|
||||
init_git_checkout "$dirty_dart"
|
||||
: > "$dirty_dart/untracked.txt"
|
||||
if run_sync "$dirty_checkout_workspace" >"$TMP_DIR/dirty-checkout.log" 2>&1; then
|
||||
echo "expected dirty Dart checkout to fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "Dart SDK target is a dirty git checkout" "$TMP_DIR/dirty-checkout.log"
|
||||
test -d "$dirty_dart/.git"
|
||||
test -f "$dirty_dart/untracked.txt"
|
||||
|
||||
forbidden_dart_remote_workspace="$TMP_DIR/forbidden-dart-remote"
|
||||
make_workspace "$forbidden_dart_remote_workspace"
|
||||
git -C "$forbidden_dart_remote_workspace/dart-sdk-new" remote add origin \
|
||||
https://github.com/dart-lang/sdk.git
|
||||
if run_sync "$forbidden_dart_remote_workspace" >"$TMP_DIR/forbidden-dart-remote.log" 2>&1; then
|
||||
echo "expected forbidden Dart SDK remote to fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "Dart SDK source checkout uses forbidden remote fragment" \
|
||||
"$TMP_DIR/forbidden-dart-remote.log"
|
||||
|
||||
forbidden_updater_remote_workspace="$TMP_DIR/forbidden-updater-remote"
|
||||
make_workspace "$forbidden_updater_remote_workspace"
|
||||
git -C "$forbidden_updater_remote_workspace/updater" remote add origin \
|
||||
https://github.com/shorebirdtech/updater.git
|
||||
if run_sync "$forbidden_updater_remote_workspace" >"$TMP_DIR/forbidden-updater-remote.log" 2>&1; then
|
||||
echo "expected forbidden updater source remote to fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "updater submodule source checkout uses forbidden remote fragment" \
|
||||
"$TMP_DIR/forbidden-updater-remote.log"
|
||||
|
||||
forbidden_updater_url_workspace="$TMP_DIR/forbidden-updater-url"
|
||||
make_workspace "$forbidden_updater_url_workspace"
|
||||
rm -rf "$forbidden_updater_url_workspace/updater"
|
||||
if DART_SRC="$forbidden_updater_url_workspace/dart-sdk-new" \
|
||||
UPDATER_SRC="$forbidden_updater_url_workspace/missing-updater" \
|
||||
UPDATER_URL=https://github.com/shorebirdtech/updater.git \
|
||||
"$forbidden_updater_url_workspace/scripts/sync_open_sources.sh" \
|
||||
>"$TMP_DIR/forbidden-updater-url.log" 2>&1; then
|
||||
echo "expected forbidden explicit UPDATER_URL to fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q "UPDATER_URL points at a forbidden official Shorebird updater remote" \
|
||||
"$TMP_DIR/forbidden-updater-url.log"
|
||||
|
||||
echo "sync_open_sources.sh smoke test passed"
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
"$ROOT/scripts/verify_ci_workflow.sh" \
|
||||
--require-upload-ready \
|
||||
"$ROOT/.github/workflows/open-shorebird-ci.yml"
|
||||
|
||||
echo "upload readiness check passed"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/open-shorebird-sha256.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
ARTIFACT="$TMP_DIR/artifact.txt"
|
||||
SIDECAR="$TMP_DIR/artifact.txt.sha256"
|
||||
CUSTOM_SIDECAR="$TMP_DIR/custom.sha256"
|
||||
|
||||
printf 'open-shorebird\n' > "$ARTIFACT"
|
||||
|
||||
"$ROOT/scripts/write_sha256.sh" "$ARTIFACT"
|
||||
|
||||
EXPECTED_HASH="$("$PYTHON_BIN" - "$ARTIFACT" <<'PY'
|
||||
import hashlib
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())
|
||||
PY
|
||||
)"
|
||||
EXPECTED_LINE="$EXPECTED_HASH artifact.txt"
|
||||
ACTUAL_LINE="$(cat "$SIDECAR")"
|
||||
|
||||
if [[ "$ACTUAL_LINE" != "$EXPECTED_LINE" ]]; then
|
||||
echo "unexpected sha256 sidecar: $ACTUAL_LINE" >&2
|
||||
echo "expected: $EXPECTED_LINE" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
"$ROOT/scripts/write_sha256.sh" "$ARTIFACT" "$CUSTOM_SIDECAR"
|
||||
CUSTOM_LINE="$(cat "$CUSTOM_SIDECAR")"
|
||||
|
||||
if [[ "$CUSTOM_LINE" != "$EXPECTED_LINE" ]]; then
|
||||
echo "unexpected custom sha256 sidecar: $CUSTOM_LINE" >&2
|
||||
echo "expected: $EXPECTED_LINE" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
if "$ROOT/scripts/write_sha256.sh" "$TMP_DIR/missing.txt" >/dev/null 2>&1; then
|
||||
echo "write_sha256.sh unexpectedly succeeded for a missing artifact" >&2
|
||||
exit 70
|
||||
fi
|
||||
|
||||
echo "write_sha256.sh smoke test passed"
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write an open Shorebird artifact proxy manifest.
|
||||
|
||||
The artifact proxy expects this file at:
|
||||
|
||||
/shorebird/<shorebird-engine-revision>/artifacts_manifest.yaml
|
||||
|
||||
It maps a custom Shorebird engine revision back to the upstream Flutter engine
|
||||
revision for unchanged artifacts, and lists the artifact paths that should be
|
||||
served from the open Shorebird mirror.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
DEFAULT_ARTIFACT_OVERRIDES = (
|
||||
"flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip",
|
||||
"flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip",
|
||||
"flutter_infra_release/flutter/$engine/linux-x64-release/artifacts.zip",
|
||||
"flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip",
|
||||
"flutter_infra_release/flutter/$engine/ios-release/artifacts.zip",
|
||||
"flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip",
|
||||
"flutter_infra_release/flutter/$engine/flutter-web-sdk.zip",
|
||||
"flutter_infra_release/flutter/$engine/darwin-arm64-release/FlutterMacOS.framework.zip",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--flutter-engine-revision",
|
||||
required=True,
|
||||
help="Upstream Flutter engine revision used for non-overridden artifacts.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--storage-bucket",
|
||||
default="shorebird",
|
||||
help="Bucket/path prefix under the Shorebird artifact mirror.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
help="Output path. Writes to stdout when omitted.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def yaml_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def build_manifest(flutter_engine_revision: str, storage_bucket: str) -> str:
|
||||
lines = [
|
||||
f"flutter_engine_revision: {yaml_quote(flutter_engine_revision)}",
|
||||
f"storage_bucket: {yaml_quote(storage_bucket)}",
|
||||
"artifact_overrides:",
|
||||
]
|
||||
lines.extend(
|
||||
f" - {yaml_quote(override)}" for override in DEFAULT_ARTIFACT_OVERRIDES
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
manifest = build_manifest(
|
||||
flutter_engine_revision=args.flutter_engine_revision,
|
||||
storage_bucket=args.storage_bucket,
|
||||
)
|
||||
|
||||
if args.output is None:
|
||||
sys.stdout.write(manifest)
|
||||
return 0
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(manifest, encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -7,8 +7,13 @@ INCLUDE_ENGINE_DEPS="${INCLUDE_ENGINE_DEPS:-0}"
|
||||
|
||||
case "$PLATFORM" in
|
||||
linux)
|
||||
TARGET_OS='["linux"]'
|
||||
DART_DOWNLOAD_ANDROID_DEPS="False"
|
||||
if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then
|
||||
TARGET_OS='["linux", "android"]'
|
||||
DART_DOWNLOAD_ANDROID_DEPS="True"
|
||||
else
|
||||
TARGET_OS='["linux"]'
|
||||
DART_DOWNLOAD_ANDROID_DEPS="False"
|
||||
fi
|
||||
;;
|
||||
macos)
|
||||
if [[ "$INCLUDE_ENGINE_DEPS" == "1" ]]; then
|
||||
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write a JSON manifest for CI release artifacts.
|
||||
|
||||
Every non-sidecar file under the input directory must have a sibling
|
||||
`<file>.sha256` sidecar in the format written by scripts/write_sha256.py:
|
||||
|
||||
<hex sha256> <basename>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import sys
|
||||
|
||||
|
||||
def digest_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(lambda: file.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_plain_file(path: Path) -> bool:
|
||||
return path.is_file() and not path.is_symlink()
|
||||
|
||||
|
||||
def parse_sidecar(path: Path) -> tuple[str, str]:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
parts = text.split()
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"expected '<sha256> <filename>', got {text!r}")
|
||||
digest, filename = parts
|
||||
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
|
||||
raise ValueError(f"invalid sha256 digest {digest!r}")
|
||||
return digest, filename
|
||||
|
||||
|
||||
def is_safe_relative_path(path: str) -> bool:
|
||||
if not path or path == ".":
|
||||
return False
|
||||
if "\\" in path or "\x00" in path or path.endswith("/"):
|
||||
return False
|
||||
if any(ord(character) < 32 for character in path):
|
||||
return False
|
||||
|
||||
candidate = PurePosixPath(path)
|
||||
if candidate.is_absolute():
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in candidate.parts):
|
||||
return False
|
||||
if candidate.parts and ":" in candidate.parts[0]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input_dir", type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--github-sha", default="")
|
||||
parser.add_argument(
|
||||
"--require",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="GLOB",
|
||||
help="Require at least one artifact path matching this glob. May be repeated.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
input_dir = args.input_dir
|
||||
output_path = args.output
|
||||
|
||||
if not input_dir.is_dir():
|
||||
print(f"missing input directory: {input_dir}", file=sys.stderr)
|
||||
return 66
|
||||
|
||||
artifacts = []
|
||||
errors = []
|
||||
for artifact_path in sorted(path for path in input_dir.rglob("*") if path.is_file()):
|
||||
if artifact_path.is_symlink():
|
||||
errors.append(f"{artifact_path.relative_to(input_dir).as_posix()}: symlink artifacts are not allowed")
|
||||
continue
|
||||
if artifact_path.suffix == ".sha256":
|
||||
continue
|
||||
if artifact_path.resolve() == output_path.resolve():
|
||||
continue
|
||||
|
||||
artifact_relative_path = artifact_path.relative_to(input_dir).as_posix()
|
||||
if not is_safe_relative_path(artifact_relative_path):
|
||||
errors.append(f"{artifact_relative_path}: unsafe artifact path")
|
||||
continue
|
||||
if artifact_path.stat().st_size <= 0:
|
||||
errors.append(f"{artifact_relative_path}: empty artifacts are not allowed")
|
||||
continue
|
||||
artifact_relative = PurePosixPath(artifact_relative_path)
|
||||
|
||||
sidecar_path = Path(f"{artifact_path}.sha256")
|
||||
if not is_plain_file(sidecar_path):
|
||||
errors.append(f"{artifact_relative_path}: missing .sha256 sidecar")
|
||||
continue
|
||||
|
||||
actual_digest = digest_file(artifact_path)
|
||||
try:
|
||||
sidecar_digest, sidecar_filename = parse_sidecar(sidecar_path)
|
||||
except ValueError as error:
|
||||
errors.append(f"{sidecar_path.relative_to(input_dir).as_posix()}: {error}")
|
||||
continue
|
||||
|
||||
if sidecar_digest != actual_digest:
|
||||
errors.append(
|
||||
f"{sidecar_path.relative_to(input_dir).as_posix()}: digest mismatch "
|
||||
f"{sidecar_digest} != {actual_digest}"
|
||||
)
|
||||
if sidecar_filename != artifact_path.name:
|
||||
errors.append(
|
||||
f"{sidecar_path.relative_to(input_dir).as_posix()}: filename mismatch "
|
||||
f"{sidecar_filename!r} != {artifact_path.name!r}"
|
||||
)
|
||||
|
||||
artifacts.append(
|
||||
{
|
||||
"path": artifact_relative_path,
|
||||
"artifact_group": artifact_relative.parts[0],
|
||||
"filename": artifact_path.name,
|
||||
"sha256": actual_digest,
|
||||
"size": artifact_path.stat().st_size,
|
||||
"sidecar": sidecar_path.relative_to(input_dir).as_posix(),
|
||||
}
|
||||
)
|
||||
|
||||
orphan_sidecars = []
|
||||
for sidecar_path in sorted(input_dir.rglob("*.sha256")):
|
||||
artifact_path = Path(str(sidecar_path)[: -len(".sha256")])
|
||||
if sidecar_path.is_symlink():
|
||||
orphan_sidecars.append(sidecar_path.relative_to(input_dir).as_posix())
|
||||
elif not is_plain_file(artifact_path):
|
||||
orphan_sidecars.append(sidecar_path.relative_to(input_dir).as_posix())
|
||||
if orphan_sidecars:
|
||||
errors.append(f"orphan .sha256 sidecars: {', '.join(orphan_sidecars)}")
|
||||
|
||||
artifact_paths = [artifact["path"] for artifact in artifacts]
|
||||
for required_glob in args.require:
|
||||
if not any(fnmatch.fnmatchcase(path, required_glob) for path in artifact_paths):
|
||||
errors.append(f"missing required artifact matching {required_glob!r}")
|
||||
|
||||
if errors:
|
||||
print(
|
||||
"release manifest validation failed:\n"
|
||||
+ "\n".join(f" {error}" for error in errors),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 70
|
||||
|
||||
manifest = {
|
||||
"format_version": 1,
|
||||
"github_sha": args.github_sha,
|
||||
"artifact_count": len(artifacts),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write a sha256 sidecar in the common `digest filename` format."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
||||
def digest_file(path: pathlib.Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(lambda: file.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) not in (2, 3):
|
||||
print("usage: write_sha256.py <artifact> [output]", file=sys.stderr)
|
||||
return 64
|
||||
|
||||
artifact_path = pathlib.Path(sys.argv[1])
|
||||
output_path = (
|
||||
pathlib.Path(sys.argv[2])
|
||||
if len(sys.argv) == 3
|
||||
else pathlib.Path(f"{artifact_path}.sha256")
|
||||
)
|
||||
|
||||
if not artifact_path.is_file():
|
||||
print(f"missing artifact: {artifact_path}", file=sys.stderr)
|
||||
return 66
|
||||
|
||||
output_path.write_text(
|
||||
f"{digest_file(artifact_path)} {artifact_path.name}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "$#" -ne 1 && "$#" -ne 2 ]]; then
|
||||
echo "usage: $0 <artifact> [output]" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON_BIN=python3
|
||||
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
||||
PYTHON_BIN=python
|
||||
fi
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/write_sha256.py" "$@"
|
||||
Reference in New Issue
Block a user