From 3445aa0636ea5a1b4f02474a16b544e22de794ac Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 25 Jun 2026 20:19:40 +0800 Subject: [PATCH] Support hosted verification without gh --- README.md | 3 +- docs/CI.md | 2 + scripts/verify_ci_workflow.rb | 4 + scripts/verify_hosted_full_sdk_build.sh | 242 ++++++++++++++++++++---- 4 files changed, 215 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index bf7ba1d..1a37485 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,8 @@ is untracked or if a required root/submodule checkout still has uncommitted changes. After upload, `scripts/verify_hosted_full_sdk_build.sh --repo owner/repo --ref main` dispatches the hosted full SDK workflow, waits for it, downloads artifacts, and -runs the downloaded-release verifier. +runs the downloaded-release verifier. It uses `gh` when available, or the +GitHub REST API with `GITHUB_TOKEN`/`GH_TOKEN`, `curl`, `jq`, and `unzip`. The heavy SDK/engine jobs default to custom runner labels `open-shorebird-linux-heavy` and `open-shorebird-macos-heavy`, then run an early disk-capacity preflight. Register larger/self-hosted runners with those labels diff --git a/docs/CI.md b/docs/CI.md index 16c0c91..a1b3e80 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -152,6 +152,8 @@ That helper dispatches `open-shorebird-ci.yml` with `full_sdk_build=true`, waits for the hosted workflow run to succeed, downloads all artifacts, and runs `scripts/verify_downloaded_release_artifacts.sh` against the downloaded output with the workflow run's `headSha`. +It uses GitHub CLI when `gh` is installed; otherwise it uses the GitHub REST API +with `GITHUB_TOKEN` or `GH_TOKEN` plus `curl`, `jq`, and `unzip`. Use `--linux-heavy-runner`, `--macos-heavy-runner`, and the disk-threshold flags when the repository uses custom larger/self-hosted runner labels. For local release assembly or reassembly after downloading workflow artifacts, diff --git a/scripts/verify_ci_workflow.rb b/scripts/verify_ci_workflow.rb index 6ec42e6..e92e8e9 100755 --- a/scripts/verify_ci_workflow.rb +++ b/scripts/verify_ci_workflow.rb @@ -1280,6 +1280,10 @@ assert!( verify_hosted_full_sdk_build.include?('-f full_sdk_build=true') && verify_hosted_full_sdk_build.include?('-f run_runtime_smokes=false') && verify_hosted_full_sdk_build.include?('gh run download "$run_id"') && + verify_hosted_full_sdk_build.include?('GITHUB_TOKEN or GH_TOKEN is required') && + verify_hosted_full_sdk_build.include?('api_request POST "/actions/workflows/$WORKFLOW/dispatches"') && + verify_hosted_full_sdk_build.include?('archive_download_url') && + verify_hosted_full_sdk_build.include?('unzip -q "$zip_path" -d "$artifact_dir"') && verify_hosted_full_sdk_build.include?('--json status,conclusion,url,headSha') && verify_hosted_full_sdk_build.include?('--github-sha "$run_head_sha"') && verify_hosted_full_sdk_build.include?('unable to read headSha') && diff --git a/scripts/verify_hosted_full_sdk_build.sh b/scripts/verify_hosted_full_sdk_build.sh index af6eb74..2081ca4 100755 --- a/scripts/verify_hosted_full_sdk_build.sh +++ b/scripts/verify_hosted_full_sdk_build.sh @@ -9,6 +9,10 @@ Dispatches the Open Shorebird full SDK build on GitHub Actions, waits for the workflow run to finish, downloads all artifacts, and verifies the release manifest plus assembled artifact mirror. +Uses GitHub CLI when `gh` is available. Otherwise uses the GitHub REST API with +`GITHUB_TOKEN` or `GH_TOKEN`; that fallback also requires `curl`, `jq`, and +`unzip`. + Options: --repo owner/name GitHub repository to run against. --ref branch-or-sha Ref to dispatch. Defaults to current branch. @@ -38,6 +42,7 @@ SDK_MIN_FREE_DISK_GB="" ENGINE_MIN_FREE_DISK_GB="" BASE_FLUTTER_ENGINE_REVISION="" RUN_GCLIENT_SYNC=true +GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}" while [[ "$#" -gt 0 ]]; do case "$1" in @@ -107,11 +112,6 @@ if [[ -z "$REPO" ]]; then exit 64 fi -if ! command -v gh >/dev/null 2>&1; then - echo "GitHub CLI 'gh' is required" >&2 - exit 69 -fi - if [[ -z "$REF" ]]; then REF="$(git -C "$ROOT" branch --show-current 2>/dev/null || true)" fi @@ -131,35 +131,195 @@ run_fields=( [[ -z "$ENGINE_MIN_FREE_DISK_GB" ]] || run_fields+=(-f engine_min_free_disk_gb="$ENGINE_MIN_FREE_DISK_GB") [[ -z "$BASE_FLUTTER_ENGINE_REVISION" ]] || run_fields+=(-f base_flutter_engine_revision="$BASE_FLUTTER_ENGINE_REVISION") +api_token() { + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + printf '%s' "$GITHUB_TOKEN" + elif [[ -n "${GH_TOKEN:-}" ]]; then + printf '%s' "$GH_TOKEN" + else + echo "GITHUB_TOKEN or GH_TOKEN is required when gh is not installed" >&2 + exit 69 + fi +} + +api_require_tool() { + local tool="$1" + if ! command -v "$tool" >/dev/null 2>&1; then + echo "$tool is required when gh is not installed" >&2 + exit 69 + fi +} + +api_request() { + local method="$1" + local path="$2" + local data="${3:-}" + local token + token="$(api_token)" + + local curl_args=( + -fsSL + -X "$method" + -H "Accept: application/vnd.github+json" + -H "Authorization: Bearer $token" + -H "X-GitHub-Api-Version: 2022-11-28" + ) + if [[ -n "$data" ]]; then + curl_args+=(-H "Content-Type: application/json" -d "$data") + fi + + curl "${curl_args[@]}" "$GITHUB_API_URL/repos/$REPO$path" +} + +api_dispatch_payload() { + local inputs_filter + inputs_filter='{full_sdk_build: "true", run_gclient_sync: $run_gclient_sync, run_runtime_smokes: "false"}' + local jq_args=( + --arg ref "$REF" + --arg run_gclient_sync "$RUN_GCLIENT_SYNC" + ) + if [[ -n "$LINUX_HEAVY_RUNNER" ]]; then + inputs_filter="$inputs_filter + {linux_heavy_runner: \$linux_heavy_runner}" + jq_args+=(--arg linux_heavy_runner "$LINUX_HEAVY_RUNNER") + fi + if [[ -n "$MACOS_HEAVY_RUNNER" ]]; then + inputs_filter="$inputs_filter + {macos_heavy_runner: \$macos_heavy_runner}" + jq_args+=(--arg macos_heavy_runner "$MACOS_HEAVY_RUNNER") + fi + if [[ -n "$SDK_MIN_FREE_DISK_GB" ]]; then + inputs_filter="$inputs_filter + {sdk_min_free_disk_gb: \$sdk_min_free_disk_gb}" + jq_args+=(--arg sdk_min_free_disk_gb "$SDK_MIN_FREE_DISK_GB") + fi + if [[ -n "$ENGINE_MIN_FREE_DISK_GB" ]]; then + inputs_filter="$inputs_filter + {engine_min_free_disk_gb: \$engine_min_free_disk_gb}" + jq_args+=(--arg engine_min_free_disk_gb "$ENGINE_MIN_FREE_DISK_GB") + fi + if [[ -n "$BASE_FLUTTER_ENGINE_REVISION" ]]; then + inputs_filter="$inputs_filter + {base_flutter_engine_revision: \$base_flutter_engine_revision}" + jq_args+=(--arg base_flutter_engine_revision "$BASE_FLUTTER_ENGINE_REVISION") + fi + + jq -n "${jq_args[@]}" "{ref: \$ref, inputs: ($inputs_filter)}" +} + +api_download_artifacts() { + rm -rf "$DOWNLOAD_DIR" + mkdir -p "$DOWNLOAD_DIR" + + local page=1 + local downloaded_count=0 + while true; do + local response + response="$(api_request GET "/actions/runs/$run_id/artifacts?per_page=100&page=$page")" + local artifact_count + artifact_count="$(jq '.artifacts | length' <<<"$response")" + [[ "$artifact_count" == "0" ]] && break + + while IFS= read -r artifact; do + local name + local url + name="$(jq -r '.name' <<<"$artifact")" + url="$(jq -r '.archive_download_url' <<<"$artifact")" + case "$name" in + ""|*/*|*..*) + echo "unsafe GitHub artifact name: $name" >&2 + exit 70 + ;; + esac + if [[ -z "$url" || "$url" == "null" ]]; then + echo "missing archive_download_url for artifact: $name" >&2 + exit 70 + fi + + local artifact_dir + local zip_path + artifact_dir="$DOWNLOAD_DIR/$name" + zip_path="$(mktemp "${TMPDIR:-/tmp}/github-artifact.XXXXXX")" + curl \ + -fsSL \ + -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(api_token)" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -o "$zip_path" \ + "$url" + rm -rf "$artifact_dir" + mkdir -p "$artifact_dir" + unzip -q "$zip_path" -d "$artifact_dir" + rm -f "$zip_path" + downloaded_count=$((downloaded_count + 1)) + done < <(jq -c '.artifacts[] | select(.expired | not)' <<<"$response") + + page=$((page + 1)) + done + + if [[ "$downloaded_count" -eq 0 ]]; then + echo "no non-expired artifacts were available for workflow run $run_id" >&2 + exit 70 + fi +} + +use_gh=0 +if command -v gh >/dev/null 2>&1; then + use_gh=1 +else + api_require_tool curl + api_require_tool jq + api_require_tool unzip + api_token >/dev/null +fi + echo "Dispatching $WORKFLOW on $REPO@$REF with full_sdk_build=true" start_epoch="$(date +%s)" start_iso="$(date -u -r "$start_epoch" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "@$start_epoch" +"%Y-%m-%dT%H:%M:%SZ")" -gh workflow run "$WORKFLOW" \ - --repo "$REPO" \ - --ref "$REF" \ - "${run_fields[@]}" +if [[ "$use_gh" == "1" ]]; then + gh workflow run "$WORKFLOW" \ + --repo "$REPO" \ + --ref "$REF" \ + "${run_fields[@]}" +else + api_request POST "/actions/workflows/$WORKFLOW/dispatches" "$(api_dispatch_payload)" >/dev/null +fi run_id="" for _ in {1..40}; do - run_list_args=( - --repo "$REPO" - --workflow "$WORKFLOW" - --event workflow_dispatch - --json databaseId,createdAt - --limit 20 - ) - if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then - run_list_args+=(--commit "$REF") + if [[ "$use_gh" == "1" ]]; then + run_list_args=( + --repo "$REPO" + --workflow "$WORKFLOW" + --event workflow_dispatch + --json databaseId,createdAt + --limit 20 + ) + if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then + run_list_args+=(--commit "$REF") + else + run_list_args+=(--branch "$REF") + fi + run_id="$( + gh run list \ + "${run_list_args[@]}" \ + --jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \ + | + head -n 1 + )" else - run_list_args+=(--branch "$REF") + runs_response="$(api_request GET "/actions/workflows/$WORKFLOW/runs?event=workflow_dispatch&per_page=20")" + run_id="$( + jq -r \ + --arg ref "$REF" \ + --arg start_iso "$start_iso" \ + ' + .workflow_runs + | map(select(.created_at >= $start_iso)) + | map(select(.head_branch == $ref or .head_sha == $ref)) + | sort_by(.created_at) + | reverse + | .[0].id // "" + ' \ + <<<"$runs_response" + )" fi - run_id="$( - gh run list \ - "${run_list_args[@]}" \ - --jq ".[] | select(.createdAt >= \"$start_iso\") | .databaseId" \ - | - head -n 1 - )" [[ -z "$run_id" ]] || break sleep 3 done @@ -173,12 +333,20 @@ echo "Waiting for hosted full SDK run: $run_id" deadline=$((start_epoch + TIMEOUT_MINUTES * 60)) run_head_sha="" while true; do - IFS=$'\t' read -r status conclusion url run_head_sha < <( - gh run view "$run_id" \ - --repo "$REPO" \ - --json status,conclusion,url,headSha \ - --jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv' - ) + if [[ "$use_gh" == "1" ]]; then + IFS=$'\t' read -r status conclusion url run_head_sha < <( + gh run view "$run_id" \ + --repo "$REPO" \ + --json status,conclusion,url,headSha \ + --jq '[.status, (.conclusion // ""), .url, (.headSha // "")] | @tsv' + ) + else + run_response="$(api_request GET "/actions/runs/$run_id")" + IFS=$'\t' read -r status conclusion url run_head_sha < <( + jq -r '[.status, (.conclusion // ""), .html_url, (.head_sha // "")] | @tsv' \ + <<<"$run_response" + ) + fi echo "run $run_id status=$status conclusion=${conclusion:-null} url=$url" if [[ "$status" == "completed" ]]; then @@ -195,9 +363,13 @@ while true; do sleep "$POLL_SECONDS" done -rm -rf "$DOWNLOAD_DIR" -mkdir -p "$DOWNLOAD_DIR" -gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR" +if [[ "$use_gh" == "1" ]]; then + rm -rf "$DOWNLOAD_DIR" + mkdir -p "$DOWNLOAD_DIR" + gh run download "$run_id" --repo "$REPO" --dir "$DOWNLOAD_DIR" +else + api_download_artifacts +fi if [[ -z "$run_head_sha" ]]; then echo "unable to read headSha for workflow run $run_id" >&2 exit 70