diff --git a/scripts/build_engine.sh b/scripts/build_engine.sh index 3bd90360..be50d789 100755 --- a/scripts/build_engine.sh +++ b/scripts/build_engine.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # Given a path to a Flutter engine, generate a shorebird version of it. # This is done by removing all the files that are not needed for the @@ -8,21 +8,67 @@ # Usage: # ./build_engine +# The path to the shorebird checkout. +SHOREBIRD_PATH=$(dirname $(dirname $(realpath $0))) + # The path to the Flutter engine. -ENGINE_PATH=$1 +ENGINE_PATH=$(realpath $1) # The path where the shorebird engine will be generated. OUTPUT_PATH=$2 +if [[ "$OUTPUT_PATH" = /* ]] +then + : # Absolute path +else + : + echo "Output path must be absolute" + exit 1 +fi + +echo "Building engine from $ENGINE_PATH to $OUTPUT_PATH" + +# Assume cargo-ndk is already installed: +# cargo install cargo-ndk + +# Assume the toolchains are already installed +# rustup target add \ +# aarch64-linux-android \ +# armv7-linux-androideabi \ +# x86_64-linux-android \ +# i686-linux-android + +# Build the Rust library. +cd $SHOREBIRD_PATH/updater/library +# Build both the arm64 and armv7 versions of the library. +cargo ndk \ + --target aarch64-linux-android \ + --target armv7-linux-androideabi \ + build --release + +# We assume the engine has symlinks set up for now (we should not rely on this). + +# Build the patch tool. +cd $SHOREBIRD_PATH/updater/patch +cargo build --release + +# Build the engine cd $ENGINE_PATH # Build the engine in release mode for android arm64. -./src/flutter/tools/gn --android --android-cpu=arm64 --runtime-mode=release +./src/flutter/tools/gn --android --android-cpu=arm64 --runtime-mode=release --no-goma ninja -C ./src/out/android_release_arm64 # Build the the host_release output. -./src/flutter/tools/gn --runtime-mode=release +./src/flutter/tools/gn --runtime-mode=release --no-goma ninja -C ./src/out/host_release +# Arm64 version for when we need it: +# ./src/flutter/tools/gn --runtime-mode=release --mac-cpu=arm64 +# ninja -C ./src/out/host_release_arm64 + +# Hack around host_release builds being broken on 3.7.8 stable. +# cp ./src/out/host_release_arm64/gen/const_finder.dart.snapshot ./src/out/host_release/gen/const_finder.dart.snapshot +# cp ./src/out/host_release_arm64/font-subset ./src/out/host_release/font-subset # List of all files to keep. KEEP_FILES=( @@ -51,7 +97,7 @@ KEEP_FILES=( "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/kernel_worker.dart.snapshot" "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dartdev.dill" "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/frontend_server.dart.snapshot" - "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dart2wasm_product.snapshot" + # "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dart2wasm_product.snapshot" "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dds.dart.snapshot" "flutter/prebuilts/macos-x64/dart-sdk/bin/dart" "flutter/prebuilts/macos-x64/dart-sdk/bin/utils/gen_snapshot" @@ -71,6 +117,8 @@ do cp -r $ENGINE_PATH/src/$file $TEMP_DIR/$file done +cp $SHOREBIRD_PATH/updater/target/release/patch $TEMP_DIR/patch + cd $TEMP_DIR # Zip the output. diff --git a/updater/Cargo.toml b/updater/Cargo.toml index bcbfce6a..ee7db420 100644 --- a/updater/Cargo.toml +++ b/updater/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["cli", "library"] \ No newline at end of file +members = ["cli", "library", "patch"] \ No newline at end of file diff --git a/updater/library/src/config.rs b/updater/library/src/config.rs index 7dbaeb6f..fb52b412 100644 --- a/updater/library/src/config.rs +++ b/updater/library/src/config.rs @@ -96,6 +96,8 @@ pub fn current_arch() -> &'static str { static ARCH: &str = "x86_64"; #[cfg(target_arch = "aarch64")] static ARCH: &str = "aarch64"; + #[cfg(target_arch = "arm")] + static ARCH: &str = "arm"; return ARCH; } diff --git a/updater/patch/Cargo.toml b/updater/patch/Cargo.toml new file mode 100644 index 00000000..62295a43 --- /dev/null +++ b/updater/patch/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "patch" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +# Compression and decompression of patch files. +bidiff = "1.0.0" +# Pipe is a simple in-memory pipe implementation, there might be a std way too? +pipe = "0.4.0" +# comde is a wrapper around several compression libraries. +# We only use zstd and could depend on it directly instead. +comde = {version = "0.2.3", default-features = false, features = ["zstandard"]} diff --git a/updater/patch/README.md b/updater/patch/README.md new file mode 100644 index 00000000..394c6ffd --- /dev/null +++ b/updater/patch/README.md @@ -0,0 +1,13 @@ +# patch command line tool + +This is the tool used by the `shorebird` command line to compute the patch +file for uploading to the server. + +This currently uses the rust `bidiff` crate to compute the patch file. +and could just use the `bic` command line tool included in that crate. However +we're explicitly writing our own command line to allow us to change the +underlying compression without affecting the `shorebird` command line callers. + +## Usage + + patch diff --git a/updater/patch/src/main.rs b/updater/patch/src/main.rs new file mode 100644 index 00000000..bf58a5e3 --- /dev/null +++ b/updater/patch/src/main.rs @@ -0,0 +1,52 @@ +use bidiff::DiffParams; +use std::{ + fs::{self, File}, + io::{BufWriter, Write}, + time::Instant, +}; + +use comde::com::Compressor; +use comde::zstd::ZstdCompressor; + +// Originally inspired from example in: +// https://github.com/divvun/bidiff/blob/main/crates/bic/src/main.rs +// and then hacked down to just service our needs. + +// comde is just a wrapper around various compression/decompression libraries. +// and we could just depend on the zstd crate directly if we end up using +// zstd long term. + +fn main() { + let mut args = std::env::args(); + args.next(); // skip program name + let older = args.next().expect("path to base file"); + let newer = args.next().expect("path to new file"); + let patch = args.next().expect("path to output file"); + + let start = Instant::now(); + + let older_contents = fs::read(older).expect("read base file"); + let newer_contents = fs::read(newer).expect("read new file"); + + let (mut patch_r, mut patch_w) = pipe::pipe(); + let diff_params = DiffParams::new(1, None).unwrap(); + std::thread::spawn(move || { + bidiff::simple_diff_with_params( + &older_contents[..], + &newer_contents[..], + &mut patch_w, + &diff_params, + ) + .unwrap(); + }); + + let compressor = ZstdCompressor::new(); + + let mut compatch_w = BufWriter::new(File::create(patch).expect("create patch file")); + compressor + .compress(&mut compatch_w, &mut patch_r) + .expect("compress patch"); + compatch_w.flush().expect("flush patch"); + + println!("Completed in {:?}", start.elapsed()); +}