feat: Add patch command for use by cli (#179)

scripts/build_engine.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 
 # Given a path to a Flutter engine, generate a shorebird version of it.
 # This is done by removing all the files that are not needed for the
@@ -8,21 +8,67 @@
 # Usage:
 # ./build_engine <path-to-engine> <output-directory>
 
+# The path to the shorebird checkout.
+SHOREBIRD_PATH=$(dirname $(dirname $(realpath $0)))
+
 # The path to the Flutter engine.
-ENGINE_PATH=$1
+ENGINE_PATH=$(realpath $1)
 
 # The path where the shorebird engine will be generated.
 OUTPUT_PATH=$2
 
+if [[ "$OUTPUT_PATH" = /* ]]
+then
+   : # Absolute path
+else
+   :
+    echo "Output path must be absolute"
+    exit 1
+fi
+
+echo "Building engine from $ENGINE_PATH to $OUTPUT_PATH"
+
+# Assume cargo-ndk is already installed:
+# cargo install cargo-ndk
+
+# Assume the toolchains are already installed
+# rustup target add \
+#     aarch64-linux-android \
+#     armv7-linux-androideabi \
+#     x86_64-linux-android \
+#     i686-linux-android
+
+# Build the Rust library.
+cd $SHOREBIRD_PATH/updater/library
+# Build both the arm64 and armv7 versions of the library.
+cargo ndk \
+  --target aarch64-linux-android \
+  --target armv7-linux-androideabi \
+  build --release
+
+# We assume the engine has symlinks set up for now (we should not rely on this).
+
+# Build the patch tool.
+cd $SHOREBIRD_PATH/updater/patch
+cargo build --release
+
+# Build the engine
 cd $ENGINE_PATH
 
 # Build the engine in release mode for android arm64.
-./src/flutter/tools/gn --android --android-cpu=arm64 --runtime-mode=release
+./src/flutter/tools/gn --android --android-cpu=arm64 --runtime-mode=release --no-goma
 ninja -C ./src/out/android_release_arm64
 
 # Build the the host_release output.
-./src/flutter/tools/gn --runtime-mode=release
+./src/flutter/tools/gn --runtime-mode=release --no-goma
 ninja -C ./src/out/host_release
+# Arm64 version for when we need it:
+# ./src/flutter/tools/gn --runtime-mode=release --mac-cpu=arm64
+# ninja -C ./src/out/host_release_arm64
+
+# Hack around host_release builds being broken on 3.7.8 stable.
+# cp ./src/out/host_release_arm64/gen/const_finder.dart.snapshot ./src/out/host_release/gen/const_finder.dart.snapshot
+# cp ./src/out/host_release_arm64/font-subset ./src/out/host_release/font-subset
 
 # List of all files to keep.
 KEEP_FILES=(
@@ -51,7 +97,7 @@ KEEP_FILES=(
   "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/kernel_worker.dart.snapshot"
   "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dartdev.dill"
   "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/frontend_server.dart.snapshot"
-  "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dart2wasm_product.snapshot"
+  # "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dart2wasm_product.snapshot"
   "flutter/prebuilts/macos-x64/dart-sdk/bin/snapshots/dds.dart.snapshot"
   "flutter/prebuilts/macos-x64/dart-sdk/bin/dart"
   "flutter/prebuilts/macos-x64/dart-sdk/bin/utils/gen_snapshot"
@@ -71,6 +117,8 @@ do
   cp -r $ENGINE_PATH/src/$file $TEMP_DIR/$file
 done
 
+cp $SHOREBIRD_PATH/updater/target/release/patch $TEMP_DIR/patch
+
 cd $TEMP_DIR
 
 # Zip the output.

updater/Cargo.toml

@@ -1,2 +1,2 @@
 [workspace]
-members = ["cli", "library"]
+members = ["cli", "library", "patch"]

updater/library/src/config.rs

@@ -96,6 +96,8 @@ pub fn current_arch() -> &'static str {
     static ARCH: &str = "x86_64";
     #[cfg(target_arch = "aarch64")]
     static ARCH: &str = "aarch64";
+    #[cfg(target_arch = "arm")]
+    static ARCH: &str = "arm";
     return ARCH;
 }
 

updater/patch/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "patch"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# Compression and decompression of patch files.
+bidiff = "1.0.0"
+# Pipe is a simple in-memory pipe implementation, there might be a std way too?
+pipe = "0.4.0"
+# comde is a wrapper around several compression libraries.
+# We only use zstd and could depend on it directly instead.
+comde = {version = "0.2.3", default-features = false, features = ["zstandard"]}

updater/patch/README.md

@@ -0,0 +1,13 @@
+# patch command line tool
+
+This is the tool used by the `shorebird` command line to compute the patch
+file for uploading to the server.
+
+This currently uses the rust `bidiff` crate to compute the patch file.
+and could just use the `bic` command line tool included in that crate. However
+we're explicitly writing our own command line to allow us to change the
+underlying compression without affecting the `shorebird` command line callers.
+
+## Usage
+
+    patch <old> <new> <patch>

updater/patch/src/main.rs

@@ -0,0 +1,52 @@
+use bidiff::DiffParams;
+use std::{
+    fs::{self, File},
+    io::{BufWriter, Write},
+    time::Instant,
+};
+
+use comde::com::Compressor;
+use comde::zstd::ZstdCompressor;
+
+// Originally inspired from example in:
+// https://github.com/divvun/bidiff/blob/main/crates/bic/src/main.rs
+// and then hacked down to just service our needs.
+
+// comde is just a wrapper around various compression/decompression libraries.
+// and we could just depend on the zstd crate directly if we end up using
+// zstd long term.
+
+fn main() {
+    let mut args = std::env::args();
+    args.next(); // skip program name
+    let older = args.next().expect("path to base file");
+    let newer = args.next().expect("path to new file");
+    let patch = args.next().expect("path to output file");
+
+    let start = Instant::now();
+
+    let older_contents = fs::read(older).expect("read base file");
+    let newer_contents = fs::read(newer).expect("read new file");
+
+    let (mut patch_r, mut patch_w) = pipe::pipe();
+    let diff_params = DiffParams::new(1, None).unwrap();
+    std::thread::spawn(move || {
+        bidiff::simple_diff_with_params(
+            &older_contents[..],
+            &newer_contents[..],
+            &mut patch_w,
+            &diff_params,
+        )
+        .unwrap();
+    });
+
+    let compressor = ZstdCompressor::new();
+
+    let mut compatch_w = BufWriter::new(File::create(patch).expect("create patch file"));
+    compressor
+        .compress(&mut compatch_w, &mut patch_r)
+        .expect("compress patch");
+    compatch_w.flush().expect("flush patch");
+
+    println!("Completed in {:?}", start.elapsed());
+}