feat: Add support for diff patches (#163)

Co-authored-by: Felix Angelov <felix@shorebird.dev>

updater/library/Cargo.toml

@@ -19,17 +19,28 @@ reqwest = { version = "0.11",  default-features = false, features = ["blocking",
 # Json serialization/de-serialization.
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.93"
-# Used for error handling.
+# Used for error handling for now.
 anyhow = {version = "1.0.69", features = ["backtrace"]}
 # For error!(), info!(), etc macros. `print` will not show up on Android.
 log = "0.4.14"
+# For implementing thread-local-storage of ResolvedConfig object.
 once_cell = "1.17.1"
+# For reading shorebird.yaml
 serde_yaml = "0.9.19"
-uuid = { version = "1.3.0", features = ["v4", "fast-rng", "macro-diagnostics", "serde"]}
+# For validating hashes of downloaded patch files.
 sha2 = "0.10.6"
+# For inflating compressed patch files.
+bipatch = "1.0.0"
+# comde is a wrapper around several compression libraries.
+# We only use zstd and could depend on it directly instead.
+comde = {version = "0.2.3", default-features = false, features = ["zstandard"]}
+# Pipe is a simple in-memory pipe implementation, there might be a std way too?
+pipe = "0.4.0"
 
 [target.'cfg(target_os = "android")'.dependencies]
+# For logging to Android logcat.
 android_logger = "0.13.0"
+# Send panics to log (instead of stderr), thus logcat on Android.
 log-panics = { version = "2", features = ["with-backtrace"]}
 
 

updater/library/include/updater.h

@@ -59,13 +59,13 @@ void shorebird_init(const struct AppParameters *c_params,
                     const char *c_yaml);
 
 /**
- * Return the active version of the app, or NULL if there is no active version.
+ * Return the active patch number, or NULL if there is no active patch.
  */
-SHOREBIRD_EXPORT char *shorebird_active_version(void);
+SHOREBIRD_EXPORT char *shorebird_active_patch_number(void);
 
 /**
- * Return the path to the active version of the app, or NULL if there is no
- * active version.
+ * Return the path to the active patch for the app, or NULL if there is no
+ * active patch.
  */
 SHOREBIRD_EXPORT char *shorebird_active_path(void);
 
@@ -84,6 +84,11 @@ SHOREBIRD_EXPORT bool shorebird_check_for_update(void);
  */
 SHOREBIRD_EXPORT void shorebird_update(void);
 
+/**
+ * Report that the app failed to launch.  This will cause the updater to
+ * attempt to roll back to the previous version if this version has not
+ * been launched successfully before.
+ */
 SHOREBIRD_EXPORT void shorebird_report_failed_launch(void);
 
 #ifdef __cplusplus

updater/library/src/c_api.rs

@@ -66,10 +66,9 @@ pub extern "C" fn shorebird_init(c_params: *const AppParameters, c_yaml: *const
     }
 }
 
-/// Return the active version of the app, or NULL if there is no active version.
-// TODO: This should probably be renamed to `shorebird_active_patch_number`.
+/// Return the active patch number, or NULL if there is no active patch.
 #[no_mangle]
-pub extern "C" fn shorebird_active_version() -> *mut c_char {
+pub extern "C" fn shorebird_active_patch_number() -> *mut c_char {
     let patch = updater::active_patch();
     match patch {
         Some(v) => {
@@ -80,8 +79,8 @@ pub extern "C" fn shorebird_active_version() -> *mut c_char {
     }
 }
 
-/// Return the path to the active version of the app, or NULL if there is no
-/// active version.
+/// Return the path to the active patch for the app, or NULL if there is no
+/// active patch.
 #[no_mangle]
 // rename to shorebird_patch_path
 pub extern "C" fn shorebird_active_path() -> *mut c_char {

updater/library/src/cache.rs

@@ -266,7 +266,7 @@ impl UpdaterState {
             .into());
         }
 
-        // Move the patch into the slot.
+        // Move the artifact into the slot.
         let artifact_path = slot_dir.join("dlc.vmcode");
         std::fs::rename(&patch.path, &artifact_path)?;
 
@@ -363,7 +363,7 @@ mod tests {
     }
 
     #[test]
-    fn dont_install_known_bad_patch() {
+    fn do_not_install_known_bad_patch() {
         let tmp_dir = TempDir::new("example").unwrap();
         let mut state = test_state(&tmp_dir);
         let bad_patch = fake_patch(&tmp_dir, 1);

updater/library/src/network.rs

@@ -4,7 +4,7 @@
 use serde::{Deserialize, Serialize};
 use std::fs::File;
 use std::io::Write;
-use std::path::PathBuf;
+use std::path::Path;
 use std::string::ToString;
 
 use crate::cache::UpdaterState;
@@ -16,9 +16,17 @@ fn patches_check_url(base_url: &str) -> String {
 
 #[derive(Debug, Deserialize)]
 pub struct Patch {
+    /// The patch number.  Starts at 1 for each new release and increases
+    /// monotonically.
     pub number: usize,
+    /// The hash of the final uncompressed patch file.
     pub hash: String,
+    /// The URL to download the patch file from.
     pub download_url: String,
+    /// Whether the artifact is a diff (modern) or full (legacy) artifact.
+    /// Will eventually be removed once we no longer support legacy artifacts.
+    #[serde(default)]
+    pub is_diff: bool,
 }
 
 #[derive(Debug, Serialize)]
@@ -74,7 +82,7 @@ pub fn send_patch_check_request(
     return Ok(response);
 }
 
-pub fn download_to_path(url: &str, path: &PathBuf) -> anyhow::Result<()> {
+pub fn download_to_path(url: &str, path: &Path) -> anyhow::Result<()> {
     // Download the file at the given url to the given path.
     let client = reqwest::blocking::Client::new();
     let response = client.get(url).send()?;

updater/library/src/updater.rs

@@ -7,7 +7,7 @@ use crate::config::{set_config, with_config, ResolvedConfig};
 use crate::logging::init_logging;
 use crate::network::{download_to_path, send_patch_check_request};
 use crate::yaml::YamlConfig;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 pub enum UpdateStatus {
     NoUpdate,
@@ -109,8 +109,17 @@ fn update_internal(config: &ResolvedConfig) -> anyhow::Result<UpdateStatus> {
     let patch = response.patch.ok_or(UpdateError::BadServerResponse)?;
 
     let download_dir = PathBuf::from(&config.cache_dir);
-    let download_path = download_dir.join(patch.number.to_string());
+    let mut download_path = download_dir.join(patch.number.to_string());
     download_to_path(&patch.download_url, &download_path)?;
+
+    // Inflate the patch from a diff if needed.
+    if patch.is_diff {
+        let base_path = PathBuf::from(&config.original_libapp_path);
+        let output_path = download_dir.join(format!("{}.full", patch.number.to_string()));
+        inflate(&download_path, &base_path, &output_path)?;
+        download_path = output_path;
+    }
+
     // Check the hash before moving into place.
     // Move/state update should be "atomic".
     // Consider supporting allowing the system to download for us (e.g. iOS).
@@ -125,6 +134,46 @@ fn update_internal(config: &ResolvedConfig) -> anyhow::Result<UpdateStatus> {
     return Ok(UpdateStatus::UpdateInstalled);
 }
 
+fn inflate(patch_path: &Path, base_path: &Path, output_path: &Path) -> anyhow::Result<()> {
+    info!("Patch is compressed, inflating...");
+    use comde::de::Decompressor;
+    use comde::zstd::ZstdDecompressor;
+    use std::fs::File;
+    use std::io::{BufReader, BufWriter};
+
+    // Open all our files first for error clarity.  Otherwise we might see
+    // PipeReader/Writer errors instead of file open errors.
+    info!("Reading base file: {:?}", base_path);
+    let base_r = File::open(base_path)?;
+    let compressed_patch_r = BufReader::new(File::open(patch_path)?);
+    let output_file_w = File::create(&output_path)?;
+
+    // Set up a pipe to connect the writing from the decompression thread
+    // to the reading of the decompressed patch data on this thread.
+    let (patch_r, patch_w) = pipe::pipe();
+
+    let decompress = ZstdDecompressor::new();
+    // Spawn a thread to run the decompression in parallel to the patching.
+    // decompress.copy will block on the pipe being full (I think) and then
+    // when it returns the thread will exit.
+    std::thread::spawn(move || {
+        let result = decompress.copy(compressed_patch_r, patch_w);
+        // If this thread fails, undoubtedly the main thread will fail too.
+        // Most important is to not crash.
+        if let Err(err) = result {
+            error!("Decompression thread failed: {err}");
+        }
+    });
+
+    // Do the patch, using the uncompressed patch data from the pipe.
+    let mut fresh_r = bipatch::Reader::new(patch_r, base_r)?;
+
+    // Write out the resulting patched file to the new location.
+    let mut output_w = BufWriter::new(output_file_w);
+    std::io::copy(&mut fresh_r, &mut output_w)?;
+    Ok(())
+}
+
 /// Reads the current patch from the cache and returns it.
 pub fn active_patch() -> Option<PatchInfo> {
     return with_config(|config| {