From 0ba204e5c0270f7f6998a4e29317c368dbe9ace3 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 24 Mar 2023 12:59:35 -0700 Subject: [PATCH] feat: Add support for diff patches (#163) Co-authored-by: Felix Angelov --- updater/library/Cargo.toml | 15 +++++++-- updater/library/include/updater.h | 13 +++++--- updater/library/src/c_api.rs | 9 +++--- updater/library/src/cache.rs | 4 +-- updater/library/src/network.rs | 12 +++++-- updater/library/src/updater.rs | 53 +++++++++++++++++++++++++++++-- 6 files changed, 89 insertions(+), 17 deletions(-) diff --git a/updater/library/Cargo.toml b/updater/library/Cargo.toml index ce1716a1..f193d3a1 100644 --- a/updater/library/Cargo.toml +++ b/updater/library/Cargo.toml @@ -19,17 +19,28 @@ reqwest = { version = "0.11", default-features = false, features = ["blocking", # Json serialization/de-serialization. serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.93" -# Used for error handling. +# Used for error handling for now. anyhow = {version = "1.0.69", features = ["backtrace"]} # For error!(), info!(), etc macros. `print` will not show up on Android. log = "0.4.14" +# For implementing thread-local-storage of ResolvedConfig object. once_cell = "1.17.1" +# For reading shorebird.yaml serde_yaml = "0.9.19" -uuid = { version = "1.3.0", features = ["v4", "fast-rng", "macro-diagnostics", "serde"]} +# For validating hashes of downloaded patch files. sha2 = "0.10.6" +# For inflating compressed patch files. +bipatch = "1.0.0" +# comde is a wrapper around several compression libraries. +# We only use zstd and could depend on it directly instead. +comde = {version = "0.2.3", default-features = false, features = ["zstandard"]} +# Pipe is a simple in-memory pipe implementation, there might be a std way too? +pipe = "0.4.0" [target.'cfg(target_os = "android")'.dependencies] +# For logging to Android logcat. android_logger = "0.13.0" +# Send panics to log (instead of stderr), thus logcat on Android. log-panics = { version = "2", features = ["with-backtrace"]} diff --git a/updater/library/include/updater.h b/updater/library/include/updater.h index c6530456..9002c1bb 100644 --- a/updater/library/include/updater.h +++ b/updater/library/include/updater.h @@ -59,13 +59,13 @@ void shorebird_init(const struct AppParameters *c_params, const char *c_yaml); /** - * Return the active version of the app, or NULL if there is no active version. + * Return the active patch number, or NULL if there is no active patch. */ -SHOREBIRD_EXPORT char *shorebird_active_version(void); +SHOREBIRD_EXPORT char *shorebird_active_patch_number(void); /** - * Return the path to the active version of the app, or NULL if there is no - * active version. + * Return the path to the active patch for the app, or NULL if there is no + * active patch. */ SHOREBIRD_EXPORT char *shorebird_active_path(void); @@ -84,6 +84,11 @@ SHOREBIRD_EXPORT bool shorebird_check_for_update(void); */ SHOREBIRD_EXPORT void shorebird_update(void); +/** + * Report that the app failed to launch. This will cause the updater to + * attempt to roll back to the previous version if this version has not + * been launched successfully before. + */ SHOREBIRD_EXPORT void shorebird_report_failed_launch(void); #ifdef __cplusplus diff --git a/updater/library/src/c_api.rs b/updater/library/src/c_api.rs index b4ce786a..22734384 100644 --- a/updater/library/src/c_api.rs +++ b/updater/library/src/c_api.rs @@ -66,10 +66,9 @@ pub extern "C" fn shorebird_init(c_params: *const AppParameters, c_yaml: *const } } -/// Return the active version of the app, or NULL if there is no active version. -// TODO: This should probably be renamed to `shorebird_active_patch_number`. +/// Return the active patch number, or NULL if there is no active patch. #[no_mangle] -pub extern "C" fn shorebird_active_version() -> *mut c_char { +pub extern "C" fn shorebird_active_patch_number() -> *mut c_char { let patch = updater::active_patch(); match patch { Some(v) => { @@ -80,8 +79,8 @@ pub extern "C" fn shorebird_active_version() -> *mut c_char { } } -/// Return the path to the active version of the app, or NULL if there is no -/// active version. +/// Return the path to the active patch for the app, or NULL if there is no +/// active patch. #[no_mangle] // rename to shorebird_patch_path pub extern "C" fn shorebird_active_path() -> *mut c_char { diff --git a/updater/library/src/cache.rs b/updater/library/src/cache.rs index 4b1b3d46..6ebe3522 100644 --- a/updater/library/src/cache.rs +++ b/updater/library/src/cache.rs @@ -266,7 +266,7 @@ impl UpdaterState { .into()); } - // Move the patch into the slot. + // Move the artifact into the slot. let artifact_path = slot_dir.join("dlc.vmcode"); std::fs::rename(&patch.path, &artifact_path)?; @@ -363,7 +363,7 @@ mod tests { } #[test] - fn dont_install_known_bad_patch() { + fn do_not_install_known_bad_patch() { let tmp_dir = TempDir::new("example").unwrap(); let mut state = test_state(&tmp_dir); let bad_patch = fake_patch(&tmp_dir, 1); diff --git a/updater/library/src/network.rs b/updater/library/src/network.rs index 645459c8..e82b218e 100644 --- a/updater/library/src/network.rs +++ b/updater/library/src/network.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::Write; -use std::path::PathBuf; +use std::path::Path; use std::string::ToString; use crate::cache::UpdaterState; @@ -16,9 +16,17 @@ fn patches_check_url(base_url: &str) -> String { #[derive(Debug, Deserialize)] pub struct Patch { + /// The patch number. Starts at 1 for each new release and increases + /// monotonically. pub number: usize, + /// The hash of the final uncompressed patch file. pub hash: String, + /// The URL to download the patch file from. pub download_url: String, + /// Whether the artifact is a diff (modern) or full (legacy) artifact. + /// Will eventually be removed once we no longer support legacy artifacts. + #[serde(default)] + pub is_diff: bool, } #[derive(Debug, Serialize)] @@ -74,7 +82,7 @@ pub fn send_patch_check_request( return Ok(response); } -pub fn download_to_path(url: &str, path: &PathBuf) -> anyhow::Result<()> { +pub fn download_to_path(url: &str, path: &Path) -> anyhow::Result<()> { // Download the file at the given url to the given path. let client = reqwest::blocking::Client::new(); let response = client.get(url).send()?; diff --git a/updater/library/src/updater.rs b/updater/library/src/updater.rs index 9789612d..bc4c16f6 100644 --- a/updater/library/src/updater.rs +++ b/updater/library/src/updater.rs @@ -7,7 +7,7 @@ use crate::config::{set_config, with_config, ResolvedConfig}; use crate::logging::init_logging; use crate::network::{download_to_path, send_patch_check_request}; use crate::yaml::YamlConfig; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; pub enum UpdateStatus { NoUpdate, @@ -109,8 +109,17 @@ fn update_internal(config: &ResolvedConfig) -> anyhow::Result { let patch = response.patch.ok_or(UpdateError::BadServerResponse)?; let download_dir = PathBuf::from(&config.cache_dir); - let download_path = download_dir.join(patch.number.to_string()); + let mut download_path = download_dir.join(patch.number.to_string()); download_to_path(&patch.download_url, &download_path)?; + + // Inflate the patch from a diff if needed. + if patch.is_diff { + let base_path = PathBuf::from(&config.original_libapp_path); + let output_path = download_dir.join(format!("{}.full", patch.number.to_string())); + inflate(&download_path, &base_path, &output_path)?; + download_path = output_path; + } + // Check the hash before moving into place. // Move/state update should be "atomic". // Consider supporting allowing the system to download for us (e.g. iOS). @@ -125,6 +134,46 @@ fn update_internal(config: &ResolvedConfig) -> anyhow::Result { return Ok(UpdateStatus::UpdateInstalled); } +fn inflate(patch_path: &Path, base_path: &Path, output_path: &Path) -> anyhow::Result<()> { + info!("Patch is compressed, inflating..."); + use comde::de::Decompressor; + use comde::zstd::ZstdDecompressor; + use std::fs::File; + use std::io::{BufReader, BufWriter}; + + // Open all our files first for error clarity. Otherwise we might see + // PipeReader/Writer errors instead of file open errors. + info!("Reading base file: {:?}", base_path); + let base_r = File::open(base_path)?; + let compressed_patch_r = BufReader::new(File::open(patch_path)?); + let output_file_w = File::create(&output_path)?; + + // Set up a pipe to connect the writing from the decompression thread + // to the reading of the decompressed patch data on this thread. + let (patch_r, patch_w) = pipe::pipe(); + + let decompress = ZstdDecompressor::new(); + // Spawn a thread to run the decompression in parallel to the patching. + // decompress.copy will block on the pipe being full (I think) and then + // when it returns the thread will exit. + std::thread::spawn(move || { + let result = decompress.copy(compressed_patch_r, patch_w); + // If this thread fails, undoubtedly the main thread will fail too. + // Most important is to not crash. + if let Err(err) = result { + error!("Decompression thread failed: {err}"); + } + }); + + // Do the patch, using the uncompressed patch data from the pipe. + let mut fresh_r = bipatch::Reader::new(patch_r, base_r)?; + + // Write out the resulting patched file to the new location. + let mut output_w = BufWriter::new(output_file_w); + std::io::copy(&mut fresh_r, &mut output_w)?; + Ok(()) +} + /// Reads the current patch from the cache and returns it. pub fn active_patch() -> Option { return with_config(|config| {