From f75725116205429b00188f07109b3d44f7725b2e Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 6 Mar 2023 15:06:31 -0800 Subject: [PATCH] chore: Fix the rust build (#27) --- .gitignore | 17 ++ Cargo.toml | 2 + README.md | 38 ++++ UPDATER_DEMO.md | 128 +++++++++++ cli/Cargo.toml | 10 + cli/src/main.rs | 61 ++++++ dart_cli/.gitignore | 3 + dart_cli/README.md | 1 + dart_cli/analysis_options.yaml | 30 +++ dart_cli/bin/dart_cli.dart | 129 +++++++++++ dart_cli/lib/bindings.dart | 60 ++++++ dart_cli/lib/updater.dart | 86 ++++++++ dart_cli/pubspec.lock | 381 +++++++++++++++++++++++++++++++++ dart_cli/pubspec.yaml | 16 ++ library/.gitignore | 14 ++ library/Cargo.toml | 28 +++ library/README.md | 19 ++ library/include/updater.h | 20 ++ library/src/c_api.rs | 72 +++++++ library/src/lib.rs | 12 ++ library/src/updater.rs | 336 +++++++++++++++++++++++++++++ 21 files changed, 1463 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 UPDATER_DEMO.md create mode 100644 cli/Cargo.toml create mode 100644 cli/src/main.rs create mode 100644 dart_cli/.gitignore create mode 100644 dart_cli/README.md create mode 100644 dart_cli/analysis_options.yaml create mode 100644 dart_cli/bin/dart_cli.dart create mode 100644 dart_cli/lib/bindings.dart create mode 100644 dart_cli/lib/updater.dart create mode 100644 dart_cli/pubspec.lock create mode 100644 dart_cli/pubspec.yaml create mode 100644 library/.gitignore create mode 100644 library/Cargo.toml create mode 100644 library/README.md create mode 100644 library/include/updater.h create mode 100644 library/src/c_api.rs create mode 100644 library/src/lib.rs create mode 100644 library/src/updater.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1aaec94 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# Cache directory from updater cli. +updater_cache \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..bcbfce6 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["cli", "library"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2002d7a --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# Updater library + +This is the C/Rust side of the Shorebird code push system. This is built +in Rust with a C API for easy calling from other languages, most notably +for linking into libflutter.so. + +# Parts +* cli: Test the updater library via the Rust API (for development). +* dart_cli: Test ffi wrapping of updater library. +* library: The rust library that does the actual update work. + +# TODO: +* Remove all non-MVP code. +* Add an async API. +* Add support for "channels" (e.g. beta, stable, etc). +* Write tests for state management. +* Make state management/filesystem management atomic (and tested). +* Move updater values out of the params into post body? +* Support hashing values and check them? +* Add "validate" command to validate state. +* Write a mode that runs the updater first and then launches whatever is downloaded? +* Use cbindgen to generate the C api header file. + https://github.com/eqrion/cbindgen/blob/master/docs.md + + +# Rust +We use normal rust idioms (e.g. Result) inside the library and then bridge those +to C via an explicit stable C API (explicit enums, null pointers for optional +arguments, etc). The reason for this is that it lets the Rust code feel natural +and also gives us maximum flexibility in the future for exposing more in the C +API without having to refactor the internals of the library. + +## Notes +* https://github.com/RubberDuckEng/safe_wren has an example of building a rust library and exposing it with a C api. + +## Other update systems +* https://theupdateframework.io/ +* https://fuchsia.dev/fuchsia-src/concepts/packages/software_update_system diff --git a/UPDATER_DEMO.md b/UPDATER_DEMO.md new file mode 100644 index 0000000..7ea9c91 --- /dev/null +++ b/UPDATER_DEMO.md @@ -0,0 +1,128 @@ +To replicate the updater demo, you'll need a copy of the Flutter Engine. + +These are _not_ how Shorebird will work, but this is what I hacked together +for the demo video. Writing these down so others can replicate if desired. + +# Building the updater library for Android + +The best way I found was to install: +https://github.com/bbqsrc/cargo-ndk + +``` +rustup install beta +cargo +beta install cargo-ndk +rustup +beta target add \ + aarch64-linux-android \ + armv7-linux-androideabi \ + x86_64-linux-android \ + i686-linux-android +cargo +beta ndk --target aarch64-linux-android build --release +``` + +# Setting up to build the Flutter Engine: + +https://github.com/flutter/flutter/wiki/Setting-up-the-Engine-development-environment +https://github.com/flutter/flutter/wiki/Compiling-the-engine + +I would consider building a clean engine first and testing that you have +that working before trying Shorebird's modified engine. + +The hacked up version of the engine used for my demo can be found here: +https://github.com/shorebirdtech/engine/tree/codepush + +# Symlink in the Rust binaries + +I symlinked the results of the rust build into the engine/src directory: + +``` +cd flutter +mkdir updater +cd updater +ln -s $HOME/Documents/GitHub/shorebird_private/shorebird/updater/library/include/updater.h +mkdir android_aarch64 +cd android_aarch64 +ln -s $HOME/Documents/GitHub/shorebird_private/shorebird/updater/target/aarch64-linux-android/release/libupdater.a +``` + +# Building Flutter Engine + +``` +./flutter/tools/gn --android --android-cpu arm64 --runtime-mode=release +ninja -C out/android_release_arm64 +``` + +The linking step for android_release_arm64 is _much_ longer than other platforms +we may need to use unopt or debug builds for faster iteration. + +I also add `&& say "done"` to the end of the ninja command so I know when it's +done (because it takes minutes). + +# Running the updater + +From updater_demo: + +``` +flutter run --local-engine-src-path $HOME/Documents/GitHub/engine/src --local-engine=android_release_arm64 --release +``` + +Only need to do that once, once it's installed on the phone then you don't +need `flutter run` anymore. + +# Building the replacement libraries + +For the demo I used "android.a" and "android.b" which were just copies of +libapp.so files which Flutter had built for me. + +Once you've built the Flutter app in the way you want it: + +``` +cp build/app/intermediates/stripped_native_libs/release/out/lib/arm64-v8a/libapp.so android.a +``` + +You could dig them out of the apk, but that intermediate directory should be +the correct file and is much easier. + +`flutter build apk -t lib/main_b.dart` should build the app in the way I used +in my demo (I built it with `flutter run` and modifying main.dart directly, but +that command should work too). + +# shorebird command line + +I hadn't yet modified `shorebird` to include the `publisher` functionality, +so I had this in my path: + +``` +#!/bin/bash +dart run $HOME/Documents/Github/shorebird_private/shorebird/updater/publisher/bin/publisher.dart publish $2 +``` + +The right solution is to remove the old shorebird functionality and integrate publisher. + +# Ports + +Because updater_server as running locally I also had to forward ports from my +host into the emulator: + +``` +adb reverse tcp:8080 tcp:8080 +``` + +# Running the updater_server + +In a separate terminal: + +``` +cd shorebird/updater/updater_server +dart run +``` + +# The demo + +The demo was then just launching the app on the emulator (manually) +and then using the `shorebird` command line to publish the new libraries +to change what code the app ran: + +``` +shorebird publish android.a +shorebird publish android.b +``` diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 0000000..dff817f --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "4.1.6", features = ["derive"] } +updater = { path = "../library" } \ No newline at end of file diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 0000000..3d992ac --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,61 @@ +extern crate updater; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command(author, version, about, long_about = None, arg_required_else_help=true)] +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Commands { + Check {}, + Current {}, + Update {}, +} + +fn main() { + let cli = Cli::parse(); + + let config = updater::AppConfig { + client_id: "demo".to_string(), + cache_dir: None, + // base_url: "http://localhost:8080", + // channel: "stable", + }; + + // You can check for the existence of subcommands, and if found use their + // matches just as you would the top level cmd + match &cli.command { + Some(Commands::Check {}) => { + let needs_update = updater::check_for_update(&config); + println!("Checking for update..."); + if needs_update { + println!("Update needed."); + } else { + println!("No update needed."); + } + } + Some(Commands::Current {}) => { + let version = updater::active_version(&config); + println!("Current version info:"); + match version { + Some(v) => { + println!("path: {:?}", v.path); + println!("hash: {:?}", v.hash); + println!("version: {:?}", v.version); + } + None => { + println!("None"); + } + } + } + Some(Commands::Update {}) => { + let status = updater::update(&config); + println!("Update: {}", status); + } + None => {} + } +} diff --git a/dart_cli/.gitignore b/dart_cli/.gitignore new file mode 100644 index 0000000..3a85790 --- /dev/null +++ b/dart_cli/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/dart_cli/README.md b/dart_cli/README.md new file mode 100644 index 0000000..7fb9249 --- /dev/null +++ b/dart_cli/README.md @@ -0,0 +1 @@ +Command line application to test ffi wrapping of updater library. \ No newline at end of file diff --git a/dart_cli/analysis_options.yaml b/dart_cli/analysis_options.yaml new file mode 100644 index 0000000..dee8927 --- /dev/null +++ b/dart_cli/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/dart_cli/bin/dart_cli.dart b/dart_cli/bin/dart_cli.dart new file mode 100644 index 0000000..533a780 --- /dev/null +++ b/dart_cli/bin/dart_cli.dart @@ -0,0 +1,129 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:dart_cli/updater.dart'; + +void main(List args) async { + // This might pass a path or flavor (debug/release) later. + Updater.loadLibrary(); + + var clientId = 'my-client-id'; + var cacheDir = 'updater_cache'; + var updater = Updater(clientId, cacheDir); + + final runner = CommandRunner('updater', 'Updater CLI') + ..addCommand(CheckForUpdate(updater)) + ..addCommand(PrintVersion(updater)) + ..addCommand(PrintPath(updater)) + ..addCommand(Update(updater)) + ..addCommand(Run(updater)); + await runner.run(args); +} + +class CheckForUpdate extends Command { + final Updater updater; + CheckForUpdate(this.updater); + + @override + final name = 'check'; + + @override + final description = 'Check for an update.'; + + @override + void run() { + var result = updater.checkForUpdate(); + if (result) { + print('Update available'); + } else { + print('No update available'); + } + } +} + +class PrintVersion extends Command { + final Updater updater; + PrintVersion(this.updater); + + @override + final name = 'version'; + + @override + final description = 'Print current installed version.'; + + @override + void run() { + print(updater.activeVersion()); + } +} + +class PrintPath extends Command { + final Updater updater; + PrintPath(this.updater); + + @override + final name = 'path'; + + @override + final description = 'Print current installed path.'; + + @override + void run() { + print(updater.activePath()); + } +} + +class Update extends Command { + final Updater updater; + Update(this.updater); + + @override + final name = 'update'; + + @override + final description = 'Update to the latest version.'; + + @override + void run() { + updater.update(); + } +} + +class Run extends Command { + final Updater updater; + Run(this.updater) { + argParser.addFlag('update', abbr: 'u', help: 'Update before running.'); + } + + @override + final name = 'run'; + + @override + final description = 'Run the active version.'; + + @override + void run() async { + // This is a basic demo of what this might look like. + // Real callers wouldn't likely do this from Dart as there is no need + // to have two copies of the Dart VM running. + + if (argResults!['update']) { + updater.update(); + } + + var path = updater.activePath(); + if (path == null) { + print('No active version (should run the bundled version)'); + return; + } + // Should this run update first? + print('Running $path'); + // Is there a portable way to just "exec" and replace the current process? + var process = await Process.start(Platform.executable, ['run', path]); + process.stdout.transform(utf8.decoder).forEach(stdout.write); + process.stderr.transform(utf8.decoder).forEach(stderr.write); + // TODO: Handle stdin. + exit(await process.exitCode); + } +} diff --git a/dart_cli/lib/bindings.dart b/dart_cli/lib/bindings.dart new file mode 100644 index 0000000..69ae67c --- /dev/null +++ b/dart_cli/lib/bindings.dart @@ -0,0 +1,60 @@ +// This entire file could be easily autogenerated. + +import 'dart:ffi' as ffi; +import 'dart:io' show Directory, Platform; + +import 'package:ffi/ffi.dart'; +import 'package:path/path.dart' as path; + +typedef _GetBoolFunc = ffi.Bool Function( + ffi.Pointer clientId, ffi.Pointer cacheDir); +typedef GetBool = bool Function( + ffi.Pointer clientId, ffi.Pointer cacheDir); + +typedef _GetStringFunc = ffi.Pointer Function( + ffi.Pointer clientId, ffi.Pointer cacheDir); +typedef GetString = ffi.Pointer Function( + ffi.Pointer clientId, ffi.Pointer cacheDir); + +typedef _GetVoidFunc = ffi.Void Function( + ffi.Pointer clientId, ffi.Pointer cacheDir); +typedef GetVoid = void Function( + ffi.Pointer clientId, ffi.Pointer cacheDir); + +typedef _FreeStringFunc = ffi.Void Function(ffi.Pointer str); +typedef FreeString = void Function(ffi.Pointer str); + +class UpdaterBindings { + final ffi.DynamicLibrary _updater; + + late GetBool checkForUpdate; + late GetString activeVersion; + late GetString activePath; + late FreeString freeString; + late GetVoid update; + + static ffi.DynamicLibrary loadLibrary(String directory, String name) { + if (Platform.isMacOS) { + return ffi.DynamicLibrary.open(path.join(directory, 'lib$name.dylib')); + } + if (Platform.isWindows) { + return ffi.DynamicLibrary.open(path.join(directory, '$name.dll')); + } + // Assume everything else follows the Linux pattern. + return ffi.DynamicLibrary.open(path.join(directory, 'lib$name.so')); + } + + UpdaterBindings() + : _updater = loadLibrary( + path.join(Directory.current.path, 'target', 'debug'), "updater") { + checkForUpdate = + _updater.lookupFunction<_GetBoolFunc, GetBool>('check_for_update'); + activeVersion = + _updater.lookupFunction<_GetStringFunc, GetString>('active_version'); + activePath = + _updater.lookupFunction<_GetStringFunc, GetString>('active_path'); + freeString = + _updater.lookupFunction<_FreeStringFunc, FreeString>('free_string'); + update = _updater.lookupFunction<_GetVoidFunc, GetVoid>('update'); + } +} diff --git a/dart_cli/lib/updater.dart b/dart_cli/lib/updater.dart new file mode 100644 index 0000000..dfc7938 --- /dev/null +++ b/dart_cli/lib/updater.dart @@ -0,0 +1,86 @@ +// This eventually moves to its own package. +import 'dart:ffi' as ffi; + +import 'package:ffi/ffi.dart'; + +import 'bindings.dart'; + +class Updater { + final String clientId; + final String cacheDir; + + Updater(this.clientId, this.cacheDir); + + static UpdaterBindings? _bindings; + + static loadLibrary() { + if (_bindings != null) { + throw Exception('Library already loaded.'); + } + _bindings = UpdaterBindings(); + } + + static UpdaterBindings get bindings { + if (_bindings == null) { + throw Exception('Must call loadLibrary() first.'); + } + return _bindings!; + } + + // Currently bindings passes context as two separate char*, if it ever + // uses a struct instead at least we only have one place to change. + T _callWithContext( + T Function(ffi.Pointer clientId, ffi.Pointer cacheDir) f) { + // Will this leak if the second toNativeUtf8 throws an exception? + var clientId = this.clientId.toNativeUtf8(); + var cacheDir = this.cacheDir.toNativeUtf8(); + try { + return f(clientId, cacheDir); + } finally { + calloc.free(clientId); + calloc.free(cacheDir); + } + } + + bool checkForUpdate() { + return _callWithContext(bindings.checkForUpdate); + } + + void update() { + return _callWithContext(bindings.update); + } + + String? activeVersion() { + return _callWithContext((clientId, cacheDir) { + ffi.Pointer cVersion = ffi.Pointer.fromAddress(0); + try { + cVersion = bindings.activeVersion(clientId, cacheDir); + if (cVersion.address == 0) { + return null; + } + return cVersion.toDartString(); + } finally { + // Can toDartString ever throw an exception, such that this finally + // block is necessary? + bindings.freeString(cVersion); + } + }); + } + + String? activePath() { + return _callWithContext((clientId, cacheDir) { + ffi.Pointer cVersion = ffi.Pointer.fromAddress(0); + try { + cVersion = bindings.activePath(clientId, cacheDir); + if (cVersion.address == 0) { + return null; + } + return cVersion.toDartString(); + } finally { + // Can toDartString ever throw an exception, such that this finally + // block is necessary? + bindings.freeString(cVersion); + } + }); + } +} diff --git a/dart_cli/pubspec.lock b/dart_cli/pubspec.lock new file mode 100644 index 0000000..a2bdb39 --- /dev/null +++ b/dart_cli/pubspec.lock @@ -0,0 +1,381 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "569ddca58d535e601dd1584afa117710abc999d036c0cd2c51777fb257df78e8" + url: "https://pub.dev" + source: hosted + version: "53.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "10927c4b7c7c88b1adbca278c3d5531db92e2f4b4abf04e2919a800af965f3f5" + url: "https://pub.dev" + source: hosted + version: "5.5.0" + args: + dependency: "direct main" + description: + name: args + sha256: "4cab82a83ffef80b262ddedf47a0a8e56ee6fbf7fe21e6e768b02792034dd440" + url: "https://pub.dev" + source: hosted + version: "2.4.0" + async: + dependency: transitive + description: + name: async + sha256: bfe67ef28df125b7dddcea62755991f807aa39a2492a23e1550161692950bbe0 + url: "https://pub.dev" + source: hosted + version: "2.10.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "6cfb5af12253eaf2b368f07bacc5a80d1301a071c73360d746b7f2e32d762c66" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + collection: + dependency: transitive + description: + name: collection + sha256: "4a07be6cb69c84d677a6c3096fcf960cc3285a8330b4603e0d463d15d9bd934c" + url: "https://pub.dev" + source: hosted + version: "1.17.1" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" + coverage: + dependency: transitive + description: + name: coverage + sha256: "2fb815080e44a09b85e0f2ca8a820b15053982b2e714b59267719e8a9ff17097" + url: "https://pub.dev" + source: hosted + version: "1.6.3" + crypto: + dependency: transitive + description: + name: crypto + sha256: aa274aa7774f8964e4f4f38cc994db7b6158dd36e9187aaceaddc994b35c6c67 + url: "https://pub.dev" + source: hosted + version: "3.0.2" + ffi: + dependency: "direct main" + description: + name: ffi + sha256: a38574032c5f1dd06c4aee541789906c12ccaab8ba01446e800d9c5b79c4a978 + url: "https://pub.dev" + source: hosted + version: "2.0.1" + file: + dependency: transitive + description: + name: file + sha256: "1b92bec4fc2a72f59a8e15af5f52cd441e4a7860b49499d69dfa817af20e925d" + url: "https://pub.dev" + source: hosted + version: "6.1.4" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: "408e3ca148b31c20282ad6f37ebfa6f4bdc8fede5b74bc2f08d9d92b55db3612" + url: "https://pub.dev" + source: hosted + version: "3.2.0" + glob: + dependency: transitive + description: + name: glob + sha256: "4515b5b6ddb505ebdd242a5f2cc5d22d3d6a80013789debfbda7777f47ea308c" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: "97486f20f9c2f7be8f514851703d0119c3596d14ea63227af6f7a481ef2b2f8b" + url: "https://pub.dev" + source: hosted + version: "3.2.1" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "2aa08ce0341cc9b354a498388e30986515406668dbcc4f7c950c3e715496693b" + url: "https://pub.dev" + source: hosted + version: "4.0.2" + io: + dependency: transitive + description: + name: io + sha256: "2ec25704aba361659e10e3e5f5d672068d332fc8ac516421d483a11e5cbd061e" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + js: + dependency: transitive + description: + name: js + sha256: f2c445dce49627136094980615a031419f7f3eb393237e4ecd97ac15dea343f3 + url: "https://pub.dev" + source: hosted + version: "0.6.7" + lints: + dependency: "direct dev" + description: + name: lints + sha256: "5e4a9cd06d447758280a8ac2405101e0e2094d2a1dbdd3756aec3fe7775ba593" + url: "https://pub.dev" + source: hosted + version: "2.0.1" + logging: + dependency: transitive + description: + name: logging + sha256: "04094f2eb032cbb06c6f6e8d3607edcfcb0455e2bb6cbc010cb01171dcb64e6d" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + matcher: + dependency: transitive + description: + name: matcher + sha256: c94db23593b89766cda57aab9ac311e3616cf87c6fa4e9749df032f66f30dcb8 + url: "https://pub.dev" + source: hosted + version: "0.12.14" + meta: + dependency: transitive + description: + name: meta + sha256: "12307e7f0605ce3da64cf0db90e5fcab0869f3ca03f76be6bb2991ce0a55e82b" + url: "https://pub.dev" + source: hosted + version: "1.9.0" + mime: + dependency: transitive + description: + name: mime + sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + url: "https://pub.dev" + source: hosted + version: "1.0.4" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "8ebdbaa3b96d5285d068f80772390d27c21e1fa10fb2df6627b1b9415043608d" + url: "https://pub.dev" + source: hosted + version: "2.0.1" + package_config: + dependency: transitive + description: + name: package_config + sha256: "1c5b77ccc91e4823a5af61ee74e6b972db1ef98c2ff5a18d3161c982a55448bd" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + path: + dependency: "direct main" + description: + name: path + sha256: "8829d8a55c13fc0e37127c29fedf290c102f4e40ae94ada574091fe0ff96c917" + url: "https://pub.dev" + source: hosted + version: "1.8.3" + pool: + dependency: transitive + description: + name: pool + sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a" + url: "https://pub.dev" + source: hosted + version: "1.5.1" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "307de764d305289ff24ad257ad5c5793ce56d04947599ad68b3baa124105fc17" + url: "https://pub.dev" + source: hosted + version: "2.1.3" + shelf: + dependency: transitive + description: + name: shelf + sha256: c24a96135a2ccd62c64b69315a14adc5c3419df63b4d7c05832a346fdb73682c + url: "https://pub.dev" + source: hosted + version: "1.4.0" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: aef74dc9195746a384843102142ab65b6a4735bb3beea791e63527b88cc83306 + url: "https://pub.dev" + source: hosted + version: "3.0.1" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: e792b76b96a36d4a41b819da593aff4bdd413576b3ba6150df5d8d9996d2e74c + url: "https://pub.dev" + source: hosted + version: "1.1.1" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: a988c0e8d8ffbdb8a28aa7ec8e449c260f3deb808781fe1284d22c5bba7156e8 + url: "https://pub.dev" + source: hosted + version: "1.0.3" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: "84cf769ad83aa6bb61e0aa5a18e53aea683395f196a6f39c4c881fb90ed4f7ae" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "708b3f6b97248e5781f493b765c3337db11c5d2c81c3094f10904bfa8004c703" + url: "https://pub.dev" + source: hosted + version: "0.10.12" + source_span: + dependency: transitive + description: + name: source_span + sha256: dd904f795d4b4f3b870833847c461801f6750a9fa8e61ea5ac53f9422b31f250 + url: "https://pub.dev" + source: hosted + version: "1.9.1" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: c3c7d8edb15bee7f0f74debd4b9c5f3c2ea86766fe4178eb2a18eb30a0bdaed5 + url: "https://pub.dev" + source: hosted + version: "1.11.0" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "83615bee9045c1d322bbbd1ba209b7a749c2cbcdcb3fdd1df8eb488b3279c1c8" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: a29248a84fbb7c79282b40b8c72a1209db169a2e0542bce341da992fe1bc7e84 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + test: + dependency: "direct dev" + description: + name: test + sha256: "5301f54eb6fe945daa99bc8df6ece3f88b5ceaa6f996f250efdaaf63e22886be" + url: "https://pub.dev" + source: hosted + version: "1.23.1" + test_api: + dependency: transitive + description: + name: test_api + sha256: "6182294da5abf431177fccc1ee02401f6df30f766bc6130a0852c6b6d7ee6b2d" + url: "https://pub.dev" + source: hosted + version: "0.4.18" + test_core: + dependency: transitive + description: + name: test_core + sha256: d2e9240594b409565524802b84b7b39341da36dd6fd8e1660b53ad928ec3e9af + url: "https://pub.dev" + source: hosted + version: "0.4.24" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: "26f87ade979c47a150c9eaab93ccd2bebe70a27dc0b4b29517f2904f04eb11a5" + url: "https://pub.dev" + source: hosted + version: "1.3.1" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: a4040e9852e56bf8a3c5a2e08a56f6facd76e75500cf2a922ce5d52394c4998a + url: "https://pub.dev" + source: hosted + version: "11.0.1" + watcher: + dependency: transitive + description: + name: watcher + sha256: "6a7f46926b01ce81bfc339da6a7f20afbe7733eff9846f6d6a5466aa4c6667c0" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: ca49c0bc209c687b887f30527fb6a9d80040b072cc2990f34b9bec3e7663101b + url: "https://pub.dev" + source: hosted + version: "2.3.0" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "67d3a8b6c79e1987d19d848b0892e582dbb0c66c57cc1fef58a177dd2aa2823d" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + yaml: + dependency: transitive + description: + name: yaml + sha256: "23812a9b125b48d4007117254bca50abb6c712352927eece9e155207b1db2370" + url: "https://pub.dev" + source: hosted + version: "3.1.1" +sdks: + dart: ">=2.19.0 <3.0.0" diff --git a/dart_cli/pubspec.yaml b/dart_cli/pubspec.yaml new file mode 100644 index 0000000..548ee66 --- /dev/null +++ b/dart_cli/pubspec.yaml @@ -0,0 +1,16 @@ +name: dart_cli +description: A sample command-line application. +version: 1.0.0 +# repository: https://github.com/my_org/my_repo + +environment: + sdk: ">=2.19.0 <4.0.0" + +dependencies: + args: ^2.4.0 + ffi: ^2.0.1 + path: ^1.8.3 + +dev_dependencies: + lints: ^2.0.0 + test: ^1.21.0 diff --git a/library/.gitignore b/library/.gitignore new file mode 100644 index 0000000..ada8be9 --- /dev/null +++ b/library/.gitignore @@ -0,0 +1,14 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb \ No newline at end of file diff --git a/library/Cargo.toml b/library/Cargo.toml new file mode 100644 index 0000000..cf9707b --- /dev/null +++ b/library/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "updater" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +# "lib" is used by the "cli" target for testing from Rust +# "cdylib" is used by the "dart_cli" target for testing from Dart +# "staticlib" is used by the engine build for linking into libflutter.so +crate-type = ["lib", "cdylib", "staticlib"] + +[dependencies] +# Used for exposing C API +libc = "0.2.98" +# Used for networking. +reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] } +# Json serialization/de-serialization. +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.93" +# Used for creating custom errors. +thiserror = "1.0" +# Used for error handling. +anyhow = {version = "1.0.69", features = ["backtrace"]} +# Used for logging. +android_logger = "0.13.0" +log = "0.4.14" diff --git a/library/README.md b/library/README.md new file mode 100644 index 0000000..c468976 --- /dev/null +++ b/library/README.md @@ -0,0 +1,19 @@ +# Shorebird CodePush Updater + +The rust library that does the actual update work. + +## Building for Android + +The best way I found was to install: +https://github.com/bbqsrc/cargo-ndk + +``` +rustup install beta +cargo +beta install cargo-ndk +rustup +beta target add \ + aarch64-linux-android \ + armv7-linux-androideabi \ + x86_64-linux-android \ + i686-linux-android +cargo +beta ndk --target aarch64-linux-android build --release +``` diff --git a/library/include/updater.h b/library/include/updater.h new file mode 100644 index 0000000..e378512 --- /dev/null +++ b/library/include/updater.h @@ -0,0 +1,20 @@ +#ifndef updater_h +#define updater_h + +#ifdef __cplusplus +extern "C" +{ +#endif + + char *active_version(const char *client_id, const char *cache_dir); + char *active_path(const char *client_id, const char *cache_dir); + bool check_for_update(const char *client_id, const char *cache_dir); + void update(const char *client_id, const char *cache_dir); + + void free_string(char *str); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* updater_h */ diff --git a/library/src/c_api.rs b/library/src/c_api.rs new file mode 100644 index 0000000..4ca2943 --- /dev/null +++ b/library/src/c_api.rs @@ -0,0 +1,72 @@ +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +use crate::updater; + +fn app_config_from_c(c_client_id: *const c_char, c_cache_dir: *const c_char) -> updater::AppConfig { + let client_id = unsafe { CStr::from_ptr(c_client_id) }.to_str().unwrap(); + let cache_dir = if c_cache_dir == std::ptr::null() { + None + } else { + Some(unsafe { CStr::from_ptr(c_cache_dir).to_str().unwrap() }.to_string()) + }; + + updater::AppConfig { + client_id: client_id.to_string(), + cache_dir: cache_dir, + } +} + +#[no_mangle] +pub extern "C" fn active_version( + c_client_id: *const c_char, + c_cache_dir: *const c_char, +) -> *mut c_char { + let config = app_config_from_c(c_client_id, c_cache_dir); + let version = updater::active_version(&config); + match version { + Some(v) => { + let c_version = CString::new(v.version).unwrap(); + c_version.into_raw() + } + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "C" fn active_path( + c_client_id: *const c_char, + c_cache_dir: *const c_char, +) -> *mut c_char { + let config = app_config_from_c(c_client_id, c_cache_dir); + let version = updater::active_version(&config); + match version { + Some(v) => { + let c_version = CString::new(v.path).unwrap(); + c_version.into_raw() + } + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "C" fn free_string(c_string: *mut c_char) { + unsafe { + if c_string.is_null() { + return; + } + drop(CString::from_raw(c_string)); + } +} + +#[no_mangle] +pub extern "C" fn check_for_update(c_client_id: *const c_char, c_cache_dir: *const c_char) -> bool { + let config = app_config_from_c(c_client_id, c_cache_dir); + return updater::check_for_update(&config); +} + +#[no_mangle] +pub extern "C" fn update(c_client_id: *const c_char, c_cache_dir: *const c_char) { + let config = app_config_from_c(c_client_id, c_cache_dir); + updater::update(&config); +} diff --git a/library/src/lib.rs b/library/src/lib.rs new file mode 100644 index 0000000..70c1cad --- /dev/null +++ b/library/src/lib.rs @@ -0,0 +1,12 @@ +// Declare that the c_api.rs file exists and is a public sub-namespace. +// C doesn't care about the namespaces, but Rust does. +pub mod c_api; + +// Declare that the updater.rs file/module exists, but don't make it public. +mod updater; + +// Take all public items from the updater namespace and make them public. +pub use self::updater::*; + +#[macro_use] +extern crate log; diff --git a/library/src/updater.rs b/library/src/updater.rs new file mode 100644 index 0000000..31c92eb --- /dev/null +++ b/library/src/updater.rs @@ -0,0 +1,336 @@ +use std::collections::HashMap; +use std::fmt::{Display, Formatter}; +use std::fs::File; +use std::io::{BufReader, BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::string::ToString; + +use android_logger::Config; +use log::LevelFilter; + +use serde::{Deserialize, Serialize}; +// use thiserror::Error; + +// #[derive(Error, Debug)] +// pub enum UpdateError { +// #[error("update server disconnected")] +// NetworkFailure(#[from] std::io::Error), +// #[error("unknown error")] +// Unknown, +// } + +pub enum UpdateStatus { + NoUpdate, + UpdateAvailable, + UpdateDownloaded, + UpdateInstalled, + UpdateHadError, +} + +impl Display for UpdateStatus { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + UpdateStatus::NoUpdate => write!(f, "No update"), + UpdateStatus::UpdateAvailable => write!(f, "Update available"), + UpdateStatus::UpdateDownloaded => write!(f, "Update downloaded"), + UpdateStatus::UpdateInstalled => write!(f, "Update installed"), + UpdateStatus::UpdateHadError => write!(f, "Update had error"), + } + } +} + +pub struct AppConfig { + // provided from the application + pub client_id: String, + pub cache_dir: Option, + // typically default=shorebird, but provided by the app as override? + // pub base_url: Option<&'a str>, + // typically default=stable, but provided by the app as override. + // pub channel: Option<&'a str>, + // Other needs: + // Architecture? Or engine can get that itself? + // fallback path? Or engine just returns null and caller figures that out? +} + +pub struct VersionInfo { + pub path: String, + pub version: String, + pub hash: String, +} + +#[derive(Deserialize, Serialize, Default, Clone)] +struct Slot { + path: String, + version: String, + hash: String, +} + +#[derive(Deserialize, Serialize)] +struct UpdaterState { + current_slot_index: usize, + slots: Vec, +} + +impl Default for UpdaterState { + fn default() -> Self { + Self { + current_slot_index: 0, + slots: Vec::new(), + } + } +} + +struct ResolvedConfig { + client_id: String, + base_url: String, + channel: String, + cache_dir: String, +} + +fn load_state(cache_dir: &str) -> anyhow::Result { + // Load UpdaterState from disk + let path = Path::new(cache_dir).join("state.json"); + let file = File::open(path)?; + let reader = BufReader::new(file); + let state = serde_json::from_reader(reader)?; + Ok(state) +} + +fn save_state(state: &UpdaterState, cache_dir: &str) -> anyhow::Result<()> { + // Save UpdaterState to disk + std::fs::create_dir_all(cache_dir)?; + let path = Path::new(cache_dir).join("state.json"); + let file = File::create(path)?; + let writer = BufWriter::new(file); + serde_json::to_writer_pretty(writer, &state)?; + Ok(()) +} + +fn resolve_config(config: &AppConfig) -> ResolvedConfig { + // Resolve the config + // If there is no base_url, use the default. + // If there is no channel, use the default. + return ResolvedConfig { + client_id: config.client_id.to_string(), + base_url: "https://shorebird-code-push-api-cypqazu4da-uc.a.run.app".to_string(), + cache_dir: config + .cache_dir + .as_deref() + .unwrap_or("updater_cache") + .to_owned(), + channel: "stable".to_string(), + }; +} + +fn updates_url(config: &ResolvedConfig) -> String { + return format!("{}/api/v1/updates", config.base_url); +} + +#[derive(Deserialize)] +struct Update { + version: String, + hash: String, + download_url: String, +} + +#[derive(Deserialize)] +struct UpdateResponse { + update_available: bool, + #[serde(default)] + update: Option, +} + +pub fn check_for_update(app_config: &AppConfig) -> bool { + let config = resolve_config(app_config); + // Load UpdaterState from disk + // If there is no state, make an empty state. + let state = load_state(&config.cache_dir).unwrap_or_default(); + // Check the current slot. + let version = current_version_internal(&state); + // Send info from app + current slot to server. + let response_result = send_update_request(&config, version); + match response_result { + Err(err) => { + error!("Failed update check: {err}"); + return false; + } + Ok(response) => { + return response.update_available; + } + } +} + +fn send_update_request( + config: &ResolvedConfig, + version: Option, +) -> anyhow::Result { + #[cfg(target_os = "macos")] + static PLATFORM: &str = "macos"; + #[cfg(target_os = "linux")] + static PLATFORM: &str = "linux"; + #[cfg(target_os = "windows")] + static PLATFORM: &str = "windows"; + #[cfg(target_os = "android")] + static PLATFORM: &str = "android"; + + #[cfg(target_arch = "x86")] + static ARCH: &str = "x86"; + #[cfg(target_arch = "x86_64")] + static ARCH: &str = "x86_64"; + #[cfg(target_arch = "aarch64")] + static ARCH: &str = "aarch64"; + + // Send the request to the server. + let client = reqwest::blocking::Client::new(); + let mut body = HashMap::new(); + body.insert("client_id", config.client_id.clone()); + body.insert("channel", config.channel.clone()); + if let Some(version) = version { + body.insert("version", version.version); + body.insert("hash", version.hash); + } + body.insert("platform", PLATFORM.to_string()); + body.insert("arch", ARCH.to_string()); + let response = client + .post(&updates_url(config)) + .json(&body) + .send()? + .json()?; + return Ok(response); +} + +fn current_version_internal(state: &UpdaterState) -> Option { + // If there is no state, return None. + if state.slots.is_empty() { + return None; + } + let slot = &state.slots[state.current_slot_index]; + // Otherwise return the version info from the current slot. + return Some(VersionInfo { + path: slot.path.clone(), + version: slot.version.clone(), + hash: slot.hash.clone(), + }); +} + +pub fn active_version(config: &AppConfig) -> Option { + let config = resolve_config(config); + let state = load_state(&config.cache_dir).unwrap_or_default(); + return current_version_internal(&state); +} + +fn unused_slot(state: &UpdaterState) -> usize { + // Assume we only use two slots and pick the one that's not current. + if state.slots.is_empty() { + return 0; + } + if state.current_slot_index == 0 { + return 1; + } + return 0; +} + +fn set_slot(state: &mut UpdaterState, index: usize, slot: Slot) { + if state.slots.len() < index + 1 { + // Make sure we're not filling with empty slots. + assert!(state.slots.len() == index); + state.slots.resize(index + 1, Slot::default()); + } + // Set the given slot to the given version. + state.slots[index] = slot +} + +fn download_file_to_path(url: &str, path: &PathBuf) -> anyhow::Result<()> { + // Download the file at the given url to the given path. + let client = reqwest::blocking::Client::new(); + let response = client.get(url).send()?; + let mut bytes = response.bytes()?; + + // Ensure the download directory exists. + std::fs::create_dir_all(path.parent().unwrap())?; + + let mut file = File::create(path)?; + file.write_all(&mut bytes)?; + Ok(()) +} + +fn download_into_slot( + config: &ResolvedConfig, + update_response: &UpdateResponse, + state: &mut UpdaterState, + slot_index: usize, +) -> anyhow::Result<()> { + // Download the new version into the given slot. + let path = Path::new(&config.cache_dir) + .join(format!("slot_{}", slot_index)) + .join("libapp.txt"); + + // TODO: Shouldn't crash on malformed response. + let update = update_response.update.as_ref().unwrap(); + + // We should download into a separate place and move into place. + // That would allow us to check the hash before moving into place. + // Would also allow the move/state update to be "atomic" or at least allow + // us to carefully guard against state corruption. + // Would also let us support when we need to allow the system to download for us (e.g. iOS). + download_file_to_path(&update.download_url, &path)?; + // Check the hash against the download? + + // Update the state to include the new version. + set_slot( + state, + slot_index, + Slot { + path: path.to_str().unwrap().to_string(), + version: update.version.clone(), + hash: update.hash.clone(), + }, + ); + save_state(&state, &config.cache_dir)?; + + return Ok(()); +} + +fn update_internal(config: &ResolvedConfig) -> anyhow::Result { + // Load the state from disk. + let mut state = load_state(&config.cache_dir).unwrap_or_default(); + let version = current_version_internal(&state); + // Check for update. + let response = send_update_request(&config, version)?; + if !response.update_available { + return Ok(UpdateStatus::NoUpdate); + } + // If needed, download the new version. + let slot = unused_slot(&mut state); + download_into_slot(&config, &response, &mut state, slot)?; + // Install the new version. + state.current_slot_index = slot; + save_state(&state, &config.cache_dir)?; + // Set the state to "restart required". + return Ok(UpdateStatus::UpdateInstalled); +} + +fn init_logging() { + android_logger::init_once( + Config::default() + // `flutter` tool ignores non-flutter tagged logs. + .with_tag("flutter") + .with_max_level(LevelFilter::Debug), + ); + debug!("Logging initialized"); +} + +pub fn update(app_config: &AppConfig) -> UpdateStatus { + init_logging(); + + let config = resolve_config(&app_config); + let result = update_internal(&config); + match result { + Err(err) => { + error!("Problem updating: {err}"); + error!("{}", err.backtrace()); + return UpdateStatus::UpdateHadError; + } + Ok(status) => status, + } +}