From 79c5685cfd3890dc72482883ed913c3301e49d7f Mon Sep 17 00:00:00 2001 From: Felix Angelov Date: Thu, 6 Apr 2023 09:55:38 -0500 Subject: [PATCH] refactor(artifact_proxy): use dart config (#241) --- packages/artifact_proxy/README.md | 2 +- packages/artifact_proxy/bin/server.dart | 4 +- packages/artifact_proxy/config.yaml | 25 ------ .../artifact_proxy/lib/artifact_proxy.dart | 90 +------------------ packages/artifact_proxy/lib/config.dart | 38 ++++++++ .../lib/src/artifact_proxy.dart | 86 ++++++++++++++++++ .../artifact_proxy/lib/src/models/models.dart | 1 + .../lib/src/models/proxy_config.dart | 35 ++++++++ packages/artifact_proxy/pubspec.lock | 2 +- packages/artifact_proxy/pubspec.yaml | 1 - .../test/artifact_proxy_test.dart | 20 ++--- 11 files changed, 175 insertions(+), 129 deletions(-) delete mode 100644 packages/artifact_proxy/config.yaml create mode 100644 packages/artifact_proxy/lib/config.dart create mode 100644 packages/artifact_proxy/lib/src/artifact_proxy.dart create mode 100644 packages/artifact_proxy/lib/src/models/models.dart create mode 100644 packages/artifact_proxy/lib/src/models/proxy_config.dart diff --git a/packages/artifact_proxy/README.md b/packages/artifact_proxy/README.md index 19877443..da4411e0 100644 --- a/packages/artifact_proxy/README.md +++ b/packages/artifact_proxy/README.md @@ -9,7 +9,7 @@ Storage (either Shorebird's bucket or the official Flutter buckets). ## Usage -Uses `config.yaml` to configure the engine revisions and artifact overrides. +Uses `config.dart` to configure the engine revisions and artifact overrides. ```bash # Run locally with hot-reload enabled. diff --git a/packages/artifact_proxy/bin/server.dart b/packages/artifact_proxy/bin/server.dart index 7f222dca..0002bd54 100644 --- a/packages/artifact_proxy/bin/server.dart +++ b/packages/artifact_proxy/bin/server.dart @@ -3,15 +3,13 @@ import 'dart:io'; import 'package:artifact_proxy/artifact_proxy.dart'; +import 'package:artifact_proxy/config.dart'; import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart' as shelf_io; import 'package:shelf_hotreload/shelf_hotreload.dart'; -import 'package:yaml/yaml.dart'; Future main() async { - const configPath = 'config.yaml'; final isDev = Platform.environment['DEV'] == 'true'; - final config = loadYaml(File(configPath).readAsStringSync()) as Map; final handler = artifactProxyHandler(config: config); final ip = InternetAddress.anyIPv6; final port = int.parse(Platform.environment['PORT'] ?? '8080'); diff --git a/packages/artifact_proxy/config.yaml b/packages/artifact_proxy/config.yaml deleted file mode 100644 index 6dd4d6a4..00000000 --- a/packages/artifact_proxy/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -engine_mappings: - 79f4c5321a581f580a9bda01ec372cbf4a53aa53: - flutter_engine_revision: 9aa7816315095c86410527932918c718cb35e7d6 - shorebird_storage_bucket: download.shorebird.dev - shorebird_artifact_overrides: - # artifacts.zip - - flutter_infra_release/flutter/$engine/android-arm-64-release/artifacts.zip - - flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip - - flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip - - # embedding release - - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom - - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar - - # arm64_v8a release - - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom - - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar - - # armeabi_v7a release - - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom - - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar - - # x86_64 release - - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom - - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar diff --git a/packages/artifact_proxy/lib/artifact_proxy.dart b/packages/artifact_proxy/lib/artifact_proxy.dart index bbf0e11d..1fbedd6d 100644 --- a/packages/artifact_proxy/lib/artifact_proxy.dart +++ b/packages/artifact_proxy/lib/artifact_proxy.dart @@ -1,88 +1,2 @@ -// ignore_for_file: avoid_print - -import 'package:collection/collection.dart'; -import 'package:shelf/shelf.dart'; - -/// A [Handler] that proxies artifact requests to the correct location. -/// This is determined based on the [config]. -Handler artifactProxyHandler({required Map config}) { - final engineMappings = config['engine_mappings'] as Map; - final shorebirdEngineRevisions = engineMappings.keys.cast(); - - return (Request request) { - final path = request.url.path; - final shorebirdEngineRevision = shorebirdEngineRevisions.firstWhereOrNull( - path.contains, - ); - - final normalizedPath = shorebirdEngineRevision != null - ? path.replaceAll(shorebirdEngineRevision, r'$engine') - : path; - - if (shorebirdEngineRevision == null) { - final location = getFlutterArtifactLocation(artifactPath: normalizedPath); - print('No engine revision detected, forwarding to: $location'); - return Response.found(location); - } - - final engineMapping = engineMappings[shorebirdEngineRevision] as Map; - final shorebirdOverrides = - engineMapping['shorebird_artifact_overrides'] as List; - final flutterEngineRevision = - engineMapping['flutter_engine_revision'] as String; - final shorebirdStorageBucket = - engineMapping['shorebird_storage_bucket'] as String; - final shouldOverride = shorebirdOverrides.contains(normalizedPath); - - if (shouldOverride) { - final location = getShorebirdArtifactLocation( - artifactPath: normalizedPath, - engine: shorebirdEngineRevision, - bucket: shorebirdStorageBucket, - ); - print('Shorebird artifact detected, forwarding to: $location'); - return Response.found(location); - } - - final location = getFlutterArtifactLocation( - artifactPath: normalizedPath, - engine: flutterEngineRevision, - ); - print('Flutter artifact detected, forwarding to: $location'); - return Response.found(location); - }; -} - -/// Returns the location of the artifact at [artifactPath] using the -/// specified [engine] revision for original Flutter artifacts. -String getFlutterArtifactLocation({ - required String artifactPath, - String? engine, -}) { - final adjustedPath = engine != null - ? artifactPath.replaceAll(r'$engine', engine) - : artifactPath; - - final isChromeInfra = adjustedPath.contains('flutter_infra_release/cipd'); - - /// TODO(felangel): remove this after 3.8 is released. - if (isChromeInfra) { - return adjustedPath.replaceAll( - 'flutter_infra_release/cipd', - 'https://chrome-infra-packages.appspot.com/dl', - ); - } - - return 'https://storage.googleapis.com/$adjustedPath'; -} - -/// Returns the location of the artifact at [artifactPath] using the -/// specified [engine] revision for Shorebird artifacts. -String getShorebirdArtifactLocation({ - required String artifactPath, - required String engine, - required String bucket, -}) { - final adjustedPath = artifactPath.replaceAll(r'$engine', engine); - return 'https://storage.googleapis.com/$bucket/$adjustedPath'; -} +export 'src/artifact_proxy.dart'; +export 'src/models/models.dart'; diff --git a/packages/artifact_proxy/lib/config.dart b/packages/artifact_proxy/lib/config.dart new file mode 100644 index 00000000..271c3f24 --- /dev/null +++ b/packages/artifact_proxy/lib/config.dart @@ -0,0 +1,38 @@ +import 'package:artifact_proxy/artifact_proxy.dart'; + +/// The proxy configuration used by the server. +const config = ProxyConfig( + engineMappings: { + ...flutter_3_7_8, + }, +); + +/// Flutter 3.7.8 +const flutter_3_7_8 = { + '79f4c5321a581f580a9bda01ec372cbf4a53aa53': EngineMapping( + flutterEngineRevision: '9aa7816315095c86410527932918c718cb35e7d6', + shorebirdStorageBucket: 'download.shorebird.dev', + shorebirdArtifactOverrides: { + // artifacts.zip + r'flutter_infra_release/flutter/$engine/android-arm-64-release/artifacts.zip', + r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip', + r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip', + + // embedding release + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom', + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar', + + // arm64_v8a release + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom', + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar', + + // armeabi_v7a release + r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom', + r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar', + + // x86_64 release + r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom', + r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar', + }, + ), +}; diff --git a/packages/artifact_proxy/lib/src/artifact_proxy.dart b/packages/artifact_proxy/lib/src/artifact_proxy.dart new file mode 100644 index 00000000..f96ad2b7 --- /dev/null +++ b/packages/artifact_proxy/lib/src/artifact_proxy.dart @@ -0,0 +1,86 @@ +// ignore_for_file: avoid_print + +import 'package:artifact_proxy/artifact_proxy.dart'; +import 'package:collection/collection.dart'; +import 'package:shelf/shelf.dart'; + +/// A [Handler] that proxies artifact requests to the correct location. +/// This is determined based on the [config]. +Handler artifactProxyHandler({required ProxyConfig config}) { + final engineMappings = config.engineMappings; + final shorebirdEngineRevisions = engineMappings.keys.cast(); + + return (Request request) { + final path = request.url.path; + final shorebirdEngineRevision = shorebirdEngineRevisions.firstWhereOrNull( + path.contains, + ); + + final normalizedPath = shorebirdEngineRevision != null + ? path.replaceAll(shorebirdEngineRevision, r'$engine') + : path; + + if (shorebirdEngineRevision == null) { + final location = getFlutterArtifactLocation(artifactPath: normalizedPath); + print('No engine revision detected, forwarding to: $location'); + return Response.found(location); + } + + final engineMapping = engineMappings[shorebirdEngineRevision]!; + final shorebirdOverrides = engineMapping.shorebirdArtifactOverrides; + final flutterEngineRevision = engineMapping.flutterEngineRevision; + final shorebirdStorageBucket = engineMapping.shorebirdStorageBucket; + final shouldOverride = shorebirdOverrides.contains(normalizedPath); + + if (shouldOverride) { + final location = getShorebirdArtifactLocation( + artifactPath: normalizedPath, + engine: shorebirdEngineRevision, + bucket: shorebirdStorageBucket, + ); + print('Shorebird artifact detected, forwarding to: $location'); + return Response.found(location); + } + + final location = getFlutterArtifactLocation( + artifactPath: normalizedPath, + engine: flutterEngineRevision, + ); + print('Flutter artifact detected, forwarding to: $location'); + return Response.found(location); + }; +} + +/// Returns the location of the artifact at [artifactPath] using the +/// specified [engine] revision for original Flutter artifacts. +String getFlutterArtifactLocation({ + required String artifactPath, + String? engine, +}) { + final adjustedPath = engine != null + ? artifactPath.replaceAll(r'$engine', engine) + : artifactPath; + + final isChromeInfra = adjustedPath.contains('flutter_infra_release/cipd'); + + // TODO(felangel): remove this after 3.8 is released. + if (isChromeInfra) { + return adjustedPath.replaceAll( + 'flutter_infra_release/cipd', + 'https://chrome-infra-packages.appspot.com/dl', + ); + } + + return 'https://storage.googleapis.com/$adjustedPath'; +} + +/// Returns the location of the artifact at [artifactPath] using the +/// specified [engine] revision for Shorebird artifacts. +String getShorebirdArtifactLocation({ + required String artifactPath, + required String engine, + required String bucket, +}) { + final adjustedPath = artifactPath.replaceAll(r'$engine', engine); + return 'https://storage.googleapis.com/$bucket/$adjustedPath'; +} diff --git a/packages/artifact_proxy/lib/src/models/models.dart b/packages/artifact_proxy/lib/src/models/models.dart new file mode 100644 index 00000000..6c941b96 --- /dev/null +++ b/packages/artifact_proxy/lib/src/models/models.dart @@ -0,0 +1 @@ +export 'proxy_config.dart'; diff --git a/packages/artifact_proxy/lib/src/models/proxy_config.dart b/packages/artifact_proxy/lib/src/models/proxy_config.dart new file mode 100644 index 00000000..88bdac28 --- /dev/null +++ b/packages/artifact_proxy/lib/src/models/proxy_config.dart @@ -0,0 +1,35 @@ +/// A map of shorebird engine revision to the corresponding [EngineMapping]. +typedef EngineMappings = Map; + +/// {@template proxy_config} +/// Contains all the information needed to proxy requests. +/// {@endtemplate} +class ProxyConfig { + /// {@macro proxy_config} + const ProxyConfig({required this.engineMappings}); + + /// The registered engine mappings. + final EngineMappings engineMappings; +} + +/// {@template engine_mapping} +/// Contains all the information needed to proxy requests for a specific +/// shorebird engine revision. +/// {@endtemplate} +class EngineMapping { + /// {@macro engine_mapping} + const EngineMapping({ + required this.flutterEngineRevision, + required this.shorebirdStorageBucket, + required this.shorebirdArtifactOverrides, + }); + + /// The flutter engine revision that this engine mapping is based on. + final String flutterEngineRevision; + + /// The storage bucket that contains the shorebird artifacts. + final String shorebirdStorageBucket; + + /// The list of shorebird artifacts that should be overridden. + final Set shorebirdArtifactOverrides; +} diff --git a/packages/artifact_proxy/pubspec.lock b/packages/artifact_proxy/pubspec.lock index 1b3f4ebc..6a93ff6b 100644 --- a/packages/artifact_proxy/pubspec.lock +++ b/packages/artifact_proxy/pubspec.lock @@ -394,7 +394,7 @@ packages: source: hosted version: "1.2.0" yaml: - dependency: "direct main" + dependency: transitive description: name: yaml sha256: "23812a9b125b48d4007117254bca50abb6c712352927eece9e155207b1db2370" diff --git a/packages/artifact_proxy/pubspec.yaml b/packages/artifact_proxy/pubspec.yaml index 4689c93e..09c3f39c 100644 --- a/packages/artifact_proxy/pubspec.yaml +++ b/packages/artifact_proxy/pubspec.yaml @@ -11,7 +11,6 @@ dependencies: collection: ^1.17.1 shelf: ^1.4.0 shelf_hotreload: ^1.4.0 - yaml: ^3.1.1 dev_dependencies: mocktail: ^0.3.0 diff --git a/packages/artifact_proxy/test/artifact_proxy_test.dart b/packages/artifact_proxy/test/artifact_proxy_test.dart index 5bfba47a..308904f6 100644 --- a/packages/artifact_proxy/test/artifact_proxy_test.dart +++ b/packages/artifact_proxy/test/artifact_proxy_test.dart @@ -8,17 +8,17 @@ void main() { const shorebirdEngineRevision = 'ff32625d5bda6d3eb2eb131e30a4c26ed4960002'; const flutterEngineRevision = 'ec975089acb540fc60752606a3d3ba809dd1528b'; const shorebirdStorageBucket = 'download.shorebird.dev'; - const config = { - 'engine_mappings': { - shorebirdEngineRevision: { - 'flutter_engine_revision': flutterEngineRevision, - 'shorebird_storage_bucket': shorebirdStorageBucket, - 'shorebird_artifact_overrides': [ + const config = ProxyConfig( + engineMappings: { + shorebirdEngineRevision: EngineMapping( + flutterEngineRevision: flutterEngineRevision, + shorebirdStorageBucket: shorebirdStorageBucket, + shorebirdArtifactOverrides: { r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip' - ] - } - } - }; + }, + ), + }, + ); Request buildRequest(String path) { return Request('GET', Uri.parse('http://localhost').replace(path: path));