diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 700aedea..5a569f43 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -58,6 +58,7 @@ jobs: env: SHOREBIRD_HOSTED_URL: https://api-dev.shorebird.dev + SHOREBIRD_HOSTED_URL_FALLBACK: https://api-dev.shorebird.cloud steps: - name: 📚 Git Checkout @@ -140,6 +141,7 @@ jobs: env: SHOREBIRD_HOSTED_URL: ${{ matrix.branch == 'stable' && 'https://api.shorebird.dev' || 'https://api-dev.shorebird.dev' }} + SHOREBIRD_HOSTED_URL_FALLBACK: ${{ matrix.branch == 'stable' && 'https://api.shorebird.cloud' || 'https://api-dev.shorebird.cloud' }} steps: - name: 📚 Git Checkout diff --git a/cspell.config.yaml b/cspell.config.yaml index 01eb5c3b..fc356e32 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -46,6 +46,7 @@ words: - eseidel - exitcode - exportoptions + - failovers - felangel - FLUSHALL - genhtml @@ -83,10 +84,12 @@ words: - mocktail - mset # From ./packages/redis_client - multioption + - multiparts - noaudio # From .github dir, doesn't show up in "**" check? - NOAUTH - nonobvious # From analysis_options.yaml - nserror + - NXDOMAIN - Oltman - orri # Arm64 instruction, Or Register with Immediate - parseable diff --git a/packages/shorebird_cli/integration_test/shorebird_cli_integration_test.dart b/packages/shorebird_cli/integration_test/shorebird_cli_integration_test.dart index 21ebe5ca..01dffef8 100644 --- a/packages/shorebird_cli/integration_test/shorebird_cli_integration_test.dart +++ b/packages/shorebird_cli/integration_test/shorebird_cli_integration_test.dart @@ -240,6 +240,100 @@ void main() { }, timeout: const Timeout(Duration(minutes: 15)), ); + + group('fallback API endpoint', () { + // RFC 2606 reserves `.invalid` for testing; resolvers return NXDOMAIN + // immediately, producing a fast transport-level failure. + final unreachable = Uri.parse('https://api.fallback-test.invalid'); + final reachable = Uri.parse(shorebirdHostedURL); + + /// Exercises one read against the API. A successful return or a + /// [CodePushException] both prove the connection layer succeeded + /// (the latter is just a 4xx HTTP response). A [SocketException] + /// proves it did not. + Future hitApi(CodePushClient client) async { + try { + await client.getCurrentUser(); + } on CodePushException { + // HTTP-level error means the request reached an origin. Acceptable + // for these tests which only verify connection-layer routing. + } + } + + test('uses primary when reachable, never contacts fallback', () async { + final client = runWithOverrides( + () => CodePushClient( + httpClient: Auth().client, + hostedUri: reachable, + fallbackHostedUri: unreachable, + ), + ); + // If the primary fails for any reason, the fallback is unreachable + // and the call would surface a SocketException. No exception means + // the primary succeeded. + await hitApi(client); + }); + + test('falls over to fallback when primary is unreachable', () async { + final client = runWithOverrides( + () => CodePushClient( + httpClient: Auth().client, + hostedUri: unreachable, + fallbackHostedUri: reachable, + ), + ); + // Primary will fail with a SocketException at DNS resolution. + // Fallback should succeed. + await hitApi(client); + }); + + test('surfaces transport error when both endpoints are unreachable', () { + final client = runWithOverrides( + () => CodePushClient( + httpClient: Auth().client, + hostedUri: Uri.parse('https://primary.fallback-test.invalid'), + fallbackHostedUri: Uri.parse( + 'https://fallback.fallback-test.invalid', + ), + ), + ); + expect(client.getCurrentUser(), throwsA(isA())); + }); + + final shorebirdHostedURLFallback = + Platform.environment['SHOREBIRD_HOSTED_URL_FALLBACK']; + test( + 'cross-provider: GCP primary and Cloudflare fallback both serve the API', + () async { + // Confirms the configured Cloudflare-fronted endpoint actually + // serves the same API as the GCP-direct endpoint, end-to-end. + // Verifies each endpoint independently by routing through it as + // the primary against a bogus fallback. + final viaPrimary = runWithOverrides( + () => CodePushClient( + httpClient: Auth().client, + hostedUri: reachable, + fallbackHostedUri: unreachable, + ), + ); + await hitApi(viaPrimary); + + final viaFallback = runWithOverrides( + () => CodePushClient( + httpClient: Auth().client, + hostedUri: Uri.parse(shorebirdHostedURLFallback!), + fallbackHostedUri: unreachable, + ), + ); + await hitApi(viaFallback); + }, + skip: + shorebirdHostedURLFallback == null || + shorebirdHostedURLFallback.isEmpty + ? 'SHOREBIRD_HOSTED_URL_FALLBACK is not set' + : null, + ); + }); } Future isPatchAvailable({ diff --git a/packages/shorebird_cli/lib/src/auth/auth.dart b/packages/shorebird_cli/lib/src/auth/auth.dart index 64baa46f..c3ab50c5 100644 --- a/packages/shorebird_cli/lib/src/auth/auth.dart +++ b/packages/shorebird_cli/lib/src/auth/auth.dart @@ -285,6 +285,7 @@ class Auth { final codePushClient = _buildCodePushClient( httpClient: this.client, hostedUri: shorebirdEnv.hostedUri, + fallbackHostedUri: shorebirdEnv.fallbackHostedUri, ); final user = await codePushClient.getCurrentUser(); diff --git a/packages/shorebird_cli/lib/src/code_push_client_wrapper.dart b/packages/shorebird_cli/lib/src/code_push_client_wrapper.dart index 53af236a..048a8516 100644 --- a/packages/shorebird_cli/lib/src/code_push_client_wrapper.dart +++ b/packages/shorebird_cli/lib/src/code_push_client_wrapper.dart @@ -74,6 +74,7 @@ ScopedRef codePushClientWrapperRef = create(() { codePushClient: CodePushClient( httpClient: auth.client, hostedUri: shorebirdEnv.hostedUri, + fallbackHostedUri: shorebirdEnv.fallbackHostedUri, customHeaders: {'x-cli-version': packageVersion}, ), ); diff --git a/packages/shorebird_cli/lib/src/http_client/http_client.dart b/packages/shorebird_cli/lib/src/http_client/http_client.dart index 675061f4..f138a976 100644 --- a/packages/shorebird_cli/lib/src/http_client/http_client.dart +++ b/packages/shorebird_cli/lib/src/http_client/http_client.dart @@ -1,5 +1,6 @@ import 'package:http/http.dart' as http; import 'package:scoped_deps/scoped_deps.dart'; +import 'package:shorebird_code_push_client/shorebird_code_push_client.dart'; import 'package:shorebird_cli/src/http_client/logging_client.dart'; import 'package:shorebird_cli/src/http_client/retrying_client.dart'; import 'package:shorebird_cli/src/http_client/tracing_client.dart'; @@ -9,9 +10,15 @@ export 'retrying_client.dart'; export 'tracing_client.dart'; /// A reference to a [http.Client] instance. +/// +/// The bottom of the stack is [CodePushClient.buildDefaultHttpClient], which +/// applies a connection-level timeout so unreachable hosts surface as +/// transport errors instead of hanging. final httpClientRef = create( () => TracingClient( - httpClient: retryingHttpClient(LoggingClient(httpClient: http.Client())), + httpClient: retryingHttpClient( + LoggingClient(httpClient: CodePushClient.buildDefaultHttpClient()), + ), ), ); diff --git a/packages/shorebird_cli/lib/src/network_checker.dart b/packages/shorebird_cli/lib/src/network_checker.dart index 16eb70d0..d5f4aa2b 100644 --- a/packages/shorebird_cli/lib/src/network_checker.dart +++ b/packages/shorebird_cli/lib/src/network_checker.dart @@ -37,6 +37,7 @@ class NetworkChecker { /// The URLs to check for network reachability. static final urlsToCheck = [ 'https://api.shorebird.dev', + 'https://api.shorebird.cloud', 'https://console.shorebird.dev', 'https://oauth2.googleapis.com', 'https://storage.googleapis.com', diff --git a/packages/shorebird_cli/lib/src/shorebird_command.dart b/packages/shorebird_cli/lib/src/shorebird_command.dart index 15a4ae8e..099005af 100644 --- a/packages/shorebird_cli/lib/src/shorebird_command.dart +++ b/packages/shorebird_cli/lib/src/shorebird_command.dart @@ -23,7 +23,11 @@ typedef UnzipFn = Future Function(String zipFilePath, String outputDir); /// Signature for a function which builds a [CodePushClient]. typedef CodePushClientBuilder = - CodePushClient Function({required http.Client httpClient, Uri? hostedUri}); + CodePushClient Function({ + required http.Client httpClient, + Uri? hostedUri, + Uri? fallbackHostedUri, + }); /// Signature for a function which starts a process (e.g. [Process.start]). typedef StartProcess = diff --git a/packages/shorebird_cli/lib/src/shorebird_env.dart b/packages/shorebird_cli/lib/src/shorebird_env.dart index 1a0d9139..b623f562 100644 --- a/packages/shorebird_cli/lib/src/shorebird_env.dart +++ b/packages/shorebird_cli/lib/src/shorebird_env.dart @@ -282,6 +282,17 @@ class ShorebirdEnv { } } + /// The fallback URL for the Shorebird code push server, used by + /// [CodePushClient] when a request to the primary [hostedUri] fails at the + /// transport layer. Overrides the [CodePushClient] default. Set via the + /// `SHOREBIRD_HOSTED_URL_FALLBACK` environment variable. If unset or empty, + /// returns null and [CodePushClient] uses its default fallback. + Uri? get fallbackHostedUri { + final raw = platform.environment['SHOREBIRD_HOSTED_URL_FALLBACK']; + if (raw == null || raw.isEmpty) return null; + return Uri.tryParse(raw); + } + /// Whether the CLI can accept user input via stdin. /// /// Returns `false` when stdin is not a terminal, when running on CI, or diff --git a/packages/shorebird_cli/test/src/auth/auth_test.dart b/packages/shorebird_cli/test/src/auth/auth_test.dart index 3fb120bd..179ab409 100644 --- a/packages/shorebird_cli/test/src/auth/auth_test.dart +++ b/packages/shorebird_cli/test/src/auth/auth_test.dart @@ -219,9 +219,14 @@ void main() { () => Auth( credentialsDir: credentialsDir, httpClient: httpClient, - buildCodePushClient: ({Uri? hostedUri, http.Client? httpClient}) { - return codePushClient; - }, + buildCodePushClient: + ({ + Uri? hostedUri, + Uri? fallbackHostedUri, + http.Client? httpClient, + }) { + return codePushClient; + }, obtainCredentialsViaLoopbackLogin: ({ required http.Client httpClient, diff --git a/packages/shorebird_cli/test/src/shorebird_env_test.dart b/packages/shorebird_cli/test/src/shorebird_env_test.dart index 64762c64..b3707311 100644 --- a/packages/shorebird_cli/test/src/shorebird_env_test.dart +++ b/packages/shorebird_cli/test/src/shorebird_env_test.dart @@ -815,6 +815,36 @@ base_url: https://example.com'''); }); }); + group('fallbackHostedUri', () { + test('returns parsed uri when env var is set', () { + when(() => platform.environment).thenReturn({ + 'SHOREBIRD_HOSTED_URL_FALLBACK': 'https://fallback.example.com', + }); + expect( + runWithOverrides(() => shorebirdEnv.fallbackHostedUri), + equals(Uri.parse('https://fallback.example.com')), + ); + }); + + test('returns null when env var is unset', () { + when(() => platform.environment).thenReturn({}); + expect( + runWithOverrides(() => shorebirdEnv.fallbackHostedUri), + isNull, + ); + }); + + test('returns null when env var is empty', () { + when( + () => platform.environment, + ).thenReturn({'SHOREBIRD_HOSTED_URL_FALLBACK': ''}); + expect( + runWithOverrides(() => shorebirdEnv.fallbackHostedUri), + isNull, + ); + }); + }); + group('canAcceptUserInput', () { late Stdin stdin; diff --git a/packages/shorebird_code_push_client/lib/src/code_push_client.dart b/packages/shorebird_code_push_client/lib/src/code_push_client.dart index 58484feb..80c81cc9 100644 --- a/packages/shorebird_code_push_client/lib/src/code_push_client.dart +++ b/packages/shorebird_code_push_client/lib/src/code_push_client.dart @@ -1,7 +1,9 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'package:http/http.dart' as http; +import 'package:http/io_client.dart'; import 'package:path/path.dart' as p; import 'package:shorebird_code_push_client/shorebird_code_push_client.dart'; import 'package:shorebird_code_push_client/src/version.dart'; @@ -59,25 +61,208 @@ class CodePushUpgradeRequiredException extends CodePushException { }); } -/// A wrapper around [http.Client] that ensures all outbound requests -/// are consistent. -/// For example, all requests include the standard `x-version` header. -class _CodePushHttpClient extends http.BaseClient { - _CodePushHttpClient(this._client, this._headers); - - final http.Client _client; +/// A thin [http.BaseClient] decorator that injects the given headers on +/// every outgoing request. Forwards everything else to the wrapped client. +class _HeaderInjectingClient extends http.BaseClient { + _HeaderInjectingClient({ + required http.Client inner, + required Map headers, + }) : _inner = inner, + _headers = headers; + final http.Client _inner; final Map _headers; @override Future send(http.BaseRequest request) { request.headers.addAll(_headers); - return _client.send(request); + return _inner.send(request); } @override void close() { - _client.close(); + _inner.close(); + super.close(); + } +} + +/// Routes requests by their URL host. Requests whose host is in the +/// `hostsThroughPrimary` set are forwarded to the primary client. All +/// other requests are forwarded to the passthrough client. +/// +/// Used to direct API calls through a failover-aware client while letting +/// requests aimed at unrelated hosts (signed GCS upload URLs, third-party +/// assets, etc.) bypass that machinery and go straight out. +class _HostRouter extends http.BaseClient { + _HostRouter({ + required http.Client primaryClient, + required http.Client passthroughClient, + required this.hostsThroughPrimary, + }) : _primaryClient = primaryClient, + _passthroughClient = passthroughClient; + + final http.Client _primaryClient; + final http.Client _passthroughClient; + final Set hostsThroughPrimary; + + @override + Future send(http.BaseRequest request) { + return hostsThroughPrimary.contains(request.url.host) + ? _primaryClient.send(request) + : _passthroughClient.send(request); + } + + /// Closes only [_primaryClient]. Callers are expected to share the + /// underlying transport with [_passthroughClient] (which transitively + /// gets closed via the primary), or to manage the passthrough's + /// lifetime themselves. + @override + void close() { + _primaryClient.close(); + super.close(); + } +} + +/// A [http.BaseClient] decorator that adds primary/fallback failover. +/// +/// Holds a [_preferredHost] (initially [primaryHost]) and an +/// [_alternateHost] (initially [fallbackHost]). Each request is sent to +/// [_preferredHost]. On a transport-level failure the request is retried +/// once against [_alternateHost], and if that succeeds the two are swapped +/// so the working host becomes preferred for subsequent calls. A session +/// that fell over to the fallback can self-heal back to the primary if +/// the fallback later fails and the primary has recovered. +/// +/// This client does not decide which requests are eligible for failover. +/// Pair with [_HostRouter] to send only the appropriate requests through +/// it. +class _FailoverClient extends http.BaseClient { + _FailoverClient({ + required http.Client inner, + required this.primaryHost, + required this.fallbackHost, + }) : _inner = inner, + _preferredHost = primaryHost, + _alternateHost = fallbackHost; + + final http.Client _inner; + + /// The host of the primary API endpoint, e.g. `api.shorebird.dev`. + final String primaryHost; + + /// The host of the fallback API endpoint, e.g. `api.shorebird.cloud`. + final String fallbackHost; + + /// The host the next request will be sent to. Updated only when a + /// failover succeeds, so steady-state requests pay no per-call overhead + /// for the host decision. + String _preferredHost; + + /// The host that requests fall over to when [_preferredHost] fails at + /// the transport layer. + String _alternateHost; + + @override + Future send(http.BaseRequest request) async { + // Capture the host pair locally so this request keeps using the same + // routing even if a concurrent in-flight request swaps the fields. + final preferredAtStart = _preferredHost; + final alternateAtStart = _alternateHost; + + try { + return await _inner.send(_routedTo(request, preferredAtStart)); + } on Exception catch (e) { + if (!_isTransportFailure(e)) rethrow; + } + + final response = await _inner.send(_routedTo(request, alternateAtStart)); + // Swap only if no concurrent send already did. The check and swap are + // atomic in Dart's single-isolate model because there is no `await` + // between them. Invariant: at most one swap per concurrent burst of + // failovers, so the post-burst state is deterministic regardless of + // how many requests piled up. + if (identical(_preferredHost, preferredAtStart)) { + _swapHosts(); + } + return response; + } + + void _swapHosts() { + final previousPreferred = _preferredHost; + _preferredHost = _alternateHost; + _alternateHost = previousPreferred; + } + + http.BaseRequest _routedTo(http.BaseRequest original, String host) { + if (original.url.host == host) return original; + return _rewriteHost(original, host); + } + + static bool _isTransportFailure(Exception e) => + e is SocketException || + e is HandshakeException || + e is TimeoutException || + e is http.ClientException; + + /// Returns a copy of [original] with its host swapped to [newHost]. + /// + /// Only supports request types we actually send to API hosts: + /// [http.Request] for JSON calls, and [http.MultipartRequest] for + /// field-only metadata POSTs (e.g. createPatchArtifact, which uses + /// multipart as a form-data envelope and carries no files). Real file + /// uploads target signed GCS URLs and do not reach this path because + /// [_HostRouter] sends them straight through. + static http.BaseRequest _rewriteHost( + http.BaseRequest original, + String newHost, + ) { + final newUri = original.url.replace(host: newHost); + if (original is http.Request) { + return http.Request(original.method, newUri) + ..headers.addAll(original.headers) + ..bodyBytes = original.bodyBytes + ..encoding = original.encoding + ..followRedirects = original.followRedirects + ..maxRedirects = original.maxRedirects + ..persistentConnection = original.persistentConnection; + } + if (original is http.MultipartRequest) { + // MultipartFile streams are single-use. Retrying a request whose + // files have already been consumed would silently send an empty + // body. API-host multiparts today are fields-only (file uploads + // target signed GCS URLs via _HostRouter's passthrough path), so + // enforce that invariant rather than hoping callers preserve it. + // Unreachable through CodePushClient's public API; ignored for + // the same reason as the StateError below. + // coverage:ignore-start + if (original.files.isNotEmpty) { + throw StateError( + 'Cannot fail over a MultipartRequest with attached files: ' + 'MultipartFile streams are single-use.', + ); + } + // coverage:ignore-end + return http.MultipartRequest(original.method, newUri) + ..headers.addAll(original.headers) + ..fields.addAll(original.fields) + ..followRedirects = original.followRedirects + ..maxRedirects = original.maxRedirects + ..persistentConnection = original.persistentConnection; + } + // Defensive guard. Unreachable through CodePushClient's public API + // because every request issued internally is either an http.Request + // or an http.MultipartRequest. Marked ignore so the unreachable + // branch does not block the 100% patch coverage check. + // coverage:ignore-start + throw StateError( + 'Cannot rewrite host on request of type ${original.runtimeType}', + ); + // coverage:ignore-end + } + + @override + void close() { + _inner.close(); super.close(); } } @@ -87,15 +272,65 @@ class _CodePushHttpClient extends http.BaseClient { /// {@endtemplate} class CodePushClient { /// {@macro code_push_client} - CodePushClient({ + factory CodePushClient({ http.Client? httpClient, Uri? hostedUri, + Uri? fallbackHostedUri, Map? customHeaders, - }) : _httpClient = _CodePushHttpClient(httpClient ?? http.Client(), { - ...standardHeaders, - ...?customHeaders, - }), - hostedUri = hostedUri ?? Uri.https('api.shorebird.dev'); + }) { + final resolvedHosted = hostedUri ?? defaultHostedUri; + final resolvedFallback = fallbackHostedUri ?? defaultFallbackHostedUri; + final transport = httpClient ?? buildDefaultHttpClient(); + final apiClient = resolvedHosted.host == resolvedFallback.host + ? transport + : _FailoverClient( + inner: transport, + primaryHost: resolvedHosted.host, + fallbackHost: resolvedFallback.host, + ); + final router = _HostRouter( + primaryClient: apiClient, + passthroughClient: transport, + hostsThroughPrimary: {resolvedHosted.host, resolvedFallback.host}, + ); + final wrapped = _HeaderInjectingClient( + inner: router, + headers: {...standardHeaders, ...?customHeaders}, + ); + return CodePushClient._( + httpClient: wrapped, + hostedUri: resolvedHosted, + fallbackHostedUri: resolvedFallback, + ); + } + + CodePushClient._({ + required http.Client httpClient, + required this.hostedUri, + required this.fallbackHostedUri, + }) : _httpClient = httpClient; + + /// The default primary URI for the Shorebird CodePush API. + static final Uri defaultHostedUri = Uri.https('api.shorebird.dev'); + + /// The default fallback URI used when the primary is unreachable. + static final Uri defaultFallbackHostedUri = Uri.https('api.shorebird.cloud'); + + /// How long to wait for a TCP/TLS handshake before treating the endpoint + /// as unreachable and falling back. Applies only to connection setup, not + /// to response time. Healthy handshakes complete in well under a second. + static const defaultConnectionTimeout = Duration(seconds: 3); + + /// Builds the default [http.Client] used when no client is supplied. The + /// client is configured with [defaultConnectionTimeout] so that an + /// unreachable primary surfaces as a transport-level error rather than + /// hanging. + static http.Client buildDefaultHttpClient({ + Duration connectionTimeout = defaultConnectionTimeout, + }) { + final inner = HttpClient()..connectionTimeout = connectionTimeout; + return IOClient(inner); + } /// The standard headers applied to all requests. static const standardHeaders = {'x-version': packageVersion}; @@ -108,6 +343,10 @@ class CodePushClient { /// The hosted uri for the Shorebird CodePush API. final Uri hostedUri; + /// The fallback hosted uri used when [hostedUri] is unreachable at the + /// transport layer. Defaults to `https://api.shorebird.cloud`. + final Uri fallbackHostedUri; + Uri get _v1 => Uri.parse('$hostedUri/api/v1'); /// Fetches the currently logged-in user. diff --git a/packages/shorebird_code_push_client/test/src/code_push_client_test.dart b/packages/shorebird_code_push_client/test/src/code_push_client_test.dart index fec38a67..8c43e72b 100644 --- a/packages/shorebird_code_push_client/test/src/code_push_client_test.dart +++ b/packages/shorebird_code_push_client/test/src/code_push_client_test.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -2150,6 +2151,271 @@ void main() { verify(() => httpClient.close()).called(1); }); }); + + group('fallback retry', () { + late CodePushClient client; + + setUp(() { + client = CodePushClient( + httpClient: httpClient, + hostedUri: Uri.https('primary.example.com'), + fallbackHostedUri: Uri.https('fallback.example.com'), + ); + }); + + http.StreamedResponse okResponse() => http.StreamedResponse( + Stream.value(utf8.encode(json.encode({'apps': []}))), + HttpStatus.ok, + ); + + test( + 'does not contact fallback when primary succeeds', + () async { + when( + () => httpClient.send(any()), + ).thenAnswer((_) async => okResponse()); + + await client.getApps(); + + final sent = verify(() => httpClient.send(captureAny())).captured; + expect(sent, hasLength(1)); + expect( + (sent.single as http.BaseRequest).url.host, + equals('primary.example.com'), + ); + }, + ); + + test( + 'retries on fallback host when primary throws SocketException', + () async { + var callCount = 0; + when(() => httpClient.send(any())).thenAnswer((invocation) async { + callCount++; + final req = + invocation.positionalArguments.first as http.BaseRequest; + if (req.url.host == 'primary.example.com') { + throw const SocketException('blocked'); + } + return okResponse(); + }); + + await client.getApps(); + + expect(callCount, equals(2)); + final sent = verify( + () => httpClient.send(captureAny()), + ).captured.cast(); + expect(sent[0].url.host, equals('primary.example.com')); + expect(sent[1].url.host, equals('fallback.example.com')); + }, + ); + + test( + 'sticks to fallback after a successful retry', + () async { + when(() => httpClient.send(any())).thenAnswer((invocation) async { + final req = + invocation.positionalArguments.first as http.BaseRequest; + if (req.url.host == 'primary.example.com') { + throw const SocketException('blocked'); + } + return okResponse(); + }); + + await client.getApps(); + await client.getApps(); + + final sent = verify( + () => httpClient.send(captureAny()), + ).captured.cast(); + // First call: primary (throws) + fallback (ok). Second call: + // fallback only, primary skipped due to sticky host. + expect(sent.map((r) => r.url.host), [ + 'primary.example.com', + 'fallback.example.com', + 'fallback.example.com', + ]); + }, + ); + + test( + 'does not retry on non-transport errors (4xx response)', + () async { + when(() => httpClient.send(any())).thenAnswer( + (_) async => http.StreamedResponse( + const Stream.empty(), + HttpStatus.forbidden, + ), + ); + + try { + await client.getApps(); + } on Exception { + // expected + } + + final sent = verify(() => httpClient.send(captureAny())).captured; + expect(sent, hasLength(1)); + }, + ); + + test( + 'surfaces error when both primary and fallback fail', + () async { + when( + () => httpClient.send(any()), + ).thenAnswer((_) async => throw const SocketException('blocked')); + + await expectLater( + client.getApps(), + throwsA(isA()), + ); + }, + ); + + test( + 'falls back from sticky host when it later fails (self-heal)', + () async { + // Phase 1: primary blocked, fallback works. Sticky becomes fallback. + when(() => httpClient.send(any())).thenAnswer((invocation) async { + final req = + invocation.positionalArguments.first as http.BaseRequest; + if (req.url.host == 'primary.example.com') { + throw const SocketException('blocked'); + } + return okResponse(); + }); + await client.getApps(); + + // Phase 2: fallback now broken, primary recovered. The session + // should fall over from the now-failing sticky to the recovered + // primary, and re-stick to primary. + when(() => httpClient.send(any())).thenAnswer((invocation) async { + final req = + invocation.positionalArguments.first as http.BaseRequest; + if (req.url.host == 'fallback.example.com') { + throw const SocketException('cf down'); + } + return okResponse(); + }); + await client.getApps(); + + // A third call should now go directly to primary, no failover. + await client.getApps(); + + final hosts = verify( + () => httpClient.send(captureAny()), + ).captured.cast().map((r) => r.url.host).toList(); + expect(hosts, [ + // Phase 1 cold: primary tried, fails, falls over to fallback. + 'primary.example.com', + 'fallback.example.com', + // Phase 2: sticky=fallback tried, fails, falls over to primary. + 'fallback.example.com', + 'primary.example.com', + // Third call: sticky=primary, no failover needed. + 'primary.example.com', + ]); + }, + ); + + // Each transport-failure exception type lives on its own line of + // _isTransportFailure. Because || short-circuits, only the matched + // type's line and the lines before it execute. Test all four to + // cover the full chain. + for (final entry in { + 'HandshakeException': const HandshakeException('tls failure'), + 'TimeoutException': TimeoutException('connect timeout'), + 'http.ClientException': http.ClientException('client error'), + }.entries) { + test('falls over when primary throws ${entry.key}', () async { + when(() => httpClient.send(any())).thenAnswer((invocation) async { + final req = + invocation.positionalArguments.first as http.BaseRequest; + if (req.url.host == 'primary.example.com') { + throw entry.value; + } + return okResponse(); + }); + + await client.getApps(); + + final hosts = verify( + () => httpClient.send(captureAny()), + ).captured.cast().map((r) => r.url.host).toList(); + expect(hosts, ['primary.example.com', 'fallback.example.com']); + }); + } + + test( + 'rewrites a MultipartRequest preserving fields when failing over', + () async { + final tempDir = Directory.systemTemp.createTempSync(); + addTearDown(() => tempDir.deleteSync(recursive: true)); + final fixture = File(path.join(tempDir.path, 'patch.txt')) + ..writeAsStringSync('contents'); + + when(() => httpClient.send(any())).thenAnswer((invocation) async { + final req = + invocation.positionalArguments.first as http.BaseRequest; + if (req.url.host == 'primary.example.com') { + throw const SocketException('blocked'); + } + if (req.url.host == 'fallback.example.com') { + return http.StreamedResponse( + Stream.value( + utf8.encode( + json.encode({ + 'id': 1, + 'patch_id': 0, + 'arch': 'aarch64', + 'platform': 'android', + 'hash': 'test-hash', + 'size': 8, + 'url': 'https://upload.gcs.example.com/x', + }), + ), + ), + HttpStatus.ok, + ); + } + // GCS upload passthrough. + return http.StreamedResponse( + const Stream.empty(), + HttpStatus.ok, + ); + }); + + await client.createPatchArtifact( + appId: 'app-id', + artifactPath: fixture.path, + patchId: 0, + arch: 'aarch64', + platform: ReleasePlatform.android, + hash: 'test-hash', + ); + + final sent = verify( + () => httpClient.send(captureAny()), + ).captured.cast(); + // Three sends: primary metadata (threw), fallback metadata + // (rewritten copy, succeeded), GCS upload (passthrough, no + // rewrite). + expect(sent.length, equals(3)); + expect(sent[0], isA()); + expect(sent[0].url.host, equals('primary.example.com')); + expect(sent[1], isA()); + expect(sent[1].url.host, equals('fallback.example.com')); + expect(sent[1].method, equals('POST')); + expect( + (sent[1] as http.MultipartRequest).fields, + containsPair('arch', 'aarch64'), + ); + expect(sent[2].url.host, equals('upload.gcs.example.com')); + }, + ); + }); }); }