import 'app_logger.dart'; import 'media_server_http_client.dart'; import '../exceptions/media_server_exceptions.dart'; /// [MediaServerHttpClient] with endpoint failover, shared by both backends /// (the single implementation of what used to be `PlexClient._getWithFailover` /// and `_JellyfinFailoverHttpClient`). /// /// Semantics — decided once, here ([retryTransientMediaServerCall]'s doc /// cross-references this): /// /// - **Failover is for GETs and explicitly replay-safe POSTs.** Mutations /// (POST/PUT/DELETE) fail fast by default on both backends: replaying a /// mutation against a second endpoint when the first was flaky-but-alive /// risks double-application. A POST whose replay is harmless may opt in /// with `allowEndpointFailover: true` (Plex play-queue creation does — an /// orphaned duplicate queue on the server is inert). /// - **Trigger:** a transient transport failure /// ([MediaServerHttpException.isTransient]) or a 5xx — whether thrown or /// returned as a response. 4xx answers never trigger failover. /// - **One authenticated retry per cascade.** Candidate validation may skip /// rejected endpoints before that retry. A failed retry (transport error or /// error status) resets the list to the preferred endpoint and fires /// [onAllEndpointsExhausted]; the next cascade starts from the best candidate /// again. Concurrent requests are generation-stamped so a request raced by a /// switch doesn't cascade a second time. /// - **Persistence is two-phase:** the switch is applied with /// `persist: false` for the retry, and only a successful retry persists the /// winner (`persist: true`). /// - **Retry interplay:** [retryTransientMediaServerCall] is for /// *slow-but-working* endpoints (per-surface timeout budgets); surfaces /// that wrap it pass `allowEndpointFailover: false` so a slow row doesn't /// move the whole client off an otherwise working endpoint. Failover is for /// *dead* endpoints. /// /// Endpoint orchestration diagnostics never contain raw endpoint literals. /// Backends still register configured endpoints before construction to protect /// unavoidable lower-level HTTP diagnostics. class FailoverHttpClient extends MediaServerHttpClient { /// [prioritizedEndpoints] may be empty (failover disabled — plain client /// behavior). A single-entry list still arms [onAllEndpointsExhausted]: /// the lone endpoint failing *is* exhaustion, and the owning manager uses /// that to flip server status and reconnect. FailoverHttpClient({ super.client, required super.baseUrl, required super.defaultHeaders, super.connectTimeout, super.receiveTimeout, required this.logLabel, required List prioritizedEndpoints, required this.onEndpointSwitch, this.onAllEndpointsExhausted, this.validateCandidate, }) : _endpointManager = prioritizedEndpoints.isNotEmpty ? EndpointFailoverManager(prioritizedEndpoints) : null; /// Backend name for log lines ('Plex' / 'Jellyfin') — keeps failover logs /// greppable per backend now that the implementation is shared. final String logLabel; final EndpointFailoverManager? _endpointManager; /// Applies a base-URL change on the owning client. The callback must update /// this client's [baseUrl] alongside its own config/connection snapshot /// (the two-phase protocol calls it with `persist: false` before the retry /// and `persist: true` only after a success — persistence must not be gated /// on the URL having changed, since the second call sees it already applied). final Future Function(String newBaseUrl, {required bool persist}) onEndpointSwitch; /// Optional trust gate run after a fallback is selected but before any /// switch callback, base-URL mutation, or authenticated retry. The active /// request's abort controller must cancel validation as well as the retry. final Future Function(String candidateBaseUrl, AbortController? abort)? validateCandidate; /// Fired when a cascade ends without a working endpoint (or the retry /// itself fails). The owning manager debounces this into a server-offline /// flip + reconnection. final void Function()? onAllEndpointsExhausted; bool _failoverSwitching = false; /// Endpoints currently configured, preferred-first (test/diagnostic view). List get endpoints => _endpointManager?.endpoints ?? const []; /// Replace the endpoint list after a connection refresh, keeping /// [currentBaseUrl] active when provided (it must be present in the list). void resetEndpoints(List prioritizedEndpoints, {String? currentBaseUrl}) { _endpointManager?.reset(prioritizedEndpoints, currentBaseUrl: currentBaseUrl); } @override Future get( String path, { Map? queryParameters, Map? headers, Duration? timeout, AbortController? abort, bool allowEndpointFailover = true, }) => _sendWithFailover( verb: 'GET', allowEndpointFailover: allowEndpointFailover, abort: abort, send: () => super.get(path, queryParameters: queryParameters, headers: headers, timeout: timeout, abort: abort), ); /// POSTs fail fast by default (see the class doc); only replay-safe POSTs /// pass `allowEndpointFailover: true`. @override Future post( String path, { Map? queryParameters, Map? headers, Object? body, Duration? timeout, AbortController? abort, bool allowEndpointFailover = false, }) => _sendWithFailover( verb: 'POST', allowEndpointFailover: allowEndpointFailover, abort: abort, send: () => super.post( path, queryParameters: queryParameters, headers: headers, body: body, timeout: timeout, abort: abort, ), ); Future _sendWithFailover({ required String verb, required bool allowEndpointFailover, required Future Function() send, AbortController? abort, }) async { final generation = _endpointManager?.generation; final MediaServerResponse response; try { response = await send(); } on MediaServerHttpException catch (e) { if (!allowEndpointFailover || !_shouldAttemptFailover(exception: e) || !_canFailover(generation)) { rethrow; } final retried = await _failoverOnce(verb: verb, send: send, abort: abort); if (retried == null) rethrow; return retried; } if (!allowEndpointFailover || !_shouldAttemptFailover(statusCode: response.statusCode) || !_canFailover(generation)) { return response; } return await _failoverOnce(verb: verb, send: send, abort: abort) ?? response; } bool _canFailover(int? requestGeneration) { final manager = _endpointManager; return manager != null && !_failoverSwitching && requestGeneration == manager.generation; } bool _shouldAttemptFailover({MediaServerHttpException? exception, int? statusCode}) { if (exception != null) { if (exception.isTransient) return true; statusCode = exception.statusCode; } return statusCode != null && statusCode >= 500 && statusCode <= 599; } /// One step of the cascade: validate candidates in priority order, move to /// the first accepted endpoint, and retry the authenticated request once. /// /// Returns the retry's response on success. Returns `null` when no accepted /// fallback exists (after firing [onAllEndpointsExhausted]) — the caller /// surfaces its original failure. A retry that answers with an error status /// is returned as-is (the caller's status handling applies), and a retry that /// throws rethrows; both count as exhaustion: the list resets to the /// preferred endpoint so the next cascade starts from the best candidate. Future _failoverOnce({ required String verb, required Future Function() send, AbortController? abort, }) async { final manager = _endpointManager!; if (!manager.hasFallback) { await _resetToPreferred(manager); onAllEndpointsExhausted?.call(); return null; } final endpoints = manager.endpoints; final currentIndex = endpoints.indexOf(manager.current); if (currentIndex < 0 || currentIndex >= endpoints.length - 1) return null; final candidateGeneration = manager.generation; _failoverSwitching = true; try { final validator = validateCandidate; String? selectedBaseUrl; for (var candidateIndex = currentIndex + 1; candidateIndex < endpoints.length; candidateIndex++) { final candidateBaseUrl = endpoints[candidateIndex]; var accepted = validator == null; if (validator != null) { try { accepted = await validator(candidateBaseUrl, abort); } catch (error) { if (error is MediaServerHttpException && error.isCancellation) rethrow; accepted = false; } } if (accepted) { selectedBaseUrl = candidateBaseUrl; break; } } if (selectedBaseUrl == null) { // Validation happens before moving the cursor, so the last accepted // endpoint remains authoritative for both the manager and live client. onAllEndpointsExhausted?.call(); return null; } abort?.throwIfAborted(); if (manager.generation != candidateGeneration || manager.current != endpoints[currentIndex]) { return null; } String? movedBaseUrl; do { movedBaseUrl = manager.moveToNext(); } while (movedBaseUrl != null && movedBaseUrl != selectedBaseUrl); if (movedBaseUrl != selectedBaseUrl) return null; appLogger.i('Switching $logLabel endpoint after $verb failure'); await onEndpointSwitch(selectedBaseUrl, persist: false); final response = await send(); if (response.statusCode < 400) { appLogger.i('$logLabel endpoint failover retry succeeded'); await onEndpointSwitch(selectedBaseUrl, persist: true); return response; } await _resetToPreferred(manager); onAllEndpointsExhausted?.call(); return response; } catch (error) { await _resetToPreferred(manager); if (error is! MediaServerHttpException || !error.isCancellation) { onAllEndpointsExhausted?.call(); } rethrow; } finally { _failoverSwitching = false; } } Future _resetToPreferred(EndpointFailoverManager manager) async { final resetBaseUrl = manager.resetToFirst(); if (resetBaseUrl != null) { await onEndpointSwitch(resetBaseUrl, persist: false); } } } /// Maintains the list of endpoints we can cycle through when one fails. class EndpointFailoverManager { EndpointFailoverManager(List urls) { _setEndpoints(urls); } late List _endpoints; int _currentIndex = 0; /// Incremented every time the active endpoint changes. Requests stamped with /// an older generation should not trigger additional failover cascades. int _generation = 0; int get generation => _generation; List get endpoints => List.unmodifiable(_endpoints); String get current => _endpoints[_currentIndex]; bool get hasFallback => _currentIndex < _endpoints.length - 1; /// Move to the next endpoint, returning its URL or null if exhausted. String? moveToNext() { if (!hasFallback) return null; _currentIndex++; _generation++; return _endpoints[_currentIndex]; } /// Reset back to the first (preferred) endpoint. Called when all endpoints /// are exhausted so the next failure cycle starts from the best candidate. String? resetToFirst() { if (_currentIndex != 0) { _currentIndex = 0; _generation++; appLogger.d('Failover endpoint list reset to first candidate'); return _endpoints[_currentIndex]; } return null; } /// Replace the endpoint list and optionally set the active endpoint. void reset(List urls, {String? currentBaseUrl}) { _setEndpoints(urls); if (currentBaseUrl != null) { final index = _endpoints.indexOf(currentBaseUrl); _currentIndex = index >= 0 ? index : 0; } else { _currentIndex = 0; } _generation++; } void _setEndpoints(List urls) { final sanitized = []; final seen = {}; for (final url in urls) { if (url.isEmpty || seen.contains(url)) continue; seen.add(url); sanitized.add(url); } if (sanitized.isEmpty) { throw ArgumentError('At least one endpoint is required'); } _endpoints = sanitized; _currentIndex = _currentIndex.clamp(0, _endpoints.length - 1); } }