From 9ec775b35ba268accb19e990c4a0d7f4b68d4d72 Mon Sep 17 00:00:00 2001 From: euzu <33094714+euzu@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:36:23 +0500 Subject: [PATCH] Feature/multi crate (#837) Refactored to multi crate project --- .cargo/config.toml | 14 + .github/workflows/ci.yaml | 9 + .gitignore | 10 +- CHANGELOG.md | 10 +- Cargo.lock | 1202 +- Cargo.toml | 67 +- Makefile | 32 +- README.md | 2 +- backend/Cargo.lock | 3752 ------- backend/{ => app}/Cargo.toml | 83 +- backend/{ => app}/build.rs | 0 backend/{ => app}/src/api/api_utils.rs | 1227 +- backend/app/src/api/auth_middleware.rs | 129 + backend/{ => app}/src/api/config_file.rs | 45 +- .../src/api/endpoints/api_playlist_utils.rs | 107 +- .../api/endpoints/custom_video_stream_api.rs | 208 +- .../src/api/endpoints/download_api.rs | 1879 ++-- .../api/endpoints/extract_accept_header.rs | 0 .../src/api/endpoints/hdhomerun_api.rs | 30 +- .../{ => app}/src/api/endpoints/hls_api.rs | 1761 ++- .../api/endpoints/hls_terminal_response.rs | 56 +- .../src/api/endpoints/library_api.rs | 45 +- .../{ => app}/src/api/endpoints/log_ws_api.rs | 21 +- .../{ => app}/src/api/endpoints/m3u_api.rs | 94 +- backend/{ => app}/src/api/endpoints/mod.rs | 0 .../src/api/endpoints/provider_resolve_api.rs | 67 +- .../{ => app}/src/api/endpoints/rbac_api.rs | 144 +- .../src/api/endpoints/recording_api.rs | 6 +- .../src/api/endpoints/recording_media_api.rs | 14 +- .../src/api/endpoints/stream_history_api.rs | 279 +- .../{ => app}/src/api/endpoints/user_api.rs | 13 +- .../src/api/endpoints/user_visibility.rs | 6 +- backend/{ => app}/src/api/endpoints/v1_api.rs | 4 +- .../src/api/endpoints/v1_api_config.rs | 177 +- .../src/api/endpoints/v1_api_playlist.rs | 179 +- .../src/api/endpoints/v1_api_user.rs | 61 +- .../{ => app}/src/api/endpoints/web_index.rs | 33 +- .../src/api/endpoints/websocket_api.rs | 2 +- .../{ => app}/src/api/endpoints/xmltv_api.rs | 441 +- .../{ => app}/src/api/endpoints/xtream_api.rs | 244 +- .../{ => app}/src/api/endpoints/xtream_url.rs | 2 +- .../src/api/hdhomerun_proprietary.rs | 0 backend/{ => app}/src/api/http_layers.rs | 4 +- backend/{ => app}/src/api/main_api.rs | 182 +- backend/{ => app}/src/api/mod.rs | 3 +- backend/{ => app}/src/api/model/app_state.rs | 131 +- backend/app/src/api/model/app_state_view.rs | 99 + .../src/api/model/hls_provisioning.rs | 154 +- backend/app/src/api/model/mod.rs | 96 + backend/{ => app}/src/api/model/proxy/mod.rs | 3 +- .../api/model/streams/active_client_stream.rs | 97 +- .../src/api/model/streams/client_stream.rs | 0 .../api/model/streams/custom_video_stream.rs | 0 .../src/api/model/streams/metering_stream.rs | 97 + backend/app/src/api/model/streams/mod.rs | 48 + .../src/api/model/streams/provider_stream.rs | 92 +- .../model/streams/provider_stream_factory.rs | 205 +- .../api/model/streams/provisioning_stream.rs | 4 +- .../src/api/model/streams/throttled_stream.rs | 7 +- .../api/model/streams/timed_client_stream.rs | 52 +- backend/{ => app}/src/api/panel_api.rs | 325 +- backend/{ => app}/src/api/serve.rs | 29 +- backend/{ => app}/src/api/setup_api.rs | 121 +- backend/{ => app}/src/api/static_headers.rs | 0 backend/{ => app}/src/api/sys_usage.rs | 96 +- .../{ => app}/src/api/tasks/config_watch.rs | 21 +- .../{ => app}/src/api/tasks/hdhomerun_ssdp.rs | 0 .../{ => app}/src/api/tasks/library_scan.rs | 0 backend/app/src/api/tasks/mod.rs | 28 + backend/{ => app}/src/api/tasks/scheduler.rs | 55 +- .../{ => app}/src/api/tasks/xtream_expiry.rs | 109 +- backend/{ => app}/src/main.rs | 103 +- backend/{ => app}/src/tools/flags_builder.rs | 5 + .../{ => app}/tests/provider_dns_https_sni.rs | 2 +- .../{ => app}/tests/proxy_redirect_auth.rs | 19 +- backend/auth/Cargo.toml | 26 + .../{src/auth => auth/src}/access_token.rs | 11 +- .../auth => auth/src}/api_user_context.rs | 76 +- backend/{src/auth => auth/src}/auth_basic.rs | 26 +- backend/{src/auth => auth/src}/auth_bearer.rs | 20 +- .../{src/auth => auth/src}/authenticator.rs | 116 +- backend/auth/src/lib.rs | 19 + backend/{src/auth => auth/src}/password.rs | 27 +- .../{src/auth => auth/src}/recording_auth.rs | 230 +- backend/{src/auth => auth/src}/user.rs | 0 backend/btree/Cargo.toml | 35 + backend/btree/src/codec.rs | 40 + .../bplustree => btree/src}/common.rs | 89 +- backend/btree/src/lib.rs | 78 + .../bplustree => btree/src}/sorted_index.rs | 48 +- .../bplustree => btree/src}/stress.rs | 23 +- .../repository/bplustree => btree/src}/v2.rs | 187 +- .../bplustree => btree/src}/v3/format.rs | 133 +- .../bplustree => btree/src}/v3/migration.rs | 68 +- .../bplustree => btree/src}/v3/mod.rs | 13 +- .../bplustree => btree/src}/v3/page.rs | 151 +- .../bplustree => btree/src}/v3/publish.rs | 89 +- .../bplustree => btree/src}/v3/tree.rs | 716 +- .../bplustree => btree/src}/v3/wal.rs | 191 +- backend/config-loader/Cargo.toml | 26 + .../src/lib.rs} | 585 +- .../src/runtime_config_report.rs | 96 +- .../config-loader/tests/temp_dir_override.rs | 13 + backend/core/Cargo.toml | 61 + backend/core/src/lib.rs | 14 + .../src}/model/batch_result_collector.rs | 8 +- backend/{ => core}/src/model/config/api.rs | 16 +- .../{ => core}/src/model/config/api_proxy.rs | 139 +- .../{ => core}/src/model/config/api_user.rs | 78 +- backend/{ => core}/src/model/config/app.rs | 164 +- backend/{ => core}/src/model/config/base.rs | 157 +- backend/{ => core}/src/model/config/cache.rs | 4 +- backend/{ => core}/src/model/config/epg.rs | 14 +- .../src/model/config/epg_smart_match.rs | 12 +- .../{ => core}/src/model/config/favourites.rs | 11 +- backend/{ => core}/src/model/config/geoip.rs | 27 +- .../{ => core}/src/model/config/hdhomerun.rs | 20 +- .../src/model/config/healthcheck.rs | 0 backend/{ => core}/src/model/config/input.rs | 50 +- .../{ => core}/src/model/config/ip_check.rs | 16 +- .../{ => core}/src/model/config/library.rs | 36 +- backend/{ => core}/src/model/config/log.rs | 6 +- backend/{ => core}/src/model/config/macros.rs | 6 +- .../src/model/config/media_tools.rs | 15 +- .../{ => core}/src/model/config/messaging.rs | 45 +- .../src/model/config/metadata_update.rs | 32 +- backend/{ => core}/src/model/config/mod.rs | 70 +- .../{ => core}/src/model/config/panel_api.rs | 67 +- backend/core/src/model/config/proxy.rs | 22 + .../src/model/config/qos_aggregation.rs | 0 .../{ => core}/src/model/config/rate_limit.rs | 16 +- backend/{ => core}/src/model/config/rename.rs | 6 +- .../src/model/config/reverse_proxy.rs | 145 +- .../{ => core}/src/model/config/schedule.rs | 14 +- backend/{ => core}/src/model/config/sort.rs | 22 +- backend/{ => core}/src/model/config/source.rs | 119 +- backend/{ => core}/src/model/config/stream.rs | 31 +- .../src/model/config/stream_history.rs | 6 +- backend/{ => core}/src/model/config/target.rs | 45 +- backend/{ => core}/src/model/config/trakt.rs | 7 +- .../{ => core}/src/model/config/trakt_api.rs | 63 +- .../src/model/config/video_download.rs | 60 +- .../{ => core}/src/model/config/web_auth.rs | 86 +- backend/{ => core}/src/model/config/web_ui.rs | 67 +- backend/core/src/model/custom_stream_flags.rs | 9 + .../auth => core/src/model}/fingerprint.rs | 34 +- backend/{ => core}/src/model/healthcheck.rs | 2 +- backend/{ => core}/src/model/input_source.rs | 17 +- backend/{ => core}/src/model/mapping.rs | 31 +- backend/{ => core}/src/model/messaging.rs | 28 +- backend/core/src/model/mod.rs | 34 + backend/core/src/model/playlist_filter.rs | 41 + backend/core/src/model/playlist_key.rs | 19 + .../src/model/provider.rs} | 136 +- backend/core/src/model/proxy_redirect.rs | 73 + backend/{ => core}/src/model/readiness.rs | 38 +- backend/core/src/model/stalker_record.rs | 16 + .../api => core/src}/model/stream_error.rs | 6 +- .../{ => core}/src/model/stream_history.rs | 70 +- .../api => core/src}/model/update_guard.rs | 0 backend/core/src/model/update_task.rs | 99 + .../src/model/user_api_request.rs} | 139 +- backend/{ => core}/src/model/xmltv.rs | 116 +- backend/{ => core}/src/model/xtream.rs | 105 +- .../src/model/xtream_response.rs} | 0 backend/core/src/response_macros.rs | 12 + .../{ => core}/src/utils/atomic_json_store.rs | 56 +- backend/{ => core}/src/utils/binary_utils.rs | 2 +- .../model => core/src/utils}/byte_range.rs | 17 +- .../compression/compressed_file_reader.rs | 0 .../compressed_file_reader_async.rs | 0 .../utils/compression/compression_utils.rs | 0 .../{ => core}/src/utils/compression/mod.rs | 2 +- backend/{ => core}/src/utils/crypto_utils.rs | 14 +- backend/{ => core}/src/utils/epg_id.rs | 28 +- backend/{ => core}/src/utils/epg_parser.rs | 52 +- backend/{ => core}/src/utils/ffmpeg.rs | 429 +- backend/core/src/utils/file/config_reader.rs | 54 + .../src/utils/file/env_resolving_reader.rs | 15 +- .../src/utils/file/file_lock_manager.rs | 44 +- .../{ => core}/src/utils/file/file_utils.rs | 258 +- .../src/utils/file/mapping_reader.rs | 93 +- backend/core/src/utils/file/mod.rs | 12 + .../src/utils/file/multi_file_reader.rs | 0 .../src/utils/file/template_reader.rs | 39 +- backend/{ => core}/src/utils/hls_duration.rs | 4 +- backend/{ => core}/src/utils/json_utils.rs | 7 +- backend/{ => core}/src/utils/logging.rs | 82 +- backend/{ => core}/src/utils/lru_cache.rs | 40 +- backend/{ => core}/src/utils/mod.rs | 85 +- .../src/utils/network/content_coding.rs | 134 +- .../src/utils/network/ip_checker.rs | 21 +- backend/core/src/utils/network/mod.rs | 15 + .../src/utils/network/persist_pipe.rs} | 8 +- .../{ => core}/src/utils/network/request.rs | 133 +- backend/{ => core}/src/utils/ordinal.rs | 0 .../src/utils/provider_resolve_token.rs | 9 +- .../{ => core}/src/utils/recording_paths.rs | 51 +- backend/core/src/utils/request_headers.rs | 21 + .../core/src/utils/response_compression.rs | 28 + backend/{ => core}/src/utils/step_measure.rs | 14 +- backend/{ => core}/src/utils/sys_utils.rs | 10 +- backend/{ => core}/src/utils/telegram.rs | 35 +- backend/core/src/utils/time_utils.rs | 15 + backend/{ => core}/src/utils/trakt/client.rs | 91 +- backend/{ => core}/src/utils/trakt/errors.rs | 7 +- backend/{ => core}/src/utils/trakt/mod.rs | 5 +- backend/dvr/Cargo.toml | 35 + .../{src/api/model => dvr/src}/download.rs | 332 +- backend/dvr/src/lib.rs | 21 + backend/dvr/src/recording/mod.rs | 35 + .../recording/recording_catalog_access.rs | 123 +- .../src}/recording/recording_conflict.rs | 36 +- backend/dvr/src/recording/recording_ctx.rs | 31 + .../recording/recording_currently_airing.rs | 7 +- .../src}/recording/recording_deletion.rs | 146 +- .../src}/recording/recording_disk.rs | 116 +- .../src}/recording/recording_edit.rs | 40 +- .../src}/recording/recording_notification.rs | 10 +- .../recording_notification_adapter.rs | 13 +- .../src}/recording/recording_observability.rs | 20 +- .../src}/recording/recording_occurrence.rs | 73 +- .../src}/recording/recording_quota.rs | 255 +- .../recording/recording_reconciliation.rs | 75 +- .../src}/recording/recording_retention.rs | 269 +- .../recording/recording_rule_scheduler.rs | 114 +- .../src}/recording/recording_rule_service.rs | 34 +- .../src}/recording/recording_security.rs | 24 +- .../src}/recording/recording_service.rs | 428 +- .../recording/recording_source_resolution.rs | 56 + .../recording/recording_supervisor/health.rs | 26 +- .../recording/recording_supervisor/mod.rs | 103 +- .../recording/recording_supervisor/outbox.rs | 90 +- .../recording_supervisor/retention.rs | 100 +- .../recording/recording_supervisor/startup.rs | 76 +- .../src}/recording/recording_worker.rs | 186 +- .../src}/recording/recording_worker_runner.rs | 266 +- .../src}/recording/recording_ws.rs | 65 +- backend/hls/Cargo.toml | 57 + backend/hls/src/availability/mod.rs | 354 + .../hls/src/availability/recovery_pressure.rs | 319 + backend/hls/src/availability/reevaluation.rs | 609 + .../hls/src/availability/terminal_cutover.rs | 670 ++ .../availability/terminal_pending_owner.rs | 543 + backend/hls/src/availability/tests.rs | 2653 +++++ .../src}/availability_reevaluation.rs | 384 +- .../hls_cache => hls/src}/backpressure.rs | 2 +- .../api/model/hls_cache => hls/src}/cache.rs | 122 +- .../hls_cache => hls/src}/critical_handoff.rs | 24 +- .../model/hls_cache => hls/src}/cutover.rs | 29 +- .../model/hls_cache => hls/src}/deadline.rs | 0 .../src}/deterministic_conflict.rs | 8 +- .../api/model/hls_cache => hls/src}/gc.rs | 287 +- .../model/proxy => hls/src}/header_policy.rs | 16 +- .../model/hls_cache => hls/src}/headers.rs | 9 +- backend/hls/src/hls_ctx.rs | 139 + .../api/model/hls_cache => hls/src}/ids.rs | 9 +- .../parser/hls => hls/src}/initial_strip.rs | 37 +- .../api/model/hls_cache => hls/src}/lease.rs | 421 +- backend/hls/src/lib.rs | 359 + .../model/hls_cache => hls/src}/lifecycle.rs | 4 +- .../model/hls_cache => hls/src}/manager.rs | 915 +- .../src}/manifest_acceptance.rs | 226 +- .../hls_cache => hls/src}/manifest_commit.rs | 0 backend/hls/src/manifest_fetch/episode.rs | 408 + backend/hls/src/manifest_fetch/error.rs | 297 + backend/hls/src/manifest_fetch/fetch.rs | 294 + backend/hls/src/manifest_fetch/fingerprint.rs | 501 + backend/hls/src/manifest_fetch/http.rs | 371 + backend/hls/src/manifest_fetch/mod.rs | 76 + backend/hls/src/manifest_fetch/model.rs | 328 + backend/hls/src/manifest_fetch/quality.rs | 348 + backend/hls/src/manifest_fetch/recovery.rs | 647 ++ .../hls/src/manifest_fetch/selection_log.rs | 206 + backend/hls/src/manifest_fetch/tests.rs | 637 ++ .../src}/manifest_origin_binding.rs | 13 +- .../src}/manifest_snapshot.rs | 12 +- .../api/model/hls_cache => hls/src}/map.rs | 4 +- .../hls_cache => hls/src}/map_fetcher.rs | 43 +- .../hls_cache => hls/src}/master_playlist.rs | 74 +- .../hls_cache => hls/src}/media_reserve.rs | 56 +- .../hls_cache => hls/src}/observability.rs | 185 +- .../api/model/hls_cache => hls/src}/origin.rs | 99 +- .../hls_cache => hls/src}/origin_progress.rs | 31 +- .../api/model/hls_cache => hls/src}/paths.rs | 0 .../model/hls_cache => hls/src}/playback.rs | 86 +- .../src}/post_refresh_availability.rs | 33 +- .../model/hls_cache => hls/src}/prefetch.rs | 8 +- .../src}/prepared_terminal_bundle.rs | 221 +- .../api/model/hls_cache => hls/src}/qos.rs | 6 +- .../hls_cache => hls/src}/recovery_timing.rs | 205 +- backend/hls/src/refresh/commit.rs | 866 ++ backend/hls/src/refresh/failure.rs | 266 + backend/hls/src/refresh/mod.rs | 1268 +++ backend/hls/src/refresh/switch_staging.rs | 877 ++ backend/hls/src/refresh/tests.rs | 6547 +++++++++++ backend/hls/src/refresh/timing.rs | 138 + .../model/hls_cache => hls/src}/renderer.rs | 51 +- .../hls_cache => hls/src}/resource_fetch.rs | 32 +- .../src}/resource_identity.rs | 97 +- .../model/hls_cache => hls/src}/response.rs | 167 +- .../src}/runtime_custom_tail.rs | 272 +- .../hls_cache => hls/src}/segment_fetcher.rs | 218 +- .../hls_cache => hls/src}/segment_repair.rs | 100 +- .../hls_cache => hls/src}/segment_watchdog.rs | 41 +- .../model/hls_cache => hls/src}/session.rs | 171 +- .../hls_cache => hls/src}/session_store.rs | 39 +- .../src}/startup_observability.rs | 54 +- .../hls_cache => hls/src}/terminal_commit.rs | 492 +- .../hls_cache => hls/src}/terminal_pending.rs | 40 +- .../hls_cache => hls/src}/terminal_tail.rs | 366 +- .../model/hls_cache => hls/src}/timeline.rs | 116 +- .../model/hls_cache => hls/src}/transient.rs | 24 +- .../src}/transient_fetcher.rs | 60 +- .../hls => hls/src}/transient_manifest.rs | 20 +- backend/iptv/Cargo.toml | 33 + backend/{src/iptv/mod.rs => iptv/src/lib.rs} | 0 backend/{src/iptv => iptv/src}/m3u/client.rs | 22 +- backend/iptv/src/m3u/mod.rs | 6 + .../{src/iptv => iptv/src}/stalker/auth.rs | 122 +- .../{src/iptv => iptv/src}/stalker/catalog.rs | 224 +- .../{src/iptv => iptv/src}/stalker/client.rs | 297 +- .../iptv => iptv/src}/stalker/cmd_parser.rs | 23 +- .../iptv => iptv/src}/stalker/cookie_jar.rs | 93 +- backend/{src/iptv => iptv/src}/stalker/epg.rs | 159 +- .../{src/iptv => iptv/src}/stalker/error.rs | 13 +- backend/{src/iptv => iptv/src}/stalker/mod.rs | 1 + .../stalker.rs => iptv/src/stalker/parser.rs} | 135 +- .../iptv => iptv/src}/stalker/playback.rs | 69 +- .../{src/iptv => iptv/src}/stalker/presets.rs | 8 +- .../{src/iptv => iptv/src}/stalker/profile.rs | 18 +- .../{src/iptv => iptv/src}/stalker/recipes.rs | 36 +- .../{src/iptv => iptv/src}/stalker/session.rs | 34 +- .../iptv => iptv/src}/stalker/url_factory.rs | 24 +- backend/{src/iptv => iptv/src}/xtream/mod.rs | 428 +- backend/library/Cargo.toml | 34 + backend/library/src/lib.rs | 12 + .../{ => library}/src/library/classifier.rs | 60 +- backend/{ => library}/src/library/metadata.rs | 43 +- .../src/library/metadata_iterator.rs | 17 +- .../src/library/metadata_resolver.rs | 135 +- .../src/library/metadata_storage.rs | 90 +- backend/{ => library}/src/library/mod.rs | 26 +- .../{ => library}/src/library/nfo_reader.rs | 139 +- .../{ => library}/src/library/processor.rs | 223 +- backend/{ => library}/src/library/scanner.rs | 143 +- .../{ => library}/src/library/thumbnail.rs | 18 +- backend/{ => library}/src/library/tmdb.rs | 144 +- .../{ => library}/src/library/tmdb_client.rs | 22 +- .../src/media_enrichment/facts.rs | 7 +- .../{ => library}/src/media_enrichment/mod.rs | 0 .../src/media_enrichment/parsed_title.rs | 0 .../src/media_enrichment/policy.rs | 6 +- .../src/media_enrichment/tmdb.rs | 7 +- .../src/media_enrichment/xtream.rs | 25 +- backend/{ => library}/src/ptt/constants.rs | 5 +- backend/{ => library}/src/ptt/handlers.rs | 321 +- backend/{ => library}/src/ptt/mod.rs | 8 +- backend/{ => library}/src/ptt/models.rs | 2 +- backend/{ => library}/src/ptt/parser.rs | 89 +- backend/{ => library}/src/ptt/transformers.rs | 39 +- backend/media-server/Cargo.toml | 28 + .../src}/catalog.rs | 65 +- .../src}/client.rs | 28 +- .../src}/emby/dto.rs | 2 +- .../src}/emby/mod.rs | 0 .../src}/errors.rs | 11 +- backend/media-server/src/jellyfin/dto.rs | 27 + .../src}/jellyfin/mod.rs | 0 .../mod.rs => media-server/src/lib.rs} | 0 .../src}/playback.rs | 113 +- .../src}/playlist_mapper.rs | 95 +- .../src}/plex/client.rs | 280 +- .../src}/plex/dto.rs | 2 +- .../src}/plex/mapper.rs | 87 +- .../src}/plex/mod.rs | 0 .../src}/redaction.rs | 10 +- .../src}/test_fixtures.rs | 0 .../src}/types.rs | 27 +- backend/messaging/Cargo.toml | 26 + .../messaging.rs => messaging/src/lib.rs} | 201 +- backend/metadata/Cargo.toml | 32 + backend/metadata/src/ctx.rs | 35 + backend/metadata/src/lib.rs | 21 + .../src/manager.rs} | 776 +- backend/mpegts/Cargo.toml | 28 + backend/mpegts/src/lib.rs | 14 + .../src}/transport_stream_buffer.rs | 642 +- .../hls_cache => mpegts/src}/ts_inspector.rs | 657 +- backend/parser/Cargo.toml | 31 + .../parser/hls.rs => parser/src/hls/mod.rs} | 74 +- .../src}/hls/origin_manifest.rs | 2 +- .../parser => parser/src}/ics/dummy.rs | 2 +- .../parser => parser/src}/ics/event.rs | 20 +- .../parser => parser/src}/ics/mod.rs | 22 +- .../parser => parser/src}/ics/time.rs | 0 backend/parser/src/lib.rs | 16 + .../processing/parser => parser/src}/m3u.rs | 214 +- .../m3u => parser/src/m3u_format}/archive.rs | 43 +- .../m3u => parser/src/m3u_format}/catchup.rs | 256 +- .../src/m3u_format}/catchup_token.rs | 24 +- backend/parser/src/m3u_format/mod.rs | 14 + .../parser => parser/src}/xtream.rs | 442 +- backend/processing/Cargo.toml | 50 + .../utils/network => processing/src}/epg.rs | 74 +- backend/processing/src/fetched_playlist.rs | 41 + .../processing => processing/src}/geoip.rs | 36 +- .../src}/input_cache.rs | 31 +- backend/processing/src/lib.rs | 30 + backend/processing/src/metadata_sink.rs | 42 + backend/processing/src/parser/mod.rs | 9 + .../src}/parser/xmltv.rs | 151 +- .../src}/playlist_watch.rs | 58 +- .../src}/processor/deduplicate.rs | 61 +- .../src}/processor/epg.rs | 52 +- .../src}/processor/library.rs | 316 +- .../src}/processor/mod.rs | 77 +- .../src}/processor/playlist.rs | 586 +- .../src}/processor/probe_handle_guard.rs | 34 +- .../src}/processor/resolve_options.rs | 10 +- .../src}/processor/sort.rs | 46 +- .../src}/processor/stalker.rs | 153 +- .../src}/processor/stalker_refresh.rs | 163 +- .../src}/processor/stream_probe.rs | 264 +- .../src}/processor/trakt.rs | 127 +- .../src}/processor/xtream.rs | 97 +- .../src}/processor/xtream_series.rs | 169 +- .../src}/processor/xtream_vod.rs | 182 +- backend/repository/Cargo.toml | 54 + .../src}/alias_repository.rs | 131 +- .../utils => repository/src}/db_viewer.rs | 57 +- .../src}/epg_repository.rs | 170 +- .../{src/utils => repository/src}/geoip.rs | 78 +- .../src}/identity_registry.rs | 173 +- .../mod.rs => repository/src/lib.rs} | 78 +- .../src}/library_repository.rs | 72 +- .../src}/live_stream_metadata_repository.rs | 166 +- .../src}/m3u_playlist_iterator.rs | 22 +- .../src}/m3u_repository.rs | 166 +- .../repository/src/metadata_retry_record.rs | 42 + backend/repository/src/network_access.rs | 126 + .../repository/src/playlist_cache_loader.rs | 54 + .../src}/playlist_mem_cache.rs | 12 +- .../src}/playlist_repository.rs | 496 +- .../src}/playlist_scratch.rs | 27 +- .../src}/playlist_source.rs | 490 +- .../src}/playlist_stream.rs | 70 +- .../src}/provider_dns_repository.rs | 150 +- .../src}/qos_snapshot_repository.rs | 159 +- .../src}/recording_rule_repository.rs | 12 +- .../src}/stalker_generation_repository.rs | 71 +- .../src}/stalker_repository.rs | 111 +- .../src/startup_migration.rs} | 60 +- .../repository => repository/src}/storage.rs | 220 +- backend/repository/src/storage_const.rs | 28 + .../src}/stream_history/archive.rs | 38 +- .../src}/stream_history/async_iterator.rs | 8 +- .../src}/stream_history/mod.rs | 15 +- .../src}/stream_history/reader.rs | 146 +- .../src}/stream_history/storage.rs | 39 +- .../src/stream_history/viewer.rs} | 186 +- .../src}/stream_history/writer.rs | 91 +- .../src}/strm_repository.rs | 79 +- .../src}/target_id_mapping.rs | 78 +- .../src}/user_repository.rs | 46 +- .../src}/xtream_playlist_iterator.rs | 66 +- .../src}/xtream_repository.rs | 1017 +- backend/session/Cargo.toml | 37 + .../src}/active_provider_manager.rs | 342 +- .../src}/active_user_manager.rs | 202 +- backend/session/src/admission.rs | 503 + .../src}/admission_strategy.rs | 78 +- .../src}/connection_manager.rs | 351 +- .../model => session/src}/event_manager.rs | 51 +- backend/session/src/lib.rs | 39 + backend/session/src/meter.rs | 104 + .../src}/provider_dns_manager.rs | 42 +- .../src}/provider_lineup_manager.rs | 113 +- .../src}/qos_aggregation_manager.rs | 225 +- .../src/response_headers.rs} | 12 +- .../{src/api/model => session/src}/stream.rs | 19 +- backend/session/src/stream_ctx.rs | 24 + backend/session/src/stream_options.rs | 49 + .../src}/streams/buffered_stream.rs | 24 +- backend/session/src/streams/mod.rs | 4 + .../src}/streams/shared_stream_manager.rs | 243 +- .../src/api/model/hls_cache/availability.rs | 5318 --------- .../src/api/model/hls_cache/manifest_fetch.rs | 3891 ------- backend/src/api/model/hls_cache/mod.rs | 292 - backend/src/api/model/hls_cache/refresh.rs | 9980 ----------------- backend/src/api/model/mod.rs | 65 - backend/src/api/model/recording/mod.rs | 31 - .../src/api/model/streams/metering_stream.rs | 195 - backend/src/api/model/streams/mod.rs | 25 - backend/src/api/tasks/mod.rs | 11 - backend/src/auth/mod.rs | 34 - backend/src/iptv/m3u/mod.rs | 9 - backend/src/media_server/jellyfin/dto.rs | 26 - backend/src/model/config/proxy.rs | 30 - backend/src/model/mod.rs | 21 - backend/src/model/playlist.rs | 68 - backend/src/modules.rs | 21 - backend/src/processing/mod.rs | 5 - backend/src/processing/parser/mod.rs | 6 - backend/src/repository/bplustree/mod.rs | 11 - backend/src/repository/storage_const.rs | 30 - backend/src/utils/file/mod.rs | 14 - backend/src/utils/network/mod.rs | 12 - backend/src/utils/time_utils.rs | 23 - bin/check-workspace-deps-test.sh | 94 + bin/check-workspace-deps.sh | 203 + bin/release.sh | 8 +- bin/test.sh | 17 +- docker/Dockerfile | 80 +- docker/cross.Dockerfile | 96 +- docs/src/configuration/config.md | 6 +- .../connection-handling-runtime-internals.md | 12 +- ...n-handling-session-implementation-notes.md | 14 +- .../configuration/hls-cache-state-machine.md | 18 +- frontend/Cargo.toml | 15 +- .../src/app/components/accordion_panel.rs | 2 +- .../src/app/components/api_user/playlist.rs | 14 +- .../components/api_user/target_playlist.rs | 4 +- frontend/src/app/components/authentication.rs | 2 +- frontend/src/app/components/breadcrumbs.rs | 2 +- frontend/src/app/components/cell_value.rs | 8 +- frontend/src/app/components/collapse_panel.rs | 2 +- .../components/config/admission_strategies.rs | 3 +- .../app/components/config/config_update.rs | 8 +- .../src/app/components/config/config_view.rs | 6 +- .../app/components/config/log_config_view.rs | 6 +- .../config/messaging_config_view.rs | 18 +- .../config/reverse_proxy_config_view.rs | 29 +- .../config/schedules_config_view.rs | 2 +- .../components/config/video_config_view.rs | 3 +- frontend/src/app/components/content_dialog.rs | 4 +- frontend/src/app/components/country.rs | 2 +- frontend/src/app/components/custom_dialog.rs | 2 +- .../app/components/dashboard/action_card.rs | 8 +- .../components/dashboard/dashboard_view.rs | 2 +- .../dashboard/discord_action_card.rs | 2 +- .../dashboard/documentation_action_card.rs | 2 +- .../dashboard/github_action_card.rs | 2 +- .../dashboard/ipinfo_action_card.rs | 6 +- .../app/components/dashboard/log_console.rs | 2 +- .../playlist_progress_status_card.rs | 2 +- .../src/app/components/dashboard/sparkline.rs | 31 +- .../app/components/dashboard/stats_view.rs | 10 +- .../app/components/dashboard/status_card.rs | 2 +- .../dashboard/stream_display/mod.rs | 9 +- .../dashboard/stream_history_view.rs | 21 +- .../app/components/dashboard/streams_view.rs | 6 +- .../components/dashboard/user_action_card.rs | 2 +- frontend/src/app/components/date_input.rs | 3 +- frontend/src/app/components/downloads.rs | 6 +- .../app/components/drop_down_icon_button.rs | 2 +- .../src/app/components/field_explanation.rs | 27 +- .../app/components/filter/filter_editor.rs | 4 +- frontend/src/app/components/health_banner.rs | 2 +- frontend/src/app/components/hide_content.rs | 4 +- frontend/src/app/components/home.rs | 15 +- frontend/src/app/components/input.rs | 2 +- .../src/app/components/key_value_editor.rs | 2 +- .../src/app/components/language_picker.rs | 3 +- .../src/app/components/loading_indicator.rs | 8 +- frontend/src/app/components/login.rs | 2 +- frontend/src/app/components/no_content.rs | 2 +- .../components/particle_flow_background.rs | 37 +- .../components/playlist/epg_config_view.rs | 2 +- .../playlist/epg_source_selector.rs | 11 +- .../src/app/components/playlist/epg_view.rs | 14 +- .../playlist/input/input_options.rs | 15 +- .../app/components/playlist/input_table.rs | 10 +- .../components/playlist/mapper_script_view.rs | 11 +- .../components/playlist/playlist_explorer.rs | 78 +- .../playlist/playlist_source_selector.rs | 11 +- .../playlist/playlist_update_view.rs | 10 +- .../playlist/source_selector_common.rs | 2 +- .../playlist/target/target_rename.rs | 4 +- .../app/components/playlist/target_table.rs | 6 +- frontend/src/app/components/popup_menu.rs | 2 +- .../src/app/components/radio_button_group.rs | 2 +- .../app/components/rbac/group_management.rs | 6 +- .../recording/recording_edit_view.rs | 14 +- .../components/recording/recording_form.rs | 6 +- .../recording/recording_library_view.rs | 17 +- .../recording/recording_rule_form.rs | 4 +- .../recording/recording_rules_view.rs | 6 +- .../recording/recording_task_edit_form.rs | 4 +- frontend/src/app/components/search.rs | 14 +- frontend/src/app/components/select.rs | 2 +- .../src/app/components/setup/step_welcome.rs | 14 +- frontend/src/app/components/sidebar.rs | 4 +- .../components/source_editor/block_view.rs | 8 +- .../components/source_editor/editor_view.rs | 42 +- .../components/source_editor/input_form.rs | 6 +- .../source_editor/input_form/common.rs | 2 +- .../source_editor/input_form/stalker.rs | 18 +- .../source_editor/output_hdhomerun_form.rs | 30 +- .../source_editor/output_m3u_form.rs | 26 +- .../source_editor/output_strm_form.rs | 50 +- .../source_editor/output_xtream_form.rs | 34 +- .../src/app/components/source_editor/rules.rs | 8 +- .../components/source_editor/target_form.rs | 144 +- frontend/src/app/components/svg_icon.rs | 2 +- frontend/src/app/components/table.rs | 4 +- frontend/src/app/components/tabset.rs | 4 +- frontend/src/app/components/theme_picker.rs | 2 +- frontend/src/app/components/toastr.rs | 2 +- frontend/src/app/components/userlist/edit.rs | 4 +- frontend/src/app/components/userlist/list.rs | 2 +- .../userlist/proxy_user_credentials_form.rs | 10 +- .../src/app/components/userlist/user_table.rs | 13 +- frontend/src/app/mod.rs | 7 +- frontend/src/hooks/use_log_stream.rs | 4 +- frontend/src/hooks/use_server_status.rs | 2 +- frontend/src/hooks/use_service_context.rs | 6 +- frontend/src/hooks/use_websocket_status.rs | 2 +- frontend/src/lib.rs | 40 + frontend/src/provider/dialog_provider.rs | 6 +- frontend/src/services/auth_service.rs | 8 +- frontend/src/services/config_service.rs | 22 +- frontend/src/services/flags_service.rs | 2 +- frontend/src/services/playlist_service.rs | 4 +- frontend/src/services/recording_service.rs | 4 +- frontend/src/services/websocket_service.rs | 22 +- frontend/src/utils/format.rs | 2 +- shared/Cargo.toml | 9 +- shared/src/defaults/hdhomerun.rs | 2 +- shared/src/defaults/library.rs | 3 + shared/src/defaults/mod.rs | 12 +- shared/src/defaults/primitives.rs | 2 +- shared/src/defaults/qos.rs | 2 +- shared/src/foundation/filter.rs | 8 +- shared/src/foundation/mapper.rs | 15 +- shared/src/lib.rs | 36 + shared/src/model/cluster_flags.rs | 2 +- shared/src/model/config/api_proxy.rs | 4 +- shared/src/model/config/api_user.rs | 2 +- shared/src/model/config/base.rs | 13 +- shared/src/model/config/byte_size.rs | 2 +- shared/src/model/config/epg.rs | 6 +- shared/src/model/config/hls.rs | 4 +- shared/src/model/config/input.rs | 18 +- .../src/model/config/media_server_catalog.rs | 4 +- shared/src/model/config/messaging.rs | 20 +- shared/src/model/config/network_access.rs | 6 +- shared/src/model/config/reverse_proxy.rs | 18 +- shared/src/model/config/stream.rs | 8 +- shared/src/model/config/target.rs | 23 +- shared/src/model/config/video_download.rs | 6 +- shared/src/model/config/web_ui.rs | 15 +- shared/src/model/custom_video_stream_type.rs | 70 + shared/src/model/epg.rs | 6 +- shared/src/model/info_doc_utils.rs | 2 +- shared/src/model/mapping.rs | 2 +- shared/src/model/messaging.rs | 2 +- shared/src/model/mod.rs | 17 +- shared/src/model/pagination.rs | 9 +- shared/src/model/playlist.rs | 14 +- shared/src/model/playlist_document.rs | 12 +- shared/src/model/playlist_info_document.rs | 54 +- shared/src/model/provider_saturation.rs | 11 +- .../src/model}/recording_math.rs | 4 +- shared/src/model/regex_cache.rs | 2 +- shared/src/model/stalker.rs | 8 +- shared/src/model/stalker_item.rs | 8 +- shared/src/model/stream_info.rs | 2 +- shared/src/model/web_socket.rs | 13 +- shared/src/utils/flags.rs | 4 +- shared/src/utils/hdhomerun_utils.rs | 2 +- shared/src/utils/json_utils.rs | 2 +- shared/src/utils/net_utils.rs | 2 +- shared/src/utils/number_utils.rs | 4 +- shared/src/utils/recording_filename.rs | 4 +- shared/src/utils/request.rs | 2 +- shared/src/utils/serde_utils.rs | 8 +- shared/src/utils/string_interner.rs | 20 +- shared/src/utils/string_utils.rs | 8 +- .../fixtures/hls/channel_unavailable.ts | Bin .../hls/hls_session_or_lease_expired.ts | Bin .../fixtures/hls/low_priority_preempted.ts | Bin .../fixtures/hls/panel_api_provisioning.ts | Bin .../hls/provider_connections_exhausted.ts | Bin .../fixtures/hls/user_account_expired.ts | Bin .../hls/user_connections_exhausted.ts | Bin 686 files changed, 42407 insertions(+), 49530 deletions(-) create mode 100644 .cargo/config.toml delete mode 100644 backend/Cargo.lock rename backend/{ => app}/Cargo.toml (58%) rename backend/{ => app}/build.rs (100%) rename backend/{ => app}/src/api/api_utils.rs (90%) create mode 100644 backend/app/src/api/auth_middleware.rs rename backend/{ => app}/src/api/config_file.rs (91%) rename backend/{ => app}/src/api/endpoints/api_playlist_utils.rs (82%) rename backend/{ => app}/src/api/endpoints/custom_video_stream_api.rs (89%) rename backend/{ => app}/src/api/endpoints/download_api.rs (63%) rename backend/{ => app}/src/api/endpoints/extract_accept_header.rs (100%) rename backend/{ => app}/src/api/endpoints/hdhomerun_api.rs (95%) rename backend/{ => app}/src/api/endpoints/hls_api.rs (94%) rename backend/{ => app}/src/api/endpoints/hls_terminal_response.rs (87%) rename backend/{ => app}/src/api/endpoints/library_api.rs (86%) rename backend/{ => app}/src/api/endpoints/log_ws_api.rs (94%) rename backend/{ => app}/src/api/endpoints/m3u_api.rs (95%) rename backend/{ => app}/src/api/endpoints/mod.rs (100%) rename backend/{ => app}/src/api/endpoints/provider_resolve_api.rs (87%) rename backend/{ => app}/src/api/endpoints/rbac_api.rs (88%) rename backend/{ => app}/src/api/endpoints/recording_api.rs (99%) rename backend/{ => app}/src/api/endpoints/recording_media_api.rs (98%) rename backend/{ => app}/src/api/endpoints/stream_history_api.rs (85%) rename backend/{ => app}/src/api/endpoints/user_api.rs (96%) rename backend/{ => app}/src/api/endpoints/user_visibility.rs (94%) rename backend/{ => app}/src/api/endpoints/v1_api.rs (99%) rename backend/{ => app}/src/api/endpoints/v1_api_config.rs (88%) rename backend/{ => app}/src/api/endpoints/v1_api_playlist.rs (95%) rename backend/{ => app}/src/api/endpoints/v1_api_user.rs (87%) rename backend/{ => app}/src/api/endpoints/web_index.rs (95%) rename backend/{ => app}/src/api/endpoints/websocket_api.rs (99%) rename backend/{ => app}/src/api/endpoints/xmltv_api.rs (86%) rename backend/{ => app}/src/api/endpoints/xtream_api.rs (94%) rename backend/{ => app}/src/api/endpoints/xtream_url.rs (97%) rename backend/{ => app}/src/api/hdhomerun_proprietary.rs (100%) rename backend/{ => app}/src/api/http_layers.rs (64%) rename backend/{ => app}/src/api/main_api.rs (89%) rename backend/{ => app}/src/api/mod.rs (92%) rename backend/{ => app}/src/api/model/app_state.rs (90%) create mode 100644 backend/app/src/api/model/app_state_view.rs rename backend/{ => app}/src/api/model/hls_provisioning.rs (87%) create mode 100644 backend/app/src/api/model/mod.rs rename backend/{ => app}/src/api/model/proxy/mod.rs (74%) rename backend/{ => app}/src/api/model/streams/active_client_stream.rs (97%) rename backend/{ => app}/src/api/model/streams/client_stream.rs (100%) rename backend/{ => app}/src/api/model/streams/custom_video_stream.rs (100%) create mode 100644 backend/app/src/api/model/streams/metering_stream.rs create mode 100644 backend/app/src/api/model/streams/mod.rs rename backend/{ => app}/src/api/model/streams/provider_stream.rs (86%) rename backend/{ => app}/src/api/model/streams/provider_stream_factory.rs (95%) rename backend/{ => app}/src/api/model/streams/provisioning_stream.rs (92%) rename backend/{ => app}/src/api/model/streams/throttled_stream.rs (94%) rename backend/{ => app}/src/api/model/streams/timed_client_stream.rs (68%) rename backend/{ => app}/src/api/panel_api.rs (94%) rename backend/{ => app}/src/api/serve.rs (94%) rename backend/{ => app}/src/api/setup_api.rs (92%) rename backend/{ => app}/src/api/static_headers.rs (100%) rename backend/{ => app}/src/api/sys_usage.rs (93%) rename backend/{ => app}/src/api/tasks/config_watch.rs (95%) rename backend/{ => app}/src/api/tasks/hdhomerun_ssdp.rs (100%) rename backend/{ => app}/src/api/tasks/library_scan.rs (100%) create mode 100644 backend/app/src/api/tasks/mod.rs rename backend/{ => app}/src/api/tasks/scheduler.rs (92%) rename backend/{ => app}/src/api/tasks/xtream_expiry.rs (90%) rename backend/{ => app}/src/main.rs (77%) rename backend/{ => app}/src/tools/flags_builder.rs (96%) rename backend/{ => app}/tests/provider_dns_https_sni.rs (98%) rename backend/{ => app}/tests/proxy_redirect_auth.rs (93%) create mode 100644 backend/auth/Cargo.toml rename backend/{src/auth => auth/src}/access_token.rs (93%) rename backend/{src/auth => auth/src}/api_user_context.rs (77%) rename backend/{src/auth => auth/src}/auth_basic.rs (74%) rename backend/{src/auth => auth/src}/auth_bearer.rs (74%) rename backend/{src/auth => auth/src}/authenticator.rs (73%) create mode 100644 backend/auth/src/lib.rs rename backend/{src/auth => auth/src}/password.rs (63%) rename backend/{src/auth => auth/src}/recording_auth.rs (84%) rename backend/{src/auth => auth/src}/user.rs (100%) create mode 100644 backend/btree/Cargo.toml create mode 100644 backend/btree/src/codec.rs rename backend/{src/repository/bplustree => btree/src}/common.rs (74%) create mode 100644 backend/btree/src/lib.rs rename backend/{src/repository/bplustree => btree/src}/sorted_index.rs (92%) rename backend/{src/repository/bplustree => btree/src}/stress.rs (97%) rename backend/{src/repository/bplustree => btree/src}/v2.rs (94%) rename backend/{src/repository/bplustree => btree/src}/v3/format.rs (94%) rename backend/{src/repository/bplustree => btree/src}/v3/migration.rs (86%) rename backend/{src/repository/bplustree => btree/src}/v3/mod.rs (92%) rename backend/{src/repository/bplustree => btree/src}/v3/page.rs (87%) rename backend/{src/repository/bplustree => btree/src}/v3/publish.rs (88%) rename backend/{src/repository/bplustree => btree/src}/v3/tree.rs (92%) rename backend/{src/repository/bplustree => btree/src}/v3/wal.rs (93%) create mode 100644 backend/config-loader/Cargo.toml rename backend/{src/utils/file/config_reader.rs => config-loader/src/lib.rs} (72%) rename backend/{ => config-loader}/src/runtime_config_report.rs (85%) create mode 100644 backend/config-loader/tests/temp_dir_override.rs create mode 100644 backend/core/Cargo.toml create mode 100644 backend/core/src/lib.rs rename backend/{src/api => core/src}/model/batch_result_collector.rs (87%) rename backend/{ => core}/src/model/config/api.rs (58%) rename backend/{ => core}/src/model/config/api_proxy.rs (55%) rename backend/{ => core}/src/model/config/api_user.rs (86%) rename backend/{ => core}/src/model/config/app.rs (84%) rename backend/{ => core}/src/model/config/base.rs (78%) rename backend/{ => core}/src/model/config/cache.rs (97%) rename backend/{ => core}/src/model/config/epg.rs (94%) rename backend/{ => core}/src/model/config/epg_smart_match.rs (90%) rename backend/{ => core}/src/model/config/favourites.rs (91%) rename backend/{ => core}/src/model/config/geoip.rs (70%) rename backend/{ => core}/src/model/config/hdhomerun.rs (85%) rename backend/{ => core}/src/model/config/healthcheck.rs (100%) rename backend/{ => core}/src/model/config/input.rs (96%) rename backend/{ => core}/src/model/config/ip_check.rs (81%) rename backend/{ => core}/src/model/config/library.rs (88%) rename backend/{ => core}/src/model/config/log.rs (94%) rename backend/{ => core}/src/model/config/macros.rs (95%) rename backend/{ => core}/src/model/config/media_tools.rs (74%) rename backend/{ => core}/src/model/config/messaging.rs (86%) rename backend/{ => core}/src/model/config/metadata_update.rs (90%) rename backend/{ => core}/src/model/config/mod.rs (100%) rename backend/{ => core}/src/model/config/panel_api.rs (83%) create mode 100644 backend/core/src/model/config/proxy.rs rename backend/{ => core}/src/model/config/qos_aggregation.rs (100%) rename backend/{ => core}/src/model/config/rate_limit.rs (59%) rename backend/{ => core}/src/model/config/rename.rs (92%) rename backend/{ => core}/src/model/config/reverse_proxy.rs (84%) rename backend/{ => core}/src/model/config/schedule.rs (61%) rename backend/{ => core}/src/model/config/sort.rs (79%) rename backend/{ => core}/src/model/config/source.rs (88%) rename backend/{ => core}/src/model/config/stream.rs (88%) rename backend/{ => core}/src/model/config/stream_history.rs (91%) rename backend/{ => core}/src/model/config/target.rs (90%) rename backend/{ => core}/src/model/config/trakt.rs (96%) rename backend/{ => core}/src/model/config/trakt_api.rs (68%) rename backend/{ => core}/src/model/config/video_download.rs (89%) rename backend/{ => core}/src/model/config/web_auth.rs (91%) rename backend/{ => core}/src/model/config/web_ui.rs (73%) create mode 100644 backend/core/src/model/custom_stream_flags.rs rename backend/{src/auth => core/src/model}/fingerprint.rs (74%) rename backend/{ => core}/src/model/healthcheck.rs (100%) rename backend/{ => core}/src/model/input_source.rs (94%) rename backend/{ => core}/src/model/mapping.rs (78%) rename backend/{ => core}/src/model/messaging.rs (82%) create mode 100644 backend/core/src/model/mod.rs create mode 100644 backend/core/src/model/playlist_filter.rs create mode 100644 backend/core/src/model/playlist_key.rs rename backend/{src/api/model/provider_config.rs => core/src/model/provider.rs} (77%) create mode 100644 backend/core/src/model/proxy_redirect.rs rename backend/{ => core}/src/model/readiness.rs (78%) create mode 100644 backend/core/src/model/stalker_record.rs rename backend/{src/api => core/src}/model/stream_error.rs (96%) rename backend/{ => core}/src/model/stream_history.rs (86%) rename backend/{src/api => core/src}/model/update_guard.rs (100%) create mode 100644 backend/core/src/model/update_task.rs rename backend/{src/api/model/request.rs => core/src/model/user_api_request.rs} (80%) rename backend/{ => core}/src/model/xmltv.rs (82%) rename backend/{ => core}/src/model/xtream.rs (73%) rename backend/{src/api/model/xtream.rs => core/src/model/xtream_response.rs} (100%) create mode 100644 backend/core/src/response_macros.rs rename backend/{ => core}/src/utils/atomic_json_store.rs (89%) rename backend/{ => core}/src/utils/binary_utils.rs (100%) rename backend/{src/api/model => core/src/utils}/byte_range.rs (90%) rename backend/{ => core}/src/utils/compression/compressed_file_reader.rs (100%) rename backend/{ => core}/src/utils/compression/compressed_file_reader_async.rs (100%) rename backend/{ => core}/src/utils/compression/compression_utils.rs (100%) rename backend/{ => core}/src/utils/compression/mod.rs (100%) rename backend/{ => core}/src/utils/crypto_utils.rs (95%) rename backend/{ => core}/src/utils/epg_id.rs (87%) rename backend/{ => core}/src/utils/epg_parser.rs (85%) rename backend/{ => core}/src/utils/ffmpeg.rs (82%) create mode 100644 backend/core/src/utils/file/config_reader.rs rename backend/{ => core}/src/utils/file/env_resolving_reader.rs (77%) rename backend/{ => core}/src/utils/file/file_lock_manager.rs (92%) rename backend/{ => core}/src/utils/file/file_utils.rs (72%) rename backend/{ => core}/src/utils/file/mapping_reader.rs (82%) create mode 100644 backend/core/src/utils/file/mod.rs rename backend/{ => core}/src/utils/file/multi_file_reader.rs (100%) rename backend/{ => core}/src/utils/file/template_reader.rs (79%) rename backend/{ => core}/src/utils/hls_duration.rs (72%) rename backend/{ => core}/src/utils/json_utils.rs (71%) rename backend/{ => core}/src/utils/logging.rs (82%) rename backend/{ => core}/src/utils/lru_cache.rs (92%) rename backend/{ => core}/src/utils/mod.rs (60%) rename backend/{ => core}/src/utils/network/content_coding.rs (89%) rename backend/{ => core}/src/utils/network/ip_checker.rs (87%) create mode 100644 backend/core/src/utils/network/mod.rs rename backend/{src/api/model/streams/persist_pipe_stream.rs => core/src/utils/network/persist_pipe.rs} (97%) rename backend/{ => core}/src/utils/network/request.rs (97%) rename backend/{ => core}/src/utils/ordinal.rs (100%) rename backend/{ => core}/src/utils/provider_resolve_token.rs (95%) rename backend/{ => core}/src/utils/recording_paths.rs (94%) create mode 100644 backend/core/src/utils/request_headers.rs create mode 100644 backend/core/src/utils/response_compression.rs rename backend/{ => core}/src/utils/step_measure.rs (89%) rename backend/{ => core}/src/utils/sys_utils.rs (95%) rename backend/{ => core}/src/utils/telegram.rs (95%) create mode 100644 backend/core/src/utils/time_utils.rs rename backend/{ => core}/src/utils/trakt/client.rs (87%) rename backend/{ => core}/src/utils/trakt/errors.rs (77%) rename backend/{ => core}/src/utils/trakt/mod.rs (99%) create mode 100644 backend/dvr/Cargo.toml rename backend/{src/api/model => dvr/src}/download.rs (93%) create mode 100644 backend/dvr/src/lib.rs create mode 100644 backend/dvr/src/recording/mod.rs rename backend/{src/api/model => dvr/src}/recording/recording_catalog_access.rs (81%) rename backend/{src/api/model => dvr/src}/recording/recording_conflict.rs (93%) create mode 100644 backend/dvr/src/recording/recording_ctx.rs rename backend/{src/api/model => dvr/src}/recording/recording_currently_airing.rs (97%) rename backend/{src/api/model => dvr/src}/recording/recording_deletion.rs (88%) rename backend/{src/api/model => dvr/src}/recording/recording_disk.rs (81%) rename backend/{src/api/model => dvr/src}/recording/recording_edit.rs (94%) rename backend/{src/api/model => dvr/src}/recording/recording_notification.rs (97%) rename backend/{src/api/model => dvr/src}/recording/recording_notification_adapter.rs (96%) rename backend/{src/api/model => dvr/src}/recording/recording_observability.rs (94%) rename backend/{src/api/model => dvr/src}/recording/recording_occurrence.rs (91%) rename backend/{src/api/model => dvr/src}/recording/recording_quota.rs (74%) rename backend/{src/api/model => dvr/src}/recording/recording_reconciliation.rs (89%) rename backend/{src/api/model => dvr/src}/recording/recording_retention.rs (72%) rename backend/{src/api/model => dvr/src}/recording/recording_rule_scheduler.rs (85%) rename backend/{src/api/model => dvr/src}/recording/recording_rule_service.rs (94%) rename backend/{src/api/model => dvr/src}/recording/recording_security.rs (96%) rename backend/{src/api/model => dvr/src}/recording/recording_service.rs (87%) create mode 100644 backend/dvr/src/recording/recording_source_resolution.rs rename backend/{src/api/model => dvr/src}/recording/recording_supervisor/health.rs (74%) rename backend/{src/api/model => dvr/src}/recording/recording_supervisor/mod.rs (62%) rename backend/{src/api/model => dvr/src}/recording/recording_supervisor/outbox.rs (84%) rename backend/{src/api/model => dvr/src}/recording/recording_supervisor/retention.rs (73%) rename backend/{src/api/model => dvr/src}/recording/recording_supervisor/startup.rs (83%) rename backend/{src/api/model => dvr/src}/recording/recording_worker.rs (84%) rename backend/{src/api/model => dvr/src}/recording/recording_worker_runner.rs (77%) rename backend/{src/api/model => dvr/src}/recording/recording_ws.rs (86%) create mode 100644 backend/hls/Cargo.toml create mode 100644 backend/hls/src/availability/mod.rs create mode 100644 backend/hls/src/availability/recovery_pressure.rs create mode 100644 backend/hls/src/availability/reevaluation.rs create mode 100644 backend/hls/src/availability/terminal_cutover.rs create mode 100644 backend/hls/src/availability/terminal_pending_owner.rs create mode 100644 backend/hls/src/availability/tests.rs rename backend/{src/api/model/hls_cache => hls/src}/availability_reevaluation.rs (79%) rename backend/{src/api/model/hls_cache => hls/src}/backpressure.rs (96%) rename backend/{src/api/model/hls_cache => hls/src}/cache.rs (96%) rename backend/{src/api/model/hls_cache => hls/src}/critical_handoff.rs (94%) rename backend/{src/api/model/hls_cache => hls/src}/cutover.rs (94%) rename backend/{src/api/model/hls_cache => hls/src}/deadline.rs (100%) rename backend/{src/api/model/hls_cache => hls/src}/deterministic_conflict.rs (81%) rename backend/{src/api/model/hls_cache => hls/src}/gc.rs (94%) rename backend/{src/api/model/proxy => hls/src}/header_policy.rs (94%) rename backend/{src/api/model/hls_cache => hls/src}/headers.rs (97%) create mode 100644 backend/hls/src/hls_ctx.rs rename backend/{src/api/model/hls_cache => hls/src}/ids.rs (92%) rename backend/{src/processing/parser/hls => hls/src}/initial_strip.rs (89%) rename backend/{src/api/model/hls_cache => hls/src}/lease.rs (93%) create mode 100644 backend/hls/src/lib.rs rename backend/{src/api/model/hls_cache => hls/src}/lifecycle.rs (98%) rename backend/{src/api/model/hls_cache => hls/src}/manager.rs (89%) rename backend/{src/api/model/hls_cache => hls/src}/manifest_acceptance.rs (92%) rename backend/{src/api/model/hls_cache => hls/src}/manifest_commit.rs (100%) create mode 100644 backend/hls/src/manifest_fetch/episode.rs create mode 100644 backend/hls/src/manifest_fetch/error.rs create mode 100644 backend/hls/src/manifest_fetch/fetch.rs create mode 100644 backend/hls/src/manifest_fetch/fingerprint.rs create mode 100644 backend/hls/src/manifest_fetch/http.rs create mode 100644 backend/hls/src/manifest_fetch/mod.rs create mode 100644 backend/hls/src/manifest_fetch/model.rs create mode 100644 backend/hls/src/manifest_fetch/quality.rs create mode 100644 backend/hls/src/manifest_fetch/recovery.rs create mode 100644 backend/hls/src/manifest_fetch/selection_log.rs create mode 100644 backend/hls/src/manifest_fetch/tests.rs rename backend/{src/api/model/hls_cache => hls/src}/manifest_origin_binding.rs (86%) rename backend/{src/api/model/hls_cache => hls/src}/manifest_snapshot.rs (98%) rename backend/{src/api/model/hls_cache => hls/src}/map.rs (98%) rename backend/{src/api/model/hls_cache => hls/src}/map_fetcher.rs (97%) rename backend/{src/api/model/hls_cache => hls/src}/master_playlist.rs (83%) rename backend/{src/api/model/hls_cache => hls/src}/media_reserve.rs (96%) rename backend/{src/api/model/hls_cache => hls/src}/observability.rs (78%) rename backend/{src/api/model/hls_cache => hls/src}/origin.rs (92%) rename backend/{src/api/model/hls_cache => hls/src}/origin_progress.rs (92%) rename backend/{src/api/model/hls_cache => hls/src}/paths.rs (100%) rename backend/{src/api/model/hls_cache => hls/src}/playback.rs (81%) rename backend/{src/api/model/hls_cache => hls/src}/post_refresh_availability.rs (93%) rename backend/{src/api/model/hls_cache => hls/src}/prefetch.rs (97%) rename backend/{src/api/model/hls_cache => hls/src}/prepared_terminal_bundle.rs (90%) rename backend/{src/api/model/hls_cache => hls/src}/qos.rs (96%) rename backend/{src/api/model/hls_cache => hls/src}/recovery_timing.rs (84%) create mode 100644 backend/hls/src/refresh/commit.rs create mode 100644 backend/hls/src/refresh/failure.rs create mode 100644 backend/hls/src/refresh/mod.rs create mode 100644 backend/hls/src/refresh/switch_staging.rs create mode 100644 backend/hls/src/refresh/tests.rs create mode 100644 backend/hls/src/refresh/timing.rs rename backend/{src/api/model/hls_cache => hls/src}/renderer.rs (97%) rename backend/{src/api/model/hls_cache => hls/src}/resource_fetch.rs (98%) rename backend/{src/api/model/hls_cache => hls/src}/resource_identity.rs (74%) rename backend/{src/api/model/hls_cache => hls/src}/response.rs (94%) rename backend/{src/api/model/hls_cache => hls/src}/runtime_custom_tail.rs (79%) rename backend/{src/api/model/hls_cache => hls/src}/segment_fetcher.rs (95%) rename backend/{src/api/model/hls_cache => hls/src}/segment_repair.rs (97%) rename backend/{src/api/model/hls_cache => hls/src}/segment_watchdog.rs (96%) rename backend/{src/api/model/hls_cache => hls/src}/session.rs (92%) rename backend/{src/api/model/hls_cache => hls/src}/session_store.rs (94%) rename backend/{src/api/model/hls_cache => hls/src}/startup_observability.rs (91%) rename backend/{src/api/model/hls_cache => hls/src}/terminal_commit.rs (84%) rename backend/{src/api/model/hls_cache => hls/src}/terminal_pending.rs (94%) rename backend/{src/api/model/hls_cache => hls/src}/terminal_tail.rs (90%) rename backend/{src/api/model/hls_cache => hls/src}/timeline.rs (95%) rename backend/{src/api/model/hls_cache => hls/src}/transient.rs (98%) rename backend/{src/api/model/hls_cache => hls/src}/transient_fetcher.rs (97%) rename backend/{src/processing/parser/hls => hls/src}/transient_manifest.rs (98%) create mode 100644 backend/iptv/Cargo.toml rename backend/{src/iptv/mod.rs => iptv/src/lib.rs} (100%) rename backend/{src/iptv => iptv/src}/m3u/client.rs (68%) create mode 100644 backend/iptv/src/m3u/mod.rs rename backend/{src/iptv => iptv/src}/stalker/auth.rs (81%) rename backend/{src/iptv => iptv/src}/stalker/catalog.rs (88%) rename backend/{src/iptv => iptv/src}/stalker/client.rs (81%) rename backend/{src/iptv => iptv/src}/stalker/cmd_parser.rs (88%) rename backend/{src/iptv => iptv/src}/stalker/cookie_jar.rs (79%) rename backend/{src/iptv => iptv/src}/stalker/epg.rs (82%) rename backend/{src/iptv => iptv/src}/stalker/error.rs (94%) rename backend/{src/iptv => iptv/src}/stalker/mod.rs (98%) rename backend/{src/processing/parser/stalker.rs => iptv/src/stalker/parser.rs} (88%) rename backend/{src/iptv => iptv/src}/stalker/playback.rs (86%) rename backend/{src/iptv => iptv/src}/stalker/presets.rs (97%) rename backend/{src/iptv => iptv/src}/stalker/profile.rs (96%) rename backend/{src/iptv => iptv/src}/stalker/recipes.rs (91%) rename backend/{src/iptv => iptv/src}/stalker/session.rs (84%) rename backend/{src/iptv => iptv/src}/stalker/url_factory.rs (89%) rename backend/{src/iptv => iptv/src}/xtream/mod.rs (62%) create mode 100644 backend/library/Cargo.toml create mode 100644 backend/library/src/lib.rs rename backend/{ => library}/src/library/classifier.rs (91%) rename backend/{ => library}/src/library/metadata.rs (91%) rename backend/{ => library}/src/library/metadata_iterator.rs (73%) rename backend/{ => library}/src/library/metadata_resolver.rs (81%) rename backend/{ => library}/src/library/metadata_storage.rs (91%) rename backend/{ => library}/src/library/mod.rs (100%) rename backend/{ => library}/src/library/nfo_reader.rs (69%) rename backend/{ => library}/src/library/processor.rs (84%) rename backend/{ => library}/src/library/scanner.rs (78%) rename backend/{ => library}/src/library/thumbnail.rs (77%) rename backend/{ => library}/src/library/tmdb.rs (81%) rename backend/{ => library}/src/library/tmdb_client.rs (97%) rename backend/{ => library}/src/media_enrichment/facts.rs (96%) rename backend/{ => library}/src/media_enrichment/mod.rs (100%) rename backend/{ => library}/src/media_enrichment/parsed_title.rs (100%) rename backend/{ => library}/src/media_enrichment/policy.rs (94%) rename backend/{ => library}/src/media_enrichment/tmdb.rs (87%) rename backend/{ => library}/src/media_enrichment/xtream.rs (93%) rename backend/{ => library}/src/ptt/constants.rs (98%) rename backend/{ => library}/src/ptt/handlers.rs (92%) rename backend/{ => library}/src/ptt/mod.rs (77%) rename backend/{ => library}/src/ptt/models.rs (99%) rename backend/{ => library}/src/ptt/parser.rs (79%) rename backend/{ => library}/src/ptt/transformers.rs (75%) create mode 100644 backend/media-server/Cargo.toml rename backend/{src/media_server => media-server/src}/catalog.rs (91%) rename backend/{src/media_server => media-server/src}/client.rs (84%) rename backend/{src/media_server => media-server/src}/emby/dto.rs (97%) rename backend/{src/media_server => media-server/src}/emby/mod.rs (100%) rename backend/{src/media_server => media-server/src}/errors.rs (96%) create mode 100644 backend/media-server/src/jellyfin/dto.rs rename backend/{src/media_server => media-server/src}/jellyfin/mod.rs (100%) rename backend/{src/media_server/mod.rs => media-server/src/lib.rs} (100%) rename backend/{src/media_server => media-server/src}/playback.rs (84%) rename backend/{src/media_server => media-server/src}/playlist_mapper.rs (94%) rename backend/{src/media_server => media-server/src}/plex/client.rs (85%) rename backend/{src/media_server => media-server/src}/plex/dto.rs (99%) rename backend/{src/media_server => media-server/src}/plex/mapper.rs (90%) rename backend/{src/media_server => media-server/src}/plex/mod.rs (100%) rename backend/{src/media_server => media-server/src}/redaction.rs (97%) rename backend/{src/media_server => media-server/src}/test_fixtures.rs (100%) rename backend/{src/media_server => media-server/src}/types.rs (94%) create mode 100644 backend/messaging/Cargo.toml rename backend/{src/messaging.rs => messaging/src/lib.rs} (82%) create mode 100644 backend/metadata/Cargo.toml create mode 100644 backend/metadata/src/ctx.rs create mode 100644 backend/metadata/src/lib.rs rename backend/{src/api/model/metadata_update_manager.rs => metadata/src/manager.rs} (89%) create mode 100644 backend/mpegts/Cargo.toml create mode 100644 backend/mpegts/src/lib.rs rename backend/{src/api/model/streams => mpegts/src}/transport_stream_buffer.rs (85%) rename backend/{src/api/model/hls_cache => mpegts/src}/ts_inspector.rs (86%) create mode 100644 backend/parser/Cargo.toml rename backend/{src/processing/parser/hls.rs => parser/src/hls/mod.rs} (87%) rename backend/{src/processing/parser => parser/src}/hls/origin_manifest.rs (99%) rename backend/{src/processing/parser => parser/src}/ics/dummy.rs (99%) rename backend/{src/processing/parser => parser/src}/ics/event.rs (97%) rename backend/{src/processing/parser => parser/src}/ics/mod.rs (93%) rename backend/{src/processing/parser => parser/src}/ics/time.rs (100%) create mode 100644 backend/parser/src/lib.rs rename backend/{src/processing/parser => parser/src}/m3u.rs (89%) rename backend/{src/iptv/m3u => parser/src/m3u_format}/archive.rs (87%) rename backend/{src/iptv/m3u => parser/src/m3u_format}/catchup.rs (83%) rename backend/{src/iptv/m3u => parser/src/m3u_format}/catchup_token.rs (84%) create mode 100644 backend/parser/src/m3u_format/mod.rs rename backend/{src/processing/parser => parser/src}/xtream.rs (65%) create mode 100644 backend/processing/Cargo.toml rename backend/{src/utils/network => processing/src}/epg.rs (86%) create mode 100644 backend/processing/src/fetched_playlist.rs rename backend/{src/processing => processing/src}/geoip.rs (75%) rename backend/{src/processing => processing/src}/input_cache.rs (82%) create mode 100644 backend/processing/src/lib.rs create mode 100644 backend/processing/src/metadata_sink.rs create mode 100644 backend/processing/src/parser/mod.rs rename backend/{src/processing => processing/src}/parser/xmltv.rs (96%) rename backend/{src/processing => processing/src}/playlist_watch.rs (74%) rename backend/{src/processing => processing/src}/processor/deduplicate.rs (79%) rename backend/{src/processing => processing/src}/processor/epg.rs (98%) rename backend/{src/processing => processing/src}/processor/library.rs (69%) rename backend/{src/processing => processing/src}/processor/mod.rs (82%) rename backend/{src/processing => processing/src}/processor/playlist.rs (88%) rename backend/{src/processing => processing/src}/processor/probe_handle_guard.rs (86%) rename backend/{src/processing => processing/src}/processor/resolve_options.rs (64%) rename backend/{src/processing => processing/src}/processor/sort.rs (96%) rename backend/{src/processing => processing/src}/processor/stalker.rs (84%) rename backend/{src/processing => processing/src}/processor/stalker_refresh.rs (86%) rename backend/{src/processing => processing/src}/processor/stream_probe.rs (73%) rename backend/{src/processing => processing/src}/processor/trakt.rs (87%) rename backend/{src/processing => processing/src}/processor/xtream.rs (82%) rename backend/{src/processing => processing/src}/processor/xtream_series.rs (90%) rename backend/{src/processing => processing/src}/processor/xtream_vod.rs (88%) create mode 100644 backend/repository/Cargo.toml rename backend/{src/repository => repository/src}/alias_repository.rs (94%) rename backend/{src/utils => repository/src}/db_viewer.rs (88%) rename backend/{src/repository => repository/src}/epg_repository.rs (78%) rename backend/{src/utils => repository/src}/geoip.rs (64%) rename backend/{src/repository => repository/src}/identity_registry.rs (83%) rename backend/{src/repository/mod.rs => repository/src/lib.rs} (61%) rename backend/{src/repository => repository/src}/library_repository.rs (87%) rename backend/{src/repository => repository/src}/live_stream_metadata_repository.rs (84%) rename backend/{src/repository => repository/src}/m3u_playlist_iterator.rs (97%) rename backend/{src/repository => repository/src}/m3u_repository.rs (86%) create mode 100644 backend/repository/src/metadata_retry_record.rs create mode 100644 backend/repository/src/network_access.rs create mode 100644 backend/repository/src/playlist_cache_loader.rs rename backend/{src/api/model => repository/src}/playlist_mem_cache.rs (95%) rename backend/{src/repository => repository/src}/playlist_repository.rs (78%) rename backend/{src/repository => repository/src}/playlist_scratch.rs (77%) rename backend/{src/repository => repository/src}/playlist_source.rs (83%) rename backend/{src/repository => repository/src}/playlist_stream.rs (76%) rename backend/{src/repository => repository/src}/provider_dns_repository.rs (79%) rename backend/{src/repository => repository/src}/qos_snapshot_repository.rs (87%) rename backend/{src/repository => repository/src}/recording_rule_repository.rs (95%) rename backend/{src/repository => repository/src}/stalker_generation_repository.rs (91%) rename backend/{src/repository => repository/src}/stalker_repository.rs (90%) rename backend/{src/repository/bplustree/migration.rs => repository/src/startup_migration.rs} (97%) rename backend/{src/repository => repository/src}/storage.rs (79%) create mode 100644 backend/repository/src/storage_const.rs rename backend/{src/repository => repository/src}/stream_history/archive.rs (93%) rename backend/{src/repository => repository/src}/stream_history/async_iterator.rs (95%) rename backend/{src/repository => repository/src}/stream_history/mod.rs (94%) rename backend/{src/repository => repository/src}/stream_history/reader.rs (87%) rename backend/{src/repository => repository/src}/stream_history/storage.rs (95%) rename backend/{src/utils/stream_history_viewer.rs => repository/src/stream_history/viewer.rs} (85%) rename backend/{src/repository => repository/src}/stream_history/writer.rs (91%) rename backend/{src/repository => repository/src}/strm_repository.rs (97%) rename backend/{src/repository => repository/src}/target_id_mapping.rs (88%) rename backend/{src/repository => repository/src}/user_repository.rs (95%) rename backend/{src/repository => repository/src}/xtream_playlist_iterator.rs (88%) rename backend/{src/repository => repository/src}/xtream_repository.rs (78%) create mode 100644 backend/session/Cargo.toml rename backend/{src/api/model => session/src}/active_provider_manager.rs (92%) rename backend/{src/api/model => session/src}/active_user_manager.rs (98%) create mode 100644 backend/session/src/admission.rs rename backend/{src/api/model => session/src}/admission_strategy.rs (69%) rename backend/{src/api/model => session/src}/connection_manager.rs (87%) rename backend/{src/api/model => session/src}/event_manager.rs (92%) create mode 100644 backend/session/src/lib.rs create mode 100644 backend/session/src/meter.rs rename backend/{src/api/model => session/src}/provider_dns_manager.rs (90%) rename backend/{src/api/model => session/src}/provider_lineup_manager.rs (94%) rename backend/{src/api/model => session/src}/qos_aggregation_manager.rs (86%) rename backend/{src/api/model/model_utils.rs => session/src/response_headers.rs} (98%) rename backend/{src/api/model => session/src}/stream.rs (90%) create mode 100644 backend/session/src/stream_ctx.rs create mode 100644 backend/session/src/stream_options.rs rename backend/{src/api/model => session/src}/streams/buffered_stream.rs (95%) create mode 100644 backend/session/src/streams/mod.rs rename backend/{src/api/model => session/src}/streams/shared_stream_manager.rs (90%) delete mode 100644 backend/src/api/model/hls_cache/availability.rs delete mode 100644 backend/src/api/model/hls_cache/manifest_fetch.rs delete mode 100644 backend/src/api/model/hls_cache/mod.rs delete mode 100644 backend/src/api/model/hls_cache/refresh.rs delete mode 100644 backend/src/api/model/mod.rs delete mode 100644 backend/src/api/model/recording/mod.rs delete mode 100644 backend/src/api/model/streams/metering_stream.rs delete mode 100644 backend/src/api/model/streams/mod.rs delete mode 100644 backend/src/api/tasks/mod.rs delete mode 100644 backend/src/auth/mod.rs delete mode 100644 backend/src/iptv/m3u/mod.rs delete mode 100644 backend/src/media_server/jellyfin/dto.rs delete mode 100644 backend/src/model/config/proxy.rs delete mode 100644 backend/src/model/mod.rs delete mode 100644 backend/src/model/playlist.rs delete mode 100644 backend/src/modules.rs delete mode 100644 backend/src/processing/mod.rs delete mode 100644 backend/src/processing/parser/mod.rs delete mode 100644 backend/src/repository/bplustree/mod.rs delete mode 100644 backend/src/repository/storage_const.rs delete mode 100644 backend/src/utils/file/mod.rs delete mode 100644 backend/src/utils/network/mod.rs delete mode 100644 backend/src/utils/time_utils.rs create mode 100755 bin/check-workspace-deps-test.sh create mode 100755 bin/check-workspace-deps.sh create mode 100644 shared/src/model/custom_video_stream_type.rs rename {backend/src/api/model/recording => shared/src/model}/recording_math.rs (97%) rename {backend/tests => test}/fixtures/hls/channel_unavailable.ts (100%) rename {backend/tests => test}/fixtures/hls/hls_session_or_lease_expired.ts (100%) rename {backend/tests => test}/fixtures/hls/low_priority_preempted.ts (100%) rename {backend/tests => test}/fixtures/hls/panel_api_provisioning.ts (100%) rename {backend/tests => test}/fixtures/hls/provider_connections_exhausted.ts (100%) rename {backend/tests => test}/fixtures/hls/user_account_expired.ts (100%) rename {backend/tests => test}/fixtures/hls/user_connections_exhausted.ts (100%) diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..12f60697a --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,14 @@ +[target.armv7-unknown-linux-gnueabihf] +linker = "arm-linux-gnueabihf-gcc" + +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" + +[target.x86_64-pc-windows-gnu] +linker = "x86_64-w64-mingw32-gcc" + +[target.x86_64-apple-darwin] +linker = "x86_64-apple-darwin14-clang" # Requires osxcross + +[target.aarch64-apple-darwin] +linker = "aarch64-apple-darwin20.4-clang" # Requires osxcross diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9e0a71eb6..bceab7a3d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,9 +41,18 @@ jobs: # This now works with cargo +nightly fmt run: make fmt-check + - name: Check workspace architecture + run: make architecture-check + - name: Lint with Clippy run: make lint + - name: Validate native Docker workspace skeleton + run: docker build --target workspace-skeleton -f docker/Dockerfile . + + - name: Validate cross Docker workspace skeleton + run: docker build --target rust-toolchain -f docker/cross.Dockerfile . + docs: name: Documentation Build runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index c71d447bd..7a1303b44 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,7 @@ artifacts /.vscode .envrc -backend/Cargo.lock +backend/app/Cargo.lock # Ignore markdown files *.md @@ -58,3 +58,11 @@ backend/Cargo.lock # Ignore ai documentation files # no ai files + +# Local refactoring measurements (clippy inventories, cargo metadata, timings). +# These are engineering artifacts of the modularization work, not project docs. +/baselines/ + +# Scratch directory some backend tests create relative to the crate root. +# Test output must never become tracked source. +/backend/app/tmp/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f55f5c8e..adb38d5c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -282,7 +282,7 @@ - **Disk-Space Alerts**: - Added a threshold-based disk-usage monitor (Normal / Warn / Critical) implemented in - `backend/src/api/sys_usage.rs::DiskAlertMonitor`. The background sampler ticks every 2 seconds (fixed, + `backend/app/src/api/sys_usage.rs::DiskAlertMonitor`. The background sampler ticks every 2 seconds (fixed, not configurable — see `SYSTEM_USAGE_INTERVAL` in that file) and the state machine decides when to emit a `DiskAlert` to the existing messaging channels (Telegram, Discord, Pushover, REST). - The state machine emits a `DiskAlert` whenever the level is non-Normal **and** (`state_changed` **or** @@ -559,7 +559,7 @@ total on SSD); peak RAM cost drops proportionally. - **Positional file write helper**: new `write_all_at_offset` in - `backend/src/repository/bplustree/common.rs` mirrors the existing + `backend/btree/src/common.rs` mirrors the existing `read_exact_at_offset`. Uses `FileExt::write_all_at` on Unix and an explicit short-write retry loop on Windows. Previously a bug in this area would have left the database file silently truncated or scrambled; the short-write loop is the same pattern used by @@ -826,9 +826,9 @@ `grace_instant_stream`, `grace_hold_stream`. - **config.yml (`messaging.disk_alert`)**: - Added optional `disk_alert` block to enable disk-usage alerts via the existing messaging channels. The - background monitor in `backend/src/api/sys_usage.rs` samples the current working directory's mount on + background monitor in `backend/app/src/api/sys_usage.rs` samples the current working directory's mount on every fixed 2-second tick and feeds each sample to the `DiskAlertMonitor` state machine - (`backend/src/api/sys_usage.rs::DiskAlertMonitor`). + (`backend/app/src/api/sys_usage.rs::DiskAlertMonitor`). - Fields: - `warn_percent` (`f64`, default `80.0`): percent-used at or above which the `Warn` level is reached. Must be in `[0, 100]`. @@ -891,7 +891,7 @@ - **B+Tree v3 persistence engine**: - Consolidated the facade, v2 compatibility reader, v3 engine, migration, WAL, sorted index, and stress tests under - `backend/src/repository/bplustree/`. + `backend/btree/src/`. - Added checksummed 4 KiB Slotted Pages, WAL-before-data in-place updates, verified atomic full replacement, typed v1/v2 startup migration, identity-bound sorted indexes, and corruption-reporting iterators. - Reused mmap mappings, page validation, and decoded internal routes across cheap `BPlusTreeQuery` clones while diff --git a/Cargo.lock b/Cargo.lock index 38ee01347..821256333 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -118,7 +118,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -129,7 +129,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -140,9 +140,9 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -177,7 +177,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -189,7 +189,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] @@ -201,14 +201,14 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -224,7 +224,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -327,7 +327,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -383,9 +383,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2b_simd" @@ -400,11 +400,10 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "arrayref", "arrayvec", "cc", "cfg-if", @@ -432,9 +431,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.3" +version = "8.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -443,9 +442,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -459,9 +458,9 @@ checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cast" @@ -572,9 +571,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -582,9 +581,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -594,14 +593,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -723,9 +722,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] @@ -742,25 +741,6 @@ dependencies = [ "winnow 0.7.15", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -812,7 +792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -845,7 +825,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -856,7 +836,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -875,9 +855,40 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] [[package]] name = "der-parser" @@ -920,7 +931,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -930,7 +941,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -951,7 +962,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -995,7 +1006,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1069,14 +1080,14 @@ checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "env_filter" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -1084,9 +1095,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -1108,7 +1119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1124,11 +1135,11 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", ] [[package]] @@ -1169,12 +1180,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foldhash" version = "0.2.0" @@ -1232,7 +1237,7 @@ dependencies = [ "shared", "strum", "strum_macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", @@ -1270,9 +1275,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -1285,9 +1290,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -1295,15 +1300,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1312,19 +1317,19 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -1345,15 +1350,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -1363,9 +1368,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -1396,7 +1401,7 @@ checksum = "913dce4c5f06c2ea40fc178c06f777ac89fc6b1383e90c254fafb1abe4ba3c82" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "uuid", ] @@ -1429,17 +1434,15 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", "wasm-bindgen", ] @@ -1657,7 +1660,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1677,7 +1680,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.4", + "rand 0.9.5", "smallvec", "spinning_top", "web-time", @@ -1725,9 +1728,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "6.4.1" +version = "6.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d43ccdfe15a81ab0a8af639e90254227c9a46afd9c5f5b6ec7efaa345c4b0f00" +checksum = "75c54236f9045c8004a77942bebc52145b4844639db934a5c70fe08617fbe61a" dependencies = [ "derive_builder", "log", @@ -1736,7 +1739,7 @@ dependencies = [ "pest_derive", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1745,15 +1748,6 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -1762,7 +1756,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1773,7 +1767,7 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1827,9 +1821,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -1867,9 +1861,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -2055,12 +2049,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -2105,7 +2093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "699c1b6d335e63d0ba5c1e1c7f647371ce989c3bcbe1f7ed2b85fa56e3bd1a21" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2126,7 +2114,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd5b3eaf1a28b758ac0faa5a4254e8ab2705605496f1b1f3fbbc3988ad73d199" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "inotify-sys", "libc", ] @@ -2169,10 +2157,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "log", "portable-atomic", @@ -2181,14 +2171,24 @@ dependencies = [ ] [[package]] -name = "jiff-static" -version = "0.2.23" +name = "jiff-core" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2225,9 +2225,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.100" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -2276,17 +2276,11 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -2323,9 +2317,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lol_html" @@ -2333,17 +2327,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cssparser", "encoding_rs", - "foldhash 0.2.0", + "foldhash", "hashbrown 0.17.1", "memchr", "mime", "precomputed-hash", "selectors", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2378,9 +2372,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" @@ -2506,7 +2500,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "fsevent-sys", "inotify", "kqueue", @@ -2524,7 +2518,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -2542,7 +2536,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2620,7 +2614,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -2719,9 +2713,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" dependencies = [ "memchr", "ucd-trie", @@ -2729,9 +2723,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.6" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" dependencies = [ "pest", "pest_generator", @@ -2739,25 +2733,24 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.6" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "pest_meta" -version = "2.8.6" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" dependencies = [ "pest", - "sha2 0.10.9", ] [[package]] @@ -2830,7 +2823,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2843,7 +2836,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2890,7 +2883,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -2968,7 +2961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3007,9 +3000,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3054,7 +3047,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3068,7 +3061,7 @@ checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.4.2", + "getrandom 0.4.3", "lru-slab", "rand 0.10.2", "rand_pcg", @@ -3077,7 +3070,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3099,9 +3092,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3129,9 +3122,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -3144,7 +3137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -3197,34 +3190,14 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.0", -] - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "bitflags 2.13.1", ] [[package]] name = "rcgen" -version = "0.14.8" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4" dependencies = [ "pem", "ring", @@ -3240,14 +3213,14 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -3257,9 +3230,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -3268,9 +3241,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwasm" @@ -3377,9 +3350,9 @@ dependencies = [ [[package]] name = "rphonetic" -version = "3.0.6" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "281c97095b4c3d5de9fae5abc43299a9d0035a9ffa8fb513371da7e641f7f3e2" +checksum = "a5908062cb5f4bef7e5b4492ec0704f04d65914a876a854096f64fe4fb2bc270" dependencies = [ "document-features", "either", @@ -3388,6 +3361,7 @@ dependencies = [ "nom 8.0.0", "regex", "serde", + "thiserror 2.0.20", ] [[package]] @@ -3442,18 +3416,18 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", @@ -3504,7 +3478,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3567,7 +3541,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3590,7 +3564,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cssparser", "derive_more", "log", @@ -3611,9 +3585,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -3651,42 +3625,42 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] name = "serde_html_form" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0946d52b4b7e28823148aebbeceb901012c595ad737920d504fa8634bb099e6f" +checksum = "8f0346d7a342ab90f405cfc08f25d15075f944f42fcabbc5eac923829fa6d228" dependencies = [ "form_urlencoded", "indexmap", "itoa", - "ryu", "serde_core", + "zmij", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", @@ -3725,7 +3699,7 @@ checksum = "ec3a1e7d2eadec84deabd46ae061bf480a91a6bce74d25dad375bd656f2e19d8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3760,17 +3734,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -3787,7 +3750,7 @@ name = "shared" version = "3.3.91" dependencies = [ "base64 0.23.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "blake3", "brotli", "bytes", @@ -3797,7 +3760,7 @@ dependencies = [ "dashmap", "deunicode", "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "hex", "indexmap", "js-sys", @@ -3814,7 +3777,7 @@ dependencies = [ "serde_tuple", "strum", "strum_macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "url", "zeroize", ] @@ -3864,9 +3827,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "smptera-format-identifiers-rust" @@ -3879,12 +3842,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3923,7 +3886,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3944,9 +3907,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -3970,7 +3944,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -3993,7 +3967,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4015,10 +3989,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4032,11 +4006,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -4047,18 +4021,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.4", ] [[package]] @@ -4121,9 +4095,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -4144,7 +4118,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4159,9 +4133,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -4183,13 +4157,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -4284,7 +4259,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -4329,7 +4304,7 @@ dependencies = [ "governor", "http 1.4.0", "pin-project", - "thiserror 2.0.18", + "thiserror 2.0.20", "tonic", "tower", "tracing", @@ -4355,7 +4330,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4390,65 +4365,48 @@ dependencies = [ "clap", "crc32fast", "cron", - "ctr", "dashmap", - "deunicode", "env_logger", - "fancy-regex", "fastrand", - "filetime", - "flate2", - "fs2", "futures", - "handlebars", "http-body-util", "hyper", "hyper-util", "iana-time-zone", "indexmap", - "ipnet", "jsonwebtoken", "libc", "log", "lol_html", "lru", - "lz4_flex", - "memchr", - "memmap2", "mime", "mime_guess", - "mpeg2ts-reader", "notify", "parking_lot", "paste", "path-clean", "pest", - "pest_derive", "quick-xml", - "rand 0.9.4", - "rayon", + "rand 0.9.5", "rcgen", "regex", "reqwest", "rmp-serde", - "rpassword", "rphonetic", - "rust-argon2", "rustls", "serde", "serde-saphyr", "serde_html_form", "serde_json", - "sha2 0.11.0", + "sha2", "shared", "smallvec", "socket2", "strsim", "strum", - "strum_macros", "sysinfo", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-rustls", "tokio-stream", @@ -4456,14 +4414,453 @@ dependencies = [ "tower", "tower-http", "tower_governor", + "tuliprox-auth", + "tuliprox-btree", + "tuliprox-config-loader", + "tuliprox-core", + "tuliprox-dvr", + "tuliprox-hls", + "tuliprox-iptv", + "tuliprox-library", + "tuliprox-media-server", + "tuliprox-messaging", + "tuliprox-metadata", + "tuliprox-mpegts", + "tuliprox-parser", + "tuliprox-processing", + "tuliprox-repository", + "tuliprox-session", "url", "uuid", "vergen", - "winapi", "windows-sys 0.61.2", "zeroize", ] +[[package]] +name = "tuliprox-auth" +version = "3.3.91" +dependencies = [ + "arc-swap", + "axum", + "base64 0.23.1", + "blake3", + "chrono", + "jsonwebtoken", + "log", + "rand 0.9.5", + "rpassword", + "rust-argon2", + "serde_json", + "shared", + "tuliprox-core", + "tuliprox-repository", + "zeroize", +] + +[[package]] +name = "tuliprox-btree" +version = "3.3.91" +dependencies = [ + "crc32fast", + "fs2", + "log", + "lz4_flex", + "memmap2", + "parking_lot", + "rand 0.9.5", + "rmp-serde", + "serde", + "smallvec", + "tempfile", + "uuid", +] + +[[package]] +name = "tuliprox-config-loader" +version = "3.3.91" +dependencies = [ + "arc-swap", + "chrono", + "log", + "serde", + "serde-saphyr", + "serde_json", + "shared", + "tempfile", + "tokio", + "tuliprox-core", + "tuliprox-repository", + "url", + "windows-sys 0.61.2", +] + +[[package]] +name = "tuliprox-core" +version = "3.3.91" +dependencies = [ + "aes", + "arc-swap", + "async-compression", + "axum", + "base64 0.23.1", + "blake3", + "bytes", + "chrono", + "chrono-tz", + "ctr", + "env_logger", + "flate2", + "futures", + "http-body-util", + "indexmap", + "ipnet", + "jsonwebtoken", + "libc", + "log", + "mime", + "parking_lot", + "paste", + "path-clean", + "quick-xml", + "rand 0.9.5", + "regex", + "reqwest", + "rmp-serde", + "serde", + "serde-saphyr", + "serde_html_form", + "serde_json", + "shared", + "strum", + "sysinfo", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-util", + "tuliprox-media-server", + "tuliprox-mpegts", + "url", + "zeroize", +] + +[[package]] +name = "tuliprox-dvr" +version = "3.3.91" +dependencies = [ + "arc-swap", + "bytes", + "chrono", + "chrono-tz", + "futures", + "libc", + "log", + "reqwest", + "serde", + "serde_json", + "shared", + "tempfile", + "tokio", + "tokio-util", + "tuliprox-auth", + "tuliprox-core", + "tuliprox-messaging", + "tuliprox-repository", + "tuliprox-session", + "url", + "windows-sys 0.61.2", +] + +[[package]] +name = "tuliprox-hls" +version = "3.3.91" +dependencies = [ + "arc-swap", + "async-compression", + "axum", + "base64 0.23.1", + "blake3", + "brotli", + "bytes", + "chrono", + "dashmap", + "fastrand", + "filetime", + "flate2", + "futures", + "http-body-util", + "log", + "lru", + "rand 0.9.5", + "reqwest", + "serde", + "serde_json", + "sha2", + "shared", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-util", + "tuliprox-core", + "tuliprox-mpegts", + "tuliprox-parser", + "tuliprox-session", + "url", + "zeroize", +] + +[[package]] +name = "tuliprox-iptv" +version = "3.3.91" +dependencies = [ + "base64 0.23.1", + "bytes", + "chrono", + "futures", + "log", + "parking_lot", + "reqwest", + "serde", + "serde_json", + "shared", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tuliprox-core", + "tuliprox-messaging", + "tuliprox-parser", + "tuliprox-repository", + "url", +] + +[[package]] +name = "tuliprox-library" +version = "3.3.91" +dependencies = [ + "blake3", + "bytes", + "chrono", + "fancy-regex", + "filetime", + "log", + "paste", + "path-clean", + "quick-xml", + "regex", + "reqwest", + "serde", + "serde_json", + "shared", + "strsim", + "tempfile", + "tokio", + "tuliprox-core", + "url", + "uuid", +] + +[[package]] +name = "tuliprox-media-server" +version = "3.3.91" +dependencies = [ + "bytes", + "futures", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "shared", + "tokio", + "url", +] + +[[package]] +name = "tuliprox-messaging" +version = "3.3.91" +dependencies = [ + "arc-swap", + "chrono", + "handlebars", + "log", + "mime", + "reqwest", + "serde", + "serde_json", + "shared", + "tokio", + "tuliprox-core", + "url", +] + +[[package]] +name = "tuliprox-metadata" +version = "3.3.91" +dependencies = [ + "arc-swap", + "chrono", + "dashmap", + "fastrand", + "log", + "parking_lot", + "reqwest", + "rmp-serde", + "serde", + "serde_json", + "shared", + "tempfile", + "tokio", + "tokio-util", + "tuliprox-core", + "tuliprox-processing", + "tuliprox-repository", + "tuliprox-session", +] + +[[package]] +name = "tuliprox-mpegts" +version = "3.3.91" +dependencies = [ + "aes", + "bytes", + "futures", + "mpeg2ts-reader", + "sha2", + "thiserror 2.0.20", + "tokio", + "zeroize", +] + +[[package]] +name = "tuliprox-parser" +version = "3.3.91" +dependencies = [ + "aes", + "base64 0.23.1", + "chrono", + "chrono-tz", + "ctr", + "indexmap", + "log", + "rand 0.9.5", + "regex", + "serde", + "serde_json", + "shared", + "tempfile", + "tokio", + "tokio-util", + "tuliprox-core", + "url", +] + +[[package]] +name = "tuliprox-processing" +version = "3.3.91" +dependencies = [ + "arc-swap", + "chrono", + "futures", + "indexmap", + "log", + "lru", + "parking_lot", + "paste", + "path-clean", + "quick-xml", + "rand 0.9.5", + "regex", + "reqwest", + "rmp-serde", + "rphonetic", + "serde", + "serde_json", + "shared", + "strsim", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tuliprox-core", + "tuliprox-iptv", + "tuliprox-library", + "tuliprox-media-server", + "tuliprox-messaging", + "tuliprox-parser", + "tuliprox-repository", + "tuliprox-session", +] + +[[package]] +name = "tuliprox-repository" +version = "3.3.91" +dependencies = [ + "arc-swap", + "base64 0.23.1", + "bytes", + "chrono", + "crc32fast", + "env_logger", + "filetime", + "fs2", + "futures", + "indexmap", + "libc", + "log", + "lz4_flex", + "parking_lot", + "paste", + "rmp-serde", + "serde", + "serde_json", + "shared", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tuliprox-btree", + "tuliprox-core", + "tuliprox-parser", + "url", + "uuid", + "windows-sys 0.61.2", +] + +[[package]] +name = "tuliprox-session" +version = "3.3.91" +dependencies = [ + "arc-swap", + "axum", + "bytes", + "chrono", + "crc32fast", + "dashmap", + "futures", + "jsonwebtoken", + "log", + "lru", + "rand 0.9.5", + "reqwest", + "serde", + "shared", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-util", + "tuliprox-core", + "tuliprox-mpegts", + "tuliprox-repository", + "url", +] + [[package]] name = "tungstenite" version = "0.29.0" @@ -4475,9 +4872,9 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.4", + "rand 0.9.5", "sha1", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -4516,12 +4913,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "untrusted" version = "0.7.1" @@ -4566,11 +4957,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.3" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "wasm-bindgen", ] @@ -4639,20 +5030,11 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" -version = "0.2.123" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -4665,9 +5047,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.73" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -4675,9 +5057,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.123" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4685,31 +5067,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.123" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.123" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.73" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28a0782b173cf2e98f62aacb487c5c021b7c5925df46098675f28e1fa85e159" +checksum = "895a2607575412a4eda1df892084a375ea10dfeadc4d7d2ab87b854e4ddc7ba1" dependencies = [ "async-trait", "cast", @@ -4729,30 +5111,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.73" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee997551ce1ad5adda03f7ce37ec34b4140fe9f547fd07b46d55901d1ba1a06b" +checksum = "4288cb0ebe215033bf949ae1fd046726daa4c32a157f24b9dc6ac387a52aa759" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "wasm-bindgen-test-shared" -version = "0.2.123" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0f005eb61f765eff31a79ef71df7e21819731268a868df41192c1e41d6d3e5" - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] +checksum = "33ff1c1b360982e93b6d8ea9c04836f71dba0817a16f91e229cf3a51bdd9d987" [[package]] name = "wasm-logger" @@ -4765,18 +5137,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -4790,23 +5150,11 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.13.0", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.100" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -4853,7 +5201,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4915,7 +5263,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -4926,7 +5274,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5228,88 +5576,6 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.13.0", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] [[package]] name = "writeable" @@ -5331,7 +5597,7 @@ dependencies = [ "oid-registry", "ring", "rusticata-macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -5360,7 +5626,7 @@ dependencies = [ "rustversion", "serde", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokise", "tracing", @@ -5399,7 +5665,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5429,7 +5695,7 @@ checksum = "8399f1d134ab8e69abc7cded19f114621d520bd9c79c5a0f34091bf664d7325b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5451,7 +5717,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] @@ -5472,7 +5738,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5492,28 +5758,28 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -5546,7 +5812,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 07d1114fb..b6bd3332b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["backend", "frontend", "shared"] +members = ["backend/*", "frontend", "shared"] resolver = "2" [workspace.package] @@ -7,34 +7,69 @@ version = "3.3.91" edition = "2021" rust-version = "1.95.0" +[workspace.lints.clippy] +# Shared policy for every workspace crate (modularization plan, Phase 0). +# Extracted crates must not silently lose the lint level that previously +# existed only as a crate attribute on backend/app/src/main.rs. +all = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +wildcard_imports = "deny" + +# Allows carried over verbatim from backend/app/src/main.rs. They were already +# accepted for the backend; moving the policy to the workspace must not change +# which warnings the backend emits. +module_name_repetitions = "allow" +must_use_candidate = "allow" +return_self_not_must_use = "allow" +missing_errors_doc = "allow" +large_futures = "allow" + [workspace.dependencies] -serde = { version = "1.0.228", features = ["derive", "rc"] } -serde_json = { version = "1.0.150" } -regex = "1.12.3" -log = "0.4.32" +shared = { version = "3", path = "shared" } +tuliprox-hls = { version = "3", path = "backend/hls" } +tuliprox-auth = { version = "3", path = "backend/auth" } +tuliprox-dvr = { version = "3", path = "backend/dvr" } +tuliprox = { version = "3", path = "backend/app" } +tuliprox-mpegts = { version = "3", path = "backend/mpegts" } +tuliprox-iptv = { version = "3", path = "backend/iptv" } +tuliprox-parser = { version = "3", path = "backend/parser" } +tuliprox-metadata = { version = "3", path = "backend/metadata" } +tuliprox-messaging = { version = "3", path = "backend/messaging" } +tuliprox-processing = { version = "3", path = "backend/processing" } +tuliprox-btree = { version = "3", path = "backend/btree" } +tuliprox-config-loader = { version = "3", path = "backend/config-loader" } +tuliprox-session = { version = "3", path = "backend/session" } +tuliprox-media-server = { version = "3", path = "backend/media-server" } +tuliprox-repository = { version = "3", path = "backend/repository" } +tuliprox-core = { version = "3", path = "backend/core" } +tuliprox-library = { version = "3", path = "backend/library" } +serde = { version = "1.0.229", features = ["derive", "rc"] } +serde_json = { version = "1.0.151" } +regex = "1.13.1" +log = "0.4.34" chrono = "0.4.45" -bytes = "1.11.1" +bytes = "1.12.1" base64 = "0.23.1" -blake3 = "1.8.5" -fastrand = "2.4.1" +blake3 = "1.8.7" +fastrand = "2.5.0" lz4_flex = "0.14.0" -brotli = "8.0.3" +brotli = "8.0.4" indexmap = "2.14.0" dashmap = "6.2.1" -arc-swap = "1.9.1" +arc-swap = "1.9.2" paste = "1.0.15" -pest = "2.8.6" -pest_derive = "2.8.6" -zeroize = "1.8.2" +pest = "2.9.0" +pest_derive = "2.9.0" +zeroize = "1.9.0" deunicode = "1.6.2" path-clean = "1.0.1" url = "2.5.8" cron = "0.17.0" -futures = "0.3.32" +futures = "0.3.34" serde-saphyr = "0.0.26" sha2 = "0.11.0" -thiserror = "2.0.18" -tokio = { version = "1.52.3" } +thiserror = "2.0.20" +tokio = { version = "1.53.1" } strum = "0.28.0" strum_macros = "0.28.0" diff --git a/Makefile b/Makefile index b973a566a..74de3cc7a 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,10 @@ BOLD := \033[1m .DEFAULT_GOAL := help +# Number of CPUs (portable): try GNU `nproc`, then POSIX `getconf`, then macOS `sysctl` +CPU_COUNT := $(shell nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) +CARGO_BUILD_JOBS := $(CPU_COUNT) + .PHONY: help help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make $(AQUA)$(RESET)\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " $(AQUA)%-26s$(RESET) %s\n", $$1, $$2 } /^##@/ { printf "\n$(BOLD)%s$(RESET)\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -104,19 +108,31 @@ markdownlint: ## Install markdownlint-cli2 (requires npm) ##@ Development: .PHONY: test -test: ## Run all workspace tests (Stable) - @echo "==> Running tests (stable)" - $(CARGO_STABLE) test --workspace +test: ## Run all workspace tests (Stable) — use detected CPU count for parallelism + @echo "==> Running tests (stable) with $(CPU_COUNT) jobs/threads" + @TMPDIR="$${TMPDIR:-/tmp}" RUST_TEST_THREADS=$(CPU_COUNT) ./bin/test.sh -j$(CPU_COUNT) --workspace -- --test-threads=$(CPU_COUNT) + +.PHONY: build +build: ## Build the entire workspace in parallel using detected CPU count + @echo "==> Building workspace with $(CARGO_BUILD_JOBS) jobs" + @TMPDIR="$${TMPDIR:-/tmp}" $(CARGO_STABLE) build -j$(CARGO_BUILD_JOBS) --workspace + +.PHONY: architecture-check +architecture-check: ## Verify workspace dependency direction + @echo "==> Checking the architecture gate itself" + ./bin/check-workspace-deps-test.sh + @echo "==> Checking workspace dependency direction" + ./bin/check-workspace-deps.sh .PHONY: lint -lint: ## Run clippy linter (Stable) - @echo "==> Running clippy (stable)" - $(CARGO_STABLE) clippy --workspace -- -D warnings +lint: ## Run clippy linter (Nightly) + @echo "==> Running clippy (nightly)" + $(CARGO_NIGHTLY) clippy --workspace -- -D warnings .PHONY: lint-fix -lint-fix: ## Automatically fix clippy suggestions (Stable) +lint-fix: ## Automatically fix clippy suggestions (Nightly) @echo "==> Applying clippy auto-fixes" - $(CARGO_STABLE) clippy --fix --workspace --allow-dirty --allow-staged -- -D clippy::uninlined_format_args + $(CARGO_NIGHTLY) clippy --fix --workspace --allow-dirty --allow-staged -- -D clippy::uninlined_format_args .PHONY: fmt fmt: ## Format all code using nightly rules (Compact) diff --git a/README.md b/README.md index 1b57c08d0..d9624ad11 100644 --- a/README.md +++ b/README.md @@ -442,7 +442,7 @@ Open the Browser and continue setup. ## Project layout -- `backend/`: main server and processing pipeline +- `backend/`: backend crates (`app/` contains the main server and processing pipeline) - `frontend/`: Yew Web UI - `shared/`: DTOs and shared logic - `config/`: example configuration diff --git a/backend/Cargo.lock b/backend/Cargo.lock deleted file mode 100644 index 3c93e698b..000000000 --- a/backend/Cargo.lock +++ /dev/null @@ -1,3752 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.6.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" - -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.59.0", -] - -[[package]] -name = "anyhow" -version = "1.0.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" - -[[package]] -name = "arc-swap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-compression" -version = "0.4.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d615619615a650c571269c00dca41db04b9210037fa76ed8239f70404ab56985" -dependencies = [ - "brotli", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "zstd", - "zstd-safe", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "axum" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" -dependencies = [ - "axum-core", - "axum-macros", - "bytes", - "form_urlencoded", - "futures-util", - "http 1.3.1", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" -dependencies = [ - "bytes", - "futures-core", - "http 1.3.1", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-macros" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" - -[[package]] -name = "blake2b_simd" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e903a20b159e944f91ec8499fe1e55651480c541ea0a584f5d967c49ad9d99" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq", -] - -[[package]] -name = "blake3" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "brotli" -version = "8.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bumpalo" -version = "3.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "castaway" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" - -[[package]] -name = "cc" -version = "1.2.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac" -dependencies = [ - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chrono" -version = "0.4.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "clap" -version = "4.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" - -[[package]] -name = "colorchoice" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "cron" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" -dependencies = [ - "chrono", - "once_cell", - "winnow", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "curl" -version = "0.4.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2d5c8f48d9c0c23250e52b55e82a6ab4fdba6650c931f5a0a57a43abda812b" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "windows-sys 0.59.0", -] - -[[package]] -name = "curl-sys" -version = "0.4.82+curl-8.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4d63638b5ec65f1a4ae945287b3fd035be4554bbaf211901159c9a2a74fb5be" -dependencies = [ - "cc", - "libc", - "libnghttp2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "windows-sys 0.59.0", -] - -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core", - "quote", - "syn", -] - -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "deranged" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" -dependencies = [ - "powerfmt", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn", -] - -[[package]] -name = "deunicode" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "document-features" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" -dependencies = [ - "litrs", -] - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "enum-iterator" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c280b9e6b3ae19e152d8e31cf47f18389781e119d4013a2a2bb0180e5facc635" -dependencies = [ - "enum-iterator-derive", -] - -[[package]] -name = "enum-iterator-derive" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "env_filter" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "filetime" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" -dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.59.0", -] - -[[package]] -name = "flate2" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "forwarded-header-value" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" -dependencies = [ - "nonempty", - "thiserror 1.0.69", -] - -[[package]] -name = "fsevent-sys" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" -dependencies = [ - "libc", -] - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "governor" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be93b4ec2e4710b04d9264c0c7350cdd62a8c20e5e4ac732552ebb8f0debe8eb" -dependencies = [ - "cfg-if", - "dashmap", - "futures-sink", - "futures-timer", - "futures-util", - "getrandom 0.3.3", - "no-std-compat", - "nonzero_ext", - "parking_lot", - "portable-atomic", - "quanta", - "rand", - "smallvec", - "spinning_top", - "web-time", -] - -[[package]] -name = "h2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.3.1", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http 1.3.1", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http 1.3.1", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "http-range-header" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http 1.3.1", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http 1.3.1", - "hyper", - "hyper-util", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http 1.3.1", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "system-configuration", - "tokio", - "tower-service", - "tracing", - "windows-registry", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" - -[[package]] -name = "icu_properties" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "potential_utf", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" - -[[package]] -name = "icu_provider" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" -dependencies = [ - "displaydoc", - "icu_locale_core", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" -dependencies = [ - "equivalent", - "hashbrown 0.15.4", -] - -[[package]] -name = "inotify" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" -dependencies = [ - "bitflags 2.9.1", - "inotify-sys", - "libc", -] - -[[package]] -name = "inotify-sys" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" -dependencies = [ - "libc", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "isahc" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" -dependencies = [ - "async-channel", - "castaway", - "crossbeam-utils", - "curl", - "curl-sys", - "encoding_rs", - "event-listener", - "futures-lite", - "http 0.2.12", - "log", - "mime", - "once_cell", - "polling", - "serde", - "serde_json", - "slab", - "sluice", - "tracing", - "tracing-futures", - "url", - "waker-fn", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "jiff" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] - -[[package]] -name = "jiff-static" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "jobserver" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" -dependencies = [ - "getrandom 0.3.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "jsonwebtoken" -version = "9.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" -dependencies = [ - "base64 0.22.1", - "js-sys", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - -[[package]] -name = "kqueue" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" -dependencies = [ - "kqueue-sys", - "libc", -] - -[[package]] -name = "kqueue-sys" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" -dependencies = [ - "bitflags 1.3.2", - "libc", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.172" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" - -[[package]] -name = "libnghttp2-sys" -version = "0.1.11+1.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6c24e48a7167cffa7119da39d577fa482e66c688a4aac016bee862e1a713c4" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.9.1", - "libc", - "redox_syscall", -] - -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" - -[[package]] -name = "litemap" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" - -[[package]] -name = "litrs" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" - -[[package]] -name = "lock_api" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "no-std-compat" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" - -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - -[[package]] -name = "nonempty" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" - -[[package]] -name = "nonzero_ext" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" - -[[package]] -name = "notify" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" -dependencies = [ - "bitflags 2.9.1", - "filetime", - "fsevent-sys", - "inotify", - "kqueue", - "libc", - "log", - "mio", - "notify-types", - "walkdir", - "windows-sys 0.59.0", -] - -[[package]] -name = "notify-types" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" - -[[package]] -name = "openssl" -version = "0.10.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" -dependencies = [ - "bitflags 2.9.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-src" -version = "300.5.0+3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" -dependencies = [ - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.6", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "path-clean" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" - -[[package]] -name = "pem" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" -dependencies = [ - "base64 0.22.1", - "serde", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pest" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" -dependencies = [ - "memchr", - "thiserror 2.0.12", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pest_meta" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" -dependencies = [ - "once_cell", - "pest", - "sha2", -] - -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "portable-atomic" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "potential_utf" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quanta" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" -dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", -] - -[[package]] -name = "quinn" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.12", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" -dependencies = [ - "bytes", - "getrandom 0.3.3", - "lru-slab", - "rand", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.12", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "raw-cpuid" -version = "11.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" -dependencies = [ - "bitflags 2.9.1", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "redox_syscall" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" -dependencies = [ - "bitflags 2.9.1", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "reqwest" -version = "0.12.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" -dependencies = [ - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http 1.3.1", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-tls", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-native-tls", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.16", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rpassword" -version = "7.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d4c8b64f049c6721ec8ccec37ddfc3d641c4a7fca57e8f2a89de509c73df39" -dependencies = [ - "libc", - "rtoolbox", - "windows-sys 0.59.0", -] - -[[package]] -name = "rphonetic" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51016596e51b46757c38a94e15a56b293ad110fe37a6fadfea0083910c52b482" -dependencies = [ - "document-features", - "either", - "enum-iterator", - "lazy_static", - "nom", - "regex", - "serde", -] - -[[package]] -name = "rtoolbox" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cc970b249fbe527d6e02e0a227762c9108b2f49d81094fe357ffc6d14d7f6f" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "rust-argon2" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d9848531d60c9cbbcf9d166c885316c24bc0e2a9d3eba0956bb6cbbd79bc6e8" -dependencies = [ - "base64 0.21.7", - "blake2b_simd", - "constant_time_eq", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - -[[package]] -name = "rustelebot" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8bbb00699ceced1255b72aad9b418ee29c3acf7da4597ea4d81bae0bd97b235" -dependencies = [ - "futures", - "isahc", - "serde", - "serde_json", - "url", -] - -[[package]] -name = "rustix" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" -dependencies = [ - "bitflags 2.9.1", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustls" -version = "0.23.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" - -[[package]] -name = "ruzstd" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640bec8aad418d7d03c72ea2de10d5c646a598f9883c7babc160d91e3c1b26c" -dependencies = [ - "twox-hash", -] - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.9.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" -dependencies = [ - "itoa", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "simple_asn1" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 2.0.12", - "time", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "sluice" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" -dependencies = [ - "async-channel", - "futures-core", - "futures-io", -] - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "spinning_top" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" -dependencies = [ - "lock_api", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags 2.9.1", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" -dependencies = [ - "fastrand 2.3.0", - "getrandom 0.3.3", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl 2.0.12", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "time" -version = "0.3.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" -dependencies = [ - "deranged", - "itoa", - "libc", - "num-conv", - "num_threads", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" - -[[package]] -name = "time-macros" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinystr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.45.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "socket2", - "tokio-macros", - "windows-sys 0.52.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - -[[package]] -name = "tokio-util" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "async-compression", - "base64 0.22.1", - "bitflags 2.9.1", - "bytes", - "futures-core", - "futures-util", - "http 1.3.1", - "http-body", - "http-body-util", - "http-range-header", - "httpdate", - "iri-string", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite", - "tokio", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tower_governor" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84e6672c7510df74859726427edea641674dad1aeeb30057b87335b1ba23b843" -dependencies = [ - "axum", - "forwarded-header-value", - "governor", - "http 1.3.1", - "pin-project", - "thiserror 2.0.12", - "tower", - "tracing", -] - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tuliprox" -version = "3.1.4" -dependencies = [ - "arc-swap", - "axum", - "base64 0.21.7", - "bincode", - "bitflags 2.9.1", - "blake3", - "bytes", - "chrono", - "clap", - "cron", - "dashmap", - "deunicode", - "enum-iterator", - "env_logger", - "filetime", - "flate2", - "futures", - "hyper", - "hyper-util", - "indexmap", - "jsonwebtoken", - "libc", - "log", - "mime", - "notify", - "openssl", - "paste", - "path-clean", - "pest", - "pest_derive", - "quick-xml", - "rand", - "rayon", - "regex", - "reqwest", - "rpassword", - "rphonetic", - "rust-argon2", - "rustelebot", - "ruzstd", - "serde", - "serde_json", - "serde_yaml", - "socket2", - "strsim", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tower-http", - "tower_governor", - "url", - "vergen", - "winapi", - "zeroize", -] - -[[package]] -name = "twox-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "unicase" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" - -[[package]] -name = "unicode-ident" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - -[[package]] -name = "url" -version = "2.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "vergen" -version = "9.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b2bf58be11fc9414104c6d3a2e464163db5ef74b12296bda593cac37b6e4777" -dependencies = [ - "anyhow", - "derive_builder", - "rustversion", - "time", - "vergen-lib", -] - -[[package]] -name = "vergen-lib" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166" -dependencies = [ - "anyhow", - "derive_builder", - "rustversion", -] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - -[[package]] -name = "waker-fn" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-link" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" - -[[package]] -name = "windows-registry" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winnow" -version = "0.6.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.1", -] - -[[package]] -name = "writeable" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" - -[[package]] -name = "yoke" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" - -[[package]] -name = "zerotrie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/backend/Cargo.toml b/backend/app/Cargo.toml similarity index 58% rename from backend/Cargo.toml rename to backend/app/Cargo.toml index 360b9a2da..4600e9bd3 100644 --- a/backend/Cargo.toml +++ b/backend/app/Cargo.toml @@ -6,87 +6,88 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true + [dependencies] lol_html = "2.9.0" -handlebars = "6.4.1" -shared = { version = "3", path = "../shared" } +shared = { workspace = true } +tuliprox-core = { workspace = true } +tuliprox-hls = { workspace = true } +tuliprox-iptv = { workspace = true } +tuliprox-library = { workspace = true } +tuliprox-messaging = { workspace = true } +tuliprox-parser = { workspace = true } +tuliprox-processing = { workspace = true } +tuliprox-auth = { workspace = true } +tuliprox-config-loader = { workspace = true } +tuliprox-dvr = { workspace = true } +tuliprox-repository = { workspace = true } +tuliprox-session = { workspace = true } +tuliprox-media-server = { workspace = true } +tuliprox-metadata = { workspace = true } +tuliprox-mpegts = { workspace = true } serde.workspace = true serde-saphyr.workspace = true serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } quick-xml = { version = "0.41", features = ["async-tokio", "serialize"] } regex.workspace = true -clap = { version = "4.6.1", features = ["derive"] } +clap = { version = "4.6.6", features = ["derive"] } reqwest = { version = "0.13.4", features = ["json", "stream", "rustls", "socks", "form", "query"] } url.workspace = true chrono.workspace = true iana-time-zone = "0.1.65" cron.workspace = true axum = { version = "0.8.9" , features = ["macros", "default", "ws"]} -serde_html_form = "0.4.0" +serde_html_form = "0.4.1" tower = "0.5.3" tower-http = { version = "0.6.11", features = ["cors", "auth", "fs", "compression-full", "trace"] } tower_governor = { version = "0.8.0", features = ["axum"] } jsonwebtoken = { version = "10.4.0", default-features = false, features = ["aws_lc_rs"] } -rust-argon2 = "3.0.0" futures.workspace = true path-clean.workspace = true pest.workspace = true -pest_derive.workspace = true thiserror.workspace = true aes = "0.8.4" -ctr = "0.9.2" -deunicode.workspace = true mime = "0.3.17" log.workspace = true -env_logger = "0.11.10" +env_logger = "0.11.11" rmp-serde = "1.3.1" -rand = "0.9.4" +rand = "0.9.5" fastrand.workspace = true -rpassword = "7.5.4" -flate2 = "1.1.9" blake3.workspace = true bytes.workspace = true -tokio-stream = { version = "0.1.18", features = ["sync"] } +tokio-stream = { version = "0.1.19", features = ["sync"] } tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs", "process", "signal", "time", "macros"] } #console-subscriber = "0" #tracing = "0.1" #tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } -tokio-util = { version = "0.7.18", features = ["io", "io-util"] } +tokio-util = { version = "0.7.19", features = ["io", "io-util"] } tempfile = "3.27.0" -filetime = "0.2.29" strsim = "0.11.1" -rphonetic = "3.0.6" -rayon = "1.12.0" +rphonetic = "3.1.0" base64.workspace = true sha2.workspace = true paste.workspace = true indexmap.workspace = true parking_lot = "0.12.5" -lz4_flex.workspace = true brotli.workspace = true -memmap2 = "0.9.11" -smallvec = "1.15.1" -fs2 = "0.4.3" +smallvec = "1.15.2" notify = "8.2.0" arc-swap.workspace = true dashmap.workspace = true -hyper = "1.10.1" +hyper = "1.11.0" hyper-util = "0.1.20" -socket2 = { version = "0.6.4", features = ["all"] } -async-compression = { version = "0.4.42", features = ["tokio", "brotli", "deflate", "gzip", "zlib", "zstd"] } +socket2 = { version = "0.6.5", features = ["all"] } +async-compression = { version = "0.4.43", features = ["tokio", "brotli", "deflate", "gzip", "zlib", "zstd"] } chrono-tz = "0.10.4" zeroize.workspace = true -uuid = { version = "1.23.3", features = ["v4"] } -fancy-regex = "0.17.0" +uuid = { version = "1.26.0", features = ["v4"] } mime_guess = "2.0.5" -crc32fast = "1.5.0" -memchr = "2.8.1" -mpeg2ts-reader = "0.18.2" -http-body-util = "0.1.3" -lru = "0.16.3" -ipnet = "2" +crc32fast = "1.5.1" +http-body-util = "0.1.5" +lru = "0.16.4" strum.workspace = true -strum_macros.workspace = true [target.'cfg(unix)'.dependencies] libc = "0.2" @@ -94,8 +95,7 @@ libc = "0.2" sysinfo = { version = "0.38.4", default-features = false, features = ["system", "network"] } [target.'cfg(target_os = "windows")'.dependencies] -winapi = { version = "0.3", features = ["psapi", "minwindef", "processthreadsapi", "sysinfoapi", "fileapi", "winbase"] } -windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Threading"] } +windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_ProcessStatus", "Win32_System_SystemInformation", "Win32_System_Threading"] } #[target.'cfg(target_os = "linux")'.dependencies] #mimalloc = "0.1" #[target.'cfg(not(target_env = "msvc"))'.dependencies] @@ -106,9 +106,18 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Stora vergen = { version = "9.1.0", features = ["build"] } [dev-dependencies] +tuliprox-processing = { workspace = true, features = ["test-support"] } +tuliprox-hls = { workspace = true, features = ["test-support"] } +# The migration and playlist-stream tests build legacy v2 databases and +# sorted-index sidecars by hand; those builders live in the engine. +tuliprox-btree = { workspace = true, features = ["test-support"] } +# HLS tests assert on the segment render/finalize counters. +tuliprox-mpegts = { workspace = true, features = ["test-support"] } +# API tests build a GeoIp that answers a fixed country. +tuliprox-repository = { workspace = true, features = ["test-support"] } tokio = { workspace = true, features = ["test-util", "macros"] } -rcgen = "0.14.8" -rustls = "0.23.40" +rcgen = "0.14.9" +rustls = "0.23.43" tokio-rustls = "0.26.4" [features] diff --git a/backend/build.rs b/backend/app/build.rs similarity index 100% rename from backend/build.rs rename to backend/app/build.rs diff --git a/backend/src/api/api_utils.rs b/backend/app/src/api/api_utils.rs similarity index 90% rename from backend/src/api/api_utils.rs rename to backend/app/src/api/api_utils.rs index 5b32467e1..d94c622e7 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/app/src/api/api_utils.rs @@ -1,3 +1,7 @@ +pub use crate::repository::{ + evaluate_network_access, log_network_access_allowed_geoip_unavailable, log_network_access_denied, + NetworkAccessDecision, NetworkAccessDenyReason, +}; use crate::{ api::{ endpoints::xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext}, @@ -7,8 +11,9 @@ use crate::{ get_custom_stream_response_error_status, get_stream_response_with_headers, is_custom_video_stream_enabled, tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, ProviderAllocation, ProviderConfig, ProviderHandle, ProviderStreamCustomReason, - ProviderStreamFactoryOptions, ProviderStreamInfo, ProviderStreamState, SharedStreamManager, StreamDetails, - StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, MAX_HLS_MANIFEST_BYTES, + ProviderStreamFactoryOptions, ProviderStreamInfo, ProviderStreamState, SharedStreamCtx, + SharedStreamManager, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, + UserSession, }, }, auth::Fingerprint, @@ -18,7 +23,6 @@ use crate::{ media_server_stream_response as open_media_server_proxy_stream_response, parse_media_server_image_ref, parse_media_server_stream_ref, }, - plex::client::PlexCatalogClient, MediaServerError, MediaServerErrorKind, MediaServerHttpClient, MediaServerImageRef, }, model::{AppConfig, ConfigInput, ConfigTarget, InputUserInfo, ProxyUserCredentials}, @@ -36,7 +40,7 @@ use crate::{ use arc_swap::ArcSwapOption; use axum::{ body::Body, - http::{header, Extensions, HeaderMap, HeaderName, HeaderValue, Response, StatusCode}, + http::{header, HeaderMap, HeaderName, HeaderValue, Response, StatusCode}, response::IntoResponse, }; use bytes::{Bytes, BytesMut}; @@ -74,196 +78,13 @@ use tokio::{ sync::{Mutex, RwLock}, }; use tokio_util::io::ReaderStream; +use tuliprox_hls::api::MAX_HLS_MANIFEST_BYTES; use url::Url; -const RECENT_EVICTION_REENTRY_TTL_SECS: u64 = 3; - /// Per-`(input id, provider id)` single-flight guards so concurrent client requests /// for the same dead stalker stream trigger only one portal re-resolve at a time. type StalkerResolveGuards = HashMap<(u16, u32), Arc>>; -static STALKER_RE_RESOLVE_GUARDS: LazyLock> = - LazyLock::new(|| Mutex::new(HashMap::new())); - -#[derive(Clone, Copy)] -pub(crate) enum EvictionReentryGuard<'a> { - Session(&'a str), - SocketPlayback { virtual_id: VirtualId }, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PlaybackRequestClass { - Prepare, - Activate, - FollowUp, - Terminate, -} - -#[derive(Clone, Copy)] -pub(crate) struct PlaybackRequestFacts<'a> { - pub(crate) item_type: PlaylistItemType, - pub(crate) existing_session: Option<&'a UserSession>, - pub(crate) prepare_only: bool, - pub(crate) terminate: bool, -} - -pub(crate) fn classify_playback_request(facts: PlaybackRequestFacts<'_>) -> PlaybackRequestClass { - if facts.terminate { - return PlaybackRequestClass::Terminate; - } - if facts.prepare_only { - return PlaybackRequestClass::Prepare; - } - if let Some(session) = facts.existing_session { - // FollowUp only for sessions that are actively counted. - // PendingProvider has no counted lease yet - activation is still pending. - // Prepared/Preserved/Expired sessions are not FollowUp. - if session.lifecycle.is_counted() { - return PlaybackRequestClass::FollowUp; - } - } - let _ = facts.item_type; - PlaybackRequestClass::Activate -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn resolve_playback_request_admission( - app_state: &Arc, - user: &ProxyUserCredentials, - fingerprint: &Fingerprint, - item_type: PlaylistItemType, - user_session: Option<&UserSession>, - session_token: &str, - activate_unbound_session: bool, - eviction_reentry_guard: EvictionReentryGuard<'_>, - prepare_only: bool, - terminate: bool, -) -> (crate::api::model::ConnectionAdmission, Option, PlaybackRequestClass) { - let request_class = classify_playback_request(PlaybackRequestFacts { - item_type, - existing_session: user_session, - prepare_only, - terminate, - }); - let limits_enabled = (user.max_connections > 0 || user.soft_connections > 0) - && app_state.app_config.config.load().user_access_control; - - // Handle explicit Terminate: run termination and return exhausted permission. - // No admission strategies are evaluated — termination immediately expires the playback. - if request_class == PlaybackRequestClass::Terminate { - if let Some(session) = user_session { - app_state.active_users.terminate_session(&user.username, session.token.as_str()).await; - } - return ( - crate::api::model::ConnectionAdmission { - permission: UserConnectionPermission::Exhausted, - kind: user_session - .and_then(|session| session.connection_kind) - .or(Some(crate::api::model::ConnectionKind::Normal)), - }, - None, - request_class, - ); - } - - // Handle Prepare: no admission cost, just prepare state. Return Allowed without - // running strategies or modifying counted state. Caller handles the actual activation. - if request_class == PlaybackRequestClass::Prepare { - return ( - crate::api::model::ConnectionAdmission { - permission: UserConnectionPermission::Allowed, - kind: user_session - .and_then(|session| session.connection_kind) - .or(Some(crate::api::model::ConnectionKind::Normal)), - }, - None, - request_class, - ); - } - - if request_class == PlaybackRequestClass::FollowUp || !limits_enabled { - return ( - crate::api::model::ConnectionAdmission { - permission: user_session.map_or(UserConnectionPermission::Allowed, |session| session.permission), - kind: user_session - .and_then(|session| session.connection_kind) - .or(Some(crate::api::model::ConnectionKind::Normal)), - }, - None, - request_class, - ); - } - - let result = resolve_admission_with_strategies( - app_state, - &user.username, - user.max_connections, - user.soft_connections, - &fingerprint.client_ip, - &fingerprint.addr, - true, - Some(session_token), - activate_unbound_session, - eviction_reentry_guard, - ) - .await; - - (result.admission, result.grace_mode, request_class) -} - -async fn should_suppress_eviction_for_recent_request( - app_state: &Arc, - username: &str, - client_ip: &str, - guard: EvictionReentryGuard<'_>, - target_addr: &std::net::SocketAddr, -) -> bool { - match guard { - EvictionReentryGuard::Session(session_token) => app_state - .active_users - .recently_evicted_session_protected_addr(session_token) - .await - .is_some_and(|protected_addr| protected_addr == *target_addr), - EvictionReentryGuard::SocketPlayback { virtual_id } => app_state - .active_users - .recent_socket_reentry_protected_addr(username, client_ip, virtual_id) - .await - .is_some_and(|protected_addr| protected_addr == *target_addr), - } -} - -async fn get_admission_for_request( - app_state: &Arc, - username: &str, - max_connections: u32, - soft_connections: u16, - is_session_request: bool, - session_token: Option<&str>, - activate_unbound_session: bool, -) -> crate::api::model::ConnectionAdmission { - if is_session_request { - if activate_unbound_session { - app_state - .get_connection_admission_for_session_activation( - username, - max_connections, - soft_connections, - session_token.unwrap_or_default(), - ) - .await - } else { - app_state - .get_connection_admission_for_session( - username, - max_connections, - soft_connections, - session_token.unwrap_or_default(), - ) - .await - } - } else { - app_state.get_connection_admission(username, max_connections, soft_connections).await - } -} +static STALKER_RE_RESOLVE_GUARDS: LazyLock> = LazyLock::new(|| Mutex::new(HashMap::new())); pub(crate) fn resolve_request_url_for_logging<'a>(input: &ConfigInput, stream_url: &'a str) -> Cow<'a, str> { if is_media_server_playback_url(input, stream_url) { @@ -364,7 +185,7 @@ pub(crate) fn admission_failure_response( error!("Unsupported admission failure reason: {reason:?}"); return StatusCode::INTERNAL_SERVER_ERROR.into_response(); }; - create_custom_video_stream_response(app_state, &fingerprint.addr, video_type).into_response() + create_custom_video_stream_response(&app_state.provider_stream_ctx(), &fingerprint.addr, video_type).into_response() } #[macro_export] @@ -433,14 +254,6 @@ macro_rules! internal_server_error { }; } -#[macro_export] -macro_rules! try_unwrap_body { - ($body:expr) => { - $body - .map_or_else(|_| axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), |resp| resp.into_response()) - }; -} - #[macro_export] macro_rules! try_result_or_status { ($option:expr, $status:expr, $msg_is_error:expr, $msg:expr) => { @@ -485,22 +298,35 @@ macro_rules! try_result_not_found { } use crate::{ - api::panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}, + api::{ + panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}, + static_headers::CT_OCTET, + }, utils::LRUResourceCache, }; pub use internal_server_error; use shared::{ defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}, error::TuliproxError, - model::{AdmissionStrategy, ConnectFailureReason, FailureStage, GeoIpUnavailablePolicy}, + model::{ConnectFailureReason, FailureStage}, }; pub use try_option_bad_request; pub use try_option_forbidden; pub use try_result_bad_request; pub use try_result_not_found; pub use try_result_or_status; -pub use try_unwrap_body; -use crate::api::static_headers::CT_OCTET; +// Moved to `tuliprox-core` so crates outside `api` can build responses too. +pub use tuliprox_core::try_unwrap_body; +// Admission moved to `tuliprox-session`, where the types it decides over +// already live. Re-exported so api call sites keep their names. +pub(crate) use tuliprox_core::utils::request_headers::{get_headers_from_request, HeaderFilter}; +pub(crate) use tuliprox_session::{ + admission::{ + classify_playback_request, connection_priority_for_kind, resolve_admission_with_strategies, + resolve_playback_request_admission, EvictionReentryGuard, PlaybackRequestClass, PlaybackRequestFacts, + }, + stream_options::{get_stream_options, StreamOptions}, +}; pub fn get_server_time() -> String { chrono::offset::Local::now().with_timezone(&chrono::Local).format("%Y-%m-%d %H:%M:%S %Z").to_string() @@ -521,21 +347,11 @@ pub fn get_build_time() -> Option { .map(|datetime| datetime.format("%Y-%m-%d %H:%M:%S %Z").to_string()) } -#[derive(Clone, Copy, Debug)] -pub(crate) struct DisableResponseCompression; - -pub(crate) fn mark_response_as_uncompressed(response: &mut Response) { - response.extensions_mut().insert(DisableResponseCompression); -} - -#[cfg(test)] -pub(crate) fn should_compress_response(response: &Response) -> bool { - should_compress_response_extensions(response.extensions()) -} - -pub(crate) fn should_compress_response_extensions(extensions: &Extensions) -> bool { - extensions.get::().is_none() -} +// Response-compression opt-out moved to `tuliprox_core::utils`; re-exported so +// api call sites keep their names. +pub(crate) use tuliprox_core::utils::response_compression::{ + mark_response_as_uncompressed, should_compress_response_extensions, +}; #[derive(Clone, Copy, Debug, Default)] struct StreamMeteringConfig { @@ -620,126 +436,6 @@ pub fn get_user_target<'a>( get_user_target_by_credentials(username, password, api_req, app_state) } -/// Result of a policy-aware network access evaluation. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum NetworkAccessDecision { - /// Request is allowed (matched CIDR or country with `GeoIP` available). - Allowed, - /// Request is allowed because `GeoIP` is unavailable and policy is Allow. - AllowedGeoIpUnavailable, - /// Request is denied with a typed reason. - Denied(NetworkAccessDenyReason), -} - -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum NetworkAccessDenyReason { - NoCidrMatch, - NoCountryMatch, - GeoIpUnavailable, - CountryUnknown, - MalformedClientIp, -} - -impl NetworkAccessDenyReason { - pub const fn as_str(self) -> &'static str { - match self { - Self::NoCidrMatch => "no_cidr_match", - Self::NoCountryMatch => "no_country_match", - Self::GeoIpUnavailable => "geoip_unavailable", - Self::CountryUnknown => "country_unknown", - Self::MalformedClientIp => "malformed_client_ip", - } - } -} - -/// Logs a network access denial with structured context for operator debugging. -/// Do NOT log passwords or secrets. -#[allow(clippy::uninlined_format_args)] -pub fn log_network_access_denied(username: &str, client_ip: &str, reason: &str) { - let sanitized_username = sanitize_sensitive_info(username); - let sanitized_client_ip = sanitize_sensitive_info(client_ip); - warn!( - target: "network_access", - "Network access denied: user=\"{}\" client_ip=\"{}\" reason={}", - sanitized_username, - sanitized_client_ip, - reason - ); -} - -/// Logs a network access allowed-without-GeoIP event for explicit-risk observability. -#[allow(clippy::uninlined_format_args)] -pub fn log_network_access_allowed_geoip_unavailable(username: &str, client_ip: &str) { - warn!( - target: "network_access", - "Network access allowed because GeoIP is unavailable and reverse_proxy.geoip.unavailable_policy=allow; user=\"{}\" client_ip=\"{}\"", - sanitize_sensitive_info(username), - sanitize_sensitive_info(client_ip) - ); -} - -/// Evaluates network access with the configured GeoIP-unavailable policy. -/// Returns a structured decision for logging and HTTP response mapping. -pub fn evaluate_network_access( - user: &ProxyUserCredentials, - client_ip: &str, - geoip: &Arc>, - geoip_unavailable_policy: GeoIpUnavailablePolicy, -) -> NetworkAccessDecision { - let Some(access) = user.network_access.as_ref() else { - return NetworkAccessDecision::Allowed; - }; - if access.is_empty() { - return NetworkAccessDecision::Allowed; - } - - let Ok(parsed_ip) = client_ip.parse::() else { - return NetworkAccessDecision::Denied(NetworkAccessDenyReason::MalformedClientIp); - }; - - // CIDR check - for net in &access.allowed_networks { - if net.contains(&parsed_ip) { - return NetworkAccessDecision::Allowed; - } - } - - // No CIDR match — check country rules - if access.allowed_countries.is_empty() { - return NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch); - } - - // Country rules exist — check if GeoIP is available - let geoip_guard = geoip.load(); - let Some(geoip_db) = geoip_guard.as_ref() else { - // GeoIP unavailable — apply policy - return match geoip_unavailable_policy { - GeoIpUnavailablePolicy::Allow => NetworkAccessDecision::AllowedGeoIpUnavailable, - GeoIpUnavailablePolicy::Deny => NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable), - }; - }; - - // GeoIP is loaded — do country lookup - match geoip_db.lookup(client_ip) { - Some(country) => { - if access.allowed_countries.iter().any(|c| c == &country) { - NetworkAccessDecision::Allowed - } else { - NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCountryMatch) - } - } - None => NetworkAccessDecision::Denied(NetworkAccessDenyReason::CountryUnknown), - } -} - -pub struct StreamOptions { - pub stream_retry: bool, - pub buffer_enabled: bool, - pub buffer_size: usize, - pub buffer_max_bytes: usize, - pub pipe_provider_stream: bool, -} - struct StreamingAcquireOptions<'a> { force_provider: Option<&'a Arc>, allow_forced_provider_fallback: bool, @@ -750,13 +446,6 @@ struct StreamingAcquireOptions<'a> { accept_requested_stream_url: bool, } -pub(crate) fn connection_priority_for_kind(user: &ProxyUserCredentials, kind: crate::api::model::ConnectionKind) -> i8 { - match kind { - crate::api::model::ConnectionKind::Normal => user.priority, - crate::api::model::ConnectionKind::Soft => user.soft_priority, - } -} - pub struct ForceStreamRequestContext<'a> { pub req_headers: &'a HeaderMap, pub input: &'a Arc, @@ -765,351 +454,6 @@ pub struct ForceStreamRequestContext<'a> { pub(crate) content_representation: crate::api::model::ProviderContentRepresentationMode, } -/// Constructs a `StreamOptions` object based on the application's reverse proxy configuration. -/// -/// This function retrieves streaming-related settings from the `AppState`: -/// - `stream_retry`: whether retrying the stream is enabled, -/// - `buffer_enabled`: whether stream buffering is enabled, -/// - `buffer_size`: the size of the stream buffer. -/// -/// If the reverse proxy or stream settings are not defined, default values are used: -/// - retry: `true` -/// - buffering: `false` -/// - buffer size: `0` -/// -/// Additionally, it computes `pipe_provider_stream` as `!stream_retry && !buffer_enabled`. -/// This means direct provider piping is enabled only when retry is disabled and buffering is disabled. -/// -/// Returns a `StreamOptions` instance with the resolved configuration. -pub(in crate::api) fn get_stream_options(app_state: &Arc) -> StreamOptions { - let (stream_retry, buffer_enabled, buffer_size, buffer_max_bytes) = app_state - .app_config - .config - .load() - .reverse_proxy - .as_ref() - .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) - .map_or((true, false, 0, crate::api::model::MAX_BUFFER_BYTES), |stream| { - let (buffer_enabled, buffer_size, buffer_max_bytes) = stream.buffer.as_ref().map_or( - (false, 0, crate::api::model::MAX_BUFFER_BYTES), - |buffer| { - let max_bytes = usize::try_from(buffer.max_bytes_mb.saturating_mul(1024 * 1024)) - .unwrap_or(crate::api::model::MAX_BUFFER_BYTES); - (buffer.enabled, buffer.size, max_bytes) - }, - ); - (stream.retry, buffer_enabled, buffer_size, buffer_max_bytes) - }); - let pipe_provider_stream = !stream_retry && !buffer_enabled; - StreamOptions { stream_retry, buffer_enabled, buffer_size, buffer_max_bytes, pipe_provider_stream } -} - -/// Metadata capturing which grace strategy was chosen and the original connection kind, -/// used to reconstruct the remaining-strategies slice on user-grace failure. -#[derive(Debug, Clone)] -pub(crate) struct GraceResolutionContext { - /// Index of the grace strategy that was actually used. - pub(crate) strategy_index: usize, - /// Full effective strategy list for stable reconstruction of the remaining slice. - pub(crate) strategies: Vec, - /// The original `ConnectionKind` from the admission decision that led to this grace. - /// Preserved so that the remaining-strategy fallback can return the correct kind - /// (e.g., `Soft`) even when the grace itself hardcoded `Normal`. - // Stored so the original admission kind remains available when follow-up - // grace fallback reconstruction starts using it again. - #[allow(dead_code)] - pub(crate) kind: Option, -} - -/// Structured result of evaluating admission strategies. -#[derive(Debug)] -pub(crate) struct AdmissionStrategyResolution { - pub(crate) admission: crate::api::model::ConnectionAdmission, - pub(crate) grace_mode: Option, - /// Present only when the request was admitted via a user-grace strategy. - pub(crate) grace_context: Option, -} - -pub(in crate::api) fn get_effective_admission_strategies(app_state: &Arc) -> Vec { - let config = app_state.app_config.config.load(); - let stream_config = config.reverse_proxy.as_ref().and_then(|rp| rp.stream.as_ref()); - match stream_config { - Some(sc) if sc.admission_strategies.is_some() => sc.admission_strategies.clone().unwrap_or_default(), - Some(sc) if sc.grace_period_millis > 0 => { - vec![if sc.grace_period_hold_stream { - AdmissionStrategy::GraceHoldStream - } else { - AdmissionStrategy::GraceInstantStream - }] - } - _ => Vec::new(), - } -} - -/// Shared strategy-evaluation loop used by both the initial admission path -/// (`resolve_admission_with_strategies`) and the remaining-strategies path -/// (`evaluate_remaining_strategies_after_grace`). -/// -/// Returns `Some(resolution)` when a Grace or a successful Eviction+Retry is found. -/// Returns `None` when every strategy in `strategies` returns `NoMatch` — the caller -/// is then responsible for constructing the final exhausted result with the correct -/// `kind` (preserved from the original admission). -#[allow(clippy::too_many_arguments)] -async fn evaluate_admission_strategy_loop<'a, F>( - app_state: &'a Arc, - username: &'a str, - max_connections: u32, - soft_connections: u16, - client_ip: &'a str, - request_addr: &'a std::net::SocketAddr, - use_session_admission: bool, - session_token: Option<&'a str>, - activate_unbound_session: bool, - eviction_reentry_guard: EvictionReentryGuard<'a>, - strategies: &'a [shared::model::AdmissionStrategy], - base_idx: usize, - admission: crate::api::model::ConnectionAdmission, - _kind_for_exhausted: Option, - build_grace_ctx: F, -) -> Option -where - F: Fn(usize) -> GraceResolutionContext, -{ - use crate::api::model::{evaluate_strategy, AdmissionDecision, StrategyContext}; - use shared::model::UserConnectionPermission; - let mut candidates = app_state.active_users.get_eviction_candidates(username, client_ip).await; - let ctx = StrategyContext { username, client_ip }; - let mut idx = 0usize; - - for strategy in strategies { - match evaluate_strategy(*strategy, &ctx, &candidates) { - AdmissionDecision::NoMatch => {} - AdmissionDecision::Grace(mode) => { - if app_state.active_users.grant_grace(username).await { - // Return a FRESH admission with GracePeriod permission (not the admission - // parameter, which may have Exhausted permission). The kind is preserved from - // the original admission. - return Some(AdmissionStrategyResolution { - admission: crate::api::model::ConnectionAdmission { - permission: UserConnectionPermission::GracePeriod, - kind: admission.kind, - }, - grace_mode: Some(mode), - grace_context: Some(build_grace_ctx(base_idx + idx)), - }); - } - debug!("Grace grant rejected for user {username}, continuing with later strategies"); - } - AdmissionDecision::Evict(target) => { - if should_suppress_eviction_for_recent_request( - app_state, - username, - client_ip, - eviction_reentry_guard, - &target.addr, - ) - .await - { - debug!( - "Skipping eviction strategy {strategy:?} for recently evicted request of user {username} targeting {}", - target.addr - ); - continue; - } - debug!("Evicting connection {} for user {username}", target.addr); - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&target.addr, *request_addr, RECENT_EVICTION_REENTRY_TTL_SECS) - .await; - app_state.connection_manager.release_connection_as_kicked(&target.addr).await; - let retry_admission = get_admission_for_request( - app_state, - username, - max_connections, - soft_connections, - use_session_admission, - session_token, - activate_unbound_session, - ) - .await; - if retry_admission.permission == UserConnectionPermission::Allowed { - return Some(AdmissionStrategyResolution { - admission: retry_admission, - grace_mode: None, - grace_context: None, - }); - } - debug!("Admission still denied after eviction for user {username}, continuing with later strategies"); - candidates = app_state.active_users.get_eviction_candidates(username, client_ip).await; - } - } - idx += 1; - } - - // All strategies returned NoMatch — caller constructs the final exhausted result. - None -} - -#[allow(clippy::too_many_arguments)] -pub(in crate::api) async fn resolve_admission_with_strategies( - app_state: &Arc, - username: &str, - max_connections: u32, - soft_connections: u16, - client_ip: &str, - request_addr: &std::net::SocketAddr, - // This controls whether an existing logical playback session may reopen while the user is already at limit. - // It is intentionally independent from whether the session is socket-bound. - use_session_admission: bool, - session_token: Option<&str>, - activate_unbound_session: bool, - eviction_reentry_guard: EvictionReentryGuard<'_>, -) -> AdmissionStrategyResolution { - use shared::model::UserConnectionPermission; - - let admission = get_admission_for_request( - app_state, - username, - max_connections, - soft_connections, - use_session_admission, - session_token, - activate_unbound_session, - ) - .await; - - if admission.permission != UserConnectionPermission::Exhausted { - return AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None }; - } - - let strategies = get_effective_admission_strategies(app_state); - if strategies.is_empty() { - debug!("No admission strategies configured, denying request for user {username}"); - return AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None }; - } - - let build_grace_ctx = |global_idx: usize| GraceResolutionContext { - strategy_index: global_idx, - strategies: strategies.clone(), - kind: admission.kind, - }; - - let _admission_guard = app_state.active_users.acquire_user_admission(username).await; - - if let Some(resolution) = evaluate_admission_strategy_loop( - app_state, - username, - max_connections, - soft_connections, - client_ip, - request_addr, - use_session_admission, - session_token, - activate_unbound_session, - eviction_reentry_guard, - &strategies, - 0, - admission, - admission.kind, - build_grace_ctx, - ) - .await - { - return resolution; - } - - debug!("No admission strategy could admit user {username}"); - AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None } -} - -/// Evaluates only the strategies that come AFTER the already-used grace strategy. -/// This is called when a user-grace has failed and the system needs to determine -/// whether a remaining eviction strategy can free a slot. -/// -/// Rules: -/// - Only `grace_context.strategies[(strategy_index + 1)..]` are evaluated -/// - `NoMatch` -> continue to next strategy -/// - `Evict` -> kick target, retry admission -/// - `Grace` -> technically possible under current config (only one grace allowed), but handled -/// - `Deny` -> final exhausted -/// - Empty remaining slice -> final exhausted -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(in crate::api) async fn evaluate_remaining_strategies_after_grace( - app_state: &Arc, - username: &str, - max_connections: u32, - soft_connections: u16, - client_ip: &str, - request_addr: &std::net::SocketAddr, - use_session_admission: bool, - session_token: Option<&str>, - activate_unbound_session: bool, - eviction_reentry_guard: EvictionReentryGuard<'_>, - grace_context: &GraceResolutionContext, - original_kind: Option, -) -> AdmissionStrategyResolution { - use shared::model::UserConnectionPermission; - - let remaining = grace_context.strategy_index + 1; - let strategies = &grace_context.strategies; - if remaining >= strategies.len() { - debug!("No remaining strategies after grace for user {username}"); - return AdmissionStrategyResolution { - admission: crate::api::model::ConnectionAdmission { - permission: UserConnectionPermission::Exhausted, - kind: original_kind, - }, - grace_mode: None, - grace_context: None, - }; - } - - // admission.kind is used only inside build_grace_ctx for the Grace case's - // GraceResolutionContext.kind. Both paths (helper early-return and caller - // exhausted construction) use original_kind, so this is safe. - let admission = - crate::api::model::ConnectionAdmission { permission: UserConnectionPermission::Exhausted, kind: original_kind }; - let build_grace_ctx = |global_idx: usize| GraceResolutionContext { - strategy_index: global_idx, - strategies: strategies.clone(), - kind: original_kind, - }; - - let _admission_guard = app_state.active_users.acquire_user_admission(username).await; - - if let Some(resolution) = evaluate_admission_strategy_loop( - app_state, - username, - max_connections, - soft_connections, - client_ip, - request_addr, - use_session_admission, - session_token, - activate_unbound_session, - eviction_reentry_guard, - &strategies[remaining..], - remaining, - admission, - original_kind, - build_grace_ctx, - ) - .await - { - return resolution; - } - - debug!("No remaining strategy could admit user {username}"); - AdmissionStrategyResolution { - admission: crate::api::model::ConnectionAdmission { - permission: UserConnectionPermission::Exhausted, - kind: original_kind, - }, - grace_mode: None, - grace_context: None, - } -} - struct SessionActivationRequest<'a> { fingerprint: &'a Fingerprint, input: &'a ConfigInput, @@ -1127,7 +471,7 @@ struct SessionActivationRequest<'a> { struct PlaybackActivationResult { admission: crate::api::model::ConnectionAdmission, grace_mode: Option, - grace_context: Option, + grace_context: Option, placeholder_transition_version: Option, } @@ -1304,7 +648,7 @@ async fn activate_session_before_stream_open( ); let result = resolve_admission_with_strategies( - app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -1899,11 +1243,7 @@ fn should_defer_provider_open_for_grace_hold( !(!item_type.is_live() && item_type.requires_provider_affinity() && is_reopen) } -fn should_refresh_stalker_playback( - input_type: InputType, - request_url_valid: bool, - status: Option, -) -> bool { +fn should_refresh_stalker_playback(input_type: InputType, request_url_valid: bool, status: Option) -> bool { input_type.is_stalker() && (!request_url_valid || status.is_some_and(|status| status.is_client_error())) } @@ -1959,20 +1299,14 @@ pub(crate) async fn resolve_initial_stalker_playback_url( if !needs_initial_stalker_resolution(input.input_type, stream_url) { return Ok(Arc::clone(stream_url)); } - re_resolve_stalker_url_singleflight( - app_state, - input, - provider_id, - stalker_stream_kind(cluster, item_type), - false, - ) - .await? - .ok_or_else(|| { - TuliproxError::RepositoryStalker(format!( - "Stalker playback URL could not be resolved for input '{}' and provider id {provider_id}", - input.name - )) - }) + re_resolve_stalker_url_singleflight(app_state, input, provider_id, stalker_stream_kind(cluster, item_type), false) + .await? + .ok_or_else(|| { + TuliproxError::RepositoryStalker(format!( + "Stalker playback URL could not be resolved for input '{}' and provider id {provider_id}", + input.name + )) + }) } #[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::fn_params_excessive_bools)] @@ -2000,7 +1334,7 @@ async fn create_stream_response_details( session_headers: Option<&HashMap>, accept_requested_stream_url: bool, grace_hold_override: Option, - grace_resolution_context: Option, + grace_resolution_context: Option, ) -> Result { let mut streaming_strategy = resolve_streaming_strategy( app_state, @@ -2104,139 +1438,144 @@ async fn create_stream_response_details( }; let (stream, stream_info, provider_session_headers, reconnect_flag) = if defer_provider_stream_until_grace_check { - debug_if_enabled!( - "Deferring provider stream open until grace check completes for {}", - sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) - ); - (None, None, HashMap::new(), None) - } else if is_media_server_stream_ref_url(request_url.as_ref()) { - match open_media_server_stream_for_input(app_state, input, request_url.as_ref(), req_headers).await { - Ok((stream, stream_info)) => (Some(stream), stream_info, HashMap::new(), None), - Err(err) => { - error!("Can't open media-server stream: {err}"); - (None, None, HashMap::new(), None) - } - } - } else { - let parsed_url = Url::parse(&request_url); - let request_url_valid = parsed_url.is_ok(); - let ((mut stream, mut stream_info, mut provider_session_headers), mut reconnect_flag) = - if let Ok(url) = parsed_url { - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let disabled_headers = app_state.get_disabled_headers(); - let mut provider_stream_factory_options = - ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { - addr: fingerprint.addr, - item_type, - share_stream, - stream_options, - stream_url: &url, - req_headers, - input_headers: streaming_strategy.input_headers.as_ref(), - session_headers, - disabled_headers: disabled_headers.as_ref(), - default_user_agent: default_user_agent.as_deref(), - username: Some(username), - client_ip: Some(&fingerprint.client_ip), - stream_channel: Some(stream_channel), - connect_failure_stage: Some(FailureStage::ProviderOpen), - content_representation, - }); - - let provider_config = input.get_resolve_provider(url.as_ref()); - provider_stream_factory_options.set_provider(provider_config); - if input.input_type.is_stalker() { - provider_stream_factory_options.require_public_destination(); - } - - let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone(); - let provider_stream = match create_provider_stream( - app_state, - &app_state.http_client.load(), - provider_stream_factory_options, - ) - .await + debug_if_enabled!( + "Deferring provider stream open until grace check completes for {}", + sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) + ); + (None, None, HashMap::new(), None) + } else if is_media_server_stream_ref_url(request_url.as_ref()) { + match open_media_server_stream_for_input(app_state, input, request_url.as_ref(), req_headers).await { - None => (None, None, HashMap::new()), - Some(response) => - (Some(response.stream), response.info, response.provider_session_headers), - }; - (provider_stream, Some(reconnect_flag)) + Ok((stream, stream_info)) => (Some(stream), stream_info, HashMap::new(), None), + Err(err) => { + error!("Can't open media-server stream: {err}"); + (None, None, HashMap::new(), None) + } + } } else { - ((None, None, HashMap::new()), None) - }; - let should_refresh_stalker = should_refresh_stalker_playback( - input.input_type, - request_url_valid, - stream_info.as_ref().map(|(_, status, _, _)| *status), - ); - if should_refresh_stalker { - let force_stalker_refresh = stream_info - .as_ref() - .is_some_and(|(_, status, _, _)| status.is_client_error()); - let kind = stalker_stream_kind(stream_channel.cluster, item_type); - let resolve_result = re_resolve_stalker_url_singleflight( - app_state, - input, - stream_channel.provider_id, - kind, - force_stalker_refresh, - ) - .await; - match resolve_result { - Ok(Some(refreshed_url)) => { - if let Ok(url) = Url::parse(&refreshed_url) { - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let disabled_headers = app_state.get_disabled_headers(); - let mut options = ProviderStreamFactoryOptions::new( - &crate::api::model::ProviderStreamFactoryParams { - addr: fingerprint.addr, - item_type, - share_stream, - stream_options, - stream_url: &url, - req_headers, - input_headers: streaming_strategy.input_headers.as_ref(), - session_headers, - disabled_headers: disabled_headers.as_ref(), - default_user_agent: default_user_agent.as_deref(), - username: Some(username), - client_ip: Some(&fingerprint.client_ip), - stream_channel: Some(stream_channel), - connect_failure_stage: Some(FailureStage::ProviderOpen), - content_representation, - }, - ); - options.set_provider(input.get_resolve_provider(url.as_ref())); - options.require_public_destination(); - let retry_reconnect_flag = options.get_reconnect_flag_clone(); - let retried = create_provider_stream( - app_state, - &app_state.http_client.load(), - options, - ) - .await; - if let Some(response) = retried { - stream = Some(response.stream); - stream_info = response.info; - provider_session_headers = response.provider_session_headers; - reconnect_flag = Some(retry_reconnect_flag); - request_url = refreshed_url; - } else { - // Keep the original stream/stream_info: the upstream response - // might still be serveable, and its status is needed for reporting. - debug!("Stalker re-resolve retry could not open a stream, keeping original provider response"); + let parsed_url = Url::parse(&request_url); + let request_url_valid = parsed_url.is_ok(); + let ((mut stream, mut stream_info, mut provider_session_headers), mut reconnect_flag) = + if let Ok(url) = parsed_url { + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let disabled_headers = app_state.get_disabled_headers(); + let mut provider_stream_factory_options = + ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { + addr: fingerprint.addr, + item_type, + share_stream, + stream_options, + stream_url: &url, + req_headers, + input_headers: streaming_strategy.input_headers.as_ref(), + session_headers, + disabled_headers: disabled_headers.as_ref(), + default_user_agent: default_user_agent.as_deref(), + username: Some(username), + client_ip: Some(&fingerprint.client_ip), + stream_channel: Some(stream_channel), + connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation, + }); + + let provider_config = input.get_resolve_provider(url.as_ref()); + provider_stream_factory_options.set_provider(provider_config); + if input.input_type.is_stalker() { + provider_stream_factory_options.require_public_destination(); + } + + let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone(); + let provider_stream = match create_provider_stream( + &app_state.provider_stream_ctx(), + &app_state.http_client.load(), + provider_stream_factory_options, + ) + .await + { + None => (None, None, HashMap::new()), + Some(response) => { + (Some(response.stream), response.info, response.provider_session_headers) + } + }; + (provider_stream, Some(reconnect_flag)) + } else { + ((None, None, HashMap::new()), None) + }; + let should_refresh_stalker = should_refresh_stalker_playback( + input.input_type, + request_url_valid, + stream_info.as_ref().map(|(_, status, _, _)| *status), + ); + if should_refresh_stalker { + let force_stalker_refresh = + stream_info.as_ref().is_some_and(|(_, status, _, _)| status.is_client_error()); + let kind = stalker_stream_kind(stream_channel.cluster, item_type); + let resolve_result = re_resolve_stalker_url_singleflight( + app_state, + input, + stream_channel.provider_id, + kind, + force_stalker_refresh, + ) + .await; + match resolve_result { + Ok(Some(refreshed_url)) => { + if let Ok(url) = Url::parse(&refreshed_url) { + let default_user_agent = + app_state.app_config.config.load().default_user_agent.clone(); + let disabled_headers = app_state.get_disabled_headers(); + let mut options = ProviderStreamFactoryOptions::new( + &crate::api::model::ProviderStreamFactoryParams { + addr: fingerprint.addr, + item_type, + share_stream, + stream_options, + stream_url: &url, + req_headers, + input_headers: streaming_strategy.input_headers.as_ref(), + session_headers, + disabled_headers: disabled_headers.as_ref(), + default_user_agent: default_user_agent.as_deref(), + username: Some(username), + client_ip: Some(&fingerprint.client_ip), + stream_channel: Some(stream_channel), + connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation, + }, + ); + options.set_provider(input.get_resolve_provider(url.as_ref())); + options.require_public_destination(); + let retry_reconnect_flag = options.get_reconnect_flag_clone(); + let retried = create_provider_stream( + &app_state.provider_stream_ctx(), + &app_state.http_client.load(), + options, + ) + .await; + if let Some(response) = retried { + stream = Some(response.stream); + stream_info = response.info; + provider_session_headers = response.provider_session_headers; + reconnect_flag = Some(retry_reconnect_flag); + request_url = refreshed_url; + } else { + // Keep the original stream/stream_info: the upstream response + // might still be serveable, and its status is needed for reporting. + debug!("Stalker re-resolve retry could not open a stream, keeping original provider response"); + } } } - } - Ok(None) => {} - Err(err) => { - warn!("Failed to refresh Stalker playback URL: {}", sanitize_sensitive_info(&err.to_string())); + Ok(None) => {} + Err(err) => { + warn!( + "Failed to refresh Stalker playback URL: {}", + sanitize_sensitive_info(&err.to_string()) + ); + } } } - } - (stream, stream_info, provider_session_headers, reconnect_flag) - }; + (stream, stream_info, provider_session_headers, reconnect_flag) + }; if log_enabled!(log::Level::Debug) { if let Some((headers, status_code, response_url, _custom_video_type)) = stream_info.as_ref() { @@ -2462,7 +1801,7 @@ async fn open_media_server_stream_for_input( let response = match input.input_type { InputType::Plex => { - let client = PlexCatalogClient::from_input(input, http_client)?; + let client = input.plex_catalog_client(http_client)?; open_media_server_proxy_stream_response(&client, &stream_ref, range).await? } InputType::Emby | InputType::Jellyfin => { @@ -2540,7 +1879,7 @@ pub async fn force_provider_stream_response( ) -> impl IntoResponse + Send { let _transition_guard = app_state.active_users.acquire_playback_transition(&ctx.user.username, &user_session.token).await; - let stream_options = get_stream_options(app_state); + let stream_options = get_stream_options(&app_state.app_config); let share_stream = false; let connection_permission = UserConnectionPermission::Allowed; let item_type = stream_channel.item_type; @@ -2824,14 +2163,14 @@ pub(crate) async fn stream_response( failure_stage: FailureStage::Admission, }); return create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) .into_response(); } - let stream_options = get_stream_options(app_state); + let stream_options = get_stream_options(&app_state.app_config); let session_state = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; let mut stream_details = match create_stream_response_details( app_state, @@ -2905,13 +2244,8 @@ pub(crate) async fn stream_response( } Err(err) => { error!("Failed to inspect catch-up payload: {err}"); - cleanup_failed_detected_catchup_hls( - app_state, - &mut stream_details, - &user.username, - session_token, - ) - .await; + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token) + .await; return StatusCode::BAD_GATEWAY.into_response(); } } @@ -3004,7 +2338,12 @@ pub(crate) async fn stream_response( // Shared Stream response let shared_headers = provider_response.as_ref().map_or_else(Vec::new, |(h, _, _, _)| h.clone()); if let Some((broadcast_stream, _shared_provider)) = SharedStreamManager::register_shared_stream( - app_state, + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, stream_url, stream, &fingerprint.addr, @@ -3155,9 +2494,7 @@ struct DetectedCatchupHlsResponseParams<'a> { fallback_stream_url: &'a str, } -async fn detected_catchup_hls_response( - params: DetectedCatchupHlsResponseParams<'_>, -) -> axum::response::Response { +async fn detected_catchup_hls_response(params: DetectedCatchupHlsResponseParams<'_>) -> axum::response::Response { let DetectedCatchupHlsResponseParams { app_state, mut stream_details, @@ -3234,26 +2571,14 @@ async fn detected_catchup_hls_response( } app_state .active_provider - .refresh_provider_reservation( - &provider, - &created_session_token, - get_catchup_session_ttl_secs(app_state), - ) + .refresh_provider_reservation(&provider, &created_session_token, get_catchup_session_ttl_secs(app_state)) .await; app_state.connection_manager.release_provider_handle(stream_details.provider_handle.take()).await; app_state .active_users - .release_unbound_session_reservation( - &user.username, - &created_session_token, - None, - false, - ) - .await; - app_state - .active_users - .clear_unbound_session_addr(&user.username, &created_session_token, &fingerprint.addr) + .release_unbound_session_reservation(&user.username, &created_session_token, None, false) .await; + app_state.active_users.clear_unbound_session_addr(&user.username, &created_session_token, &fingerprint.addr).await; catchup_hls_manifest_response(rewritten) } @@ -3279,10 +2604,7 @@ async fn cleanup_failed_detected_catchup_hls( app_state.active_provider.clear_provider_reservation(session_token).await; } -async fn probe_catchup_payload( - stream: BoxedProviderStream, - deadline: Duration, -) -> Result { +async fn probe_catchup_payload(stream: BoxedProviderStream, deadline: Duration) -> Result { tokio::time::timeout(deadline, probe_catchup_payload_inner(stream)) .await .map_err(|_| StreamError::Stream("catch-up payload probe timed out".to_string()))? @@ -3449,7 +2771,12 @@ async fn try_shared_stream_response_if_any( req_headers: &HeaderMap, ) -> Option { if let Some((stream, provider)) = SharedStreamManager::subscribe_shared_stream( - app_state, + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, stream_url, &fingerprint.addr, connection_priority_for_kind(user, connection_kind), @@ -3575,7 +2902,7 @@ pub(crate) async fn local_stream_response( failure_stage: FailureStage::Admission, }); return create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) @@ -3729,7 +3056,7 @@ pub(crate) async fn local_stream_response( ) .await; return create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) @@ -3830,18 +3157,6 @@ pub fn is_hls_stream_share_enabled(target: &ConfigTarget) -> bool { target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_hls_enabled) } -pub type HeaderFilter = Option bool + Send>>; -pub fn get_headers_from_request(req_headers: &HeaderMap, filter: &HeaderFilter) -> HashMap> { - req_headers - .iter() - .filter(|(k, _)| match &filter { - None => true, - Some(predicate) => predicate(k.as_str()), - }) - .map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())) - .collect() -} - fn get_add_cache_content( res_url: &str, mime_type: Option, @@ -3883,7 +3198,7 @@ async fn same_windows_file_identity(opened_file: &tokio::fs::File, canonical_pat #[cfg(windows)] fn windows_file_identity(file: &tokio::fs::File) -> std::io::Result<(u32, u32, u32)> { use std::os::windows::io::AsRawHandle; - use winapi::um::fileapi::{GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; + use windows_sys::Win32::Storage::FileSystem::{GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; let mut info = unsafe { std::mem::zeroed::() }; // SAFETY: `file.as_raw_handle()` is a live file handle for the duration of @@ -4071,7 +3386,7 @@ async fn open_media_server_image_resource( let response = match input.input_type { InputType::Plex => { - let client = PlexCatalogClient::from_input(&input, http_client)?; + let client = input.plex_catalog_client(http_client)?; open_media_server_proxy_image_response(&client, &image_ref).await? } InputType::Emby | InputType::Jellyfin => { @@ -4200,11 +3515,7 @@ pub fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> bool false } -pub fn is_seekable_media_request( - cluster: XtreamCluster, - req_headers: &HeaderMap, - extension: Option<&str>, -) -> bool { +pub fn is_seekable_media_request(cluster: XtreamCluster, req_headers: &HeaderMap, extension: Option<&str>) -> bool { !extension.is_some_and(|ext| ext.eq_ignore_ascii_case(HLS_EXT)) && is_seek_request(cluster, req_headers) } @@ -4250,10 +3561,7 @@ where stream_json_array_stream(data) } -pub fn stream_json_or_bin_response_try_stream( - accept: Option<&str>, - data: S, -) -> axum::response::Response +pub fn stream_json_or_bin_response_try_stream(accept: Option<&str>, data: S) -> axum::response::Response where P: serde::Serialize + Send + 'static, S: Stream> + Send + Unpin + 'static, @@ -4648,7 +3956,8 @@ mod tests { MediaToolCapabilities, NetworkAccess, ProcessTargets, ProxyUserCredentials, SourcesConfig, StreamHistoryConfig, }, - utils::{FileLockManager, GeoIp}, + repository::GeoIp, + utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::http::{HeaderMap, Response, StatusCode}; @@ -4659,8 +3968,9 @@ mod tests { defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}, foundation::Filter, model::{ - ClusterFlags, ConfigPaths, ConfigProviderDto, ConfigTargetOptions, InputFetchMethod, InputType, - PlaylistItemType, ProcessingOrder, ProviderUrlSelectionPolicy, ProxyType, StreamChannel, XtreamCluster, + AdmissionStrategy, ClusterFlags, ConfigPaths, ConfigProviderDto, ConfigTargetOptions, + GeoIpUnavailablePolicy, InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder, + ProviderUrlSelectionPolicy, ProxyType, StreamChannel, XtreamCluster, }, utils::Internable, }; @@ -4670,15 +3980,19 @@ mod tests { net::TcpListener, sync::{mpsc, RwLock}, }; + use tuliprox_core::utils::response_compression::should_compress_response; + use tuliprox_session::{ + admission::{ + evaluate_remaining_strategies_after_grace, get_effective_admission_strategies, + RECENT_EVICTION_REENTRY_TTL_SECS, + }, + GraceResolutionContext, + }; #[test] fn stalker_playback_refreshes_invalid_or_rejected_urls() { assert!(should_refresh_stalker_playback(InputType::Stalker, false, None)); - assert!(should_refresh_stalker_playback( - InputType::Stalker, - true, - Some(StatusCode::UNAUTHORIZED) - )); + assert!(should_refresh_stalker_playback(InputType::Stalker, true, Some(StatusCode::UNAUTHORIZED))); assert!(!should_refresh_stalker_playback(InputType::Stalker, true, Some(StatusCode::OK))); assert!(!should_refresh_stalker_playback(InputType::Xtream, false, None)); } @@ -4688,10 +4002,7 @@ mod tests { assert!(needs_initial_stalker_resolution(InputType::Stalker, "")); assert!(!needs_initial_stalker_resolution(InputType::Stalker, "https://stream.example/live.ts")); assert!(!needs_initial_stalker_resolution(InputType::Xtream, "")); - assert_eq!( - stalker_stream_kind(XtreamCluster::Live, PlaylistItemType::Catchup), - StalkerStreamKind::Archive - ); + assert_eq!(stalker_stream_kind(XtreamCluster::Live, PlaylistItemType::Catchup), StalkerStreamKind::Archive); } fn test_runtime_provider(url: &str, username: &str, password: &str) -> Arc { @@ -4771,11 +4082,8 @@ mod tests { #[tokio::test] async fn catchup_payload_probe_rejects_oversized_manifest() { let oversized = vec![b'x'; MAX_HLS_MANIFEST_BYTES]; - let source = stream::iter([ - Ok::<_, StreamError>(Bytes::from_static(b"#EXTM3U")), - Ok(Bytes::from(oversized)), - ]) - .boxed(); + let source = + stream::iter([Ok::<_, StreamError>(Bytes::from_static(b"#EXTM3U")), Ok(Bytes::from(oversized))]).boxed(); let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; @@ -4856,16 +4164,8 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert("range", HeaderValue::from_static("bytes=0-")); - assert!(!is_seekable_media_request( - XtreamCluster::Video, - &headers, - Some(HLS_EXT) - )); - assert!(is_seekable_media_request( - XtreamCluster::Video, - &headers, - Some(".ts") - )); + assert!(!is_seekable_media_request(XtreamCluster::Video, &headers, Some(HLS_EXT))); + assert!(is_seekable_media_request(XtreamCluster::Video, &headers, Some(".ts"))); } #[test] @@ -5676,17 +4976,14 @@ mod tests { #[tokio::test] async fn resolve_streaming_strategy_accepts_stalker_portal_url() { let app_config = create_test_provider_app_config(); - let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { - unreachable!() - }; + let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; let mut stalker_input = (*configured_input).clone(); stalker_input.input_type = InputType::Stalker; stalker_input.username = None; stalker_input.password = None; - app_config.sources.store(Arc::new(SourcesConfig { - inputs: vec![Arc::new(stalker_input)], - ..SourcesConfig::default() - })); + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(stalker_input)], ..SourcesConfig::default() })); let app_state = create_test_app_state_for_config(Arc::new(app_config)); let input_name = "provider_1".intern(); let input = app_state @@ -6643,7 +5940,7 @@ mod tests { user.max_connections = 1; let (admission, grace_mode, request_class) = resolve_playback_request_admission( - &app_state, + &app_state.admission_ctx(), &user, &fingerprint, PlaylistItemType::LiveHls, @@ -6709,7 +6006,7 @@ mod tests { assert!(before.is_some(), "session should exist before terminate"); let (admission, grace_mode, request_class) = resolve_playback_request_admission( - &app_state, + &app_state.admission_ctx(), &user, &fingerprint, PlaylistItemType::LiveHls, @@ -6989,7 +6286,7 @@ mod tests { }); assert_eq!( - get_effective_admission_strategies(&app_state), + get_effective_admission_strategies(&app_state.admission_ctx()), vec![shared::model::AdmissionStrategy::GraceHoldStream] ); } @@ -7012,7 +6309,7 @@ mod tests { admission_strategies: Some(vec![]), }); - assert!(get_effective_admission_strategies(&app_state).is_empty()); + assert!(get_effective_admission_strategies(&app_state.admission_ctx()).is_empty()); } #[tokio::test] @@ -7092,7 +6389,7 @@ mod tests { // Now the new request finds the slot exhausted and the grace strategy kicks in. let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -7185,7 +6482,7 @@ mod tests { .expect("stream should be created"); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "remaining-evict", 1, 0, @@ -7289,7 +6586,7 @@ mod tests { .expect("stream should be created"); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "remaining-skip-no-match", 1, 0, @@ -7338,7 +6635,7 @@ mod tests { let fingerprint = create_test_fingerprint(addr); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "no-remaining-strategies", 1, 0, @@ -7392,7 +6689,7 @@ mod tests { let fingerprint = create_test_fingerprint(addr); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "soft-kind-user", 1, 0, @@ -7497,7 +6794,7 @@ mod tests { .expect("stream should be created"); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "remaining-no-retry", 1, 0, @@ -7552,7 +6849,7 @@ mod tests { let fingerprint = create_test_fingerprint(addr); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "kind-mismatch-empty", 1, 0, @@ -7655,7 +6952,7 @@ mod tests { .expect("stream should be created"); let result = evaluate_remaining_strategies_after_grace( - &app_state, + &app_state.admission_ctx(), "kind-mismatch-grace", 1, 0, @@ -7784,7 +7081,7 @@ mod tests { .await; let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), "fallthrough", 1, 1, @@ -7847,7 +7144,7 @@ mod tests { .await; let session_based = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -7862,7 +7159,7 @@ mod tests { assert_eq!(session_based.admission.permission, UserConnectionPermission::Allowed); let connection_based = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -7951,7 +7248,7 @@ mod tests { app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), "loop-user", 1, 0, @@ -8080,7 +7377,7 @@ mod tests { app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), "loop-user-2", 1, 0, @@ -8204,7 +7501,7 @@ mod tests { app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), "loop-user-4", 1, 0, @@ -8298,7 +7595,7 @@ mod tests { app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), "loop-user-3", 1, 1, @@ -8479,7 +7776,12 @@ mod tests { .expect("owner allocation should exist"); let shared_stream = stream::pending::>(); let registered = SharedStreamManager::register_shared_stream( - app_state.as_ref(), + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, stream_url, shared_stream, &owner_addr, @@ -8523,8 +7825,10 @@ mod tests { .await .expect("normal stream should register"); - let admission = - app_state.get_connection_admission(&user.username, user.max_connections, user.soft_connections).await; + let admission = app_state + .active_users + .connection_admission(&user.username, user.max_connections, user.soft_connections) + .await; assert_eq!(admission.permission, UserConnectionPermission::Allowed); assert_eq!(admission.kind, Some(crate::api::model::ConnectionKind::Soft)); @@ -9033,7 +8337,7 @@ mod tests { .await; let first_admission = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -9046,7 +8350,7 @@ mod tests { ) .await; let second_admission = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -9203,7 +8507,7 @@ mod tests { let mut close_rx = app_state.connection_manager.get_close_connection_channel(); let result = resolve_admission_with_strategies( - &app_state, + &app_state.admission_ctx(), &user.username, user.max_connections, user.soft_connections, @@ -9525,7 +8829,7 @@ mod tests { ); let (ts_admission, ts_grace_mode, request_class) = resolve_playback_request_admission( - &app_state, + &app_state.admission_ctx(), &user, &ts_fingerprint, PlaylistItemType::Live, @@ -9592,10 +8896,7 @@ mod tests { assert_eq!(app_state.active_users.active_users_and_connections().await, (1, 1)); let active_streams = app_state.active_users.active_streams().await; assert_eq!(active_streams.len(), 1); - assert_eq!( - active_streams.first().and_then(|stream| stream.session_token.as_deref()), - Some(ts_token.as_str()) - ); + assert_eq!(active_streams.first().and_then(|stream| stream.session_token.as_deref()), Some(ts_token.as_str())); assert!( app_state .active_users @@ -9671,7 +8972,7 @@ mod tests { let mut details = create_stream_response_details( &app_state, - &get_stream_options(&app_state), + &get_stream_options(&app_state.app_config), stream_url, "deferred-user", &fingerprint, diff --git a/backend/app/src/api/auth_middleware.rs b/backend/app/src/api/auth_middleware.rs new file mode 100644 index 000000000..bdfe9ab05 --- /dev/null +++ b/backend/app/src/api/auth_middleware.rs @@ -0,0 +1,129 @@ +//! Axum authentication middleware. +//! +//! These handlers take `State>` and are HTTP concerns, not +//! authentication primitives. They lived in `auth`, which made that module - and +//! everything depending on it - reach up into `api`. The JWT creation and +//! verification they call still lives in `auth::authenticator`. + +use crate::{ + api::{api_utils::get_username_from_auth_header, model::AppState}, + auth::{validate_token_claims, verify_token, verify_token_admin, verify_token_api_user, AuthBearer, AuthError}, +}; +use log::warn; +use shared::model::permission::{permission_to_name, Permission}; +use std::sync::Arc; + +fn validate_request( + app_state: &Arc, + token: &str, + verify_fn: fn(&str, &[u8]) -> bool, +) -> Result<(), AuthError> { + let config = app_state.app_config.config.load(); + let Some(web_auth_config) = config.web_ui.as_ref().and_then(|c| c.auth.as_ref()) else { + return Err(AuthError::InvalidToken); + }; + let secret_key = web_auth_config.secret.as_ref(); + let token_data = verify_token(token, secret_key).ok_or(AuthError::InvalidToken)?; + validate_token_claims(&token_data.claims)?; + if !verify_fn(token, secret_key) { + return Err(AuthError::Forbidden); + } + Ok(()) +} + +/// Build a stable, recognizable rejection response. Refresh-required +/// cases carry the `X-Token-Refresh: required` header so the +/// frontend can branch on it. `Forbidden` (a successful authentication +/// that nonetheless cannot perform the action) maps to 403 so it does +/// not collapse into the "you're not authenticated" 401 path. +fn rejection_for(err: AuthError) -> axum::response::Response { + use axum::http::StatusCode; + let status = match &err { + AuthError::Forbidden => StatusCode::FORBIDDEN, + _ => StatusCode::UNAUTHORIZED, + }; + let mut builder = axum::http::Response::builder().status(status); + if err.is_token_refresh_required() { + builder = builder.header("X-Token-Refresh", "required"); + } + builder + .body(axum::body::Body::from(err.to_string())) + .unwrap_or_else(|_| axum::http::Response::new(axum::body::Body::empty())) +} + +pub async fn validator_admin( + axum::extract::State(app_state): axum::extract::State>, + AuthBearer(token): AuthBearer, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + match validate_request(&app_state, &token, verify_token_admin) { + Ok(()) => next.run(request).await, + Err(err) => rejection_for(err), + } +} + +pub async fn validator_api_user( + axum::extract::State(app_state): axum::extract::State>, + AuthBearer(token): AuthBearer, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + if let Some(username) = get_username_from_auth_header(&token, &app_state) { + if let Some(user) = app_state.app_config.get_user_credentials(&username) { + if !user.ui_enabled { + return axum::http::Response::builder() + .status(axum::http::StatusCode::FORBIDDEN) + .body(axum::body::Body::from("principal does not have the required role".to_string())) + .unwrap_or_else(|_| axum::http::Response::new(axum::body::Body::empty())); + } + } + } + match validate_request(&app_state, &token, verify_token_api_user) { + Ok(()) => next.run(request).await, + Err(err) => rejection_for(err), + } +} + +pub async fn require_permission_inner( + permission: Permission, + axum::extract::State(app_state): axum::extract::State>, + AuthBearer(token): AuthBearer, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let config = app_state.app_config.config.load(); + let Some(web_auth_config) = config.web_ui.as_ref().and_then(|c| c.auth.as_ref()) else { + return rejection_for(AuthError::InvalidToken); + }; + + let Some(token_data) = verify_token(&token, web_auth_config.secret.as_bytes()) else { + return rejection_for(AuthError::InvalidToken); + }; + if let Err(err) = validate_token_claims(&token_data.claims) { + return rejection_for(err); + } + + if !token_data.claims.permissions.contains(permission) { + let denied_permission = permission_to_name(permission).unwrap_or("unknown"); + warn!("User '{}' denied permission '{denied_permission}'", token_data.claims.username); + return rejection_for(AuthError::Forbidden); + } + + next.run(request).await +} + +/// Builds an Axum layer that enforces one permission. +/// +/// Defined here rather than in `auth` because it expands to a call into this +/// module; keeping it there meant `auth` named `api`. +#[macro_export] +macro_rules! permission_layer { + ($app_state:expr, $permission:expr ) => {{ + let app_state = ::std::sync::Arc::clone($app_state); + ::axum::middleware::from_fn_with_state(app_state, move |state, auth, request, next| { + $crate::api::auth_middleware::require_permission_inner($permission, state, auth, request, next) + }) + }}; +} +pub use permission_layer; diff --git a/backend/src/api/config_file.rs b/backend/app/src/api/config_file.rs similarity index 91% rename from backend/src/api/config_file.rs rename to backend/app/src/api/config_file.rs index 8e3ee698a..599fe73c1 100644 --- a/backend/src/api/config_file.rs +++ b/backend/app/src/api/config_file.rs @@ -1,11 +1,12 @@ use crate::{ api::model::{update_app_state_config, update_app_state_sources, AppState, EventMessage}, + config_loader::{ + prepare_sources_batch, read_sources_file, read_sources_file_from_path_with_templates, read_templates, + resolve_template_and_mapping_paths, + }, model::{Config, Mappings, ProcessTargets, SourcesConfig}, utils, - utils::{ - prepare_sources_batch, read_mappings_file_unprepared, read_mappings_file_with_templates, - read_sources_file, read_sources_file_from_path_with_templates, read_templates, - }, + utils::{read_mappings_file_unprepared, read_mappings_file_with_templates}, }; use log::{debug, error, info, warn}; use shared::{ @@ -16,7 +17,6 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use crate::utils::resolve_template_and_mapping_paths; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ConfigFile { @@ -81,7 +81,9 @@ fn refresh_forced_targets(current: Arc, sources: &SourcesConfig) match sources.validate_targets(Some(&still_valid)) { Ok(refreshed) => Arc::new(refreshed), Err(err) => { - warn!("Failed to re-validate remaining CLI-forced targets {still_valid:?}: {err}. Keeping previous IDs."); + warn!( + "Failed to re-validate remaining CLI-forced targets {still_valid:?}: {err}. Keeping previous IDs." + ); current } } @@ -112,7 +114,8 @@ impl ConfigFile { read_sources_file(paths.sources_file_path.as_str(), false, false, None, None).await?.templates; // Use robust fallbacks for mapping and template paths - let (effective_template_path, effective_mapping_path) = resolve_template_and_mapping_paths(paths, config.template_path.as_deref(), config.mapping_path.as_deref()); + let (effective_template_path, effective_mapping_path) = + resolve_template_and_mapping_paths(paths, config.template_path.as_deref(), config.mapping_path.as_deref()); let mapping_inline_templates = read_mappings_file_unprepared(effective_mapping_path.as_ref(), false)? .map(|(_, mapping)| mapping) @@ -236,7 +239,7 @@ impl ConfigFile { // ----------------------------------------------------------------- async fn load_api_proxy(app_state: &Arc) -> Result<(), TuliproxError> { - match utils::read_api_proxy_config(&app_state.app_config, true).await { + match crate::config_loader::read_api_proxy_config(&app_state.app_config, true).await { Ok(Some(api_proxy)) => { app_state.app_config.set_api_proxy(api_proxy)?; let paths = app_state.app_config.paths.load(); @@ -263,19 +266,24 @@ impl ConfigFile { let config_file = paths.config_file_path.clone(); // ── Parse ──────────────────────────────────────────────────── - let config_dto = crate::utils::read_config_file_with_options(config_file.as_str(), crate::utils::ReadConfigOptions::resolve_and_compute())?; + let config_dto = crate::config_loader::read_config_file_with_options( + config_file.as_str(), + crate::config_loader::ReadConfigOptions::resolve_and_compute(), + )?; - let current_mapping_path = paths.mapping_file_path.clone().unwrap_or_else(|| { - utils::resolve_mapping_file_path(paths.config_path.as_str(), None) - }); + let current_mapping_path = paths + .mapping_file_path + .clone() + .unwrap_or_else(|| utils::resolve_mapping_file_path(paths.config_path.as_str(), None)); let next_mapping_path = paths.mapping_file_path.clone().unwrap_or_else(|| { utils::resolve_mapping_file_path(paths.config_path.as_str(), config_dto.mapping_path.as_deref()) }); let mapping_changed = current_mapping_path != next_mapping_path; - let current_template_path = paths.template_file_path.clone().unwrap_or_else(|| { - utils::resolve_template_file_path(paths.config_path.as_str(), None) - }); + let current_template_path = paths + .template_file_path + .clone() + .unwrap_or_else(|| utils::resolve_template_file_path(paths.config_path.as_str(), None)); let next_template_path = paths.template_file_path.clone().unwrap_or_else(|| { utils::resolve_template_file_path(paths.config_path.as_str(), config_dto.template_path.as_deref()) }); @@ -300,7 +308,8 @@ impl ConfigFile { PreparedFollowUp::Sources(prepared) } else if mapping_changed { // Only mapping path changed; templates are the same -> load templates once. - let prepared_templates = Self::load_prepared_global_templates_with_config(&effective_paths, &config).await?; + let prepared_templates = + Self::load_prepared_global_templates_with_config(&effective_paths, &config).await?; let prepared = Self::prepare_mapping_reload( effective_paths.mapping_file_path.as_deref(), prepared_templates.as_deref(), @@ -346,7 +355,9 @@ impl ConfigFile { ); } - if let Err(rollback_err) = update_app_state_sources(app_state, previous_sources, Some(previous_forced_targets)).await { + if let Err(rollback_err) = + update_app_state_sources(app_state, previous_sources, Some(previous_forced_targets)).await + { error!("Failed to rollback sources after dependent reload failure: {rollback_err}"); error!( "Source rollback failed after dependent reload error; runtime state may be inconsistent. Please restart the service." diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/app/src/api/endpoints/api_playlist_utils.rs similarity index 82% rename from backend/src/api/endpoints/api_playlist_utils.rs rename to backend/app/src/api/endpoints/api_playlist_utils.rs index 4d6ae724a..67132e945 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/app/src/api/endpoints/api_playlist_utils.rs @@ -1,6 +1,11 @@ -use crate::api::model::AppState; use crate::{ - api::api_utils::{empty_json_list_response, stream_json_or_bin_response_stream, stream_json_or_bin_response_try_stream}, + api::{ + api_utils::{ + empty_json_list_response, json_or_bin_response, stream_json_or_bin_response_stream, + stream_json_or_bin_response_try_stream, + }, + model::AppState, + }, iptv::{m3u, xtream}, model::{ConfigInput, ConfigTarget}, processing::processor::{download_stalker_playlist, StalkerCluster}, @@ -11,11 +16,12 @@ use crate::{ }; use axum::response::IntoResponse; use serde_json::json; -use shared::utils::{concat_path, concat_path_leading_slash, interner_gc, obfuscate_text, Internable}; -use shared::model::{InputPersistence, M3uPlaylistItem, TargetType, UiPlaylistItem, XtreamCluster, XtreamPlaylistItem}; +use shared::{ + model::{InputPersistence, M3uPlaylistItem, TargetType, UiPlaylistItem, XtreamCluster, XtreamPlaylistItem}, + utils::{concat_path, concat_path_leading_slash, interner_gc, obfuscate_text, Internable}, +}; use std::sync::Arc; use tokio_stream::StreamExt; -use crate::api::api_utils::json_or_bin_response; pub(in crate::api::endpoints) const STALKER_RESOURCE_SCHEME: &str = "stalker://"; @@ -44,53 +50,44 @@ pub(in crate::api::endpoints) async fn get_playlist_for_target( accept: Option<&str>, ) -> impl IntoResponse + Send { let config = app_state.app_config.config.load(); - let web_ui_path = config - .web_ui - .as_ref() - .and_then(|w| w.path.as_ref()) - .map_or("", String::as_str); + let web_ui_path = config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); let encrypt_secret = app_state.get_encrypt_secret(); if let Some(target) = cfg_target { if target.has_output(TargetType::Xtream) { - let Some(channel_iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, target, cluster).await else { + let Some(channel_iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, target, cluster).await + else { return empty_json_list_response(); }; let item_filter = if cluster == XtreamCluster::Series { - |pli: &XtreamPlaylistItem| { - !pli.item_type.is_series() - } + |pli: &XtreamPlaylistItem| !pli.item_type.is_series() } else { |_pli: &XtreamPlaylistItem| true }; let converted_stream = channel_iterator.filter_map(move |entry| match entry { - Ok(item) if item_filter(&item) => Some(Ok(rewrite_resource_url( - &encrypt_secret, - &resource_url, - UiPlaylistItem::from(item), - ))), + Ok(item) if item_filter(&item) => { + Some(Ok(rewrite_resource_url(&encrypt_secret, &resource_url, UiPlaylistItem::from(item)))) + } Ok(_) => None, Err(error) => Some(Err(error)), }); return stream_json_or_bin_response_try_stream(accept, converted_stream).into_response(); } else if target.has_output(TargetType::M3u) { - let Some(channel_iterator) = iter_raw_m3u_target_playlist(&app_state.app_config, target, Some(cluster)).await else { + let Some(channel_iterator) = + iter_raw_m3u_target_playlist(&app_state.app_config, target, Some(cluster)).await + else { return empty_json_list_response(); }; let item_filter = if cluster == XtreamCluster::Series { - |pli: &M3uPlaylistItem| { - !pli.item_type.is_series() - } + |pli: &M3uPlaylistItem| !pli.item_type.is_series() } else { |_pli: &M3uPlaylistItem| true }; let converted_stream = channel_iterator.filter_map(move |res| match res { - Ok(pli) if item_filter(&pli) => Some(Ok(rewrite_resource_url( - &encrypt_secret, - &resource_url, - UiPlaylistItem::from(pli), - ))), + Ok(pli) if item_filter(&pli) => { + Some(Ok(rewrite_resource_url(&encrypt_secret, &resource_url, UiPlaylistItem::from(pli)))) + } Ok(_) => None, Err(error) => Some(Err(error)), }); @@ -118,11 +115,8 @@ fn rewrite_stalker_playback_url( input_id: u16, mut item: UiPlaylistItem, ) -> UiPlaylistItem { - let locator = format!( - "{STALKER_RESOURCE_SCHEME}{input_id}/{}/{}", - item.xtream_cluster.as_stream_type(), - item.provider_id - ); + let locator = + format!("{STALKER_RESOURCE_SCHEME}{input_id}/{}/{}", item.xtream_cluster.as_stream_type(), item.provider_id); item.url = concat_path(resource_url, &obfuscate_text(encrypt_secret, &locator)).intern(); item } @@ -135,7 +129,8 @@ pub(in crate::api::endpoints) async fn get_playlist_for_input( ) -> impl IntoResponse + Send { if let Some(input) = cfg_input { if input.input_type.is_xtream() { - let Some(channel_iterator) = iter_raw_xtream_input_playlist(&app_state.app_config, input, cluster).await else { + let Some(channel_iterator) = iter_raw_xtream_input_playlist(&app_state.app_config, input, cluster).await + else { return empty_json_list_response(); }; let converted_stream = channel_iterator.map(|entry| entry.map(UiPlaylistItem::from)); @@ -150,16 +145,15 @@ pub(in crate::api::endpoints) async fn get_playlist_for_input( // TODO refactor let stalker_cluster = stalker_cluster(cluster); let client = app_state.http_client.load(); - let (groups, errors, _, partial) = - download_stalker_playlist( - &app_state.app_config, - client.as_ref(), - input, - Some(&[stalker_cluster]), - crate::processing::processor::StalkerRefreshMode::ServerSlice, - true, - ) - .await; + let (groups, errors, _, partial) = download_stalker_playlist( + &app_state.app_config, + client.as_ref(), + input, + Some(&[stalker_cluster]), + crate::processing::processor::StalkerRefreshMode::ServerSlice, + true, + ) + .await; if groups.is_empty() { if partial { return stalker_refresh_pending_response(accept); @@ -254,18 +248,20 @@ pub(in crate::api::endpoints) async fn get_playlist_for_custom_provider( } else { // Stream the UI conversion lazily (like the target/input endpoints) instead of // collecting the whole playlist into a second Vec and serializing it all at once. - let web_ui_path = cfg.web_ui.as_ref().and_then(|web_ui| web_ui.path.as_ref()).map_or("", String::as_str); + let web_ui_path = + cfg.web_ui.as_ref().and_then(|web_ui| web_ui.path.as_ref()).map_or("", String::as_str); let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); let encrypt_secret = app_state.get_encrypt_secret(); let input_id = input.id; - let converted_stream = tokio_stream::iter(result.into_iter().flat_map(|g| g.channels).map(move |pli| { - rewrite_stalker_playback_url( - &encrypt_secret, - &resource_url, - input_id, - UiPlaylistItem::from(&pli), - ) - })); + let converted_stream = + tokio_stream::iter(result.into_iter().flat_map(|g| g.channels).map(move |pli| { + rewrite_stalker_playback_url( + &encrypt_secret, + &resource_url, + input_id, + UiPlaylistItem::from(&pli), + ) + })); stream_json_or_bin_response_stream(accept, converted_stream).into_response() } } @@ -339,9 +335,6 @@ mod tests { let rewritten = rewrite_resource_url(&secret, "/api/v1/playlist/resource", item); let expected_suffix = obfuscate_text(&secret, "https://example.com/poster.jpg"); - assert_eq!( - rewritten.logo.as_ref(), - format!("/api/v1/playlist/resource/{expected_suffix}") - ); + assert_eq!(rewritten.logo.as_ref(), format!("/api/v1/playlist/resource/{expected_suffix}")); } } diff --git a/backend/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs similarity index 89% rename from backend/src/api/endpoints/custom_video_stream_api.rs rename to backend/app/src/api/endpoints/custom_video_stream_api.rs index a7630894e..668102db5 100644 --- a/backend/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -7,24 +7,22 @@ use crate::{ }, model::{ create_custom_video_stream_response, hls_custom_video_manifest_response_with_virtual_id, - finite_hls_immutable_media_response, - parse_hls_panel_provisioning_segment_route_name, resolve_hls_standalone_custom_segment, - AppState, CustomVideoStreamType, HlsAccessLeaseId, HlsStandaloneCustomSegmentAccess, - HlsStandaloneCustomSegmentError, TransportStreamBuffer, + parse_hls_panel_provisioning_segment_route_name, AppState, CustomVideoStreamType, TransportStreamBuffer, }, }, auth::{check_network_access_only, resolve_api_user_context, verify_access_token, Fingerprint}, model::{ConfigTarget, ProxyUserCredentials}, }; use axum::{ - http::{ - header, - HeaderMap, HeaderValue, Method, StatusCode, - }, + http::{header, HeaderMap, HeaderValue, Method, StatusCode}, response::{IntoResponse, Response}, }; use serde::Deserialize; use std::{str::FromStr, sync::Arc}; +use tuliprox_hls::api::{ + finite_hls_immutable_media_response, resolve_hls_standalone_custom_segment, HlsAccessLeaseId, + HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError, +}; use url::form_urlencoded; const HLS_CVS_CONTENT_TYPE: &str = "video/mp2t"; @@ -67,7 +65,13 @@ fn resolve_cvs_user_context( return Err(Box::new(app_state.app_config.get_auth_error_status().into_response())); }; - if let Err(e) = resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), app_state) { + if let Err(e) = resolve_api_user_context( + user.clone(), + target.clone(), + fingerprint.clone(), + &app_state.app_config, + &app_state.geoip, + ) { return Err(Box::new(e.into_player_response(app_state.app_config.get_auth_error_status()))); } @@ -84,7 +88,7 @@ fn resolve_hls_cvs_user_context( return Err(Box::new(app_state.app_config.get_auth_error_status().into_response())); }; - if let Err(e) = check_network_access_only(&user, fingerprint, app_state) { + if let Err(e) = check_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip) { return Err(Box::new(e.into_player_response(app_state.app_config.get_auth_error_status()))); } @@ -100,7 +104,7 @@ fn resolve_hls_cvs_access_user( return Err(Box::new(app_state.app_config.get_auth_error_status().into_response())); }; - if let Err(e) = check_network_access_only(&user, fingerprint, app_state) { + if let Err(e) = check_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip) { return Err(Box::new(e.into_player_response(app_state.app_config.get_auth_error_status()))); } @@ -232,7 +236,12 @@ async fn cvs_api_response(context: CvsApiResponseContext<'_>) -> Response { if !verify_access_token(token, &app_state.app_config.access_token_secret) { return app_state.app_config.get_auth_error_status().into_response(); } - return create_custom_video_stream_response(app_state, &fingerprint.addr, custom_video_type).into_response(); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + custom_video_type, + ) + .into_response(); } } @@ -246,8 +255,10 @@ async fn cvs_api_response(context: CvsApiResponseContext<'_>) -> Response { match route_kind { CvsRouteKind::Hls => StatusCode::NOT_FOUND.into_response(), - CvsRouteKind::Ts => create_custom_video_stream_response(app_state, &fingerprint.addr, custom_video_type) - .into_response(), + CvsRouteKind::Ts => { + create_custom_video_stream_response(&app_state.provider_stream_ctx(), &fingerprint.addr, custom_video_type) + .into_response() + } } } @@ -265,15 +276,20 @@ async fn cvs_standalone_hls_segment_response( }; let access_lease = HlsAccessLeaseId(access_lease.to_string()); let now_ms = current_time_millis(); - let access = - match resolve_hls_standalone_custom_segment(app_state, &access_lease, asset_fingerprint, index, now_ms) { - Ok(access) => access, - Err( - HlsStandaloneCustomSegmentError::InvalidIndex - | HlsStandaloneCustomSegmentError::UnknownAccessLease - | HlsStandaloneCustomSegmentError::StaleAssetFingerprint, - ) => return StatusCode::NOT_FOUND.into_response(), - }; + let access = match resolve_hls_standalone_custom_segment( + &app_state.hls_ctx(), + &access_lease, + asset_fingerprint, + index, + now_ms, + ) { + Ok(access) => access, + Err( + HlsStandaloneCustomSegmentError::InvalidIndex + | HlsStandaloneCustomSegmentError::UnknownAccessLease + | HlsStandaloneCustomSegmentError::StaleAssetFingerprint, + ) => return StatusCode::NOT_FOUND.into_response(), + }; if let Err(response) = validate_hls_standalone_custom_access(app_state, fingerprint, &access_lease, &access, now_ms).await { @@ -301,8 +317,7 @@ async fn validate_hls_standalone_custom_access( .access_lease_response_snapshot(access_lease_id, &shared_lease.proxy_session_id, now_ms) .await; if !lease.is_some_and(|lease| { - lease.issued_at_ms == shared_lease.lease_issued_at_ms - && lease.username == access.username.as_ref() + lease.issued_at_ms == shared_lease.lease_issued_at_ms && lease.username == access.username.as_ref() }) { return Err(Box::new(StatusCode::NOT_FOUND.into_response())); } @@ -398,10 +413,7 @@ async fn cvs_provisioning_manifest_api( pub fn cvs_api_register() -> axum::Router> { axum::Router::new() - .route( - "/cvs/hls/{username}/{password}/provisioning.m3u8", - axum::routing::get(cvs_provisioning_manifest_api), - ) + .route("/cvs/hls/{username}/{password}/provisioning.m3u8", axum::routing::get(cvs_provisioning_manifest_api)) .route("/cvs/{route_kind}/{username}/{password}/{stream_type}", axum::routing::get(cvs_typed_api)) .route("/cvs/{username}/{password}/{stream_type}", axum::routing::get(cvs_api)) } @@ -492,17 +504,18 @@ mod tests { use crate::{ api::model::{ build_hls_standalone_custom_plan, hls_custom_video_manifest_response_for_access_lease, - ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, - CustomVideoStreamType, DownloadQueue, EventManager, HlsAccessLease, HlsAccessLeaseId, - HlsPlaybackFamilyKey, HlsProvisioningState, HlsProxyManager, HlsRuntimeCustomTailReason, - HlsStandaloneCustomAccess, MetadataUpdateManager, PlaylistStorageState, ProxySessionId, - SharedStreamManager, TransportStreamBuffer, UpdateGuard, + ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, CustomVideoStreamType, + DownloadQueue, EventManager, HlsAccessLease, HlsAccessLeaseId, HlsPlaybackFamilyKey, HlsProvisioningState, + HlsProxyManager, HlsRuntimeCustomTailReason, HlsStandaloneCustomAccess, MetadataUpdateManager, + PlaylistStorageState, ProxySessionId, SharedStreamManager, TransportStreamBuffer, UpdateGuard, }, model::{ ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigSource, ConfigTarget, CustomStreamResponse, MediaToolCapabilities, ProxyUserCredentials, SourcesConfig, TargetOutput, TargetUser, XtreamTargetFlagsSet, XtreamTargetOutput, }, + repository::GeoIp, + utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::{ @@ -511,12 +524,14 @@ mod tests { response::IntoResponse, Router, }; - use crate::utils::{FileLockManager, GeoIp}; + use shared::{ + foundation::Filter, + model::{ConfigPaths, InputFetchMethod, InputType, ProcessingOrder}, + }; use std::{collections::HashMap, sync::Arc}; - use tower::ServiceExt; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; - use shared::{foundation::Filter, model::{ConfigPaths, InputFetchMethod, InputType, ProcessingOrder}}; + use tower::ServiceExt; fn test_fingerprint() -> crate::auth::Fingerprint { crate::auth::Fingerprint::new( @@ -529,23 +544,41 @@ mod tests { fn test_custom_stream_response() -> CustomStreamResponse { CustomStreamResponse { channel_unavailable: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")).to_vec(), + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .to_vec(), )), user_connections_exhausted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/user_connections_exhausted.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/user_connections_exhausted.ts" + )) + .to_vec(), )), provider_connections_exhausted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/provider_connections_exhausted.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/provider_connections_exhausted.ts" + )) + .to_vec(), )), low_priority_preempted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/low_priority_preempted.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/low_priority_preempted.ts" + )) + .to_vec(), )), user_account_expired: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/user_account_expired.ts")).to_vec(), + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_account_expired.ts")) + .to_vec(), )), panel_api_provisioning: None, hls_session_or_lease_expired: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/hls_session_or_lease_expired.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/hls_session_or_lease_expired.ts" + )) + .to_vec(), )), panel_api_provisioning_hls_segments: Vec::new(), } @@ -640,9 +673,7 @@ mod tests { media_tools: Arc::new(MediaToolCapabilities::new()), }; - app_cfg - .custom_stream_response - .store(Some(Arc::new(test_custom_stream_response()))); + app_cfg.custom_stream_response.store(Some(Arc::new(test_custom_stream_response()))); app_cfg } @@ -710,7 +741,7 @@ mod tests { async fn custom_video_stream_response_returns_ok_for_channel_unavailable() { let app_state = create_test_app_state(); let response = crate::api::model::create_custom_video_stream_response( - &app_state, + &app_state.provider_stream_ctx(), &test_fingerprint().addr, crate::api::model::CustomVideoStreamType::ChannelUnavailable, ) @@ -791,10 +822,7 @@ mod tests { assert!(!body.contains("/channel_unavailable/")); } - async fn custom_manifest_body( - app_state: &Arc, - video_type: CustomVideoStreamType, - ) -> String { + async fn custom_manifest_body(app_state: &Arc, video_type: CustomVideoStreamType) -> String { let user = app_state.app_config.get_user_credentials("viewer").expect("test user"); let response = crate::api::model::hls_custom_video_manifest_response_with_virtual_id( app_state, @@ -806,10 +834,7 @@ mod tests { .await; assert_eq!(response.status(), StatusCode::OK); String::from_utf8( - axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("custom manifest body") - .to_vec(), + axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("custom manifest body").to_vec(), ) .expect("custom manifest utf8") } @@ -841,10 +866,8 @@ mod tests { let parsed = url::Url::parse(url).expect("absolute custom segment URL"); let route_start = parsed.path().rfind("/cvs/").expect("custom segment route"); let route_path = &parsed.path()[route_start..]; - let request_uri = parsed.query().map_or_else( - || route_path.to_string(), - |query| format!("{route_path}?{query}"), - ); + let request_uri = + parsed.query().map_or_else(|| route_path.to_string(), |query| format!("{route_path}?{query}")); let mut request = Request::builder().method(method).uri(request_uri); if let Some(range) = range { request = request.header(header::RANGE, range); @@ -865,11 +888,8 @@ mod tests { } fn replace_low_priority_asset(app_state: &Arc, bytes: Vec) { - let responses = app_state - .app_config - .custom_stream_response - .load_full() - .expect("custom responses before reload"); + let responses = + app_state.app_config.custom_stream_response.load_full().expect("custom responses before reload"); let mut revised = responses.as_ref().clone(); revised.low_priority_preempted = Some(TransportStreamBuffer::new(bytes)); app_state.app_config.custom_stream_response.store(Some(Arc::new(revised))); @@ -984,18 +1004,13 @@ mod tests { ) .await; let body = String::from_utf8( - axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("shared custom manifest") - .to_vec(), + axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("shared custom manifest").to_vec(), ) .expect("shared custom manifest utf8"); let url = custom_segment_urls(&body)[0]; assert_eq!(standalone_route_parts(url).0, lease_id.0); assert_eq!( - standalone_segment_response(Arc::clone(&app_state), url, Method::GET, None) - .await - .status(), + standalone_segment_response(Arc::clone(&app_state), url, Method::GET, None).await.status(), StatusCode::OK ); @@ -1044,7 +1059,7 @@ mod tests { HlsRuntimeCustomTailReason::SessionOrLeaseExpired, ] { let plan = build_hls_standalone_custom_plan( - &app_state, + &app_state.hls_ctx(), "https://example.test/iptv", HlsStandaloneCustomAccess::for_user("viewer"), reason, @@ -1053,19 +1068,13 @@ mod tests { .await .expect("standalone custom plan"); - assert_eq!( - plan.prepared_bundle.source_asset_duration_ticks_90khz, - 902_400, - "{reason:?}" - ); + assert_eq!(plan.prepared_bundle.source_asset_duration_ticks_90khz, 902_400, "{reason:?}"); assert_eq!(plan.prepared_bundle.source_asset_duration_ms, 10_027, "{reason:?}"); assert!(plan.manifest_body.contains("#EXTINF:10.027,\n"), "{reason:?}"); assert_eq!(plan.prepared_bundle.segments[0].timestamp_offset_ticks_90khz, 0); for pair in plan.prepared_bundle.segments.windows(2) { assert_eq!( - pair[1] - .timestamp_offset_ticks_90khz - .saturating_sub(pair[0].timestamp_offset_ticks_90khz), + pair[1].timestamp_offset_ticks_90khz.saturating_sub(pair[0].timestamp_offset_ticks_90khz), 902_400, "{reason:?}" ); @@ -1081,9 +1090,10 @@ mod tests { let url = custom_segment_urls(&body)[0]; let before = standalone_segment_response(Arc::clone(&app_state), url, Method::GET, None).await; assert_eq!(before.status(), StatusCode::OK); - let before = - axum::body::to_bytes(before.into_body(), usize::MAX).await.expect("original immutable body"); - let mut revised = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/low_priority_preempted.ts")).to_vec(); + let before = axum::body::to_bytes(before.into_body(), usize::MAX).await.expect("original immutable body"); + let mut revised = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/low_priority_preempted.ts")) + .to_vec(); *revised.last_mut().expect("non-empty low-priority asset") ^= 1; replace_low_priority_asset(&app_state, revised); @@ -1091,9 +1101,7 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); assert_eq!( - axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("replayed immutable body"), + axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("replayed immutable body"), before ); } @@ -1104,11 +1112,8 @@ mod tests { let body = custom_manifest_body(&app_state, CustomVideoStreamType::LowPriorityPreempted).await; let url = custom_segment_urls(&body)[0]; let (_, asset_fingerprint, _) = standalone_route_parts(url); - let wrong_fingerprint = if asset_fingerprint == "0000000000000000" { - "1111111111111111" - } else { - "0000000000000000" - }; + let wrong_fingerprint = + if asset_fingerprint == "0000000000000000" { "1111111111111111" } else { "0000000000000000" }; let invalid_url = url.replacen(&asset_fingerprint, wrong_fingerprint, 1); let response = standalone_segment_response(app_state, &invalid_url, Method::GET, None).await; @@ -1120,7 +1125,7 @@ mod tests { async fn standalone_custom_range_requests_slice_prepared_immutable_bytes() { let app_state = create_test_app_state(); let plan = build_hls_standalone_custom_plan( - &app_state, + &app_state.hls_ctx(), "https://example.test/iptv", HlsStandaloneCustomAccess::for_user("viewer"), HlsRuntimeCustomTailReason::LowPriorityPreempted, @@ -1131,26 +1136,18 @@ mod tests { let url = custom_segment_urls(&plan.manifest_body)[0]; let expected = plan.segment_bytes(0).expect("prepared segment zero"); - let response = - standalone_segment_response(Arc::clone(&app_state), url, Method::GET, Some("bytes=0-187")).await; + let response = standalone_segment_response(Arc::clone(&app_state), url, Method::GET, Some("bytes=0-187")).await; assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); assert_eq!(response.headers()[header::CONTENT_LENGTH], "188"); - assert!(response.headers()[header::CACHE_CONTROL] - .to_str() - .is_ok_and(|value| value.contains("immutable"))); + assert!(response.headers()[header::CACHE_CONTROL].to_str().is_ok_and(|value| value.contains("immutable"))); assert_eq!( - axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("range body"), + axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("range body"), expected.slice(..188) ); let head = standalone_segment_response(app_state, url, Method::HEAD, None).await; assert_eq!(head.status(), StatusCode::OK); assert_eq!(head.headers()[header::CONTENT_LENGTH], expected.len().to_string()); - assert!(axum::body::to_bytes(head.into_body(), usize::MAX) - .await - .expect("HEAD body") - .is_empty()); + assert!(axum::body::to_bytes(head.into_body(), usize::MAX).await.expect("HEAD body").is_empty()); } #[tokio::test] @@ -1165,8 +1162,7 @@ mod tests { assert_eq!(finalize_count, 0); for url in urls.iter().take(2) { - let response = - standalone_segment_response(Arc::clone(&app_state), url, Method::GET, None).await; + let response = standalone_segment_response(Arc::clone(&app_state), url, Method::GET, None).await; assert_eq!(response.status(), StatusCode::OK); assert!(!axum::body::to_bytes(response.into_body(), usize::MAX) .await diff --git a/backend/src/api/endpoints/download_api.rs b/backend/app/src/api/endpoints/download_api.rs similarity index 63% rename from backend/src/api/endpoints/download_api.rs rename to backend/app/src/api/endpoints/download_api.rs index 61846a03d..5136b7509 100644 --- a/backend/src/api/endpoints/download_api.rs +++ b/backend/app/src/api/endpoints/download_api.rs @@ -1,17 +1,20 @@ use crate::{ - api::endpoints::v1_api_playlist, - api::model::{ - mutate, mutate_optional, AppState, ActiveProviderManager, ConnectionManager, DownloadControl, DownloadKind, - DownloadQueue, DownloadState, DownloadWaitOutcome, EventManager, EventMessage, FileDownload, - FileDownloadRequest, FileRecordingRequest, PersistedFileDownload, QueueMutationError, - RecordingExecutionResult, recording_partial_path, run_recording, + api::{ + endpoints::v1_api_playlist, + model::{ + mutate, mutate_optional, + recording_notification::LifecycleEvent, + recording_notification_adapter::{build_marker, decide, message_for, DispatchDecision}, + recording_partial_path, run_recording, ActiveProviderManager, AppState, ConnectionManager, DownloadControl, + DownloadKind, DownloadQueue, DownloadState, DownloadWaitOutcome, EventManager, EventMessage, FileDownload, + FileDownloadRequest, FileRecordingRequest, PersistedFileDownload, QueueMutationError, + RecordingExecutionResult, + }, }, messaging::send_message, model::{AppConfig, MessageContent, VideoDownloadConfig}, utils::{async_file_writer, request, request::create_client, IO_BUFFER_SIZE}, }; -use crate::api::model::recording_notification::LifecycleEvent; -use crate::api::model::recording_notification_adapter::{DispatchDecision, build_marker, decide, message_for}; use axum::response::IntoResponse; use futures::stream::TryStreamExt; use log::{debug, error, info, warn}; @@ -65,10 +68,7 @@ fn recording_execution_download(app_config: &AppConfig, download: &FileDownload) .as_ref() .and_then(|metadata| metadata.source.as_ref()) .ok_or_else(|| "Recording source metadata missing".to_string())?; - let virtual_id = source - .virtual_id - .parse::() - .map_err(|_| "Recording source virtual id invalid".to_string())?; + let virtual_id = source.virtual_id.parse::().map_err(|_| "Recording source virtual id invalid".to_string())?; let url = v1_api_playlist::build_stable_recording_url( app_config, &source.target_id, @@ -113,11 +113,10 @@ fn apply_download_retry_jitter(base_secs: u64, jitter_percent: u8) -> u64 { #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] fn compute_download_retry_backoff_secs(attempts: u8, download_cfg: &VideoDownloadConfig) -> u64 { let exponent = i32::from(attempts.saturating_sub(1)); - let scaled_secs = (download_cfg.retry_backoff_initial_secs as f64) * download_cfg.retry_backoff_multiplier.powi(exponent); - let clamped_secs = scaled_secs.clamp( - download_cfg.retry_backoff_initial_secs as f64, - download_cfg.retry_backoff_max_secs as f64, - ); + let scaled_secs = + (download_cfg.retry_backoff_initial_secs as f64) * download_cfg.retry_backoff_multiplier.powi(exponent); + let clamped_secs = + scaled_secs.clamp(download_cfg.retry_backoff_initial_secs as f64, download_cfg.retry_backoff_max_secs as f64); let base_secs = clamped_secs.round() as u64; apply_download_retry_jitter(base_secs, download_cfg.retry_backoff_jitter_percent) } @@ -180,9 +179,7 @@ fn background_download_should_wait( let blocked_by_background_limit = background_limit > 0 && capacities.iter().all(|(_, current, _)| *current >= background_limit); let blocked_by_reserved_slots = reserve_slots > 0 - && capacities - .iter() - .all(|(_, current, max)| *max > 0 && current.saturating_add(reserve_slots) >= *max); + && capacities.iter().all(|(_, current, max)| *max > 0 && current.saturating_add(reserve_slots) >= *max); blocked_by_background_limit || blocked_by_reserved_slots } @@ -194,25 +191,21 @@ fn capacities_have_free_slot(capacities: &[(Arc, usize, usize)]) -> bool { #[cfg(test)] fn recording_deadline_reached(download: &FileDownload, now_ts: i64) -> bool { download.kind == crate::api::model::DownloadKind::Recording - && download - .start_at - .zip(download.duration_secs) - .is_some_and(|(start_at, duration_secs)| now_ts >= start_at.saturating_add(i64::try_from(duration_secs).unwrap_or(90))) + && download.start_at.zip(download.duration_secs).is_some_and(|(start_at, duration_secs)| { + now_ts >= start_at.saturating_add(i64::try_from(duration_secs).unwrap_or(90)) + }) } #[cfg(test)] -async fn active_download_snapshot(active: &RwLock>) -> Option { active.read().await.clone() } +async fn active_download_snapshot(active: &RwLock>) -> Option { + active.read().await.clone() +} async fn active_download_snapshot_for_worker( active: &RwLock>, worker_uuid: &str, ) -> Option { - active - .read() - .await - .as_ref() - .filter(|download| download.uuid == worker_uuid) - .cloned() + active.read().await.as_ref().filter(|download| download.uuid == worker_uuid).cloned() } async fn update_active_download_for_worker( @@ -236,11 +229,7 @@ pub async fn download_queue_snapshot(download_queue: &DownloadQueue) -> Download let finished = finished.iter().map(shared::model::FileDownloadDto::from).collect(); let active = active.as_ref().map(shared::model::FileDownloadDto::from).into_iter().collect(); - DownloadsResponse { - queue, - finished, - active, - } + DownloadsResponse { queue, finished, active } } async fn broadcast_download_queue_update(event_manager: &Arc, download_queue: &DownloadQueue) { @@ -251,9 +240,7 @@ async fn broadcast_download_queue_update(event_manager: &Arc, down let queue = queue.iter().map(shared::model::FileDownloadDto::from).collect(); let finished = finished.iter().map(shared::model::FileDownloadDto::from).collect(); let _ = event_manager.send_event(EventMessage::DownloadsDeltaUpdate(DownloadsDelta::QueueReplaced { queue })); - let _ = event_manager.send_event(EventMessage::DownloadsDeltaUpdate(DownloadsDelta::FinishedReplaced { - finished, - })); + let _ = event_manager.send_event(EventMessage::DownloadsDeltaUpdate(DownloadsDelta::FinishedReplaced { finished })); if let Some(download) = active.as_ref() { let _ = event_manager.send_event(EventMessage::DownloadsDeltaUpdate(DownloadsDelta::ActivePatched( shared::model::FileDownloadDto::from(download), @@ -338,351 +325,359 @@ async fn download_file( let worker_uuid = file_download.uuid.as_str(); let url = file_download.url.clone(); let file_path = file_download.file_path.clone(); - // Check for existing partial file for resume - let existing_size = tokio::fs::metadata(&file_path).await.map_or(0, |metadata| metadata.len()); + // Check for existing partial file for resume + let existing_size = tokio::fs::metadata(&file_path).await.map_or(0, |metadata| metadata.len()); - let mut request_builder = client.get(url.clone()); - if existing_size > 0 { - request_builder = request_builder.header("Range", format!("bytes={existing_size}-")); - } + let mut request_builder = client.get(url.clone()); + if existing_size > 0 { + request_builder = request_builder.header("Range", format!("bytes={existing_size}-")); + } - if let Some(result) = handle_download_control_without_writer(current_download_control(&control_signal)) { - return result; - } + if let Some(result) = handle_download_control_without_writer(current_download_control(&control_signal)) { + return result; + } - let send_request = request_builder.send(); - tokio::pin!(send_request); - let response_result = loop { - if let Some(cancel_token) = provider_cancel_token.as_ref() { - tokio::select! { - biased; - () = cancel_token.cancelled() => return DownloadExecutionResult::Preempted, - () = control_notify.notified() => { - if let Some(result) = - handle_download_control_without_writer(*control_signal.read().await) - { - return result; - } + let send_request = request_builder.send(); + tokio::pin!(send_request); + let response_result = loop { + if let Some(cancel_token) = provider_cancel_token.as_ref() { + tokio::select! { + biased; + () = cancel_token.cancelled() => return DownloadExecutionResult::Preempted, + () = control_notify.notified() => { + if let Some(result) = + handle_download_control_without_writer(*control_signal.read().await) + { + return result; } - response = &mut send_request => break response, } - } else { - tokio::select! { - biased; - () = control_notify.notified() => { - if let Some(result) = - handle_download_control_without_writer(*control_signal.read().await) - { - return result; - } + response = &mut send_request => break response, + } + } else { + tokio::select! { + biased; + () = control_notify.notified() => { + if let Some(result) = + handle_download_control_without_writer(*control_signal.read().await) + { + return result; + } + } + response = &mut send_request => break response, + } + } + }; + + match response_result { + Ok(response) => { + let status = response.status(); + if !status.is_success() && status != reqwest::StatusCode::PARTIAL_CONTENT { + if is_retryable_download_status(status) { + return DownloadExecutionResult::Retryable(format!( + "Download request failed for {url} with transient HTTP {status}" + )); + } + return DownloadExecutionResult::Failed(format!( + "Download request failed for {url} with HTTP {status}" + )); + } + let is_resume = status == reqwest::StatusCode::PARTIAL_CONTENT; + + let total_size = compute_download_total_size(&response, existing_size); + + if let Some(total) = total_size { + let changed = update_active_download_for_worker(&active, worker_uuid, |download| { + download.total_size = Some(total); + true + }) + .await; + if changed { + if let Some(event_manager) = event_manager { + broadcast_active_download_delta(event_manager, &active).await; } - response = &mut send_request => break response, } } - }; - match response_result { - Ok(response) => { - let status = response.status(); - if !status.is_success() && status != reqwest::StatusCode::PARTIAL_CONTENT { - if is_retryable_download_status(status) { - return DownloadExecutionResult::Retryable(format!( - "Download request failed for {url} with transient HTTP {status}" - )); - } - return DownloadExecutionResult::Failed(format!("Download request failed for {url} with HTTP {status}")); - } - let is_resume = status == reqwest::StatusCode::PARTIAL_CONTENT; + match fs::create_dir_all(&file_download.file_dir).await { + Ok(()) => { + let mut open_options = tokio::fs::OpenOptions::new(); + let file_mode = if existing_size > 0 && is_resume { + open_options.append(true) + } else { + open_options.write(true).create(true).truncate(true) + }; - let total_size = compute_download_total_size(&response, existing_size); + if let Some(file_path_str) = file_path.to_str() { + info!("{} {}", if is_resume { "Resuming" } else { "Downloading" }, file_path_str); + match file_mode.open(&file_path).await { + Ok(file) => { + let mut buf_writer = async_file_writer(file); + let mut downloaded: u64 = if is_resume { existing_size } else { 0 }; + let mut stream = response.bytes_stream(); + let mut write_counter = 0; + let mut saw_first_chunk = existing_size > 0; + let mut last_progress_log_at = Instant::now(); + let mut last_progress_logged_bytes = downloaded; + let mut last_snapshot_update_at = Instant::now(); + let mut last_snapshot_update_bytes = downloaded; + let mut control_poll = time::interval(DOWNLOAD_CONTROL_POLL_INTERVAL); + control_poll.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + control_poll.tick().await; + let deadline_at = recording_deadline_instant(&file_download); + let mut deadline_sleep = deadline_at + .map(|deadline| Box::pin(time::sleep_until(deadline)) as Pin>); - if let Some(total) = total_size { - let changed = update_active_download_for_worker(&active, worker_uuid, |download| { - download.total_size = Some(total); - true - }) - .await; - if changed { - if let Some(event_manager) = event_manager { - broadcast_active_download_delta(event_manager, &active).await; - } - } - } - - match fs::create_dir_all(&file_download.file_dir).await { - Ok(()) => { - let mut open_options = tokio::fs::OpenOptions::new(); - let file_mode = if existing_size > 0 && is_resume { - open_options.append(true) - } else { - open_options.write(true).create(true).truncate(true) - }; - - if let Some(file_path_str) = file_path.to_str() { - info!("{} {}", if is_resume { "Resuming" } else { "Downloading" }, file_path_str); - match file_mode.open(&file_path).await { - Ok(file) => { - let mut buf_writer = async_file_writer(file); - let mut downloaded: u64 = if is_resume { existing_size } else { 0 }; - let mut stream = response.bytes_stream(); - let mut write_counter = 0; - let mut saw_first_chunk = existing_size > 0; - let mut last_progress_log_at = Instant::now(); - let mut last_progress_logged_bytes = downloaded; - let mut last_snapshot_update_at = Instant::now(); - let mut last_snapshot_update_bytes = downloaded; - let mut control_poll = time::interval(DOWNLOAD_CONTROL_POLL_INTERVAL); - control_poll.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - control_poll.tick().await; - let deadline_at = recording_deadline_instant(&file_download); - let mut deadline_sleep = deadline_at.map(|deadline| { - Box::pin(time::sleep_until(deadline)) as Pin> - }); - - loop { - if deadline_at.is_some_and(|deadline| Instant::now() >= deadline) { - if let Err(err) = buf_writer.flush().await { - return DownloadExecutionResult::Failed(err.to_string()); - } - if let Err(err) = buf_writer.shutdown().await { - return DownloadExecutionResult::Failed(err.to_string()); - } - return DownloadExecutionResult::Completed; + loop { + if deadline_at.is_some_and(|deadline| Instant::now() >= deadline) { + if let Err(err) = buf_writer.flush().await { + return DownloadExecutionResult::Failed(err.to_string()); } + if let Err(err) = buf_writer.shutdown().await { + return DownloadExecutionResult::Failed(err.to_string()); + } + return DownloadExecutionResult::Completed; + } - let next_item = if let (Some(cancel_token), Some(deadline_sleep)) = - (provider_cancel_token.as_ref(), deadline_sleep.as_mut()) - { - tokio::select! { - biased; - () = cancel_token.cancelled() => return DownloadExecutionResult::Preempted, - () = control_notify.notified() => { - if let Some(result) = handle_download_control( - &active, - *control_signal.read().await, - &mut buf_writer, - ).await { - return result; - } - continue; + let next_item = if let (Some(cancel_token), Some(deadline_sleep)) = + (provider_cancel_token.as_ref(), deadline_sleep.as_mut()) + { + tokio::select! { + biased; + () = cancel_token.cancelled() => return DownloadExecutionResult::Preempted, + () = control_notify.notified() => { + if let Some(result) = handle_download_control( + &active, + *control_signal.read().await, + &mut buf_writer, + ).await { + return result; } - _ = control_poll.tick() => { - if let Some(result) = handle_download_control( - &active, - current_download_control(&control_signal), - &mut buf_writer, - ).await { - return result; - } - continue; - } - () = deadline_sleep.as_mut() => return DownloadExecutionResult::Completed, - next_item = stream.try_next() => next_item.map_err(to_io_error), + continue; } - } else if let Some(cancel_token) = provider_cancel_token.as_ref() { - tokio::select! { - biased; - () = cancel_token.cancelled() => return DownloadExecutionResult::Preempted, - () = control_notify.notified() => { - if let Some(result) = handle_download_control( - &active, - *control_signal.read().await, - &mut buf_writer, - ).await { - return result; - } - continue; + _ = control_poll.tick() => { + if let Some(result) = handle_download_control( + &active, + current_download_control(&control_signal), + &mut buf_writer, + ).await { + return result; } - _ = control_poll.tick() => { - if let Some(result) = handle_download_control( - &active, - current_download_control(&control_signal), - &mut buf_writer, - ).await { - return result; - } - continue; - } - next_item = stream.try_next() => next_item.map_err(to_io_error), + continue; } - } else if let Some(deadline_sleep) = deadline_sleep.as_mut() { - tokio::select! { - biased; - () = control_notify.notified() => { - if let Some(result) = handle_download_control( - &active, - *control_signal.read().await, - &mut buf_writer, - ).await { - return result; - } - continue; + () = deadline_sleep.as_mut() => return DownloadExecutionResult::Completed, + next_item = stream.try_next() => next_item.map_err(to_io_error), + } + } else if let Some(cancel_token) = provider_cancel_token.as_ref() { + tokio::select! { + biased; + () = cancel_token.cancelled() => return DownloadExecutionResult::Preempted, + () = control_notify.notified() => { + if let Some(result) = handle_download_control( + &active, + *control_signal.read().await, + &mut buf_writer, + ).await { + return result; } - _ = control_poll.tick() => { - if let Some(result) = handle_download_control( - &active, - current_download_control(&control_signal), - &mut buf_writer, - ).await { - return result; - } - continue; - } - () = deadline_sleep.as_mut() => return DownloadExecutionResult::Completed, - next_item = stream.try_next() => next_item.map_err(to_io_error), + continue; } - } else { - tokio::select! { - biased; - () = control_notify.notified() => { - if let Some(result) = handle_download_control( - &active, - *control_signal.read().await, - &mut buf_writer, - ).await { - return result; - } - continue; + _ = control_poll.tick() => { + if let Some(result) = handle_download_control( + &active, + current_download_control(&control_signal), + &mut buf_writer, + ).await { + return result; } - _ = control_poll.tick() => { - if let Some(result) = handle_download_control( - &active, - current_download_control(&control_signal), - &mut buf_writer, - ).await { - return result; - } - continue; - } - next_item = stream.try_next() => next_item.map_err(to_io_error), + continue; } - }; + next_item = stream.try_next() => next_item.map_err(to_io_error), + } + } else if let Some(deadline_sleep) = deadline_sleep.as_mut() { + tokio::select! { + biased; + () = control_notify.notified() => { + if let Some(result) = handle_download_control( + &active, + *control_signal.read().await, + &mut buf_writer, + ).await { + return result; + } + continue; + } + _ = control_poll.tick() => { + if let Some(result) = handle_download_control( + &active, + current_download_control(&control_signal), + &mut buf_writer, + ).await { + return result; + } + continue; + } + () = deadline_sleep.as_mut() => return DownloadExecutionResult::Completed, + next_item = stream.try_next() => next_item.map_err(to_io_error), + } + } else { + tokio::select! { + biased; + () = control_notify.notified() => { + if let Some(result) = handle_download_control( + &active, + *control_signal.read().await, + &mut buf_writer, + ).await { + return result; + } + continue; + } + _ = control_poll.tick() => { + if let Some(result) = handle_download_control( + &active, + current_download_control(&control_signal), + &mut buf_writer, + ).await { + return result; + } + continue; + } + next_item = stream.try_next() => next_item.map_err(to_io_error), + } + }; - match next_item { - Ok(item) => { - if let Some(chunk) = item { - match buf_writer.write_all(&chunk).await { - Ok(()) => { - write_counter += chunk.len(); - if write_counter >= IO_BUFFER_SIZE { - if let Err(err) = buf_writer.flush().await { - return DownloadExecutionResult::Failed(err.to_string()); - } - write_counter = 0; + match next_item { + Ok(item) => { + if let Some(chunk) = item { + match buf_writer.write_all(&chunk).await { + Ok(()) => { + write_counter += chunk.len(); + if write_counter >= IO_BUFFER_SIZE { + if let Err(err) = buf_writer.flush().await { + return DownloadExecutionResult::Failed( + err.to_string(), + ); } + write_counter = 0; + } - downloaded += chunk.len() as u64; - if saw_first_chunk { - let now = Instant::now(); - let should_log_progress = now.duration_since(last_progress_log_at) - >= DOWNLOAD_PROGRESS_LOG_INTERVAL - || downloaded.saturating_sub(last_progress_logged_bytes) + downloaded += chunk.len() as u64; + if saw_first_chunk { + let now = Instant::now(); + let should_log_progress = now + .duration_since(last_progress_log_at) + >= DOWNLOAD_PROGRESS_LOG_INTERVAL + || downloaded + .saturating_sub(last_progress_logged_bytes) >= DOWNLOAD_PROGRESS_LOG_BYTES; - if should_log_progress { - match total_size { - Some(total) if total > 0 => { - let percent = downloaded - .saturating_mul(100) - .checked_div(total) - .unwrap_or(0) - .min(100); - debug!( + if should_log_progress { + match total_size { + Some(total) if total > 0 => { + let percent = downloaded + .saturating_mul(100) + .checked_div(total) + .unwrap_or(0) + .min(100); + debug!( "Download progress for {file_path_str}: {}MB / {}MB ({}%)", bytes_to_megabytes(downloaded), bytes_to_megabytes(total), percent ); - } - _ => { - debug!( + } + _ => { + debug!( "Download progress for {file_path_str}: {}MB received", bytes_to_megabytes(downloaded) ); - } } - last_progress_log_at = now; - last_progress_logged_bytes = downloaded; + } + last_progress_log_at = now; + last_progress_logged_bytes = downloaded; if let Some(event_manager) = event_manager { - broadcast_active_download_delta(event_manager, &active).await; + broadcast_active_download_delta( + event_manager, + &active, + ) + .await; } } } else { - saw_first_chunk = true; - info!( + saw_first_chunk = true; + info!( "Receiving download data for {file_path_str}: {}MB received", bytes_to_megabytes(downloaded) ); - last_progress_log_at = Instant::now(); - last_progress_logged_bytes = downloaded; - if let Some(event_manager) = event_manager { - broadcast_active_download_delta(event_manager, &active).await; - } - } - let should_update_snapshot = downloaded - .saturating_sub(last_snapshot_update_bytes) - >= DOWNLOAD_SNAPSHOT_UPDATE_BYTES - || Instant::now().duration_since(last_snapshot_update_at) - >= DOWNLOAD_SNAPSHOT_UPDATE_INTERVAL; - if should_update_snapshot { - update_active_download_for_worker( - &active, - worker_uuid, - |download| { - download.size = downloaded; - true - }, - ) - .await; - last_snapshot_update_at = Instant::now(); - last_snapshot_update_bytes = downloaded; + last_progress_log_at = Instant::now(); + last_progress_logged_bytes = downloaded; + if let Some(event_manager) = event_manager { + broadcast_active_download_delta(event_manager, &active) + .await; } } - Err(err) => { - return DownloadExecutionResult::Failed(format!( - "Error while writing to file: {file_path_str} {err}" - )); + let should_update_snapshot = downloaded + .saturating_sub(last_snapshot_update_bytes) + >= DOWNLOAD_SNAPSHOT_UPDATE_BYTES + || Instant::now().duration_since(last_snapshot_update_at) + >= DOWNLOAD_SNAPSHOT_UPDATE_INTERVAL; + if should_update_snapshot { + update_active_download_for_worker( + &active, + worker_uuid, + |download| { + download.size = downloaded; + true + }, + ) + .await; + last_snapshot_update_at = Instant::now(); + last_snapshot_update_bytes = downloaded; } } - } else { - let megabytes = bytes_to_megabytes(downloaded); - info!("Downloaded {file_path_str}, filesize: {megabytes}MB"); - update_active_download_for_worker( - &active, - worker_uuid, - |download| { - download.size = downloaded; - true - }, - ) - .await; - if let Err(err) = buf_writer.flush().await { - return DownloadExecutionResult::Failed(err.to_string()); + Err(err) => { + return DownloadExecutionResult::Failed(format!( + "Error while writing to file: {file_path_str} {err}" + )); } - if let Err(err) = buf_writer.shutdown().await { - return DownloadExecutionResult::Failed(err.to_string()); - } - return DownloadExecutionResult::Completed; } + } else { + let megabytes = bytes_to_megabytes(downloaded); + info!("Downloaded {file_path_str}, filesize: {megabytes}MB"); + update_active_download_for_worker(&active, worker_uuid, |download| { + download.size = downloaded; + true + }) + .await; + if let Err(err) = buf_writer.flush().await { + return DownloadExecutionResult::Failed(err.to_string()); + } + if let Err(err) = buf_writer.shutdown().await { + return DownloadExecutionResult::Failed(err.to_string()); + } + return DownloadExecutionResult::Completed; } - Err(err) => return classify_download_stream_io_error(file_path_str, &err), } + Err(err) => return classify_download_stream_io_error(file_path_str, &err), } } - Err(err) => DownloadExecutionResult::Failed(format!("Error while opening file: {file_path_str} {err}")), } - } else { - DownloadExecutionResult::Failed("Error file-download file-path unknown".to_string()) + Err(err) => DownloadExecutionResult::Failed(format!( + "Error while opening file: {file_path_str} {err}" + )), } + } else { + DownloadExecutionResult::Failed("Error file-download file-path unknown".to_string()) } - Err(err) => DownloadExecutionResult::Failed(format!( - "Error while creating directory for file: {} {}", - file_download.file_dir.to_str().unwrap_or("?"), - err - )), } + Err(err) => DownloadExecutionResult::Failed(format!( + "Error while creating directory for file: {} {}", + file_download.file_dir.to_str().unwrap_or("?"), + err + )), } - Err(err) => classify_download_open_error(&url, &err), } + Err(err) => classify_download_open_error(&url, &err), + } } fn current_download_control(control_signal: &RwLock) -> DownloadControl { @@ -790,11 +785,8 @@ async fn commit_acquired_download( active.error = None; active.paused = false; active.finished = false; - let notification = mark_recording_metadata_notification( - active.recording.as_mut(), - LifecycleEvent::Started, - None, - ); + let notification = + mark_recording_metadata_notification(active.recording.as_mut(), LifecycleEvent::Started, None); Ok(Some(notification)) }) .await @@ -830,7 +822,9 @@ fn mark_recording_metadata_notification( marker_kind: Some(kind), } } - DispatchDecision::AlreadyDelivered { .. } | DispatchDecision::Suppressed { .. } => RecordingNotificationPlan::empty(), + DispatchDecision::AlreadyDelivered { .. } | DispatchDecision::Suppressed { .. } => { + RecordingNotificationPlan::empty() + } } } @@ -959,9 +953,7 @@ async fn requeue_active_download_for_capacity_wait( Ok(Some(true)) }; let result = if let Some(control) = consumed_control { - download_queue - .mutate_optional_and_clear_control(control, mutation) - .await? + download_queue.mutate_optional_and_clear_control(control, mutation).await? } else { mutate_optional(download_queue, mutation).await? }; @@ -1007,32 +999,29 @@ where .await } -async fn cancel_active_and_promote( - download_queue: &DownloadQueue, - uuid: &str, -) -> Result { +async fn cancel_active_and_promote(download_queue: &DownloadQueue, uuid: &str) -> Result { Ok(download_queue .mutate_optional_and_clear_control(DownloadControl::Cancel, |candidate| { - let Some(mut active) = candidate.active.take() else { - return Ok(None); - }; - if active.uuid != uuid { - candidate.active = Some(active); - return Ok(None); - } - active.finished = true; - active.paused = false; - active.next_retry_at = None; - active.error.get_or_insert_with(|| "Cancelled by user".to_string()); - active.state = DownloadState::Cancelled; - candidate.finished.push(active); - if !candidate.queue.is_empty() { - candidate.active = Some(candidate.queue.remove(0)); - } - Ok(Some(true)) - }) - .await? - .unwrap_or(false)) + let Some(mut active) = candidate.active.take() else { + return Ok(None); + }; + if active.uuid != uuid { + candidate.active = Some(active); + return Ok(None); + } + active.finished = true; + active.paused = false; + active.next_retry_at = None; + active.error.get_or_insert_with(|| "Cancelled by user".to_string()); + active.state = DownloadState::Cancelled; + candidate.finished.push(active); + if !candidate.queue.is_empty() { + candidate.active = Some(candidate.queue.remove(0)); + } + Ok(Some(true)) + }) + .await? + .unwrap_or(false)) } enum RetryCommit { @@ -1061,11 +1050,8 @@ async fn prepare_active_retry( failed.next_retry_at = None; failed.state = DownloadState::Failed; failed.error = Some(error.clone()); - let notification = mark_recording_metadata_notification( - failed.recording.as_mut(), - LifecycleEvent::Failed, - Some(error), - ); + let notification = + mark_recording_metadata_notification(failed.recording.as_mut(), LifecycleEvent::Failed, Some(error)); candidate.finished.push(failed); if !candidate.queue.is_empty() { candidate.active = Some(candidate.queue.remove(0)); @@ -1074,9 +1060,8 @@ async fn prepare_active_retry( } let delay_secs = compute_download_retry_backoff_secs(attempts, download_cfg); - let next_retry_at = chrono::Utc::now() - .timestamp() - .saturating_add(i64::try_from(delay_secs).unwrap_or(i64::MAX)); + let next_retry_at = + chrono::Utc::now().timestamp().saturating_add(i64::try_from(delay_secs).unwrap_or(i64::MAX)); active.next_retry_at = Some(next_retry_at); active.state = DownloadState::RetryWaiting; active.paused = false; @@ -1149,83 +1134,34 @@ pub(in crate::api) async fn ensure_download_worker_running( let app_config = Arc::new(cfg.clone()); if let Ok(client) = create_client(cfg).default_headers(headers).build() { - if let Some(active) = dq.active.read().await.as_ref() { - info!( - "Starting download worker for active download {} ({})", - active.uuid, active.filename - ); - } - tokio::spawn(async move { - 'worker: loop { - if dq.active.read().await.deref().is_some() { - if let Some(download) = dq.active.read().await.as_ref() { - if download.paused { - break; - } - } - let Some(worker_uuid) = dq.active.read().await.as_ref().map(|download| download.uuid.clone()) else { + if let Some(active) = dq.active.read().await.as_ref() { + info!("Starting download worker for active download {} ({})", active.uuid, active.filename); + } + tokio::spawn(async move { + 'worker: loop { + if dq.active.read().await.deref().is_some() { + if let Some(download) = dq.active.read().await.as_ref() { + if download.paused { break; - }; + } + } + let Some(worker_uuid) = dq.active.read().await.as_ref().map(|download| download.uuid.clone()) + else { + break; + }; - // Acquire a provider connection slot for this download. - // If the provider is at capacity, wait in the priority queue until signalled. - // Never proceeds without a slot when input_name is set — account bans otherwise. - let provider_acquire_result = { - let (input_name, priority) = { - let active = dq.active.read().await; - active.as_ref().map_or((None, 0i8), |dl| (dl.input_name.clone(), dl.priority)) - }; - if let Some(input_name) = input_name { - loop { - let capacities = active_provider.provider_capacities_for_input(&input_name).await; - if background_download_should_wait(priority, &capacities, &download_cfg) { - if let Err(err) = broadcast_worker_mutation( - &event_manager, - &dq, - set_active_download_state( - &dq, - &worker_uuid, - DownloadState::WaitingForCapacity, - None, - false, - ) - .await, - "waiting-for-capacity state", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - match dq - .slot_waiters - .wait( - Some(Arc::clone(&input_name)), - priority, - control_signal.as_ref(), - control_notify.as_ref(), - ) - .await - { - DownloadWaitOutcome::Signalled => {} - DownloadWaitOutcome::Paused => break ProviderAcquireResult::Paused, - DownloadWaitOutcome::Cancelled => break ProviderAcquireResult::Cancelled, - DownloadWaitOutcome::Restarted => break ProviderAcquireResult::Preempted, - } - continue; - } - if let Some(handle) = active_provider.acquire_connection_for_download(&input_name, priority).await { - break ProviderAcquireResult::Acquired(Some(handle)); - } - if *control_signal.read().await == DownloadControl::Cancel { - break ProviderAcquireResult::Cancelled; - } - if *control_signal.read().await == DownloadControl::Pause { - break ProviderAcquireResult::Paused; - } - if *control_signal.read().await == DownloadControl::Restart { - break ProviderAcquireResult::Preempted; - } + // Acquire a provider connection slot for this download. + // If the provider is at capacity, wait in the priority queue until signalled. + // Never proceeds without a slot when input_name is set — account bans otherwise. + let provider_acquire_result = { + let (input_name, priority) = { + let active = dq.active.read().await; + active.as_ref().map_or((None, 0i8), |dl| (dl.input_name.clone(), dl.priority)) + }; + if let Some(input_name) = input_name { + loop { + let capacities = active_provider.provider_capacities_for_input(&input_name).await; + if background_download_should_wait(priority, &capacities, &download_cfg) { if let Err(err) = broadcast_worker_mutation( &event_manager, &dq, @@ -1244,7 +1180,6 @@ pub(in crate::api) async fn ensure_download_worker_running( error!("Download worker commit failed: {err}"); break 'worker; } - // Wait for highest-priority signal — no sleep, no polling. match dq .slot_waiters .wait( @@ -1260,85 +1195,136 @@ pub(in crate::api) async fn ensure_download_worker_running( DownloadWaitOutcome::Cancelled => break ProviderAcquireResult::Cancelled, DownloadWaitOutcome::Restarted => break ProviderAcquireResult::Preempted, } + continue; } - } else { - ProviderAcquireResult::Acquired(None) - } - }; - - let provider_handle = match provider_acquire_result { - ProviderAcquireResult::Acquired(handle) => { - match commit_acquired_download(&dq, &worker_uuid).await { - Ok(Some(notification)) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - broadcast_download_queue_update(&event_manager, &dq).await; - } - Ok(None) => { - connection_manager.release_provider_handle(handle).await; - error!("Download worker active task changed after provider acquire"); - break 'worker; - } - Err(err) => { - connection_manager.release_provider_handle(handle).await; - error!("Download worker commit failed after provider acquire: {err}"); - break 'worker; - } - } - handle - } - ProviderAcquireResult::Paused => { - break; - } - ProviderAcquireResult::Cancelled => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - cancel_active_and_promote(&dq, &worker_uuid).await, - "provider-wait cancellation", - ) - .await + if let Some(handle) = + active_provider.acquire_connection_for_download(&input_name, priority).await { - error!("Download worker commit failed: {err}"); - break 'worker; + break ProviderAcquireResult::Acquired(Some(handle)); } - continue; - } - ProviderAcquireResult::Preempted => { - if let Err(err) = broadcast_required_worker_mutation( + if *control_signal.read().await == DownloadControl::Cancel { + break ProviderAcquireResult::Cancelled; + } + if *control_signal.read().await == DownloadControl::Pause { + break ProviderAcquireResult::Paused; + } + if *control_signal.read().await == DownloadControl::Restart { + break ProviderAcquireResult::Preempted; + } + if let Err(err) = broadcast_worker_mutation( &event_manager, &dq, - requeue_active_download_for_capacity_wait( + set_active_download_state( &dq, &worker_uuid, - "Reloading download service configuration", - true, - Some(DownloadControl::Restart), + DownloadState::WaitingForCapacity, + None, + false, ) .await, - "configuration-reload requeue", + "waiting-for-capacity state", ) .await { error!("Download worker commit failed: {err}"); break 'worker; } - break; + // Wait for highest-priority signal — no sleep, no polling. + match dq + .slot_waiters + .wait( + Some(Arc::clone(&input_name)), + priority, + control_signal.as_ref(), + control_notify.as_ref(), + ) + .await + { + DownloadWaitOutcome::Signalled => {} + DownloadWaitOutcome::Paused => break ProviderAcquireResult::Paused, + DownloadWaitOutcome::Cancelled => break ProviderAcquireResult::Cancelled, + DownloadWaitOutcome::Restarted => break ProviderAcquireResult::Preempted, + } } - }; + } else { + ProviderAcquireResult::Acquired(None) + } + }; - let execution_result = { - let Some(download) = active_download_snapshot_for_worker(&dq.active, &worker_uuid).await - else { - connection_manager.release_provider_handle(provider_handle).await; + let provider_handle = match provider_acquire_result { + ProviderAcquireResult::Acquired(handle) => { + match commit_acquired_download(&dq, &worker_uuid).await { + Ok(Some(notification)) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + broadcast_download_queue_update(&event_manager, &dq).await; + } + Ok(None) => { + connection_manager.release_provider_handle(handle).await; + error!("Download worker active task changed after provider acquire"); + break 'worker; + } + Err(err) => { + connection_manager.release_provider_handle(handle).await; + error!("Download worker commit failed after provider acquire: {err}"); + break 'worker; + } + } + handle + } + ProviderAcquireResult::Paused => { + break; + } + ProviderAcquireResult::Cancelled => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + cancel_active_and_promote(&dq, &worker_uuid).await, + "provider-wait cancellation", + ) + .await + { + error!("Download worker commit failed: {err}"); break 'worker; - }; - match download.kind.clone() { - DownloadKind::Download => download_file( + } + continue; + } + ProviderAcquireResult::Preempted => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + requeue_active_download_for_capacity_wait( + &dq, + &worker_uuid, + "Reloading download service configuration", + true, + Some(DownloadControl::Restart), + ) + .await, + "configuration-reload requeue", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + }; + + let execution_result = { + let Some(download) = active_download_snapshot_for_worker(&dq.active, &worker_uuid).await + else { + connection_manager.release_provider_handle(provider_handle).await; + break 'worker; + }; + match download.kind.clone() { + DownloadKind::Download => { + download_file( Arc::clone(&dq.active), download, &client, @@ -1347,345 +1333,349 @@ pub(in crate::api) async fn ensure_download_worker_running( provider_handle.as_ref().and_then(|handle| handle.cancel_token.clone()), Some(&event_manager), ) - .await, - DownloadKind::Recording => 'recording_execution: { - let execution_download = match recording_execution_download(&app_config, &download) { - Ok(execution_download) => execution_download, - Err(err) => break 'recording_execution DownloadExecutionResult::Failed(err), - }; - let progress_path = recording_partial_path(&execution_download.file_path); - let container_format = app_config - .config - .load() - .video - .as_ref() - .and_then(|video| video.download.as_ref()) - .and_then(|dl| dl.recording.as_ref()) - .map_or_else( - shared::model::RecordingContainerFormat::default, - |recording| recording.container_format, - ); - let mut recording_future = Box::pin(run_recording( - &execution_download, - &control_signal, - &control_notify, - provider_handle.as_ref().and_then(|handle| handle.cancel_token.as_ref()), - container_format, - )); - let mut progress_tick = time::interval(RECORDING_PROGRESS_UPDATE_INTERVAL); - progress_tick.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + .await + } + DownloadKind::Recording => 'recording_execution: { + let execution_download = match recording_execution_download(&app_config, &download) + { + Ok(execution_download) => execution_download, + Err(err) => break 'recording_execution DownloadExecutionResult::Failed(err), + }; + let progress_path = recording_partial_path(&execution_download.file_path); + let container_format = app_config + .config + .load() + .video + .as_ref() + .and_then(|video| video.download.as_ref()) + .and_then(|dl| dl.recording.as_ref()) + .map_or_else(shared::model::RecordingContainerFormat::default, |recording| { + recording.container_format + }); + let mut recording_future = Box::pin(run_recording( + &execution_download, + &control_signal, + &control_notify, + provider_handle.as_ref().and_then(|handle| handle.cancel_token.as_ref()), + container_format, + )); + let mut progress_tick = time::interval(RECORDING_PROGRESS_UPDATE_INTERVAL); + progress_tick.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - let result = loop { - tokio::select! { - recording_result = &mut recording_future => break recording_result, - _ = progress_tick.tick() => { - if event_manager.has_event_receivers() { - refresh_recording_progress( - &dq.active, - &worker_uuid, - &progress_path, - &event_manager, - ) - .await; - } + let result = loop { + tokio::select! { + recording_result = &mut recording_future => break recording_result, + _ = progress_tick.tick() => { + if event_manager.has_event_receivers() { + refresh_recording_progress( + &dq.active, + &worker_uuid, + &progress_path, + &event_manager, + ) + .await; } } - }; - if event_manager.has_event_receivers() { - refresh_recording_progress( - &dq.active, - &worker_uuid, - &progress_path, - &event_manager, - ) - .await; - } - - match result { - RecordingExecutionResult::Completed => DownloadExecutionResult::Completed, - RecordingExecutionResult::Paused => DownloadExecutionResult::Paused, - RecordingExecutionResult::Cancelled => DownloadExecutionResult::Cancelled, - RecordingExecutionResult::Preempted => DownloadExecutionResult::Preempted, - RecordingExecutionResult::Retryable(err) => DownloadExecutionResult::Retryable(err), - RecordingExecutionResult::Failed(err) => DownloadExecutionResult::Failed(err), - } - } - } - }; - - match execution_result - { - DownloadExecutionResult::Completed => { - connection_manager.release_provider_handle(provider_handle).await; - let measured_bytes = { - let active = dq.active.read().await; - match active.as_ref() { - Some(fd) => tokio::fs::metadata(&fd.file_path) - .await - .map_or(fd.size, |metadata| metadata.len()), - None => 0, } }; - let committed = finish_active_and_promote(&dq, &worker_uuid, |fd| { - fd.finished = true; - fd.paused = false; - fd.state = DownloadState::Completed; - fd.size = measured_bytes; - fd.error = None; - fd.next_retry_at = None; - if let Some(meta) = fd.recording.as_mut() { - meta.measured_bytes = measured_bytes; - meta.reserved_bytes = 0; - meta.completed_at = Some(chrono::Utc::now().timestamp()); - meta.partial_relative_path = None; - } - mark_recording_metadata_notification( - fd.recording.as_mut(), - LifecycleEvent::Completed, - None, - ) - }) - .await; - match committed { - Ok(Some(notification)) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - broadcast_download_queue_update(&event_manager, &dq).await; - } - Ok(None) => {} - Err(err) => { - error!("Failed to persist completed download state: {err}"); - break 'worker; - } - } - } - DownloadExecutionResult::Paused => { - connection_manager.release_provider_handle(provider_handle).await; - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - set_active_download_state(&dq, &worker_uuid, DownloadState::Paused, None, true).await, - "paused state", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - break; - } - DownloadExecutionResult::Cancelled => { - connection_manager.release_provider_handle(provider_handle).await; - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - cancel_active_and_promote(&dq, &worker_uuid).await, - "cancelled state", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - DownloadExecutionResult::Preempted => { - connection_manager.release_provider_handle(provider_handle).await; - let control = *control_signal.read().await; - match control { - DownloadControl::Restart => warn!("Active transfer is restarting to apply updated download service configuration"), - _ => warn!("Active transfer was preempted by a higher-priority stream"), - } - let reason = { - let active = dq.active.read().await; - if control == DownloadControl::Restart { - "Reloading download service configuration" - } else { - active.as_ref().map_or(DOWNLOAD_PREEMPTED_REASON, preemption_reason_for) - } - }; - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - requeue_active_download_for_capacity_wait( - &dq, + if event_manager.has_event_receivers() { + refresh_recording_progress( + &dq.active, &worker_uuid, - reason, - true, - (control == DownloadControl::Restart).then_some(DownloadControl::Restart), + &progress_path, + &event_manager, ) - .await, - "preempted requeue", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; + .await; } - if should_exit_worker_after_preempt(control) { - break; + + match result { + RecordingExecutionResult::Completed => DownloadExecutionResult::Completed, + RecordingExecutionResult::Paused => DownloadExecutionResult::Paused, + RecordingExecutionResult::Cancelled => DownloadExecutionResult::Cancelled, + RecordingExecutionResult::Preempted => DownloadExecutionResult::Preempted, + RecordingExecutionResult::Retryable(err) => { + DownloadExecutionResult::Retryable(err) + } + RecordingExecutionResult::Failed(err) => DownloadExecutionResult::Failed(err), } } - DownloadExecutionResult::Retryable(_err) => { - connection_manager.release_provider_handle(provider_handle).await; - warn!("Retrying active download after transient failure"); - let retry_commit = prepare_active_retry(&dq, &worker_uuid, &download_cfg).await; - let retry_delay_secs = match retry_commit { - Ok(Some(RetryCommit::Waiting { delay_secs, attempts })) => { - debug!("Download retry attempt {attempts} scheduled in {delay_secs}s"); - broadcast_download_queue_update(&event_manager, &dq).await; - delay_secs - } - Ok(Some(RetryCommit::Failed(notification))) => { + } + }; + + match execution_result { + DownloadExecutionResult::Completed => { + connection_manager.release_provider_handle(provider_handle).await; + let measured_bytes = { + let active = dq.active.read().await; + match active.as_ref() { + Some(fd) => tokio::fs::metadata(&fd.file_path) + .await + .map_or(fd.size, |metadata| metadata.len()), + None => 0, + } + }; + let committed = finish_active_and_promote(&dq, &worker_uuid, |fd| { + fd.finished = true; + fd.paused = false; + fd.state = DownloadState::Completed; + fd.size = measured_bytes; + fd.error = None; + fd.next_retry_at = None; + if let Some(meta) = fd.recording.as_mut() { + meta.measured_bytes = measured_bytes; + meta.reserved_bytes = 0; + meta.completed_at = Some(chrono::Utc::now().timestamp()); + meta.partial_relative_path = None; + } + mark_recording_metadata_notification( + fd.recording.as_mut(), + LifecycleEvent::Completed, + None, + ) + }) + .await; + match committed { + Ok(Some(notification)) => { spawn_recording_notification_after_persist( &app_config, &client, notification, - true, + true, ); - broadcast_download_queue_update(&event_manager, &dq).await; - if dq.active.read().await.is_some() { - continue; - } - break; - } - Ok(None) => break, - Err(err) => { - error!("Failed to persist retry state: {err}"); - break; - } - }; - let mut retry_sleep = Box::pin(time::sleep(Duration::from_secs(retry_delay_secs))); - let retry_wait_outcome = loop { - tokio::select! { - () = &mut retry_sleep => break DownloadExecutionResult::Retryable(String::new()), - () = control_notify.notified() => { - match *control_signal.read().await { - DownloadControl::Pause => break DownloadExecutionResult::Paused, - DownloadControl::Cancel => break DownloadExecutionResult::Cancelled, - DownloadControl::Restart => break DownloadExecutionResult::Preempted, - DownloadControl::None => {} - } - } - } - }; - - match retry_wait_outcome { - DownloadExecutionResult::Retryable(_) => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - requeue_active_download_for_retry(&dq, &worker_uuid, true).await, - "retry requeue", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - DownloadExecutionResult::Paused => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - set_active_download_state( - &dq, - &worker_uuid, - DownloadState::Paused, - None, - true, - ) - .await, - "paused retry state", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - break; - } - DownloadExecutionResult::Cancelled => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - cancel_active_and_promote(&dq, &worker_uuid).await, - "cancelled retry state", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - DownloadExecutionResult::Completed - | DownloadExecutionResult::Failed(_) => {} - DownloadExecutionResult::Preempted => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - &dq, - requeue_active_download_for_capacity_wait( - &dq, - &worker_uuid, - "Reloading download service configuration", - true, - Some(DownloadControl::Restart), - ) - .await, - "configuration-reload retry requeue", - ) - .await - { - error!("Download worker commit failed: {err}"); - break 'worker; - } - break; - } + broadcast_download_queue_update(&event_manager, &dq).await; } - } - DownloadExecutionResult::Failed(err) => { - connection_manager.release_provider_handle(provider_handle).await; - warn!("Download failed permanently: {err}"); - let committed = finish_active_and_promote(&dq, &worker_uuid, |fd| { - fd.finished = true; - fd.paused = false; - fd.next_retry_at = None; - fd.error = Some(err.clone()); - fd.state = DownloadState::Failed; - mark_recording_metadata_notification( - fd.recording.as_mut(), - LifecycleEvent::Failed, - Some(err), - ) - }) - .await; - match committed { - Ok(Some(notification)) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - broadcast_download_queue_update(&event_manager, &dq).await; - } - Ok(None) => {} - Err(commit_err) => { - error!("Failed to persist failed download state: {commit_err}"); - break 'worker; - } + Ok(None) => {} + Err(err) => { + error!("Failed to persist completed download state: {err}"); + break 'worker; + } + } + } + DownloadExecutionResult::Paused => { + connection_manager.release_provider_handle(provider_handle).await; + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + set_active_download_state(&dq, &worker_uuid, DownloadState::Paused, None, true) + .await, + "paused state", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + DownloadExecutionResult::Cancelled => { + connection_manager.release_provider_handle(provider_handle).await; + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + cancel_active_and_promote(&dq, &worker_uuid).await, + "cancelled state", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + DownloadExecutionResult::Preempted => { + connection_manager.release_provider_handle(provider_handle).await; + let control = *control_signal.read().await; + match control { + DownloadControl::Restart => warn!( + "Active transfer is restarting to apply updated download service configuration" + ), + _ => warn!("Active transfer was preempted by a higher-priority stream"), + } + let reason = { + let active = dq.active.read().await; + if control == DownloadControl::Restart { + "Reloading download service configuration" + } else { + active.as_ref().map_or(DOWNLOAD_PREEMPTED_REASON, preemption_reason_for) + } + }; + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + requeue_active_download_for_capacity_wait( + &dq, + &worker_uuid, + reason, + true, + (control == DownloadControl::Restart).then_some(DownloadControl::Restart), + ) + .await, + "preempted requeue", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + if should_exit_worker_after_preempt(control) { + break; + } + } + DownloadExecutionResult::Retryable(_err) => { + connection_manager.release_provider_handle(provider_handle).await; + warn!("Retrying active download after transient failure"); + let retry_commit = prepare_active_retry(&dq, &worker_uuid, &download_cfg).await; + let retry_delay_secs = match retry_commit { + Ok(Some(RetryCommit::Waiting { delay_secs, attempts })) => { + debug!("Download retry attempt {attempts} scheduled in {delay_secs}s"); + broadcast_download_queue_update(&event_manager, &dq).await; + delay_secs + } + Ok(Some(RetryCommit::Failed(notification))) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + broadcast_download_queue_update(&event_manager, &dq).await; + if dq.active.read().await.is_some() { + continue; + } + break; + } + Ok(None) => break, + Err(err) => { + error!("Failed to persist retry state: {err}"); + break; + } + }; + let mut retry_sleep = Box::pin(time::sleep(Duration::from_secs(retry_delay_secs))); + let retry_wait_outcome = loop { + tokio::select! { + () = &mut retry_sleep => break DownloadExecutionResult::Retryable(String::new()), + () = control_notify.notified() => { + match *control_signal.read().await { + DownloadControl::Pause => break DownloadExecutionResult::Paused, + DownloadControl::Cancel => break DownloadExecutionResult::Cancelled, + DownloadControl::Restart => break DownloadExecutionResult::Preempted, + DownloadControl::None => {} + } + } + } + }; + + match retry_wait_outcome { + DownloadExecutionResult::Retryable(_) => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + requeue_active_download_for_retry(&dq, &worker_uuid, true).await, + "retry requeue", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + DownloadExecutionResult::Paused => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + set_active_download_state( + &dq, + &worker_uuid, + DownloadState::Paused, + None, + true, + ) + .await, + "paused retry state", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + DownloadExecutionResult::Cancelled => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + cancel_active_and_promote(&dq, &worker_uuid).await, + "cancelled retry state", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + DownloadExecutionResult::Completed | DownloadExecutionResult::Failed(_) => {} + DownloadExecutionResult::Preempted => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + &dq, + requeue_active_download_for_capacity_wait( + &dq, + &worker_uuid, + "Reloading download service configuration", + true, + Some(DownloadControl::Restart), + ) + .await, + "configuration-reload retry requeue", + ) + .await + { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + } + } + DownloadExecutionResult::Failed(err) => { + connection_manager.release_provider_handle(provider_handle).await; + warn!("Download failed permanently: {err}"); + let committed = finish_active_and_promote(&dq, &worker_uuid, |fd| { + fd.finished = true; + fd.paused = false; + fd.next_retry_at = None; + fd.error = Some(err.clone()); + fd.state = DownloadState::Failed; + mark_recording_metadata_notification( + fd.recording.as_mut(), + LifecycleEvent::Failed, + Some(err), + ) + }) + .await; + match committed { + Ok(Some(notification)) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + broadcast_download_queue_update(&event_manager, &dq).await; + } + Ok(None) => {} + Err(commit_err) => { + error!("Failed to persist failed download state: {commit_err}"); + break 'worker; } } } - } else { - break; } + } else { + break; } - *dq.worker_running.write().await = false; - }); + } + *dq.worker_running.write().await = false; + }); } else { *download_queue.worker_running.write().await = false; return Err("Failed to build http client".to_string()); @@ -1937,7 +1927,15 @@ pub async fn queue_recording_file( } let input_name = req.input_name.map(|s| std::sync::Arc::from(s.as_str())); let priority = req.priority.unwrap_or(download_cfg.recording_priority); - match FileDownload::new_recording(req.url.as_str(), req.filename.as_str(), download_cfg, req.start_at, req.duration_secs, input_name, priority) { + match FileDownload::new_recording( + req.url.as_str(), + req.filename.as_str(), + download_cfg, + req.start_at, + req.duration_secs, + input_name, + priority, + ) { Some(recording) => { if let Some(existing) = app_state.downloads.find_duplicate(&recording).await { info!( @@ -1996,11 +1994,7 @@ pub async fn pause_download( Ok(false) => axum::Json(json!({"success": false})).into_response(), Err(err) => { error!("Failed to persist paused download state: {err}"); - ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": err.message()})), - ) - .into_response() + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.message()}))).into_response() } } } @@ -2011,14 +2005,8 @@ pub async fn resume_download( ) -> impl axum::response::IntoResponse + Send { match app_state.downloads.resume_active(&req.uuid).await { Ok(true) => { - let download_cfg = app_state - .app_config - .config - .load() - .video - .as_ref() - .and_then(|video| video.download.as_ref()) - .cloned(); + let download_cfg = + app_state.app_config.config.load().video.as_ref().and_then(|video| video.download.as_ref()).cloned(); if let Some(download_cfg) = download_cfg { let app_state = Arc::clone(&app_state); tokio::spawn(async move { @@ -2048,11 +2036,7 @@ pub async fn resume_download( Ok(false) => axum::Json(json!({"success": false})).into_response(), Err(err) => { error!("Failed to persist resumed download state: {err}"); - ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": err.message()})), - ) - .into_response() + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.message()}))).into_response() } } } @@ -2083,27 +2067,18 @@ pub async fn cancel_download( broadcast_download_queue_update(&app_state.event_manager, &app_state.downloads).await; axum::Json(json!({"success": true})).into_response() } - Ok(None) => { - match app_state.downloads.remove_from_queue(&req.uuid).await { - Ok(true) => { - broadcast_download_queue_update(&app_state.event_manager, &app_state.downloads).await; - axum::Json(json!({"success": true})).into_response() - } - Ok(false) => axum::Json(json!({"success": false})).into_response(), - Err(err) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": err.message()})), - ) - .into_response(), + Ok(None) => match app_state.downloads.remove_from_queue(&req.uuid).await { + Ok(true) => { + broadcast_download_queue_update(&app_state.event_manager, &app_state.downloads).await; + axum::Json(json!({"success": true})).into_response() } - } + Ok(false) => axum::Json(json!({"success": false})).into_response(), + Err(err) => (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.message()}))) + .into_response(), + }, Err(err) => { error!("Failed to persist cancelled download state: {err}"); - ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": err.message()})), - ) - .into_response() + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.message()}))).into_response() } } } @@ -2119,11 +2094,9 @@ pub async fn remove_download( } axum::Json(json!({"success": removed})).into_response() } - Err(err) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": err.message()})), - ) - .into_response(), + Err(err) => { + (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.message()}))).into_response() + } } } @@ -2134,10 +2107,7 @@ pub async fn retry_download( let retried = match app_state.downloads.retry_finished(&req.uuid).await { Ok(retried) => retried, Err(err) => { - return ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": err.message()})), - ) + return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.message()}))) .into_response(); } }; @@ -2170,28 +2140,24 @@ mod tests { use super::{ active_download_snapshot, active_download_snapshot_for_worker, broadcast_download_queue_update, broadcast_required_worker_mutation, broadcast_worker_mutation, cancel_active_and_promote, cancel_download, - commit_acquired_download, finish_active_and_promote, - mark_recording_notification, - parse_content_range_total, pause_download, preemption_reason_for, recording_deadline_reached, - recording_execution_download, - refresh_recording_progress, requeue_active_download_for_capacity_wait, - requeue_active_download_for_retry, resume_download, rollback_last_recording_marker, - retryable_transport_error_message, set_active_download_state, should_exit_worker_after_preempt, - DownloadActionRequest, - DOWNLOAD_PREEMPTED_REASON, RECORDING_PREEMPTED_REASON, + commit_acquired_download, finish_active_and_promote, mark_recording_notification, parse_content_range_total, + pause_download, preemption_reason_for, recording_deadline_reached, recording_execution_download, + refresh_recording_progress, requeue_active_download_for_capacity_wait, requeue_active_download_for_retry, + resume_download, retryable_transport_error_message, rollback_last_recording_marker, set_active_download_state, + should_exit_worker_after_preempt, DownloadActionRequest, DOWNLOAD_PREEMPTED_REASON, RECORDING_PREEMPTED_REASON, }; use crate::{ api::model::{ - ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, DownloadControl, - DownloadKind, DownloadQueue, DownloadState, EventManager, EventMessage, FileDownload, MetadataUpdateManager, - PlaylistStorageState, SharedStreamManager, UpdateGuard, + recording_notification::LifecycleEvent, ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, + ConnectionManager, DownloadControl, DownloadKind, DownloadQueue, DownloadState, EventManager, EventMessage, + FileDownload, MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, UpdateGuard, }, - api::model::recording_notification::LifecycleEvent, model::{ - ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, MediaToolCapabilities, - MessageContent, ProcessTargets, SourcesConfig, + ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, MediaToolCapabilities, MessageContent, + ProcessTargets, SourcesConfig, }, - utils::{FileLockManager, GeoIp}, + repository::GeoIp, + utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::response::IntoResponse; @@ -2204,10 +2170,14 @@ mod tests { utils::Internable, }; use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; - use tokio::sync::mpsc; - use tokio::sync::RwLock; + use tokio::sync::{mpsc, RwLock}; - fn make_download(kind: DownloadKind, state: DownloadState, start_at: Option, duration_secs: Option) -> FileDownload { + fn make_download( + kind: DownloadKind, + state: DownloadState, + start_at: Option, + duration_secs: Option, + ) -> FileDownload { FileDownload { uuid: "id".to_string(), file_dir: PathBuf::from("/tmp"), @@ -2232,15 +2202,8 @@ mod tests { } fn attach_recording(download: &mut FileDownload, owner: RecordingOwner, visibility: RecordingVisibility) { - let mut metadata = RecordingMetadata::new( - owner, - visibility, - RecordingSource::new("1", "42", "input-a"), - 1_000, - 1_060, - 0, - 0, - ); + let mut metadata = + RecordingMetadata::new(owner, visibility, RecordingSource::new("1", "42", "input-a"), 1_000, 1_060, 0, 0); metadata.program_title = Some("Programme".to_string()); metadata.channel_name = Some("Channel".to_string()); metadata.relative_path = Some("Channel/Programme.ts".to_string()); @@ -2257,34 +2220,18 @@ mod tests { assert!(matches!(first.message, Some(MessageContent::RecordingLifecycle(_)))); assert!(duplicate.message.is_none()); - assert_eq!( - download - .recording - .as_ref() - .map_or(0, |metadata| metadata.notification_markers.len()), - 1 - ); + assert_eq!(download.recording.as_ref().map_or(0, |metadata| metadata.notification_markers.len()), 1); } #[test] fn private_user_recording_notification_is_suppressed() { let mut download = make_download(DownloadKind::Recording, DownloadState::Completed, Some(1_000), Some(60)); - attach_recording( - &mut download, - RecordingOwner::User(UserId::from("web:alice")), - RecordingVisibility::Private, - ); + attach_recording(&mut download, RecordingOwner::User(UserId::from("web:alice")), RecordingVisibility::Private); let message = mark_recording_notification(&mut download, LifecycleEvent::Completed, None); assert!(message.message.is_none()); - assert_eq!( - download - .recording - .as_ref() - .map_or(0, |metadata| metadata.notification_markers.len()), - 0 - ); + assert_eq!(download.recording.as_ref().map_or(0, |metadata| metadata.notification_markers.len()), 0); } #[test] @@ -2296,7 +2243,8 @@ mod tests { // successful Completed followed by a Failed on the same task — this // mirrors what would happen in production across two persist rounds. let first = mark_recording_notification(&mut download, LifecycleEvent::Completed, None); - let second = mark_recording_notification(&mut download, LifecycleEvent::Failed, Some("ffmpeg exited".to_string())); + let second = + mark_recording_notification(&mut download, LifecycleEvent::Failed, Some("ffmpeg exited".to_string())); assert!(first.marker_kind.is_some()); assert!(second.marker_kind.is_some()); @@ -2305,10 +2253,7 @@ mod tests { let markers = &download.recording.as_ref().unwrap().notification_markers; assert_eq!(markers.len(), 1, "only the Completed marker should remain"); - assert!(matches!( - markers[0].kind, - shared::model::recording::NotificationMarkerKind::Completed - )); + assert!(matches!(markers[0].kind, shared::model::recording::NotificationMarkerKind::Completed)); } #[test] @@ -2338,11 +2283,8 @@ mod tests { ) .expect("valid descriptor"); attach_recording(&mut recording, RecordingOwner::LegacyAdmin, RecordingVisibility::Private); - let source = recording - .recording - .as_mut() - .and_then(|metadata| metadata.source.as_mut()) - .expect("recording source"); + let source = + recording.recording.as_mut().and_then(|metadata| metadata.source.as_mut()).expect("recording source"); source.target_id = "stable-target".to_string(); source.virtual_id = "42".to_string(); source.input_name = "provider_1".to_string(); @@ -2412,12 +2354,8 @@ mod tests { async fn terminal_transition_finishes_active_and_promotes_next_in_one_commit() { let dir = tempfile::tempdir().expect("tempdir"); let queue = DownloadQueue::new_with_state_file(Some(dir.path().join("downloads_state.json"))); - *queue.active.write().await = Some(make_download( - DownloadKind::Download, - DownloadState::Downloading, - None, - None, - )); + *queue.active.write().await = + Some(make_download(DownloadKind::Download, DownloadState::Downloading, None, None)); let mut next = make_download(DownloadKind::Download, DownloadState::Queued, None, None); next.uuid = "next".to_string(); queue.queue.lock().await.push_back(next); @@ -2441,19 +2379,18 @@ mod tests { let blocking_dir = dir.path().join("state"); std::fs::create_dir_all(&blocking_dir).expect("create blocking dir"); let queue = DownloadQueue::new_with_state_file(Some(blocking_dir)); - *queue.active.write().await = Some(make_download( - DownloadKind::Download, - DownloadState::Downloading, - None, - None, - )); + *queue.active.write().await = + Some(make_download(DownloadKind::Download, DownloadState::Downloading, None, None)); let result = requeue_active_download_for_retry(&queue, "id", false).await; assert!(result.is_err()); assert_eq!(queue.revision.load(std::sync::atomic::Ordering::SeqCst), 0); assert!(queue.queue.lock().await.is_empty()); - assert_eq!(queue.active.read().await.as_ref().map(|active| active.state.clone()), Some(DownloadState::Downloading)); + assert_eq!( + queue.active.read().await.as_ref().map(|active| active.state.clone()), + Some(DownloadState::Downloading) + ); } #[tokio::test] @@ -2516,18 +2453,16 @@ mod tests { #[tokio::test] async fn acquisition_without_provider_handle_commits_downloading_state() { let queue = DownloadQueue::new(); - *queue.active.write().await = Some(make_download( - DownloadKind::Download, - DownloadState::Queued, - None, - None, - )); + *queue.active.write().await = Some(make_download(DownloadKind::Download, DownloadState::Queued, None, None)); let notification = commit_acquired_download(&queue, "id").await.expect("acquired commit"); assert!(notification.is_some()); assert_eq!(queue.revision.load(std::sync::atomic::Ordering::SeqCst), 1); - assert_eq!(queue.active.read().await.as_ref().map(|active| active.state.clone()), Some(DownloadState::Downloading)); + assert_eq!( + queue.active.read().await.as_ref().map(|active| active.state.clone()), + Some(DownloadState::Downloading) + ); } #[tokio::test] @@ -2541,13 +2476,8 @@ mod tests { *queue.active.write().await = Some(switched); let transition = commit_acquired_download(&queue, "task-a").await.map(|notification| notification.is_some()); - let result = broadcast_required_worker_mutation( - &event_manager, - &queue, - transition, - "acquired downloading state", - ) - .await; + let result = + broadcast_required_worker_mutation(&event_manager, &queue, transition, "acquired downloading state").await; assert!(result.is_err()); assert_eq!(queue.revision.load(std::sync::atomic::Ordering::SeqCst), 0); @@ -2730,10 +2660,7 @@ mod tests { aliases: None, ..ConfigInput::default() }); - let sources = SourcesConfig { - inputs: vec![input], - ..SourcesConfig::default() - }; + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; AppConfig { config: Arc::new(ArcSwap::from_pointee(Config::default())), @@ -2823,9 +2750,7 @@ mod tests { }) } - fn create_test_app_state() -> Arc { - create_test_app_state_with_downloads(Arc::new(DownloadQueue::new())) - } + fn create_test_app_state() -> Arc { create_test_app_state_with_downloads(Arc::new(DownloadQueue::new())) } #[tokio::test] async fn pause_persist_failure_returns_error_without_event_or_memory_change() { @@ -2833,12 +2758,8 @@ mod tests { let blocking_dir = dir.path().join("state"); std::fs::create_dir_all(&blocking_dir).expect("create blocking dir"); let downloads = Arc::new(DownloadQueue::new_with_state_file(Some(blocking_dir))); - *downloads.active.write().await = Some(make_download( - DownloadKind::Download, - DownloadState::Downloading, - None, - None, - )); + *downloads.active.write().await = + Some(make_download(DownloadKind::Download, DownloadState::Downloading, None, None)); let app_state = create_test_app_state_with_downloads(Arc::clone(&downloads)); let mut events = app_state.event_manager.get_event_channel(); @@ -2942,12 +2863,7 @@ mod tests { #[tokio::test] async fn cancel_uuid_mismatch_does_not_finish_or_promote_next_task() { let queue = DownloadQueue::new(); - *queue.active.write().await = Some(make_download( - DownloadKind::Download, - DownloadState::Paused, - None, - None, - )); + *queue.active.write().await = Some(make_download(DownloadKind::Download, DownloadState::Paused, None, None)); let mut next = make_download(DownloadKind::Download, DownloadState::Queued, None, None); next.uuid = "next".to_string(); queue.queue.lock().await.push_back(next); @@ -2967,13 +2883,7 @@ mod tests { let mut events = event_manager.get_event_channel(); let queue = DownloadQueue::new(); - let changed = broadcast_worker_mutation( - &event_manager, - &queue, - Ok(false), - "test no-op mutation", - ) - .await; + let changed = broadcast_worker_mutation(&event_manager, &queue, Ok(false), "test no-op mutation").await; assert!(!changed.expect("no-op result")); assert!(events.try_recv().is_err()); @@ -3005,8 +2915,7 @@ mod tests { let mut events = event_manager.get_event_channel(); let queue = DownloadQueue::new(); - let result = - broadcast_required_worker_mutation(&event_manager, &queue, Ok(false), "terminal transition").await; + let result = broadcast_required_worker_mutation(&event_manager, &queue, Ok(false), "terminal transition").await; assert!(result.is_err()); assert!(events.try_recv().is_err()); @@ -3042,9 +2951,7 @@ mod tests { Duration::from_millis(100), pause_download( axum::extract::State(Arc::clone(&app_state)), - axum::extract::Json(DownloadActionRequest { - uuid: "handler-id".to_string(), - }), + axum::extract::Json(DownloadActionRequest { uuid: "handler-id".to_string() }), ), ) .await; @@ -3054,9 +2961,7 @@ mod tests { Duration::from_millis(100), resume_download( axum::extract::State(app_state), - axum::extract::Json(DownloadActionRequest { - uuid: "handler-id".to_string(), - }), + axum::extract::Json(DownloadActionRequest { uuid: "handler-id".to_string() }), ), ) .await; diff --git a/backend/src/api/endpoints/extract_accept_header.rs b/backend/app/src/api/endpoints/extract_accept_header.rs similarity index 100% rename from backend/src/api/endpoints/extract_accept_header.rs rename to backend/app/src/api/endpoints/extract_accept_header.rs diff --git a/backend/src/api/endpoints/hdhomerun_api.rs b/backend/app/src/api/endpoints/hdhomerun_api.rs similarity index 95% rename from backend/src/api/endpoints/hdhomerun_api.rs rename to backend/app/src/api/endpoints/hdhomerun_api.rs index de5042f43..cc7a45396 100644 --- a/backend/src/api/endpoints/hdhomerun_api.rs +++ b/backend/app/src/api/endpoints/hdhomerun_api.rs @@ -250,7 +250,13 @@ async fn lineup_status( let cfg = Arc::clone(&app_state.app_state.app_config); let network_access_denied = cfg.get_target_for_username(&app_state.device.t_username).is_none_or(|(credentials, _)| { - try_check_network_access_only(&credentials, &fingerprint, &app_state.app_state).is_err() + try_check_network_access_only( + &credentials, + &fingerprint, + &app_state.app_state.app_config, + &app_state.app_state.geoip, + ) + .is_err() }); let current_state = app_state.hd_scan_state.load(std::sync::atomic::Ordering::Acquire); if network_access_denied || current_state < 0 { @@ -336,7 +342,13 @@ async fn lineup_post( ) -> impl IntoResponse { let cfg = Arc::clone(&app_state.app_state.app_config); let allowed = cfg.get_target_for_username(&app_state.device.t_username).is_some_and(|(credentials, _)| { - try_check_network_access_only(&credentials, &fingerprint, &app_state.app_state).is_ok() + try_check_network_access_only( + &credentials, + &fingerprint, + &app_state.app_state.app_config, + &app_state.app_state.geoip, + ) + .is_ok() }); if !allowed { return axum::http::StatusCode::FORBIDDEN.into_response(); @@ -454,7 +466,12 @@ async fn auth_lineup_json( if !username.eq(&credentials.username) || !password.eq(&credentials.password) { return axum::http::StatusCode::UNAUTHORIZED.into_response(); } - if let Err(e) = try_check_network_access_only(&credentials, &fingerprint, &app_state.app_state) { + if let Err(e) = try_check_network_access_only( + &credentials, + &fingerprint, + &app_state.app_state.app_config, + &app_state.app_state.geoip, + ) { return e.into_player_response(app_state.app_state.app_config.get_auth_error_status()); } let user_credentials = Arc::clone(&credentials); @@ -469,7 +486,12 @@ async fn lineup_json( ) -> impl IntoResponse { let cfg = Arc::clone(&app_state.app_state.app_config); if let Some((credentials, target)) = cfg.get_target_for_username(&app_state.device.t_username) { - if let Err(e) = try_check_network_access_only(&credentials, &fingerprint, &app_state.app_state) { + if let Err(e) = try_check_network_access_only( + &credentials, + &fingerprint, + &app_state.app_state.app_config, + &app_state.app_state.geoip, + ) { return e.into_player_response(app_state.app_state.app_config.get_auth_error_status()); } let user_credentials = Arc::clone(&credentials); diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/app/src/api/endpoints/hls_api.rs similarity index 94% rename from backend/src/api/endpoints/hls_api.rs rename to backend/app/src/api/endpoints/hls_api.rs index 8f5dfb146..8c6fe17f8 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/app/src/api/endpoints/hls_api.rs @@ -24,63 +24,16 @@ use crate::{ ConnectFailedAttempt, EvictionReentryGuard, HeaderFilter, }, model::{ - begin_hls_origin_account_io_bounded, build_hls_origin_session_owner, build_proxy_session_id, - commit_hls_runtime_custom_tail, - cold_start_retry_after_seconds, - derive_hls_lease_manifest_snapshot, extract_hls_provider_session_headers, - fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, - fetch_hls_transient_origin_response_with_attempt_prepare, finite_hls_terminal_key_response, - force_identity_without_range, - hls_cached_manifest_options_for_requirement, hls_committed_manifest_body_for_request, - hls_custom_video_manifest_response_for_access_lease, - hls_custom_video_manifest_response_with_virtual_id, - hls_manifest_acceptance_directive_for_session, hls_manifest_commit_requirement, hls_object_body_deadline, - hls_origin_account_status, hls_provisioning_discontinuity_sequence, - hls_should_wait_for_initial_manifest_commit, hls_startup_admission_allows_snapshot, - hls_transient_object_fetch_failure, hls_transient_origin_response, hls_virtual_entry_redirect_response, - is_custom_video_stream_enabled, is_hls_provisioning_gap_segment, is_hls_provisioning_segment, - maybe_trigger_origin_refresh_with_outcome, new_hls_access_lease_id, origin_account_binding_from_allocation, - record_successful_transient_segment_fetch, record_temporary_transient_segment_fetch_failure, - register_hls_availability_reevaluation, - resolve_hls_transient_object_cache_action, safe_hls_access_lease_id, - safe_proxy_session_id, safe_session_key, safe_user_session_token, scrub_hls_origin_headers, - serve_hls_map_cache_outcome, - serve_hls_segment_cache_outcome, serve_hls_transient_object_cache_outcome, - serve_hls_transient_object_cache_response, should_remove_hls_origin_header, - start_hls_panel_provisioning_once, - try_hls_panel_provisioning_manifest_response, validate_hls_access_lease, AppState, CacheAccessState, - ConnectionHistoryMode, CustomVideoStreamType, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, - HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, - HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAccountBindingProtection, - HlsAccountOverlapTiming, HlsAvailabilityReevaluationObservation, - HlsAvailabilityReevaluationRegistration, HlsBoundAccountAcquireErrorKind, HlsCacheResponseContext, - HlsBandwidthPersistenceOutcome, HlsCachedManifestOptions, HlsCommittedManifestBody, - HlsEffectiveOriginAcquirePolicy, - GraceMode, - HlsLeaseManifestSnapshotInput, HlsLeasePlaybackMode, HlsManifestCommitRequirement, HlsMapFile, - HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, HlsMediaActivityCommitOutcome, - HlsMediaActivityMarker, HlsMediaLeaseIdentity, HlsOriginAccountBinding, HlsOriginAccountBindingMode, - HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, - HlsOriginRefreshTriggerOutcome, HlsOriginResourceClients, - HlsOriginResourceFetchError, HlsOriginSource, HlsOriginSourceKind, HlsOriginWorkClass, - HlsMasterBandwidth, HlsMasterBandwidthSelection, - HlsPanelProvisioningRedirectPaths, HlsPlaybackFamilyKey, HlsProvisioningStatus, HlsQosMeterInit, - HlsQosRuntimeConfig, HlsResourceFetchAttempt, HlsResourceServeFailure, HlsResourceServeOutcome, - HlsSegmentFile, HlsSession, - HlsSessionHandle, HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, HlsTerminalSegmentPath, - HlsRuntimeCustomTailOutcome, HlsRuntimeCustomTailReason, HlsRuntimeCustomTailRequest, - HlsTransientCacheCommitContext, - HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, HlsTransientObjectCacheAction, - HlsTransientObjectFetchFailure, HlsTransientObjectFetchFinalizer, HlsTransientOriginCacheFetchRequest, - HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, LiveHlsOriginEntry, OriginRefreshRequest, - OriginSegmentKey, ProviderAllocation, ProviderConfig as RuntimeProviderConfig, ProviderHandle, - ProxySessionId, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, SegmentDemandFetchOutcome, - SegmentFetchContext, SegmentFetchPolicy, StreamMeterHandle, - TransientObjectUnavailableState, - TransientResourceFile, TransientResourceRef, TransportStreamBuffer, UserSession, - HlsSingleVariantMasterPlaylist, trigger_origin_refresh_sync, - HLS_ACCESS_LEASE_ID_PLACEHOLDER, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, - HLS_PROVISIONING_SEGMENT_DURATION_MS, HLS_PROVISIONING_TARGET_DURATION_SECS, MAX_HLS_MANIFEST_BYTES, + hls_cache::initial_strip::{ + materialize_initial_hls_strip_view, HlsInitialStripOutcome, HlsInitialStripSkipReason, + }, + hls_custom_video_manifest_response_for_access_lease, hls_custom_video_manifest_response_with_virtual_id, + hls_provisioning_discontinuity_sequence, hls_virtual_entry_redirect_response, + is_custom_video_stream_enabled, start_hls_panel_provisioning_once, + try_hls_panel_provisioning_manifest_response, AppState, ConnectionHistoryMode, CustomVideoStreamType, + GraceMode, HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, ProviderAllocation, + ProviderConfig as RuntimeProviderConfig, ProviderHandle, StreamMeterHandle, TransportStreamBuffer, + UserSession, }, panel_api::can_provision_on_exhausted, }, @@ -89,13 +42,7 @@ use crate::{ ConfigInput, ConfigInputFlags, ConfigProvider, ConfigTarget, InputSource, ProxyUserCredentials, ReverseProxyDisabledHeaderConfig, }, - processing::parser::hls::{ - get_hls_session_token_and_url_from_token, - initial_strip::{ - materialize_initial_hls_strip_view, HlsInitialStripOutcome, HlsInitialStripSkipReason, - }, - rewrite_hls, RewriteHlsProps, - }, + processing::parser::hls::{get_hls_session_token_and_url_from_token, rewrite_hls, RewriteHlsProps}, repository::{ load_input_live_bitrate_bps, m3u_get_item_for_stream_id, persist_input_live_bitrate_bps, storage_const, xtream_get_item_for_stream_id, LiveBitratePersistenceOutcome, @@ -123,6 +70,53 @@ use shared::{ }, }; use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; +use tuliprox_hls::{ + api::{ + begin_hls_origin_account_io_bounded, build_hls_origin_session_owner, build_proxy_session_id, + cold_start_retry_after_seconds, commit_hls_runtime_custom_tail, derive_hls_lease_manifest_snapshot, + extract_hls_provider_session_headers, fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, finite_hls_terminal_key_response, + force_identity_without_range, hls_cached_manifest_options_for_requirement, + hls_committed_manifest_body_for_request, hls_manifest_acceptance_directive_for_session, + hls_manifest_commit_requirement, hls_object_body_deadline, hls_origin_account_status, + hls_should_wait_for_initial_manifest_commit, hls_startup_admission_allows_snapshot, + hls_transient_object_fetch_failure, hls_transient_origin_response, is_hls_provisioning_gap_segment, + is_hls_provisioning_segment, maybe_trigger_origin_refresh_with_outcome, new_hls_access_lease_id, + origin_account_binding_from_allocation, record_successful_transient_segment_fetch, + record_temporary_transient_segment_fetch_failure, register_hls_availability_reevaluation, + resolve_hls_transient_object_cache_action, safe_hls_access_lease_id, safe_proxy_session_id, safe_session_key, + safe_user_session_token, scrub_hls_origin_headers, serve_hls_map_cache_outcome, + serve_hls_segment_cache_outcome, serve_hls_transient_object_cache_outcome, + serve_hls_transient_object_cache_response, should_remove_hls_origin_header, trigger_origin_refresh_sync, + validate_hls_access_lease, CacheAccessState, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, + HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, + HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAccountBindingProtection, + HlsAccountOverlapTiming, HlsAvailabilityReevaluationObservation, HlsAvailabilityReevaluationRegistration, + HlsBandwidthPersistenceOutcome, HlsBoundAccountAcquireErrorKind, HlsCacheResponseContext, + HlsCachedManifestOptions, HlsCommittedManifestBody, HlsEffectiveOriginAcquirePolicy, + HlsLeaseManifestSnapshotInput, HlsLeasePlaybackMode, HlsLeaseStartupAdmissionState, HlsLogIdentity, + HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitRequirement, + HlsMapFile, HlsMasterBandwidth, HlsMasterBandwidthSelection, HlsMediaActivityCommitOutcome, + HlsMediaActivityMarker, HlsMediaLeaseIdentity, HlsOriginAccountBinding, HlsOriginAccountBindingMode, + HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, HlsOriginRefreshTriggerOutcome, + HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginSource, HlsOriginSourceKind, + HlsOriginWorkClass, HlsPlaybackFamilyKey, HlsPostRefreshRuntime, HlsQosMeterInit, HlsQosRuntimeConfig, + HlsResourceFetchAttempt, HlsResourceServeFailure, HlsResourceServeOutcome, HlsRuntimeCustomTailOutcome, + HlsRuntimeCustomTailReason, HlsRuntimeCustomTailRequest, HlsSegmentFile, HlsSession, HlsSessionHandle, + HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, HlsSingleVariantMasterPlaylist, + HlsTerminalFailedClosedReason, HlsTerminalSegmentPath, HlsTransientCacheCommitContext, + HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, HlsTransientObjectCacheAction, + HlsTransientObjectFetchFailure, HlsTransientObjectFetchFinalizer, HlsTransientOriginCacheFetchRequest, + HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, LiveHlsOriginEntry, OriginRefreshRequest, + OriginSegmentKey, ProxySessionId, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, SegmentDemandFetchOutcome, + SegmentEntry, SegmentFetchContext, SegmentFetchPolicy, TransientObjectFetchToken, + TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, TransientResourceId, + TransientResourceRef, HLS_ACCESS_LEASE_ID_PLACEHOLDER, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, + HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, HLS_PROVISIONING_TARGET_DURATION_SECS, + MAX_HLS_MANIFEST_BYTES, + }, + HlsCtx, MAX_MANUAL_REDIRECTS, +}; use url::Url; const HLS_TEMPORARY_RESOURCE_RETRY_AFTER_SECS: u64 = 1; @@ -131,8 +125,6 @@ const HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS: u64 = HLS_TEMPORARY_RESOURCE_RETRY_ /// reduce time-to-first-manifest at the cost of more wakeups per waiting client. const HLS_MANIFEST_WAIT_POLL_INTERVAL: Duration = Duration::from_millis(25); -use crate::api::model::MAX_MANUAL_REDIRECTS; - /// Recover archive EPG reference from `m3u-catchup|...|archive|{start}|{duration}` session keys. /// /// `BitTV` archive media URLs look like `2026/07/24/14/13/38-06800.ts` and lose Flussonic @@ -492,22 +484,9 @@ impl HlsInitialStripLeaseSkipReason { #[derive(Debug, Clone, Copy, Eq, PartialEq)] enum HlsInitialStripPublicationDiagnostic { - Applied { - mode: &'static str, - strip_mode: &'static str, - configured: u64, - effective: usize, - visible_segments: usize, - }, - Skipped { - mode: &'static str, - reason: HlsInitialStripSkipReason, - visible_segments: usize, - }, - SkippedForLeaseState { - mode: &'static str, - reason: HlsInitialStripLeaseSkipReason, - }, + Applied { mode: &'static str, strip_mode: &'static str, configured: u64, effective: usize, visible_segments: usize }, + Skipped { mode: &'static str, reason: HlsInitialStripSkipReason, visible_segments: usize }, + SkippedForLeaseState { mode: &'static str, reason: HlsInitialStripLeaseSkipReason }, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -526,18 +505,15 @@ fn hls_initial_strip_publication_diagnostic( HlsInitialStripPublicationStatus::Committed => {} } Some(match &materialized.initial_strip_outcome { - Some(HlsInitialStripOutcome::Applied { - mode: strip_mode, - configured, - effective, - visible_segments, - }) => HlsInitialStripPublicationDiagnostic::Applied { - mode: materialized.mode, - strip_mode, - configured: *configured, - effective: *effective, - visible_segments: *visible_segments, - }, + Some(HlsInitialStripOutcome::Applied { mode: strip_mode, configured, effective, visible_segments }) => { + HlsInitialStripPublicationDiagnostic::Applied { + mode: materialized.mode, + strip_mode, + configured: *configured, + effective: *effective, + visible_segments: *visible_segments, + } + } Some(HlsInitialStripOutcome::Skipped { reason, visible_segments }) => { HlsInitialStripPublicationDiagnostic::Skipped { mode: materialized.mode, @@ -562,13 +538,7 @@ fn log_hls_initial_strip_publication( diagnostic: HlsInitialStripPublicationDiagnostic, ) { match diagnostic { - HlsInitialStripPublicationDiagnostic::Applied { - mode, - strip_mode, - configured, - effective, - visible_segments, - } => { + HlsInitialStripPublicationDiagnostic::Applied { mode, strip_mode, configured, effective, visible_segments } => { debug!( "HLS initial strip applied: mode={} lease={} proxy_session={} strip_mode={} configured={} effective={} visible_segments={}", mode, @@ -716,10 +686,8 @@ async fn prepare_hls_resource_access( } fn hls_lease_allows_live_origin_work(lease: &HlsAccessLease) -> bool { - matches!( - lease.state, - HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle - ) && lease.playback_mode == HlsLeasePlaybackMode::Live + matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) + && lease.playback_mode == HlsLeasePlaybackMode::Live } fn hls_lease_allows_cached_segment(lease: &HlsAccessLease, proxy_seq: u64) -> bool { @@ -761,7 +729,8 @@ fn create_hls_cache_user_session_token( existing_session_token: Option<&str>, archive_reference: Option, ) -> String { - let base = hls_entry_user_session_token(fingerprint, username, virtual_id, existing_session_token, archive_reference); + let base = + hls_entry_user_session_token(fingerprint, username, virtual_id, existing_session_token, archive_reference); format!("{base}|hls-cache|{}", generate_random_string(16)) } @@ -779,7 +748,7 @@ async fn hls_cache_response_context( let qos_meter = app_state.hls_proxy.qos().meter_for_access_lease(&access_context.lease_id).await; let log_identity = { let session = session.read().await; - crate::api::model::HlsLogIdentity::from_session(&session) + HlsLogIdentity::from_session(&session) }; HlsCacheResponseContext::new( access_context.lease_id.clone(), @@ -853,14 +822,8 @@ async fn hls_proxy_segment( return StatusCode::NOT_FOUND.into_response(); } let allows_origin_work = hls_lease_allows_live_origin_work(&lease); - if let Err(response) = validate_hls_segment_entry( - &app_state, - &session, - &access_context, - &segment_file, - allows_origin_work, - ) - .await + if let Err(response) = + validate_hls_segment_entry(&app_state, &session, &access_context, &segment_file, allows_origin_work).await { return *response; } @@ -939,7 +902,9 @@ async fn demand_fetch_hls_live_segment( .await { Ok(handle) => handle, - Err(err) => return Err(Box::new(hls_origin_runtime_resource_failure_response(app_state, access_context, err))), + Err(err) => { + return Err(Box::new(hls_origin_runtime_resource_failure_response(app_state, access_context, err))) + } } } else { None @@ -1025,18 +990,17 @@ async fn hls_proxy_terminal_segment( let now_ms = current_time_millis(); let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id.clone()); let immutable_replay_plan = - app_state.hls_proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await - .and_then(|lease| match lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(plan) - if plan.matches_route(&proxy_session_id, &access_lease_id) => - { + app_state.hls_proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await.and_then( + |lease| match lease.playback_mode { + HlsLeasePlaybackMode::TerminalTail(plan) if plan.matches_route(&proxy_session_id, &access_lease_id) => { Some(plan) } HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, - }); + }, + ); let access = prepare_hls_resource_access( &app_state, &fingerprint, @@ -1061,12 +1025,9 @@ async fn hls_proxy_terminal_segment( let Some(current_lease) = current_hls_resource_lease(&app_state, &access_context).await else { return StatusCode::NOT_FOUND.into_response(); }; - let Some(plan) = terminal_tail_plan_for_current_route( - &lease, - ¤t_lease, - &proxy_session_id, - &access_context.lease_id, - ) else { + let Some(plan) = + terminal_tail_plan_for_current_route(&lease, ¤t_lease, &proxy_session_id, &access_context.lease_id) + else { return StatusCode::NOT_FOUND.into_response(); }; if method == Method::HEAD { @@ -1078,13 +1039,9 @@ async fn hls_proxy_terminal_segment( }; let response_context = hls_cache_response_context(&app_state, &session, &access_context, lease_identity, now_ms).await; - let Some(response) = terminal_segment_get_response( - &plan, - path, - headers.get(header::RANGE), - &response_context, - &proxy_session_id, - ) else { + let Some(response) = + terminal_segment_get_response(&plan, path, headers.get(header::RANGE), &response_context, &proxy_session_id) + else { return StatusCode::NOT_FOUND.into_response(); }; if is_hls_media_activity_status(response.status()) { @@ -1139,7 +1096,7 @@ async fn build_hls_segment_fetch_context( ) }; let mut origin_io = HlsOriginIoContext { - app_state: Arc::clone(app_state), + ctx: app_state.hls_ctx(), client_addr: fingerprint.addr, allow_grace: HlsOriginWorkClass::Demand.allows_grace(), priority: origin_policy.priority, @@ -1294,9 +1251,7 @@ async fn hls_proxy_resource( }; match &lease.playback_mode { HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await, - HlsLeasePlaybackMode::TerminalTail(_) => { - serve_hls_terminal_key_resource(context, &lease.playback_mode).await - } + HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await, HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { StatusCode::NOT_FOUND.into_response() } @@ -1323,11 +1278,9 @@ async fn serve_hls_terminal_key_resource( return StatusCode::NOT_FOUND.into_response(); }; let proxy_session_id = &context.access_context.proxy_session_id; - let Some(binding) = plan.terminal_key_binding( - proxy_session_id, - &context.access_context.lease_id, - &context.resource_file, - ) else { + let Some(binding) = + plan.terminal_key_binding(proxy_session_id, &context.access_context.lease_id, &context.resource_file) + else { return StatusCode::NOT_FOUND.into_response(); }; if !context.session.read().await.terminal_key_binding_is_current( @@ -1552,7 +1505,7 @@ async fn serve_hls_transient_passthrough_result( now_ms: endpoint.now_ms, log_identity: { let session = endpoint.session.read().await; - crate::api::model::HlsLogIdentity::from_session(&session) + HlsLogIdentity::from_session(&session) }, }, ) @@ -1620,7 +1573,7 @@ async fn fetch_transient_origin_response_with_provider_io( }; let log_identity = { let session = request.session.read().await; - crate::api::model::HlsLogIdentity::from_session(&session) + HlsLogIdentity::from_session(&session) }; let fetch_request = HlsTransientOriginFetchRequest { resolved_origin_uri: request.resource.resolved_origin_uri.clone(), @@ -1659,7 +1612,10 @@ fn hls_transient_origin_prepare_closure( runtime_prepare_error: &Arc>>, ) -> impl FnMut( HlsResourceFetchAttempt, -) -> futures::future::BoxFuture<'static, Result, HlsOriginResourceFetchError>> { +) -> futures::future::BoxFuture< + 'static, + Result, HlsOriginResourceFetchError>, +> { let app_state = Arc::clone(app_state); let session = Arc::clone(session); let access_context = access_context.clone(); @@ -1703,9 +1659,9 @@ struct HlsTransientEndpointCacheFetchContext<'a> { headers: &'a HeaderMap, access_context: &'a HlsAccessContext, lease_identity: HlsMediaLeaseIdentity, - resource: &'a crate::api::model::TransientResourceRef, + resource: &'a TransientResourceRef, resource_file: TransientResourceFile, - fetch_token: crate::api::model::TransientObjectFetchToken, + fetch_token: TransientObjectFetchToken, origin_headers: HeaderMap, origin_provider_session_headers: HeaderMap, range_header: Option, @@ -1827,7 +1783,7 @@ async fn wait_for_transient_object_cache_fetch(context: TransientObjectWaitConte response } -fn safe_transient_resource_id(resource_id: &crate::api::model::TransientResourceId) -> String { +fn safe_transient_resource_id(resource_id: &TransientResourceId) -> String { // Truncate at the first char boundary at or before byte 8 to avoid allocating // a temporary `String` of 8 chars (and a second UTF-8 walk via `len()`). let full = resource_id.0.as_str(); @@ -1865,7 +1821,7 @@ async fn validate_hls_proxy_access_request( .await .is_some_and(|lease| { lease.state == HlsAccessLeaseState::Pending - && lease.startup_admission == crate::api::model::HlsLeaseStartupAdmissionState::Pending + && lease.startup_admission == HlsLeaseStartupAdmissionState::Pending }); if startup_admission_pending { return Err(HlsAccessLeaseValidationError::AvailabilityPending); @@ -2128,7 +2084,7 @@ async fn validate_hls_proxy_access_context( admission_mode: HlsAccessAdmissionMode, ) -> Result { validate_hls_access_lease( - app_state, + &app_state.hls_ctx(), fingerprint, proxy_session_id, &HlsAccessLeaseId(hls_access_lease_id.to_string()), @@ -2159,14 +2115,7 @@ async fn hls_custom_video_manifest_response_for_lease( let Some(user) = app_state.app_config.get_user_credentials(&lease.username) else { return fallback_status.into_response(); }; - hls_custom_video_manifest_response_for_access_lease( - app_state, - &user, - video_type, - fallback_status, - lease, - ) - .await + hls_custom_video_manifest_response_for_access_lease(app_state, &user, video_type, fallback_status, lease).await } async fn hls_runtime_custom_tail_response( @@ -2178,7 +2127,7 @@ async fn hls_runtime_custom_tail_response( fallback_status: StatusCode, ) -> axum::response::Response { let outcome = commit_hls_runtime_custom_tail( - Arc::clone(app_state), + app_state.hls_ctx(), HlsRuntimeCustomTailRequest { session: Arc::clone(session), proxy_session_id: proxy_session_id.clone(), @@ -2188,10 +2137,7 @@ async fn hls_runtime_custom_tail_response( }, ) .await; - if matches!( - outcome, - HlsRuntimeCustomTailOutcome::Committed | HlsRuntimeCustomTailOutcome::AlreadyCommitted - ) { + if matches!(outcome, HlsRuntimeCustomTailOutcome::Committed | HlsRuntimeCustomTailOutcome::AlreadyCommitted) { let now_ms = current_time_millis(); if let Some(lease) = app_state.hls_proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await @@ -2216,27 +2162,20 @@ async fn hls_runtime_or_standalone_custom_tail_response( fallback_status: StatusCode, ) -> axum::response::Response { let now_ms = current_time_millis(); - let Some(lease) = app_state - .hls_proxy - .access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms) - .await + let Some(lease) = + app_state.hls_proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await else { return fallback_status.into_response(); }; match &lease.playback_mode { HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { - if let Some(response) = - hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) - { + if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { return response; } } HlsLeasePlaybackMode::Live if lease.last_manifest_snapshot.is_some() - && matches!( - lease.state, - HlsAccessLeaseState::Activated | HlsAccessLeaseState::PolicyRevoking - ) => + && matches!(lease.state, HlsAccessLeaseState::Activated | HlsAccessLeaseState::PolicyRevoking) => { return hls_runtime_custom_tail_response( app_state, @@ -2253,13 +2192,7 @@ async fn hls_runtime_or_standalone_custom_tail_response( } HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => {} } - hls_custom_video_manifest_response_for_lease( - app_state, - &lease, - reason.video_type(), - fallback_status, - ) - .await + hls_custom_video_manifest_response_for_lease(app_state, &lease, reason.video_type(), fallback_status).await } async fn hls_manifest_channel_unavailable_response_for_username( @@ -2285,11 +2218,7 @@ async fn hls_unpublished_lease_channel_unavailable_response( let reason = HlsRuntimeCustomTailReason::ChannelUnavailable; let lease = app_state .hls_proxy - .access_lease_response_snapshot( - access_lease_id, - proxy_session_id, - current_time_millis(), - ) + .access_lease_response_snapshot(access_lease_id, proxy_session_id, current_time_millis()) .await; if let Some(lease) = lease.filter(|lease| lease.permits_unpublished_standalone_tail(reason)) { return hls_custom_video_manifest_response_for_lease( @@ -2410,14 +2339,13 @@ async fn hls_manifest_access_lease_validation_response( fn hls_resource_access_lease_validation_response(err: &HlsAccessLeaseValidationError) -> axum::response::Response { match err { HlsAccessLeaseValidationError::AdmissionDenied { .. } - | HlsAccessLeaseValidationError::UserAccountExpired { .. } => { - StatusCode::FORBIDDEN.into_response() - } + | HlsAccessLeaseValidationError::UserAccountExpired { .. } => StatusCode::FORBIDDEN.into_response(), HlsAccessLeaseValidationError::AvailabilityPending => { hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) } - HlsAccessLeaseValidationError::UserSessionMissing { .. } - | HlsAccessLeaseValidationError::Expired => StatusCode::NOT_FOUND.into_response(), + HlsAccessLeaseValidationError::UserSessionMissing { .. } | HlsAccessLeaseValidationError::Expired => { + StatusCode::NOT_FOUND.into_response() + } } } @@ -2429,10 +2357,7 @@ async fn hls_manifest_access_context_and_state( access_lease_snapshot: Option<&HlsAccessLease>, now_ms: u64, ) -> Result<(HlsAccessContext, HlsAccessLeaseState), Box> { - app_state - .hls_proxy - .startup_observability() - .record_media_manifest_request(access_lease_id, now_ms); + app_state.hls_proxy.startup_observability().record_media_manifest_request(access_lease_id, now_ms); let access_context = match validate_hls_proxy_access_context( app_state, fingerprint, @@ -2451,13 +2376,8 @@ async fn hls_manifest_access_context_and_state( safe_proxy_session_id(proxy_session_id) ); return Err(Box::new( - hls_manifest_access_lease_validation_response( - app_state, - proxy_session_id, - access_lease_snapshot, - err, - ) - .await, + hls_manifest_access_lease_validation_response(app_state, proxy_session_id, access_lease_snapshot, err) + .await, )); } }; @@ -2518,7 +2438,7 @@ async fn hls_transient_object_unavailable_response( ) -> axum::response::Response { let state = { let session = session.read().await; - let key = crate::api::model::TransientPassthroughState::transient_object_key( + let key = TransientPassthroughState::transient_object_key( &session.proxy_session_id, &resource_file.resource_id, resource_file.extension.clone(), @@ -2556,7 +2476,7 @@ async fn fetch_and_cache_transient_origin_response( }; let log_identity = { let session = context.session.read().await; - crate::api::model::HlsLogIdentity::from_session(&session) + HlsLogIdentity::from_session(&session) }; let fetch_request = HlsTransientOriginFetchRequest { resolved_origin_uri: context.resource.resolved_origin_uri.clone(), @@ -2591,64 +2511,62 @@ async fn fetch_and_cache_transient_origin_response( context.headers, &runtime_prepare_error, ); - let final_failure = match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare( - cache_fetch_request, - prepare_attempt, - ) - .await - { - Ok(()) => { - let ready_at_ms = current_time_millis(); - let response_context = hls_cache_response_context( - context.app_state, - context.session, - context.access_context, - context.lease_identity, - ready_at_ms, - ) - .await; - let response = serve_hls_transient_object_cache_response( - Arc::clone(context.app_state.hls_proxy.segment_cache()), - Arc::clone(context.session), - context.resource_file.clone(), - context.range_header.clone(), - &response_context, - ) - .await; - if is_hls_media_activity_status(response.status()) { - register_hls_cache_stream_for_successful_media_response( + let final_failure = + match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt) + .await + { + Ok(()) => { + let ready_at_ms = current_time_millis(); + let response_context = hls_cache_response_context( context.app_state, - context.fingerprint, - context.headers, - context.access_context, context.session, + context.access_context, + context.lease_identity, + ready_at_ms, + ) + .await; + let response = serve_hls_transient_object_cache_response( + Arc::clone(context.app_state.hls_proxy.segment_cache()), + Arc::clone(context.session), + context.resource_file.clone(), + context.range_header.clone(), &response_context, ) .await; - } - record_successful_transient_segment_fetch(context.session, context.resource).await; - fetch_finalizer.complete(); - return response; - } - Err(err) => { - if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { - let runtime_prepare_error = *runtime_prepare_error.lock().await; - if let Some(runtime_err) = runtime_prepare_error { - context.session.write().await.fail_transient_object_retryable_if_current( - &context.fetch_token, - current_time_millis(), - HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, - ); - return hls_origin_runtime_resource_failure_response( + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( context.app_state, + context.fingerprint, + context.headers, context.access_context, - runtime_err, - ); + context.session, + &response_context, + ) + .await; } + record_successful_transient_segment_fetch(context.session, context.resource).await; + fetch_finalizer.complete(); + return response; } - hls_transient_object_fetch_failure(&err) - } - }; + Err(err) => { + if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { + let runtime_prepare_error = *runtime_prepare_error.lock().await; + if let Some(runtime_err) = runtime_prepare_error { + context.session.write().await.fail_transient_object_retryable_if_current( + &context.fetch_token, + current_time_millis(), + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + return hls_origin_runtime_resource_failure_response( + context.app_state, + context.access_context, + runtime_err, + ); + } + } + hls_transient_object_fetch_failure(&err) + } + }; let failed_at_ms = current_time_millis(); match final_failure { @@ -3387,7 +3305,7 @@ async fn prepare_hls_origin_runtime( } if let Some(binding) = existing_binding { if binding.is_active() { - match hls_origin_account_status(app_state, &binding) { + match hls_origin_account_status(&app_state.hls_ctx(), &binding) { stale_status @ (HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired) => { return rebind_hls_origin_account( app_state, @@ -3711,7 +3629,7 @@ async fn detach_unprotected_hls_origin_account_bindings(app_state: &Arc 0 { continue; } - if !matches!(hls_origin_account_status(app_state, binding), HlsOriginAccountStatus::Known) { + if !matches!(hls_origin_account_status(&app_state.hls_ctx(), binding), HlsOriginAccountStatus::Known) { continue; } let victim_policy = session_guard.effective_origin_acquire_policy_or_default(); @@ -4446,10 +4364,7 @@ async fn create_hls_cache_entry_master_playlist_response( match load_input_live_bitrate_bps(&app_state.app_config, input, &origin_source.stream_ref).await { Ok(known_bitrate_bps) => known_bitrate_bps, Err(err) => { - warn!( - "HLS entry live bitrate lookup failed; using fallback: input_id={} error={err}", - input.id - ); + warn!("HLS entry live bitrate lookup failed; using fallback: input_id={} error={err}", input.id); None } } @@ -4515,15 +4430,11 @@ async fn create_hls_cache_entry_master_playlist_response( safe_proxy_session_id(&proxy_session_id), safe_user_session_token(&session_token) ); - let response = hls_entry_master_playlist_response( - &proxy_session_id, - &access_lease_id, - bandwidth.bandwidth(), - server_path, - ); + let response = + hls_entry_master_playlist_response(&proxy_session_id, &access_lease_id, bandwidth.bandwidth(), server_path); app_state.hls_proxy.startup_observability().record_entry_master_response( access_lease_id.clone(), - crate::api::model::HlsLogIdentity::new(&session_key, &proxy_session_id), + HlsLogIdentity::new(&session_key, &proxy_session_id), current_time_millis(), ); debug!( @@ -4649,8 +4560,7 @@ const fn hls_canonical_owner_registration( HlsAvailabilityReevaluationRegistration::Scheduled => { HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::Scheduled) } - HlsAvailabilityReevaluationRegistration::AlreadyOwned - | HlsAvailabilityReevaluationRegistration::Superseded => { + HlsAvailabilityReevaluationRegistration::AlreadyOwned | HlsAvailabilityReevaluationRegistration::Superseded => { HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) } HlsAvailabilityReevaluationRegistration::CapacityExceeded => { @@ -4673,10 +4583,7 @@ enum HlsCanonicalOwnerResolution { Live(axum::response::Response), Terminal(axum::response::Response), Standalone(axum::response::Response), - FailedClosed { - reason: HlsCanonicalOwnerFailureReason, - response: axum::response::Response, - }, + FailedClosed { reason: HlsCanonicalOwnerFailureReason, response: axum::response::Response }, } impl HlsCanonicalOwnerResolution { @@ -4717,12 +4624,8 @@ impl HlsCanonicalOwnerFailureReason { fn response(self) -> axum::response::Response { let reason = match self { - Self::DeadlineElapsed => { - crate::api::model::HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed - } - Self::Superseded | Self::LeaseUnavailable => { - crate::api::model::HlsTerminalFailedClosedReason::LeaseStateUnavailable - } + Self::DeadlineElapsed => HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed, + Self::Superseded | Self::LeaseUnavailable => HlsTerminalFailedClosedReason::LeaseStateUnavailable, }; hls_terminal_failed_closed_response(reason) } @@ -4767,11 +4670,7 @@ fn hls_canonical_owner_lease_deadline_ms(lease: &HlsAccessLease) -> u64 { } } -fn hls_canonical_owner_request_deadline_ms( - lease: &HlsAccessLease, - wait_timeout: Duration, - now_ms: u64, -) -> u64 { +fn hls_canonical_owner_request_deadline_ms(lease: &HlsAccessLease, wait_timeout: Duration, now_ms: u64) -> u64 { let lease_deadline_ms = hls_canonical_owner_lease_deadline_ms(lease); if wait_timeout.is_zero() && lease.state == HlsAccessLeaseState::Pending { lease_deadline_ms @@ -4816,12 +4715,8 @@ async fn evaluate_hls_canonical_owner_handoff( return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); } - let current_session = context - .app_state - .hls_proxy - .sessions() - .get_by_proxy_session_id(context.proxy_session_id) - .await; + let current_session = + context.app_state.hls_proxy.sessions().get_by_proxy_session_id(context.proxy_session_id).await; if let Some(current_session) = current_session.as_ref() { let options = hls_cached_manifest_options_for_requirement( Duration::ZERO, @@ -4938,16 +4833,12 @@ async fn hls_direct_refresh_follow_up( warn!("HLS direct origin refresh state contended; scheduling current availability reevaluation"); } } - let Some(owner_key) = app_state.hls_proxy.availability_reevaluation_owner_key(session, proxy_session_id).await else { + let Some(owner_key) = app_state.hls_proxy.availability_reevaluation_owner_key(session, proxy_session_id).await + else { warn!("HLS direct origin refresh follow-up unavailable: reason=session_superseded"); return Some(hls_canonical_retry_after_response()); }; - match register_hls_availability_reevaluation( - Arc::clone(app_state), - Arc::clone(session), - owner_key, - refresh_request, - ) { + match register_hls_availability_reevaluation(app_state.hls_ctx(), Arc::clone(session), owner_key, refresh_request) { HlsAvailabilityReevaluationRegistration::Scheduled | HlsAvailabilityReevaluationRegistration::AlreadyOwned | HlsAvailabilityReevaluationRegistration::Superseded => None, @@ -4979,14 +4870,7 @@ async fn trigger_hls_canonical_manifest_refresh( match ordering { HlsManifestRefreshOrdering::Background => { let outcome = maybe_trigger_origin_refresh_with_outcome(refresh_request.clone()).await; - hls_direct_refresh_follow_up( - app_state, - session, - proxy_session_id, - refresh_request, - outcome, - ) - .await + hls_direct_refresh_follow_up(app_state, session, proxy_session_id, refresh_request, outcome).await } HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation => { // An already-owned refresh cannot be joined through this call, but @@ -4994,26 +4878,15 @@ async fn trigger_hls_canonical_manifest_refresh( // in-flight owner will still publish its eventual progress/failure. let _refresh_started = trigger_origin_refresh_sync(refresh_request).await; let now_ms = current_time_millis(); - let Some(lease) = app_state - .hls_proxy - .access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms) - .await + let Some(lease) = + app_state.hls_proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await else { - return Some(hls_terminal_failed_closed_response( - crate::api::model::HlsTerminalFailedClosedReason::LeaseStateUnavailable, - )); + return Some(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable)); }; - resolve_hls_terminal_manifest_state( - app_state, - session, - proxy_session_id, - access_lease_id, - lease, - now_ms, - ) - .await - .err() - .map(|response| *response) + resolve_hls_terminal_manifest_state(app_state, session, proxy_session_id, access_lease_id, lease, now_ms) + .await + .err() + .map(|response| *response) } } } @@ -5114,7 +4987,7 @@ async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( let session_token = create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, PlaylistItemType::LiveHls, None); let (connection_admission, _, _) = resolve_playback_request_admission( - app_state, + &app_state.admission_ctx(), user, fingerprint, PlaylistItemType::LiveHls, @@ -5389,14 +5262,16 @@ async fn hls_panel_provisioning_poll_response( ) .await } - HlsProvisioningStatus::ProviderExhausted => hls_custom_video_manifest_response_with_virtual_id( - app_state, - user, - CustomVideoStreamType::ProviderConnectionsExhausted, - StatusCode::SERVICE_UNAVAILABLE, - Some(virtual_id), - ) - .await, + HlsProvisioningStatus::ProviderExhausted => { + hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + Some(virtual_id), + ) + .await + } } } @@ -5465,12 +5340,12 @@ fn shared_hls_provisioning_segment_entry( content_length: u64, duration_ms: u64, now_ms: u64, -) -> crate::api::model::SegmentEntry { +) -> SegmentEntry { let origin_epoch = match plan.segment_kind { SharedHlsProvisioningLocalSegmentKind::Provisioning => HLS_PROVISIONING_ORIGIN_EPOCH, SharedHlsProvisioningLocalSegmentKind::Gap => HLS_PROVISIONING_GAP_ORIGIN_EPOCH, }; - crate::api::model::SegmentEntry { + SegmentEntry { origin_key: OriginSegmentKey { origin_epoch, effective_host_id: 0, @@ -5505,21 +5380,25 @@ async fn commit_shared_hls_provisioning_segments( for plan in plans { let video = provisioning_segments.get(plan.physical_index)?; let duration_ms = video.duration_ms().unwrap_or(HLS_PROVISIONING_SEGMENT_DURATION_MS); - let metadata = - match app_state.hls_proxy.segment_cache().write_bytes_and_commit(&plan.cache_key, video.as_bytes()).await { - Ok(metadata) => metadata, - Err(err) => { - let safe_proxy_session = { - let session_guard = session.read().await; - safe_proxy_session_id(&session_guard.proxy_session_id) - }; - warn!( + let metadata = match app_state + .hls_proxy + .segment_cache() + .write_bytes_and_commit(&plan.cache_key, video.as_bytes()) + .await + { + Ok(metadata) => metadata, + Err(err) => { + let safe_proxy_session = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + warn!( "HLS provisioning segment cache commit failed for shared manifest: proxy_session={} seq={} error={err}", safe_proxy_session, plan.proxy_seq ); - return None; - } - }; + return None; + } + }; committed.push((plan.clone(), metadata.size, duration_ms)); } Some(committed) @@ -5811,15 +5690,17 @@ async fn prepare_hls_canonical_manifest_origin_runtime( Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, }) => { - return Err(Box::new(hls_runtime_or_standalone_custom_tail_response( - app_state, - session, - path_proxy_session_id, - access_lease_id, - HlsRuntimeCustomTailReason::LowPriorityPreempted, - StatusCode::SERVICE_UNAVAILABLE, - ) - .await)); + return Err(Box::new( + hls_runtime_or_standalone_custom_tail_response( + app_state, + session, + path_proxy_session_id, + access_lease_id, + HlsRuntimeCustomTailReason::LowPriorityPreempted, + StatusCode::SERVICE_UNAVAILABLE, + ) + .await, + )); } Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, @@ -5988,12 +5869,9 @@ async fn try_hls_cache_canonical_manifest_response( .await; let manifest_commit_requirement = hls_manifest_commit_requirement(&session, session_outcome, handoff_previous_rendered_at_ms, now_ms).await; - let acceptance_evaluation = hls_manifest_acceptance_directive_for_session( - app_state, - &session, - path_proxy_session_id, - ) - .await; + let hls_ctx = app_state.hls_ctx(); + let acceptance_evaluation = + hls_manifest_acceptance_directive_for_session(&hls_ctx, &session, path_proxy_session_id).await; let (acceptance_directive, availability_reevaluation_owner_key) = match acceptance_evaluation { HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => (directive, None), HlsManifestAcceptanceEvaluationOutcome::StateContention { owner_key } => { @@ -6008,7 +5886,8 @@ async fn try_hls_cache_canonical_manifest_response( } }; let manifest_boundary_rendered_at_ms = handoff_previous_rendered_at_ms.unwrap_or(previous_manifest_rendered_at_ms); - let wait_timeout = hls_manifest_wait_timeout_for_requirement(app_state, &session, manifest_commit_requirement).await; + let wait_timeout = + hls_manifest_wait_timeout_for_requirement(app_state, &session, manifest_commit_requirement).await; let cached_manifest_options = hls_cached_manifest_options_for_requirement( wait_timeout, manifest_commit_requirement, @@ -6023,7 +5902,7 @@ async fn try_hls_cache_canonical_manifest_response( let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone(); let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle; let mut origin_io = HlsOriginIoContext { - app_state: Arc::clone(app_state), + ctx: hls_ctx.clone(), client_addr: fingerprint.addr, allow_grace: HlsOriginWorkClass::ManifestInteractive.allows_grace(), priority: origin_policy.priority, @@ -6065,9 +5944,7 @@ async fn try_hls_cache_canonical_manifest_response( disabled_headers: app_state.get_disabled_headers(), now_ms, origin_io: Some(origin_io), - post_refresh_runtime: Some(crate::api::model::HlsPostRefreshRuntime { - app_state: Arc::downgrade(app_state), - }), + post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }), }; let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused { refresh_ordering @@ -6075,10 +5952,7 @@ async fn try_hls_cache_canonical_manifest_response( HlsManifestRefreshOrdering::Background }; if let Some(owner_key) = availability_reevaluation_owner_key { - app_state - .connection_manager - .release_provider_handle(preacquired_provider_handle) - .await; + app_state.connection_manager.release_provider_handle(preacquired_provider_handle).await; touch_initial_manifest_access_lease_window( app_state, access_lease_id, @@ -6093,20 +5967,15 @@ async fn try_hls_cache_canonical_manifest_response( .access_lease_response_snapshot(access_lease_id, path_proxy_session_id, current_time_millis()) .await; let expected_lease_issued_at_ms = owner_wait_lease.as_ref().map(|lease| lease.issued_at_ms); - let request_deadline_ms = - owner_wait_lease.as_ref().map_or(now_ms, |lease| { - hls_canonical_owner_request_deadline_ms(lease, wait_timeout, now_ms) - }); + let request_deadline_ms = owner_wait_lease + .as_ref() + .map_or(now_ms, |lease| hls_canonical_owner_request_deadline_ms(lease, wait_timeout, now_ms)); let safe_session = { let session = session.read().await; safe_session_key(&session.key) }; - let registration = register_hls_availability_reevaluation( - Arc::clone(app_state), - Arc::clone(&session), - owner_key, - refresh_request, - ); + let registration = + register_hls_availability_reevaluation(hls_ctx, Arc::clone(&session), owner_key, refresh_request); return Some(match hls_canonical_owner_registration(registration) { HlsCanonicalOwnerRegistration::Join(registration) => { let strip = app_state.hls_proxy.strip(); @@ -6272,14 +6141,7 @@ async fn try_hls_cache_canonical_manifest_response( } } - Some( - hls_unpublished_lease_channel_unavailable_response( - app_state, - path_proxy_session_id, - access_lease_id, - ) - .await, - ) + Some(hls_unpublished_lease_channel_unavailable_response(app_state, path_proxy_session_id, access_lease_id).await) } fn hls_initial_manifest_decision_wait_timeout(app_state: &Arc) -> Duration { @@ -6368,14 +6230,11 @@ async fn hls_origin_binding_needs_reacquire(session: &HlsSessionHandle) -> bool session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached) } -fn hls_transient_origin_binding_requires_runtime_prepare( - app_state: &Arc, - binding: &HlsOriginAccountBinding, -) -> bool { +fn hls_transient_origin_binding_requires_runtime_prepare(hls_ctx: &HlsCtx, binding: &HlsOriginAccountBinding) -> bool { binding.is_detached() || (binding.is_active() && matches!( - hls_origin_account_status(app_state, binding), + hls_origin_account_status(hls_ctx, binding), HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired )) } @@ -6430,14 +6289,15 @@ async fn prepare_hls_transient_origin_io_for_authorized_resource_work( req_headers: &HeaderMap, now_ms: u64, ) -> Result, HlsOriginRuntimeAcquireError> { + let hls_ctx = app_state.hls_ctx(); let existing_binding = session.read().await.origin_account_binding.clone(); let origin_policy = hls_effective_origin_acquire_policy(session).await; let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(session).await; if let Some(binding) = existing_binding.as_ref().filter(|binding| binding.is_active()) { - match hls_origin_account_status(app_state, binding) { + match hls_origin_account_status(&hls_ctx, binding) { HlsOriginAccountStatus::Known => { let origin_io = HlsOriginIoContext { - app_state: Arc::clone(app_state), + ctx: hls_ctx.clone(), client_addr: fingerprint.addr, allow_grace: HlsOriginWorkClass::Demand.allows_grace(), priority: origin_policy.priority, @@ -6471,7 +6331,7 @@ async fn prepare_hls_transient_origin_io_for_authorized_resource_work( if !existing_binding .as_ref() - .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(app_state, binding)) + .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, binding)) { return Ok(None); } @@ -6509,7 +6369,7 @@ async fn prepare_hls_transient_origin_io_for_authorized_resource_work( return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; let origin_io = HlsOriginIoContext { - app_state: Arc::clone(app_state), + ctx: hls_ctx, client_addr: fingerprint.addr, allow_grace: HlsOriginWorkClass::Demand.allows_grace(), priority: origin_policy.priority, @@ -6619,18 +6479,9 @@ impl HlsCachedManifestViewContext<'_> { materialized: HlsMaterializedSharedManifest, strip_diagnostic: HlsInitialStripPublicationDiagnostic, ) -> axum::response::Response { - touch_pending_manifest_follow_up_window( - app_state, - session, - self.access_lease_id, - self.access_lease_state, - ) - .await; - drop(spawn_hls_runtime_bandwidth_persistence( - app_state, - session, - self.bandwidth_learning, - )); + touch_pending_manifest_follow_up_window(app_state, session, self.access_lease_id, self.access_lease_state) + .await; + drop(spawn_hls_runtime_bandwidth_persistence(app_state, session, self.bandwidth_learning)); mark_successful_canonical_manifest_activity(app_state, session, current_time_millis()).await; log_hls_initial_strip_publication(self.proxy_session_id, self.access_lease_id, strip_diagnostic); hls_response(materialized.body).into_response() @@ -6651,11 +6502,7 @@ fn spawn_hls_runtime_bandwidth_persistence( return None; }; let bitrate_bps = session_guard.begin_bandwidth_persistence(current_time_millis())?; - ( - bitrate_bps, - session_guard.proxy_session_id.clone(), - session_guard.origin_source.stream_ref.clone(), - ) + (bitrate_bps, session_guard.proxy_session_id.clone(), session_guard.origin_source.stream_ref.clone()) }; let app_config = Arc::clone(&app_state.app_config); let hls_proxy = Arc::clone(&app_state.hls_proxy); @@ -6663,9 +6510,7 @@ fn spawn_hls_runtime_bandwidth_persistence( Some(tokio::spawn(async move { let outcome = match persist_input_live_bitrate_bps(&app_config, &input, &stream_ref, bitrate_bps).await { - Ok(repository_outcome) => { - hls_bandwidth_persistence_outcome(repository_outcome, &proxy_session_id) - } + Ok(repository_outcome) => hls_bandwidth_persistence_outcome(repository_outcome, &proxy_session_id), Err(err) => { error!( "HLS runtime bandwidth persistence failed: proxy_session={} error={}", @@ -6681,10 +6526,7 @@ fn spawn_hls_runtime_bandwidth_persistence( if !Arc::ptr_eq(¤t_session, &session) { return; } - current_session - .write() - .await - .finish_bandwidth_persistence(bitrate_bps, outcome, current_time_millis()); + current_session.write().await.finish_bandwidth_persistence(bitrate_bps, outcome, current_time_millis()); })) } @@ -6801,7 +6643,8 @@ async fn try_hls_cached_manifest_response( }; if let Some((materialized, snapshot, delivered_at_ms)) = prepared { if access_lease_state == HlsAccessLeaseState::Pending - && !hls_startup_admission_allows_snapshot(app_state, session, &snapshot, delivered_at_ms).await + && !hls_startup_admission_allows_snapshot(&app_state.hls_ctx(), session, &snapshot, delivered_at_ms) + .await { if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { continue; @@ -6824,12 +6667,12 @@ async fn try_hls_cached_manifest_response( let published_at_ms = current_time_millis(); let first_startup_publication = app_state.hls_proxy.startup_observability().record_manifest_publication( - access_lease_id, - snapshot_generation, - admission_at_ms, - published_at_ms, - Arc::from(startup_snapshot.visible_proxy_seqs().collect::>()), - ); + access_lease_id, + snapshot_generation, + admission_at_ms, + published_at_ms, + Arc::from(startup_snapshot.visible_proxy_seqs().collect::>()), + ); if first_startup_publication && hls_access_manifest_uses_startup_view(access_lease_state) { app_state.hls_proxy.spawn_access_lease_repair_prewarm( Arc::clone(session), @@ -6844,11 +6687,9 @@ async fn try_hls_cached_manifest_response( } else { HlsInitialStripPublicationStatus::NotCommitted }; - let Some(strip_diagnostic) = hls_initial_strip_publication_diagnostic( - publication_status, - access_lease_state, - &materialized, - ) else { + let Some(strip_diagnostic) = + hls_initial_strip_publication_diagnostic(publication_status, access_lease_state, &materialized) + else { return Some(hls_cached_manifest_temporarily_unavailable()); }; return Some(view.finish(app_state, session, materialized, strip_diagnostic).await); @@ -7185,12 +7026,17 @@ async fn get_stream_channel( virtual_id: u32, ) -> Option { if target.has_output(TargetType::Xtream) { - if let Ok(pli) = xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await { + if let Ok(pli) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await + { return Some(pli.to_stream_channel(target.id)); } } let target_id = target.id; - m3u_get_item_for_stream_id(virtual_id, app_state, target).await.ok().map(|pli| pli.to_stream_channel(target_id)) + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .ok() + .map(|pli| pli.to_stream_channel(target_id)) } fn hls_stream_context_or_unavailable( @@ -7209,18 +7055,22 @@ pub(in crate::api) async fn resolve_hls_virtual_source_for_target( virtual_id: u32, ) -> Result { let (input_name, stream_context) = if target.has_output(TargetType::Xtream) { - if let Ok(item) = xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await { + if let Ok(item) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await + { let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; (Arc::clone(&item.input_name), stream_context) } else { - let item = - m3u_get_item_for_stream_id(virtual_id, app_state, target).await.map_err(|_| StatusCode::NOT_FOUND)?; + let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .map_err(|_| StatusCode::NOT_FOUND)?; let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; (Arc::clone(&item.input_name), stream_context) } } else { - let item = - m3u_get_item_for_stream_id(virtual_id, app_state, target).await.map_err(|_| StatusCode::NOT_FOUND)?; + let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .map_err(|_| StatusCode::NOT_FOUND)?; let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; (Arc::clone(&item.input_name), stream_context) }; @@ -7236,7 +7086,7 @@ async fn resolve_hls_origin_playlist_url( fallback_url: &str, ) -> Result { if input.input_type.is_xtream() && target.has_output(TargetType::Xtream) { - let pli = xtream_get_item_for_stream_id(virtual_id, app_state, target, None) + let pli = xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None) .await .map_err(|_| StatusCode::NOT_FOUND)?; let hls_extension = format!(".{HLS_EXT}"); @@ -7397,33 +7247,28 @@ async fn hls_manifest_preflight_refresh_ordering( now_ms: u64, ) -> Result> { match hls_manifest_terminal_preflight(session, lease, now_ms).await { - HlsManifestTerminalPreflight::ServeCommittedPlayback => Err(Box::new(hls_terminal_playback_response( - lease, - proxy_session_id, - access_lease_id, - ) - .unwrap_or_else(|| { - hls_terminal_failed_closed_response( - crate::api::model::HlsTerminalFailedClosedReason::RuntimeUnavailable, - ) - }))), + HlsManifestTerminalPreflight::ServeCommittedPlayback => { + Err(Box::new(hls_terminal_playback_response(lease, proxy_session_id, access_lease_id).unwrap_or_else( + || hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RuntimeUnavailable), + ))) + } HlsManifestTerminalPreflight::BootstrapPendingLease => Ok(HlsManifestRefreshOrdering::Background), HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation => { Ok(HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation) } - HlsManifestTerminalPreflight::EvaluateTerminal => { - resolve_hls_terminal_manifest_state( - app_state, - session, - proxy_session_id, - access_lease_id, - lease.clone(), - now_ms, - ) - .await - .map(|_| HlsManifestRefreshOrdering::Background) + HlsManifestTerminalPreflight::EvaluateTerminal => resolve_hls_terminal_manifest_state( + app_state, + session, + proxy_session_id, + access_lease_id, + lease.clone(), + now_ms, + ) + .await + .map(|_| HlsManifestRefreshOrdering::Background), + HlsManifestTerminalPreflight::FailClosed { reason } => { + Err(Box::new(hls_terminal_failed_closed_response(reason))) } - HlsManifestTerminalPreflight::FailClosed { reason } => Err(Box::new(hls_terminal_failed_closed_response(reason))), } } @@ -7446,8 +7291,7 @@ async fn hls_proxy_manifest( if let (false, Some(response)) = ( standalone_policy_response_required, hls_terminal_playback_response(lease, &proxy_session_id, &access_lease_id), - ) - { + ) { return response; } } @@ -7631,31 +7475,35 @@ async fn admit_recovered_archive_stream( stream_channel: StreamChannel, ) -> Result<(UserSession, StreamChannel, Option), Box> { if session.permission == UserConnectionPermission::Exhausted { - return Err(Box::new(hls_admission_failure_manifest_response( - app_state, - fingerprint, - user, - stream_channel, - session.provider.clone(), - req_headers, - ConnectFailureReason::UserConnectionsExhausted, - ) - .await)); + return Err(Box::new( + hls_admission_failure_manifest_response( + app_state, + fingerprint, + user, + stream_channel, + session.provider.clone(), + req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ) + .await, + )); } if app_state.active_provider.is_over_limit(&session.provider).await { - return Err(Box::new(hls_admission_failure_manifest_response( - app_state, - fingerprint, - user, - stream_channel, - session.provider.clone(), - req_headers, - ConnectFailureReason::ProviderConnectionsExhausted, - ) - .await)); + return Err(Box::new( + hls_admission_failure_manifest_response( + app_state, + fingerprint, + user, + stream_channel, + session.provider.clone(), + req_headers, + ConnectFailureReason::ProviderConnectionsExhausted, + ) + .await, + )); } let (connection_admission, grace_mode, _) = crate::api::api_utils::resolve_playback_request_admission( - app_state, + &app_state.admission_ctx(), user, fingerprint, PlaylistItemType::Catchup, @@ -7677,16 +7525,18 @@ async fn admit_recovered_archive_stream( || (connection_permission == UserConnectionPermission::GracePeriod && connection_kind.is_none()) { let provider = if session.provider.is_empty() { input.name.clone() } else { session.provider.clone() }; - return Err(Box::new(hls_admission_failure_manifest_response( - app_state, - fingerprint, - user, - stream_channel, - provider, - req_headers, - ConnectFailureReason::UserConnectionsExhausted, - ) - .await)); + return Err(Box::new( + hls_admission_failure_manifest_response( + app_state, + fingerprint, + user, + stream_channel, + provider, + req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ) + .await, + )); } Ok((session, stream_channel, grace_mode)) } @@ -7705,7 +7555,7 @@ async fn hls_api_stream_leaked_relative( relative_path: String, request_query: Option<&str>, ) -> axum::response::Response { - if let Err(e) = check_network_access_only(&user, &fingerprint, &app_state) { + if let Err(e) = check_network_access_only(&user, &fingerprint, &app_state.app_config, &app_state.geoip) { return e.into_player_response(app_state.app_config.get_auth_error_status()); } let Some(origin_url) = resolve_leaked_hls_relative_origin(&session_stream_url, &relative_path, request_query) @@ -7736,13 +7586,20 @@ async fn hls_api_stream_leaked_relative( stream_channel.epg_reference_ts = archive_reference; } } - let (session, stream_channel, grace_mode) = - match admit_recovered_archive_stream(&app_state, &fingerprint, &user, &req_headers, &input, session, stream_channel) - .await - { - Ok(admission) => admission, - Err(response) => return *response, - }; + let (session, stream_channel, grace_mode) = match admit_recovered_archive_stream( + &app_state, + &fingerprint, + &user, + &req_headers, + &input, + session, + stream_channel, + ) + .await + { + Ok(admission) => admission, + Err(response) => return *response, + }; force_provider_stream_response( &fingerprint, &app_state, @@ -7773,7 +7630,7 @@ async fn hls_api_stream_resolved( token: String, ) -> axum::response::Response { // Network access check only - permission check is done later with full stream info - if let Err(e) = check_network_access_only(&user, &fingerprint, &app_state) { + if let Err(e) = check_network_access_only(&user, &fingerprint, &app_state.app_config, &app_state.geoip) { return e.into_player_response(app_state.app_config.get_auth_error_status()); } let target_name = &target.name; @@ -7784,7 +7641,7 @@ async fn hls_api_stream_resolved( format!("Can't find input {} for target {target_name}, stream_id {virtual_id}, hls", input_id) ); - if user.permission_denied(&app_state) { + if user.permission_denied(&app_state.app_config) { let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, "", None, None).await; return hls_admission_failure_manifest_response( &app_state, @@ -7914,7 +7771,7 @@ async fn hls_api_stream_resolved( let (connection_admission, grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( - &app_state, + &app_state.admission_ctx(), &user, &fingerprint, if is_m3u_catchup_session_token(&session.token) || archive_reference.is_some() { @@ -8090,16 +7947,16 @@ pub fn hls_api_register() -> axum::Router> { #[cfg(test)] mod tests { - use super::super::hls_terminal_response::{ - hls_manifest_terminal_preflight, hls_terminal_endpoint_action, hls_terminal_failed_closed_response, - HlsManifestTerminalPreflight, HlsTerminalEndpointAction, - }; use super::{ + super::hls_terminal_response::{ + hls_manifest_terminal_preflight, hls_terminal_endpoint_action, hls_terminal_failed_closed_response, + HlsManifestTerminalPreflight, HlsTerminalEndpointAction, + }, build_hls_manifest_request_headers, extract_hls_provider_session_headers, hls_api_register, hls_availability_reevaluation_registration_failure_response, hls_canonical_owner_registration, - hls_temporary_resource_unavailable_response, m3u_archive_epg_reference_ts, HlsCanonicalOwnerRegistration, + hls_temporary_resource_unavailable_response, m3u_archive_epg_reference_ts, m3u_catchup_epg_reference_from_session_token, resolve_leaked_hls_relative_origin, - HlsCanonicalOwnerRegistrationFailure, HlsCanonicalOwnerRegistrationKind, + HlsCanonicalOwnerRegistration, HlsCanonicalOwnerRegistrationFailure, HlsCanonicalOwnerRegistrationKind, MAX_HLS_MANIFEST_BYTES, }; use crate::{ @@ -8108,38 +7965,33 @@ mod tests { build_terminal_tail_plan, build_transient_resource_id, commit_terminal_tail_if_lease_reserve_requires_cutover, finish_hls_origin_account_io, prepare_terminal_base_evidence, prepared_terminal_bundle_key, snapshot_terminal_media_asset, - ActiveProviderManager, ActiveUserManager, - AppState, CacheAccessState, CancelTokens, ConnectionKind, ConnectionManager, CreateUserSessionParams, - CustomVideoStreamType, EventManager, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, + trigger_origin_refresh_sync, ActiveProviderManager, ActiveUserManager, AppState, CacheAccessState, + CancelTokens, ConnectionKind, ConnectionManager, CreateUserSessionParams, CustomVideoStreamType, + EventManager, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsAccessLeaseTiming, - HlsAccessLeaseValidationError, HlsAvailabilityReevaluationFinishReason, - HlsAvailabilityReevaluationMode, HlsAvailabilityReevaluationRegistration, - HlsBandwidthPersistenceState, HlsEffectiveOriginAcquirePolicy, - HlsFreshManifestRequiredReason, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, - HlsLifecycleEvent, HlsLifecycleEventKey, HlsManifestCommitRequirement, HlsManifestDeliveryMode, + HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAvailabilityReevaluationFinishReason, + HlsAvailabilityReevaluationMode, HlsAvailabilityReevaluationRegistration, HlsBandwidthPersistenceState, + HlsEffectiveOriginAcquirePolicy, HlsFreshManifestRequiredReason, HlsLeaseManifestSegment, + HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, HlsLifecycleEvent, HlsLifecycleEventKey, HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, - HlsManifestAcceptanceExhaustionReason, HlsManifestAcceptanceTrigger, HlsManifestSourceRenderMarker, - HlsMapSignature, HlsMediaContainer, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, - HlsOriginAccountBinding, HlsOriginAccountBindingMode, + HlsManifestAcceptanceExhaustionReason, HlsManifestAcceptanceTrigger, HlsManifestCommitRequirement, + HlsManifestDeliveryMode, HlsManifestSourceRenderMarker, HlsMapSignature, HlsMediaContainer, + HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginIoContext, HlsOriginPathCondition, HlsOriginSource, HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsPreparedTerminalBundleState, HlsProxyManager, - HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, - HlsRecoveryWorkload, HlsSegmentFile, HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionMode, - HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, HlsSessionStoreOutcome, - HlsTerminalAssetIdentity, - HlsTerminalBaseMediaState, HlsTerminalBaseProtection, - HlsTerminalBaseSegmentAvailability, HlsTerminalMediaAsset, HlsTerminalMediaPreparationState, - HlsTerminalFailedClosedReason, HlsTerminalResolution, HlsTerminalTailBuildInput, - HlsTerminalSegmentPath, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, - HlsTransitionMarginMs, - ManualPlaylistUpdateRequest, MapCacheStatus, MapEntry, MetadataUpdateManager, OriginMapKey, - OriginSegmentFetchRef, OriginSegmentKey, PlaybackLifecycle, PlaylistStorageState, - ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, ProxyMapId, ProxySessionId, - RenderedManifest, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, SegmentFetchPriority, - SharedStreamManager, TransientObjectCacheKey, TransientObjectCacheStatus, TransientResourceId, - TransientResourceKind, TransientResourceRef, TransportStreamBuffer, UpdateGuard, UserSession, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - LiveHlsOriginEntry, OriginRefreshRequest, RetryPolicy, trigger_origin_refresh_sync, + HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsRuntimeCustomTailAssetIdentity, + HlsRuntimeCustomTailReason, HlsSegmentFile, HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionMode, + HlsSessionStoreOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, + HlsTerminalBaseSegmentAvailability, HlsTerminalFailedClosedReason, HlsTerminalMediaAsset, + HlsTerminalMediaPreparationState, HlsTerminalResolution, HlsTerminalSegmentPath, HlsTerminalTailBuildInput, + HlsTerminalTailCompatibility, HlsTerminalTailGeneration, HlsTerminalTailPlan, HlsTerminalTailProtection, + HlsTransitionMarginMs, LiveHlsOriginEntry, ManualPlaylistUpdateRequest, MapCacheStatus, MapEntry, + MetadataUpdateManager, OriginMapKey, OriginRefreshRequest, OriginSegmentFetchRef, OriginSegmentKey, + PlaybackLifecycle, PlaylistStorageState, ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, + ProxyMapId, ProxySessionId, RenderedManifest, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, + SegmentEntry, SegmentFetchPriority, SharedStreamManager, TransientObjectCacheKey, + TransientObjectCacheStatus, TransientResourceId, TransientResourceKind, TransientResourceRef, + TransportStreamBuffer, UpdateGuard, UserSession, HLS_TERMINAL_TAIL_SEGMENT_COUNT, }, auth::Fingerprint, model::{ @@ -8151,7 +8003,7 @@ mod tests { origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, rewrite_hls, RewriteHlsProps, }, - utils::GeoIp, + repository::GeoIp, }; use aes::{ cipher::{Block, BlockEncrypt, KeyInit}, @@ -8168,12 +8020,11 @@ mod tests { use shared::{ model::{ provider_saturation::build_group_lookup, ConfigPaths, ConfigProviderDto, ConfigTargetDto, - ConfigTargetOptions, ConfigTargetShareLiveStreams, HlsCacheConfigDto, - HlsManifestRecoveryBurstConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, - HlsStripConfigDto, HlsStripMode, InputType, M3uPlaylistItem, M3uTargetOutputDto, PlaylistItem, - PlaylistItemHeader, PlaylistItemType, ProviderUrlSelectionPolicy, ReverseProxyConfigDto, - StreamConfigDto, StreamProperties, TargetOutputDto, UserConnectionPermission, XtreamCluster, - XtreamTargetOutputDto, + ConfigTargetOptions, ConfigTargetShareLiveStreams, HlsCacheConfigDto, HlsManifestRecoveryBurstConfigDto, + HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripConfigDto, HlsStripMode, InputType, + M3uPlaylistItem, M3uTargetOutputDto, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + ProviderUrlSelectionPolicy, ReverseProxyConfigDto, StreamConfigDto, StreamProperties, TargetOutputDto, + UserConnectionPermission, XtreamCluster, XtreamTargetOutputDto, }, utils::Internable, }; @@ -8195,9 +8046,7 @@ mod tests { use tower::ServiceExt; fn path_has_extension(path: &str, extension: &str) -> bool { - std::path::Path::new(path) - .extension() - .is_some_and(|actual| actual.eq_ignore_ascii_case(extension)) + std::path::Path::new(path).extension().is_some_and(|actual| actual.eq_ignore_ascii_case(extension)) } #[test] @@ -8454,8 +8303,7 @@ mod tests { task_release.notified().await; publish_owner_handoff_test_manifest(&task_session).await; task_app_state.hls_proxy.notify_session_evidence_changed(&task_proxy_session_id); - let _ = ownership - .finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); + let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); task_completed.notify_one(); }, ), @@ -8506,8 +8354,7 @@ mod tests { task_release.notified().await; publish_owner_handoff_test_manifest(&task_session).await; task_app_state.hls_proxy.notify_session_evidence_changed(&task_proxy_session_id); - let _ = ownership - .finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); + let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); task_completed.notify_one(); }, ), @@ -8516,13 +8363,9 @@ mod tests { let duplicate_owner_runs = Arc::new(AtomicUsize::new(0)); let task_duplicate_owner_runs = Arc::clone(&duplicate_owner_runs); assert_eq!( - coordinator.register( - owner_key, - HlsAvailabilityReevaluationMode::RecoveryPressure, - move |_| async move { - task_duplicate_owner_runs.fetch_add(1, Ordering::SeqCst); - }, - ), + coordinator.register(owner_key, HlsAvailabilityReevaluationMode::RecoveryPressure, move |_| async move { + task_duplicate_owner_runs.fetch_add(1, Ordering::SeqCst); + },), HlsAvailabilityReevaluationRegistration::AlreadyOwned ); let deadline_ms = super::current_time_millis().saturating_add(60_000); @@ -8568,11 +8411,7 @@ mod tests { .await .expect("pending deadline lease"); assert_eq!( - super::hls_canonical_owner_request_deadline_ms( - &lease, - Duration::ZERO, - super::current_time_millis(), - ), + super::hls_canonical_owner_request_deadline_ms(&lease, Duration::ZERO, super::current_time_millis(),), lease.pending_deadline_ms().expect("pending lease deadline") ); let owner_key = fixture @@ -8593,11 +8432,7 @@ mod tests { HlsAvailabilityReevaluationRegistration::Scheduled ); let response = super::join_hls_canonical_manifest_owner( - fixture.handoff_context( - 0, - fixture.safe_session().await, - super::current_time_millis().saturating_sub(1), - ), + fixture.handoff_context(0, fixture.safe_session().await, super::current_time_millis().saturating_sub(1)), HlsCanonicalOwnerRegistrationKind::Scheduled, ) .await; @@ -8651,12 +8486,8 @@ mod tests { 60_000, )) .await; - let session = app_state - .hls_proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id) - .await - .expect("shared session"); + let session = + app_state.hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("shared session"); let owner_key = app_state .hls_proxy .availability_reevaluation_owner_key(&session, &proxy_session_id) @@ -8672,8 +8503,7 @@ mod tests { HlsAvailabilityReevaluationMode::RecoveryPressure, move |ownership| async move { task_release.notified().await; - let _ = ownership - .finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); + let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); }, ), HlsAvailabilityReevaluationRegistration::Scheduled @@ -8803,9 +8633,7 @@ mod tests { lease.state = HlsAccessLeaseState::Activated; assert_eq!( hls_manifest_terminal_preflight(&session, &lease, 2_000).await, - HlsManifestTerminalPreflight::FailClosed { - reason: HlsTerminalFailedClosedReason::LeaseStateUnavailable, - } + HlsManifestTerminalPreflight::FailClosed { reason: HlsTerminalFailedClosedReason::LeaseStateUnavailable } ); lease.last_manifest_snapshot = Some(HlsLeaseManifestSnapshot { @@ -8867,11 +8695,10 @@ mod tests { for segment in session.segments.values_mut() { segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; } - session.segments.get_mut(&1).expect("deferred segment").status = - SegmentCacheStatus::CapacityDeferred { - priority: SegmentFetchPriority::Prefetch, - deferred_at_ms: 2_000, - }; + session.segments.get_mut(&1).expect("deferred segment").status = SegmentCacheStatus::CapacityDeferred { + priority: SegmentFetchPriority::Prefetch, + deferred_at_ms: 2_000, + }; session.origin_control.target_duration_snapshot_ms = Some(8_000); session.origin_control.last_media_progress_at_ms = Some(1_000); } @@ -8932,8 +8759,7 @@ mod tests { assert_eq!(pending.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), Some("1")); assert!(pending.headers().get(header::LOCATION).is_none()); - let failed_closed = - hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RetryAttemptsExhausted); + let failed_closed = hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RetryAttemptsExhausted); assert_eq!(failed_closed.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(failed_closed.headers().get(header::RETRY_AFTER).is_none()); assert!(failed_closed.headers().get(header::LOCATION).is_none()); @@ -9132,12 +8958,11 @@ mod tests { | TestLiveBitrateRepositoryState::RepositoryIoError => unreachable!(), }; let mut item = test_m3u_hls_item(input, 12345, stream_ref, "http://origin.test/live.m3u8"); - item.additional_properties = Some(StreamProperties::Live(Box::new( - shared::model::LiveStreamProperties { + item.additional_properties = + Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { bitrate: stored_bitrate, ..Default::default() - }, - ))); + }))); let mut tree = crate::repository::BPlusTree::new(); tree.insert(Arc::clone(&item.provider_id), item); tree.store(&database_path).expect("input tree"); @@ -9168,15 +8993,11 @@ mod tests { let mut session_guard = session.write().await; session_guard.apply_origin_manifest(&manifest).expect("runtime learning timeline"); for entry in session_guard.segments.values_mut() { - entry.status = SegmentCacheStatus::Ready { - content_length: 1_000_000, - ready_at_ms: manifest_rendered_at_ms, - }; + entry.status = + SegmentCacheStatus::Ready { content_length: 1_000_000, ready_at_ms: manifest_rendered_at_ms }; } session_guard.advance_media_readiness_generation(); - session_guard - .render_and_store_manifest(manifest_rendered_at_ms) - .expect("runtime learning manifest"); + session_guard.render_and_store_manifest(manifest_rendered_at_ms).expect("runtime learning manifest"); session_guard.mark_authorized_media_access(manifest_rendered_at_ms); } session @@ -9242,10 +9063,8 @@ mod tests { let bandwidth_persistence = tokio::time::timeout(Duration::from_secs(10), async { loop { let state = session.read().await.bandwidth_persistence; - if !matches!( - state, - HlsBandwidthPersistenceState::Idle | HlsBandwidthPersistenceState::InFlight { .. } - ) { + if !matches!(state, HlsBandwidthPersistenceState::Idle | HlsBandwidthPersistenceState::InFlight { .. }) + { break state; } tokio::task::yield_now().await; @@ -9268,16 +9087,14 @@ mod tests { #[tokio::test] async fn hls_runtime_bandwidth_missing_database_retries_without_failing_media_manifest() { - let (state, _) = - hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingDatabase).await; + let (state, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingDatabase).await; assert!(matches!(state, HlsBandwidthPersistenceState::RetryAfter { .. })); } #[tokio::test] async fn hls_runtime_bandwidth_missing_item_retries_without_failing_media_manifest() { - let (state, _) = - hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingStreamItem).await; + let (state, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingStreamItem).await; assert!(matches!(state, HlsBandwidthPersistenceState::RetryAfter { .. })); } @@ -9328,10 +9145,8 @@ mod tests { input_type: InputType::M3u, ..ConfigInput::default() }; - let storage_path = crate::repository::build_input_storage_path( - &input.name, - temp.path().to_string_lossy().as_ref(), - ); + let storage_path = + crate::repository::build_input_storage_path(&input.name, temp.path().to_string_lossy().as_ref()); std::fs::create_dir_all(&storage_path).expect("input storage"); let item = test_m3u_hls_item(&input, 12345, "channel-a", "http://origin.test/live.m3u8"); let mut tree = crate::repository::BPlusTree::new(); @@ -9541,16 +9356,10 @@ mod tests { RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES, ) .await; - let uri = format!( - "/hls/shared/live/{}/{}/000002.ts", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + let uri = format!("/hls/shared/live/{}/{}/000002.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; assert_eq!(segment.status(), StatusCode::OK); - assert_eq!( - segment.headers()[header::CONTENT_LENGTH], - RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES.to_string() - ); + assert_eq!(segment.headers()[header::CONTENT_LENGTH], RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES.to_string()); assert_eq!( u64::try_from(response_body(segment).await.len()).unwrap_or(u64::MAX), RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES @@ -9561,7 +9370,10 @@ mod tests { let temp_dir = tempfile::tempdir().expect("recovery cache tempdir"); let unchanged_manifest = Arc::<[u8]>::from(regression_origin_manifest(100, 6)); let progressed_manifest = Arc::<[u8]>::from(regression_origin_manifest(101, 6)); - let segment = Arc::<[u8]>::from(include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")).as_slice()); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); let origin_phase = Arc::new(AtomicUsize::new(0)); let phase = Arc::clone(&origin_phase); let origin = spawn_test_binary_origin(Arc::new(move |path| { @@ -9587,10 +9399,8 @@ mod tests { enabled: true, ..ConfigInput::default() }; - let app_state = test_app_state_with_hls_proxy_and_inputs( - test_beast_hls_proxy(temp_dir.path()), - vec![Arc::new(input)], - ); + let app_state = + test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input)]); enable_hls_cache(&app_state); create_active_hls_user_session(&app_state).await; let manifest_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); @@ -9656,10 +9466,7 @@ mod tests { fixture.origin.manifest_request_count().saturating_sub(requests_before) >= plan.total_candidates().saturating_mul(3) ); - let uri = format!( - "/hls/shared/live/{}/{}/000003.ts", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await; assert_eq!(cached.status(), StatusCode::OK); assert!(!response_body(cached).await.is_empty()); @@ -9702,21 +9509,14 @@ mod tests { assert!(body.contains("/000006.ts")); assert!(!body.contains("/terminal/")); assert!(!body.contains("#EXT-X-ENDLIST")); - let uri = format!( - "/hls/shared/live/{}/{}/000006.ts", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + let uri = format!("/hls/shared/live/{}/{}/000006.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; assert_eq!(segment.status(), StatusCode::OK); assert!(!response_body(segment).await.is_empty()); let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("recovered lease remains stored"); assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); @@ -9742,7 +9542,10 @@ mod tests { let pinned_manifest = Arc::<[u8]>::from(regression_origin_manifest(100, 6)); let alternative_manifest = Arc::<[u8]>::from(regression_origin_manifest(200, 6)); let continued_manifest = Arc::<[u8]>::from(regression_origin_manifest(201, 6)); - let segment = Arc::<[u8]>::from(include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")).as_slice()); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); let alternative_phase = Arc::new(AtomicUsize::new(0)); let alternative_phase_for_handler = Arc::clone(&alternative_phase); let alternative_segment = Arc::clone(&segment); @@ -9758,10 +9561,8 @@ mod tests { TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&alternative_segment)) })) .await; - let alternative_url = format!( - "{}/live/user/pass/12345.m3u8", - alternative.base_url.replacen("127.0.0.1", "localhost", 1) - ); + let alternative_url = + format!("{}/live/user/pass/12345.m3u8", alternative.base_url.replacen("127.0.0.1", "localhost", 1)); let pinned_phase = Arc::new(AtomicUsize::new(0)); let burst_candidates = Arc::new(AtomicUsize::new(0)); let pinned_candidates = Arc::new(AtomicUsize::new(0)); @@ -9789,14 +9590,7 @@ mod tests { } })) .await; - StaleOriginServers { - pinned, - alternative, - pinned_phase, - alternative_phase, - burst_candidates, - pinned_candidates, - } + StaleOriginServers { pinned, alternative, pinned_phase, alternative_phase, burst_candidates, pinned_candidates } } struct StaleOriginFixture { @@ -9825,10 +9619,8 @@ mod tests { enabled: true, ..ConfigInput::default() }; - let app_state = test_app_state_with_hls_proxy_and_inputs( - test_beast_hls_proxy(temp_dir.path()), - vec![Arc::new(input)], - ); + let app_state = + test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input)]); enable_hls_cache(&app_state); create_active_hls_user_session(&app_state).await; let manifest_url = format!("{}/live/user/pass/12345.m3u8", servers.pinned.base_url); @@ -9935,10 +9727,8 @@ mod tests { assert!(!body.contains("#EXT-X-ENDLIST")); { let mut session = fixture.session.write().await; - for segment in session - .segments - .values_mut() - .filter(|segment| segment.proxy_seq > fixture.initial_visible_tail) + for segment in + session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.initial_visible_tail) { segment.duration_ms = 10_000; } @@ -9946,7 +9736,7 @@ mod tests { session.advance_media_readiness_generation(); } let directive = match crate::api::model::hls_manifest_acceptance_directive_for_session( - &fixture.app_state, + &fixture.app_state.hls_ctx(), &fixture.session, &fixture.proxy_session_id, ) @@ -9960,11 +9750,7 @@ mod tests { let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("stale progress evidence keeps the lease stored"); assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); @@ -10017,21 +9803,14 @@ mod tests { assert!(body.contains("#EXT-X-DISCONTINUITY")); assert!(!body.contains("/terminal/")); assert!(!body.contains("#EXT-X-ENDLIST")); - let uri = format!( - "/hls/shared/live/{}/{}/000006.ts", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + let uri = format!("/hls/shared/live/{}/{}/000006.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; assert_eq!(segment.status(), StatusCode::OK); assert!(!response_body(segment).await.is_empty()); let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("recovered lease remains stored"); assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); @@ -10068,10 +9847,7 @@ mod tests { assert!(body.contains("/000012.ts")); assert!(!body.contains("/terminal/")); assert!(!body.contains("#EXT-X-ENDLIST")); - let uri = format!( - "/hls/shared/live/{}/{}/000012.ts", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + let uri = format!("/hls/shared/live/{}/{}/000012.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; assert_eq!(segment.status(), StatusCode::OK); assert!(!response_body(segment).await.is_empty()); @@ -10134,18 +9910,12 @@ mod tests { let live_tail_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/000123.ts"); let live_tail_response = get_response(Arc::clone(&app_state), &live_tail_uri, Some("bytes=0-")).await; assert_eq!(live_tail_response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!( - live_tail_response.headers()[header::CONTENT_LENGTH], - LIVE_TAIL_BYTES.len().to_string() - ); + assert_eq!(live_tail_response.headers()[header::CONTENT_LENGTH], LIVE_TAIL_BYTES.len().to_string()); assert_eq!( live_tail_response.headers()[header::CONTENT_RANGE], format!("bytes 0-{}/{}", LIVE_TAIL_BYTES.len() - 1, LIVE_TAIL_BYTES.len()) ); - assert_eq!( - response_body(live_tail_response).await, - bytes::Bytes::from_static(LIVE_TAIL_BYTES) - ); + assert_eq!(response_body(live_tail_response).await, bytes::Bytes::from_static(LIVE_TAIL_BYTES)); let cursor_after = app_state .hls_proxy .access_lease_response_snapshot( @@ -10185,11 +9955,8 @@ mod tests { .cloned() .expect("configured terminal renderer"); let asset = snapshot_terminal_media_asset(&asset_buffer).expect("terminal asset snapshot"); - let key = prepared_terminal_bundle_key( - &asset, - base_manifest.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); + let key = + prepared_terminal_bundle_key(&asset, base_manifest.target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); let state = app_state.hls_proxy.start_prepared_terminal_bundle( Arc::clone(&asset), base_manifest.target_duration_ms, @@ -10209,10 +9976,7 @@ mod tests { if bundle.key == key && bundle.segments.len() == usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) )); - assert_eq!( - asset_buffer.finite_hls_render_count(), - usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) - ); + assert_eq!(asset_buffer.finite_hls_render_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); (asset_buffer, asset) } @@ -10318,10 +10082,8 @@ mod tests { session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1); let commit_guard_seq = transition_buffer_seq.saturating_add(1); - for segment in session - .segments - .values_mut() - .filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) + for segment in + session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) { if segment.proxy_seq == transition_buffer_seq { segment.duration_ms = 1_000; @@ -10339,7 +10101,7 @@ mod tests { ) -> HlsManifestAcceptanceDirective { apply_terminal_cutover_pressure(fixture).await; let directive = match crate::api::model::hls_manifest_acceptance_directive_for_session( - &fixture.app_state, + &fixture.app_state.hls_ctx(), &fixture.session, &fixture.proxy_session_id, ) @@ -10356,10 +10118,7 @@ mod tests { ); let timing = directive.timing_seed.expect("prepared acceptance timing seed"); assert_eq!(timing.required_terminal_media_key, Some(bundle_key)); - assert_eq!( - timing.terminal_media_preparation, - HlsTerminalMediaPreparationState::Ready { key: bundle_key } - ); + assert_eq!(timing.terminal_media_preparation, HlsTerminalMediaPreparationState::Ready { key: bundle_key }); directive } @@ -10400,11 +10159,7 @@ mod tests { let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("pressured live lease"); assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); @@ -10445,7 +10200,7 @@ mod tests { .expect("measured lease playback start") .saturating_add(fixture.base_manifest.playlist_duration_ms); let first = commit_terminal_tail_if_lease_reserve_requires_cutover( - &fixture.app_state, + &fixture.app_state.hls_ctx(), &fixture.session, &fixture.proxy_session_id, pressured_lease, @@ -10456,15 +10211,11 @@ mod tests { let terminal_lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("terminal lease remains stored"); let second = commit_terminal_tail_if_lease_reserve_requires_cutover( - &fixture.app_state, + &fixture.app_state.hls_ctx(), &fixture.session, &fixture.proxy_session_id, &terminal_lease, @@ -10479,14 +10230,9 @@ mod tests { (plan.generation.0, cutover_now_ms) } - async fn assert_prepared_terminal_cutover_manifest( - fixture: &PreparedTerminalCutoverFixture, - generation: u64, - ) { - let manifest_uri = format!( - "/hls/shared/live/{}/{}/manifest.m3u8", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + async fn assert_prepared_terminal_cutover_manifest(fixture: &PreparedTerminalCutoverFixture, generation: u64) { + let manifest_uri = + format!("/hls/shared/live/{}/{}/manifest.m3u8", fixture.proxy_session_id.0, fixture.lease_id.0); let response = get_response(Arc::clone(&fixture.app_state), &manifest_uri, None).await; assert_eq!(response.status(), StatusCode::OK); assert!(!response.headers().contains_key(header::LOCATION)); @@ -10495,10 +10241,7 @@ mod tests { "/{}/{}/{:06}.ts", fixture.proxy_session_id.0, fixture.lease_id.0, fixture.base_manifest.last_proxy_seq ); - let terminal_prefix = format!( - "/{}/{}/terminal/{generation}/", - fixture.proxy_session_id.0, fixture.lease_id.0 - ); + let terminal_prefix = format!("/{}/{}/terminal/{generation}/", fixture.proxy_session_id.0, fixture.lease_id.0); assert!(body.contains(&live_tail_path)); assert_eq!(body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); assert!(body.find(&live_tail_path) < body.find("#EXT-X-DISCONTINUITY\n")); @@ -10535,20 +10278,15 @@ mod tests { ) { { let mut session = fixture.session.write().await; - let recovered_manifest = normal_manifest( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:83\n#EXT-X-TARGETDURATION:12\n#EXTINF:12.0,\n83.ts\n", - ); + let recovered_manifest = + normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:83\n#EXT-X-TARGETDURATION:12\n#EXTINF:12.0,\n83.ts\n"); session.apply_origin_manifest(&recovered_manifest).expect("later shared-session recovery"); session.origin_control.record_media_progress(cutover_now_ms.saturating_add(1), 12_000); } let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("sticky terminal lease"); assert!(matches!( @@ -10597,10 +10335,7 @@ mod tests { visible_segments: Arc::from([HlsLeaseManifestSegment { proxy_seq: base_proxy_seq, duration_ms, - uri: format!( - "/iptv/hls/shared/live/{}/{}/{base_proxy_seq:06}.m4s", - proxy_session_id.0, lease_id.0 - ), + uri: format!("/iptv/hls/shared/live/{}/{}/{base_proxy_seq:06}.m4s", proxy_session_id.0, lease_id.0), discontinuity_before: false, map_ref_ready: true, encryption: None, @@ -10609,10 +10344,7 @@ mod tests { target_duration_ms: terminal_test_asset().duration_ms().saturating_add(1_000), playlist_duration_ms: duration_ms, last_visible_media_end_ms: duration_ms, - active_map: Some(HlsMapSignature { - fingerprint: [7; 32], - container: HlsMediaContainer::FragmentedMp4, - }), + active_map: Some(HlsMapSignature { fingerprint: [7; 32], container: HlsMediaContainer::FragmentedMp4 }), active_encryption: None, container: HlsMediaContainer::FragmentedMp4, }; @@ -10624,10 +10356,7 @@ mod tests { now_ms, ) .await; - let manifest_uri = format!( - "/hls/shared/live/{}/{}/manifest.m3u8", - proxy_session_id.0, lease_id.0 - ); + let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; @@ -10640,10 +10369,7 @@ mod tests { .await .expect("failed-closed lease snapshot"); assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - let terminal_uri = format!( - "/hls/shared/live/{}/{}/terminal/1/0.ts", - proxy_session_id.0, lease_id.0 - ); + let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0); let terminal_response = get_response(app_state, &terminal_uri, None).await; assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND); assert!(!terminal_response.headers().contains_key(header::LOCATION)); @@ -10670,15 +10396,11 @@ mod tests { assert_eq!(hls_session_last_media_at_ms(app_state, proxy_session_id).await, None); assert_no_hls_cache_stream_registered(app_state).await; - let head_range = - request_response(Arc::clone(app_state), Method::HEAD, segment_uri, Some("bytes=0-187")).await; + let head_range = request_response(Arc::clone(app_state), Method::HEAD, segment_uri, Some("bytes=0-187")).await; assert_eq!(head_range.status(), StatusCode::PARTIAL_CONTENT); assert_eq!(head_range.headers()[header::CONTENT_LENGTH], "188"); let expected_content_range = format!("bytes 0-187/{content_length}"); - assert_eq!( - head_range.headers()[header::CONTENT_RANGE].to_str().ok(), - Some(expected_content_range.as_str()) - ); + assert_eq!(head_range.headers()[header::CONTENT_RANGE].to_str().ok(), Some(expected_content_range.as_str())); assert!(response_body(head_range).await.is_empty()); assert_eq!(hls_session_last_media_at_ms(app_state, proxy_session_id).await, None); assert_no_hls_cache_stream_registered(app_state).await; @@ -10692,8 +10414,7 @@ mod tests { test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; let lease_id = format!("test-access-lease-{proxy_session_id}"); - let terminal_renderer = - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let terminal_renderer = terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; let renders_before_requests = terminal_renderer.finite_hls_render_count(); let finalizations_before_requests = terminal_renderer.finite_hls_finalize_count(); assert_eq!(renders_before_requests, usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); @@ -10704,13 +10425,12 @@ mod tests { let repair_before = app_state.hls_proxy.segment_repair().stats().await; let provider_connections_before = app_state.active_provider.get_provider_connections_count().await; - let head_content_length = - terminal_head_content_length(&app_state, &proxy_session_id, &segment_zero_uri).await; + let head_content_length = terminal_head_content_length(&app_state, &proxy_session_id, &segment_zero_uri).await; let segment_zero_response = get_response(Arc::clone(&app_state), &segment_zero_uri, None).await; assert_eq!(segment_zero_response.status(), StatusCode::OK); assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); - assert!(!crate::api::api_utils::should_compress_response(&segment_zero_response)); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&segment_zero_response)); assert_eq!(segment_zero_response.headers()[header::CONTENT_TYPE], "video/mp2t"); assert_eq!(segment_zero_response.headers()[header::ACCEPT_RANGES], "bytes"); assert!(segment_zero_response.headers()[header::CACHE_CONTROL] @@ -10737,7 +10457,7 @@ mod tests { let range_response = get_response(Arc::clone(&app_state), &segment_zero_uri, Some("bytes=0-187")).await; assert_eq!(range_response.status(), StatusCode::PARTIAL_CONTENT); - assert!(!crate::api::api_utils::should_compress_response(&range_response)); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&range_response)); assert_eq!(range_response.headers()[header::CONTENT_LENGTH], "188"); let expected_content_range = format!("bytes 0-187/{declared_length}"); assert_eq!( @@ -10753,7 +10473,7 @@ mod tests { let unsatisfiable_range = format!("bytes={declared_length}-"); let unsatisfiable_response = get_response(app_state, &segment_zero_uri, Some(&unsatisfiable_range)).await; assert_eq!(unsatisfiable_response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert!(!crate::api::api_utils::should_compress_response(&unsatisfiable_response)); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&unsatisfiable_response)); let expected_unsatisfied_content_range = format!("bytes */{declared_length}"); assert_eq!( unsatisfiable_response.headers()[header::CONTENT_RANGE].to_str().ok(), @@ -10798,35 +10518,25 @@ mod tests { let origin_refresh_before = session.read().await.origin_refresh.clone(); mark_hls_user_session_exhausted(&fixture.app_state).await; - let denied_live = - get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; + let denied_live = get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; assert_eq!(denied_live.status(), StatusCode::FORBIDDEN); let revoking = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("revoking lease snapshot"); assert_eq!(revoking.state, HlsAccessLeaseState::PolicyRevoking); assert_eq!(revoking.playback_mode, HlsLeasePlaybackMode::Live); assert_eq!( - revoking - .runtime_policy_revocation - .as_ref() - .map(|revocation| revocation.reason), + revoking.runtime_policy_revocation.as_ref().map(|revocation| revocation.reason), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted) ); - let pending_manifest = - get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + let pending_manifest = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; assert_eq!(pending_manifest.status(), StatusCode::SERVICE_UNAVAILABLE); let committed_plan = wait_for_runtime_policy_terminal_plan(&fixture).await; - let manifest_response = - get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + let manifest_response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; assert_eq!(manifest_response.status(), StatusCode::OK); assert!(!manifest_response.headers().contains_key(header::LOCATION)); let manifest = String::from_utf8(response_body(manifest_response).await.to_vec()) @@ -10837,11 +10547,7 @@ mod tests { let committed = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("committed runtime policy lease"); assert_eq!(committed.state, HlsAccessLeaseState::Denied); @@ -10855,32 +10561,20 @@ mod tests { "/hls/shared/live/{}/{}/terminal/{}/", fixture.proxy_session_id.0, fixture.lease_id.0, plan.generation.0 ); - assert_eq!( - manifest.matches(&terminal_prefix).count(), - usize::from(plan.segment_count) - ); + assert_eq!(manifest.matches(&terminal_prefix).count(), usize::from(plan.segment_count)); assert_runtime_policy_terminal_segments(&fixture, &terminal_prefix, plan.segment_count).await; - let replay = - get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + let replay = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; assert_eq!(replay.status(), StatusCode::OK); assert_eq!( - String::from_utf8(response_body(replay).await.to_vec()) - .expect("replayed runtime policy manifest utf8"), + String::from_utf8(response_body(replay).await.to_vec()).expect("replayed runtime policy manifest utf8"), manifest ); - assert_eq!( - get_status(Arc::clone(&fixture.app_state), &fixture.live_segment_uri).await, - StatusCode::FORBIDDEN - ); + assert_eq!(get_status(Arc::clone(&fixture.app_state), &fixture.live_segment_uri).await, StatusCode::FORBIDDEN); let retained = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("retained runtime policy plan"); assert!(matches!( @@ -10918,26 +10612,19 @@ mod tests { 60_000, ) .await, - crate::api::model::HlsAccessLeaseTouch::Denied + HlsAccessLeaseTouch::Denied ); - let pending = - get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + let pending = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; assert_eq!(pending.status(), StatusCode::SERVICE_UNAVAILABLE); let committed_plan = wait_for_runtime_policy_terminal_plan(&fixture).await; - let response = - get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + let response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; assert_eq!(response.status(), StatusCode::OK); - let body = - String::from_utf8(response_body(response).await.to_vec()).expect("policy touch manifest utf8"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("policy touch manifest utf8"); let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("touch-denied lease"); let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else { @@ -10957,40 +10644,28 @@ mod tests { let fixture = runtime_policy_endpoint_fixture(false).await; mark_hls_user_session_exhausted(&fixture.app_state).await; - let denial = - get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; + let denial = get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; assert_eq!(denial.status(), StatusCode::FORBIDDEN); let denied = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await .expect("cold denied lease"); assert_eq!(denied.state, HlsAccessLeaseState::Denied); assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::Ended); assert!(denied.runtime_policy_revocation.is_none()); - assert_eq!( - denied.runtime_policy_denial_reason(), - Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted) - ); + assert_eq!(denied.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); - let response = - get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + let response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; assert_eq!(response.status(), StatusCode::OK); - let body = - String::from_utf8(response_body(response).await.to_vec()).expect("standalone policy manifest utf8"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone policy manifest utf8"); assert!(body.contains(&format!("/cvs/hls/{}/", fixture.lease_id.0))); assert!(!body.contains("/hls-user/")); assert!(!body.contains("/hls-pass/")); assert!(!body.contains("/user_connections_exhausted/")); - assert!(!body.contains(&format!( - "/hls/shared/live/{}/{}/terminal/", - fixture.proxy_session_id.0, fixture.lease_id.0 - ))); + assert!(!body + .contains(&format!("/hls/shared/live/{}/{}/terminal/", fixture.proxy_session_id.0, fixture.lease_id.0))); assert!(body.ends_with("#EXT-X-ENDLIST\n")); } @@ -11019,7 +10694,7 @@ mod tests { .hls_proxy .deny_access_lease( &HlsAccessLeaseId(lease_id), - crate::api::model::HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail, + tuliprox_hls::HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail, ) .await; assert!(!response_body(response).await.is_empty()); @@ -11045,17 +10720,13 @@ mod tests { let out_of_bounds_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/{segment_count}.ts"); - let malformed_generation = - format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/01/0.ts"); + let malformed_generation = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/01/0.ts"); let non_numeric_generation = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/not-a-generation/0.ts"); - let overflowing_generation = format!( - "/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/18446744073709551616/0.ts" - ); - let malformed_file = - format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/00.ts"); - let wrong_extension = - format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.m4s"); + let overflowing_generation = + format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/18446744073709551616/0.ts"); + let malformed_file = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/00.ts"); + let wrong_extension = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.m4s"); for uri in [ stale_uri, @@ -11315,23 +10986,15 @@ mod tests { let now_ms = super::current_time_millis(); let terminal_before = app_state .hls_proxy - .access_lease_response_snapshot( - &HlsAccessLeaseId(terminal_lease_id.clone()), - &proxy_session_key, - now_ms, - ) + .access_lease_response_snapshot(&HlsAccessLeaseId(terminal_lease_id.clone()), &proxy_session_key, now_ms) .await .expect("terminal lease exists before the other lease"); let HlsLeasePlaybackMode::TerminalTail(terminal_plan_before) = terminal_before.playback_mode else { panic!("original lease is terminal"); }; - let terminal_path = HlsTerminalSegmentPath { - generation: terminal_plan_before.generation, - index: 0, - }; - let terminal_bytes_before = terminal_plan_before - .segment_bytes(terminal_path) - .expect("terminal segment zero is immutable"); + let terminal_path = HlsTerminalSegmentPath { generation: terminal_plan_before.generation, index: 0 }; + let terminal_bytes_before = + terminal_plan_before.segment_bytes(terminal_path).expect("terminal segment zero is immutable"); let other_lease_id = prepare_other_live_lease(&app_state, &proxy_session_key, now_ms).await; let session = app_state .hls_proxy @@ -11438,7 +11101,7 @@ mod tests { let response = super::hls_response("#EXTM3U\n".to_string()).into_response(); assert_eq!(response.headers().get(header::CONTENT_TYPE).unwrap(), "application/vnd.apple.mpegurl"); - assert!(crate::api::api_utils::should_compress_response(&response)); + assert!(tuliprox_core::utils::response_compression::should_compress_response(&response)); } #[test] @@ -11656,7 +11319,7 @@ mod tests { "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/{}/000123.ts\n", crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER ); - let lease_id = crate::api::model::HlsAccessLeaseId("access-lease".to_string()); + let lease_id = HlsAccessLeaseId("access-lease".to_string()); let materialized = super::materialize_hls_access_manifest(&body, &lease_id, Some("/iptv")); @@ -11715,18 +11378,9 @@ mod tests { #[test] fn hls_origin_source_kind_covers_xtream_m3u_and_direct_media_playlist() { - assert_eq!( - super::hls_origin_source_kind(InputType::Xtream), - crate::api::model::HlsOriginSourceKind::XtreamLive - ); - assert_eq!( - super::hls_origin_source_kind(InputType::M3u), - crate::api::model::HlsOriginSourceKind::M3uMediaPlaylist - ); - assert_eq!( - super::hls_origin_source_kind(InputType::Library), - crate::api::model::HlsOriginSourceKind::DirectMediaPlaylist - ); + assert_eq!(super::hls_origin_source_kind(InputType::Xtream), HlsOriginSourceKind::XtreamLive); + assert_eq!(super::hls_origin_source_kind(InputType::M3u), HlsOriginSourceKind::M3uMediaPlaylist); + assert_eq!(super::hls_origin_source_kind(InputType::Library), HlsOriginSourceKind::DirectMediaPlaylist); } #[test] @@ -11823,7 +11477,7 @@ mod tests { let source = super::build_hls_origin_source(&input, "stable-item"); assert_eq!(origin.session_entry_url.as_str(), "http://media.example.com/live/channel/index.m3u8"); - assert_eq!(source.source_kind, crate::api::model::HlsOriginSourceKind::M3uMediaPlaylist); + assert_eq!(source.source_kind, HlsOriginSourceKind::M3uMediaPlaylist); assert_eq!(source.session_key().stable_value(), "input:9|hls|stable-item"); } @@ -12331,9 +11985,7 @@ mod tests { cache_path: Some(cache_path.to_string_lossy().into_owned()), strip: HlsStripConfigDto { mode: HlsStripMode::Segments, value: 3 }, max_segments_prefetch: 6, - manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { - level: HlsManifestRecoveryBurstLevel::Beast, - }, + manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { level: HlsManifestRecoveryBurstLevel::Beast }, ..HlsCacheConfigDto::default() }); Arc::new(HlsProxyManager::from_hls_cache_config(Some(&config))) @@ -12363,13 +12015,25 @@ mod tests { app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { channel_unavailable: None, user_connections_exhausted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/user_connections_exhausted.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/user_connections_exhausted.ts" + )) + .to_vec(), )), provider_connections_exhausted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/provider_connections_exhausted.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/provider_connections_exhausted.ts" + )) + .to_vec(), )), low_priority_preempted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/low_priority_preempted.ts")).to_vec(), + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/low_priority_preempted.ts" + )) + .to_vec(), )), user_account_expired: None, panel_api_provisioning: None, @@ -12397,7 +12061,10 @@ mod tests { } fn test_custom_video_buffer() -> TransportStreamBuffer { - TransportStreamBuffer::new(include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")).to_vec()) + TransportStreamBuffer::new( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .to_vec(), + ) } fn enable_hls_provisioning_custom_response(app_state: &Arc) { @@ -12562,7 +12229,7 @@ mod tests { fn test_hls_origin_io_context(app_state: &Arc) -> HlsOriginIoContext { HlsOriginIoContext { - app_state: Arc::clone(app_state), + ctx: app_state.hls_ctx(), client_addr: test_fingerprint().addr, allow_grace: false, priority: 0, @@ -12703,9 +12370,10 @@ mod tests { let mut detached_binding = known_binding.clone(); detached_binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, 2_000); - assert!(!super::hls_transient_origin_binding_requires_runtime_prepare(&app_state, &known_binding)); - assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&app_state, &missing_binding)); - assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&app_state, &detached_binding)); + let hls_ctx = app_state.hls_ctx(); + assert!(!super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &known_binding)); + assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &missing_binding)); + assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &detached_binding)); } #[tokio::test] @@ -13241,11 +12909,7 @@ mod tests { store_test_sources_with_target( &fixture.app_state, input.clone(), - ConfigTarget::from(&ConfigTargetDto { - id: 1, - name: "default".to_string(), - ..Default::default() - }), + ConfigTarget::from(&ConfigTargetDto { id: 1, name: "default".to_string(), ..Default::default() }), ); let current = fixture.app_state.app_config.config.load(); fixture.app_state.app_config.config.store(Arc::new(Config { @@ -14136,10 +13800,7 @@ mod tests { } async fn mark_hls_user_session_exhausted(app_state: &Arc) { - let user = app_state - .app_config - .get_user_credentials("hls-user") - .expect("configured HLS test user"); + let user = app_state.app_config.get_user_credentials("hls-user").expect("configured HLS test user"); let addr = test_addr(); app_state .active_users @@ -14510,7 +14171,7 @@ mod tests { .await .expect("m3u hls should create shared session"); let session = session.read().await; - assert_eq!(session.origin_source.source_kind, crate::api::model::HlsOriginSourceKind::M3uMediaPlaylist); + assert_eq!(session.origin_source.source_kind, HlsOriginSourceKind::M3uMediaPlaylist); let binding = session.origin_account_binding.as_ref().expect("m3u hls input still has account binding"); assert_eq!(binding.input_name.as_ref(), "m3u-input"); assert_eq!(binding.account_name.as_ref(), "m3u-input"); @@ -14519,7 +14180,8 @@ mod tests { const AES_TEST_MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-VERSION:5\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:77\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n#EXTINF:12,\n77.ts\n#EXTINF:12,\n78.ts\n#EXTINF:12,\n79.ts\n#EXTINF:12,\n80.ts\n#EXTINF:12,\n81.ts\n#EXTINF:12,\n82.ts\n"; const AES_TEST_KEY_BYTES: &[u8] = b"0123456789abcdef"; const AES_TEST_ROTATED_KEY_BYTES: &[u8] = b"fedcba9876543210"; - const AES_TEST_PLAINTEXT_SEGMENT: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")); + const AES_TEST_PLAINTEXT_SEGMENT: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); struct AesEndpointFixture { _temp_dir: tempfile::TempDir, @@ -14724,9 +14386,9 @@ mod tests { base_availability: evidence.availability(), base_track_signature: evidence.track_signature(), base_splice_evidence: evidence.splice_evidence().cloned(), - terminal_splice_evidence: Some( - HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&fixture.asset), - ), + terminal_splice_evidence: Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test( + &fixture.asset, + )), base_timing: evidence.timing().cloned(), base_key_bindings: evidence.key_bindings(), expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable( @@ -14740,7 +14402,7 @@ mod tests { asset: Arc::clone(&fixture.asset), }) .expect("READY AES key permits safe terminal reset"); - let protection = crate::api::model::HlsTerminalTailProtection { + let protection = HlsTerminalTailProtection { generation: plan.generation, base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), key_bindings: plan.key_bindings(), @@ -14754,20 +14416,14 @@ mod tests { lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::new(plan)); leases.prepare_access_lease(lease); } - fixture - .session - .write() - .await - .install_terminal_tail_protection(fixture.access_lease_id.clone(), protection); + fixture.session.write().await.install_terminal_tail_protection(fixture.access_lease_id.clone(), protection); evidence.release(); frozen_source_cache_key } async fn assert_aes_terminal_endpoints(fixture: &AesEndpointFixture) { - let manifest_uri = format!( - "/hls/shared/live/{}/{}/manifest.m3u8", - fixture.proxy_session_id.0, fixture.access_lease_id.0 - ); + let manifest_uri = + format!("/hls/shared/live/{}/{}/manifest.m3u8", fixture.proxy_session_id.0, fixture.access_lease_id.0); let response = get_response(Arc::clone(&fixture.app_state), &manifest_uri, None).await; assert_eq!(response.status(), StatusCode::OK); let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal utf8"); @@ -14785,8 +14441,7 @@ mod tests { let range = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=4-7")).await; assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT); assert_eq!(response_body(range).await.as_ref(), &AES_TEST_KEY_BYTES[4..=7]); - let unsatisfiable = - get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=16-")).await; + let unsatisfiable = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=16-")).await; assert_eq!(unsatisfiable.status(), StatusCode::RANGE_NOT_SATISFIABLE); assert_eq!(unsatisfiable.headers()[header::CONTENT_RANGE], "bytes */16"); } @@ -14842,16 +14497,14 @@ mod tests { assert_eq!(key_response.status(), StatusCode::OK); assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_ROTATED_KEY_BYTES); assert_eq!(fixture.origin.key_request_count(), 2); - assert!( - fixture - .app_state - .hls_proxy - .segment_cache() - .metadata(frozen_source_cache_key) - .await - .expect("A metadata lookup") - .is_none() - ); + assert!(fixture + .app_state + .hls_proxy + .segment_cache() + .metadata(frozen_source_cache_key) + .await + .expect("A metadata lookup") + .is_none()); let terminal_key = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; assert_eq!(terminal_key.status(), StatusCode::OK); assert_eq!(response_body(terminal_key).await.as_ref(), AES_TEST_KEY_BYTES); @@ -14987,7 +14640,7 @@ mod tests { .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) .await .expect("lease remains available for strict cold-start handling"); - assert_eq!(snapshot.playback_mode, crate::api::model::HlsLeasePlaybackMode::Live); + assert_eq!(snapshot.playback_mode, HlsLeasePlaybackMode::Live); assert!(snapshot.last_manifest_snapshot.is_none()); } assert_eq!(rendered_bodies[0], rendered_bodies[1]); @@ -15345,15 +14998,7 @@ mod tests { user.username = "hls-user".to_string(); user.password = "hls-pass".to_string(); let entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); - SharedRequestFlowFixture { - _origin: origin, - app_state, - input, - target, - user, - origin_manifest_url, - entry_path, - } + SharedRequestFlowFixture { _origin: origin, app_state, input, target, user, origin_manifest_url, entry_path } } async fn shared_request_flow_entry( @@ -15426,8 +15071,7 @@ mod tests { build_proxy_session_id(&archive_source.session_key(), &app_state.get_encrypt_secret()) ); - let access_lease_id = - HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); let lease = app_state .hls_proxy .access_leases() @@ -15470,12 +15114,9 @@ mod tests { let reloaded_media_response = get_response(Arc::clone(app_state), &media_playlist_uri, None).await; assert_eq!(reloaded_media_response.status(), StatusCode::OK); assert!(!reloaded_media_response.headers().contains_key(header::LOCATION)); - let reloaded_media_body = - String::from_utf8(response_body(reloaded_media_response).await.to_vec()).expect("reloaded media playlist utf8"); - assert_eq!( - manifest_media_sequence(&reloaded_media_body), - manifest_media_sequence(&first_media_body) - ); + let reloaded_media_body = String::from_utf8(response_body(reloaded_media_response).await.to_vec()) + .expect("reloaded media playlist utf8"); + assert_eq!(manifest_media_sequence(&reloaded_media_body), manifest_media_sequence(&first_media_body)); assert!(reloaded_media_body.lines().any(|line| line == segment_uri)); assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 1); @@ -15484,11 +15125,7 @@ mod tests { assert!(!response_body(segment_response).await.is_empty()); let lease = app_state .hls_proxy - .access_lease_response_snapshot( - &access_lease_id, - &proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) .await .expect("request-flow access lease"); assert_eq!(lease.state, HlsAccessLeaseState::Activated); @@ -15550,7 +15187,10 @@ mod tests { async fn publication_late_fixture() -> PublicationLateFixture { let initial_manifest = Arc::<[u8]>::from(regression_origin_manifest(123, 6)); let progressed_manifest = Arc::<[u8]>::from(regression_origin_manifest(124, 6)); - let segment = Arc::<[u8]>::from(include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")).as_slice()); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); let origin_phase = Arc::new(AtomicUsize::new(0)); let origin_phase_for_handler = Arc::clone(&origin_phase); let origin = spawn_test_binary_origin(Arc::new(move |path| { @@ -15650,8 +15290,7 @@ mod tests { } let requests_before = fixture.origin.manifest_request_count(); fixture.origin_phase.store(1, Ordering::SeqCst); - let response = - get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; assert_eq!(response.status(), StatusCode::OK); assert!(!response.headers().contains_key(header::LOCATION)); let body = String::from_utf8(response_body(response).await.to_vec()).expect("refreshed media utf8"); @@ -15712,10 +15351,7 @@ mod tests { }; assert!(matches!(state, HlsPreparedTerminalBundleState::Ready { .. })); let mut session = fixture.session.write().await; - for segment in session - .segments - .values_mut() - .filter(|segment| segment.proxy_seq > base_manifest.last_proxy_seq) + for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > base_manifest.last_proxy_seq) { segment.duration_ms = 1; } @@ -15726,8 +15362,7 @@ mod tests { async fn assert_publication_late_terminal_result(fixture: &PublicationLateFixture) { let requests_before = fixture.origin.manifest_request_count(); fixture.origin_phase.store(2, Ordering::SeqCst); - let response = - get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; let lease = fixture .app_state .hls_proxy @@ -15739,7 +15374,9 @@ mod tests { .await .expect("terminal lease remains stored"); let recovery_plan = fixture.app_state.hls_proxy.manifest_recovery_burst().level.plan(); - assert!(fixture.origin.manifest_request_count().saturating_sub(requests_before) >= recovery_plan.total_candidates()); + assert!( + fixture.origin.manifest_request_count().saturating_sub(requests_before) >= recovery_plan.total_candidates() + ); match lease.playback_mode { HlsLeasePlaybackMode::TerminalTail(_) => { assert_eq!(response.status(), StatusCode::OK); @@ -15806,20 +15443,14 @@ mod tests { app_state.app_config.config.store(Arc::new(config)); let mut user = ProxyUserCredentials::default(); user.username = "hls-user".to_string(); - let input = ConfigInput { - id: 9, - name: Arc::from("m3u-input"), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let mut stored_item = - test_m3u_hls_item(&input, 70001, "channel-a", "http://origin.example.com/live.m3u8"); - stored_item.additional_properties = Some(StreamProperties::Live(Box::new( - shared::model::LiveStreamProperties { + let input = + ConfigInput { id: 9, name: Arc::from("m3u-input"), input_type: InputType::M3u, ..ConfigInput::default() }; + let mut stored_item = test_m3u_hls_item(&input, 70001, "channel-a", "http://origin.example.com/live.m3u8"); + stored_item.additional_properties = + Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { bitrate: 2_500_000, ..shared::model::LiveStreamProperties::default() - }, - ))); + }))); let input_storage = crate::repository::build_input_storage_path(&input.name, &temp.path().to_string_lossy()); std::fs::create_dir_all(&input_storage).expect("input storage"); let db_path = crate::repository::get_input_m3u_playlist_file_path(&input_storage, &input.name); @@ -15949,14 +15580,11 @@ mod tests { &proxy_session_id, &access_lease_id.0, now_ms, - crate::api::model::HlsAccessAdmissionMode::ManifestPrepare, + HlsAccessAdmissionMode::ManifestPrepare, ) .await .expect_err("denied lease must surface as admission denied"); - assert!(matches!( - err, - HlsAccessLeaseValidationError::AdmissionDenied { runtime_tail: None, .. } - )); + assert!(matches!(err, HlsAccessLeaseValidationError::AdmissionDenied { runtime_tail: None, .. })); } } @@ -16533,8 +16161,7 @@ mod tests { .await; let first_variant_uri = single_variant_uri(first_response).await; let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); - let first_access_lease_id = - HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); + let first_access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_access_lease_id).await; @@ -16612,7 +16239,7 @@ mod tests { super::hls_access_lease_ttl_ms(&app_state), ) .await, - crate::api::model::HlsAccessLeaseTouch::Touched { .. } + HlsAccessLeaseTouch::Touched { .. } )); let second_response = super::create_hls_cache_entry_master_playlist_response( @@ -16967,9 +16594,7 @@ mod tests { assert!(app_state.hls_proxy.sessions().is_empty().await); } - async fn prepare_server_path_manifest_session( - app_state: &Arc, - ) -> (HlsSessionHandle, ProxySessionId) { + async fn prepare_server_path_manifest_session(app_state: &Arc) -> (HlsSessionHandle, ProxySessionId) { let session = app_state .hls_proxy .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) @@ -17275,7 +16900,7 @@ mod tests { diagnostics, [super::HlsInitialStripPublicationDiagnostic::Skipped { mode: "normal", - reason: crate::processing::parser::hls::initial_strip::HlsInitialStripSkipReason::StripDisabled, + reason: crate::api::model::hls_cache::initial_strip::HlsInitialStripSkipReason::StripDisabled, visible_segments: 6, }] ); @@ -17323,11 +16948,7 @@ mod tests { let mut session = session.write().await; let proxy_session_id = session.proxy_session_id.clone(); let rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body( - &mut session, - normal_manifest_body(&proxy_session_id.0), - rendered_at_ms, - ); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id.0), rendered_at_ms); session.segments.get_mut(&1).expect("visible test segment").status = SegmentCacheStatus::Discovered; session.advance_media_readiness_generation(); session.mark_authorized_media_access(rendered_at_ms); @@ -17394,10 +17015,7 @@ mod tests { visible_segments: Arc::from([HlsLeaseManifestSegment { proxy_seq, duration_ms, - uri: format!( - "/hls/shared/live/{}/{}/{proxy_seq:06}.ts", - proxy_session_id.0, access_lease_id.0 - ), + uri: format!("/hls/shared/live/{}/{}/{proxy_seq:06}.ts", proxy_session_id.0, access_lease_id.0), discontinuity_before: false, map_ref_ready: true, encryption: None, @@ -17424,21 +17042,13 @@ mod tests { } async fn prepare_user_exhausted_terminal_bundle(app_state: &Arc, target_duration_ms: u64) { - let response = app_state - .app_config - .custom_stream_response - .load_full() - .expect("runtime custom responses"); + let response = app_state.app_config.custom_stream_response.load_full().expect("runtime custom responses"); let asset = response .user_connections_exhausted .as_ref() .and_then(|buffer| snapshot_terminal_media_asset(buffer).ok()) .expect("valid user-exhausted terminal asset"); - let key = prepared_terminal_bundle_key( - &asset, - target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); + let key = prepared_terminal_bundle_key(&asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); let state = app_state.hls_proxy.start_prepared_terminal_bundle( asset, target_duration_ms, @@ -17533,53 +17143,31 @@ mod tests { let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); enable_runtime_policy_custom_responses(&app_state); - let live_bytes = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")); - let proxy_session_id = - ProxySessionId(map_ready_segment_without_lease(&app_state, 123, "ts", live_bytes).await); + let live_bytes = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + let proxy_session_id = ProxySessionId(map_ready_segment_without_lease(&app_state, 123, "ts", live_bytes).await); let session = app_state .hls_proxy .sessions() .get_by_proxy_session_id(&proxy_session_id) .await .expect("runtime policy session"); - session - .write() - .await - .segments - .get_mut(&123) - .expect("runtime policy base segment") - .duration_ms = TARGET_DURATION_MS; - let lease_id = - HlsAccessLeaseId(grant_hls_proxy_lease(&app_state, &proxy_session_id.0).await); + session.write().await.segments.get_mut(&123).expect("runtime policy base segment").duration_ms = + TARGET_DURATION_MS; + let lease_id = HlsAccessLeaseId(grant_hls_proxy_lease(&app_state, &proxy_session_id.0).await); if publish_manifest { - publish_ready_test_manifest_for_lease( - &app_state, - &proxy_session_id, - &lease_id, - TARGET_DURATION_MS, - ) - .await; + publish_ready_test_manifest_for_lease(&app_state, &proxy_session_id, &lease_id, TARGET_DURATION_MS).await; let target_duration_ms = app_state .hls_proxy - .access_lease_response_snapshot( - &lease_id, - &proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::current_time_millis()) .await .and_then(|lease| lease.last_manifest_snapshot) .map(|manifest| manifest.target_duration_ms) .expect("published runtime policy target duration"); prepare_user_exhausted_terminal_bundle(&app_state, target_duration_ms).await; } - let manifest_uri = format!( - "/hls/shared/live/{}/{}/manifest.m3u8", - proxy_session_id.0, lease_id.0 - ); - let live_segment_uri = format!( - "/hls/shared/live/{}/{}/000123.ts", - proxy_session_id.0, lease_id.0 - ); + let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); + let live_segment_uri = format!("/hls/shared/live/{}/{}/000123.ts", proxy_session_id.0, lease_id.0); if publish_manifest { assert_runtime_policy_base_timing(&app_state, &proxy_session_id, &lease_id, "before serve").await; } @@ -17598,9 +17186,7 @@ mod tests { } } - async fn wait_for_runtime_policy_terminal_plan( - fixture: &RuntimePolicyEndpointFixture, - ) -> Arc { + async fn wait_for_runtime_policy_terminal_plan(fixture: &RuntimePolicyEndpointFixture) -> Arc { let plan = tokio::time::timeout(Duration::from_secs(10), async { loop { if let Some(HlsLeasePlaybackMode::TerminalTail(plan)) = fixture @@ -17626,11 +17212,7 @@ mod tests { let lease = fixture .app_state .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) .await; let state = lease.as_ref().map_or("missing", |lease| lease.state.as_log_value()); let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { @@ -17920,15 +17502,11 @@ mod tests { assert_eq!(media_uri_count(&stored_before), 6); } - fn assert_encrypted_transient_terminal_incompatibility( - snapshot: HlsLeaseManifestSnapshot, - now_ms: u64, - ) { + fn assert_encrypted_transient_terminal_incompatibility(snapshot: HlsLeaseManifestSnapshot, now_ms: u64) { let asset = terminal_test_asset(); let bundle_target_duration_ms = snapshot.target_duration_ms.max(asset.duration_ms()); let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &snapshot)); - let base_splice_evidence = - Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); let terminal_splice_evidence = base_splice_evidence.clone(); assert_eq!( build_terminal_tail_plan(HlsTerminalTailBuildInput { @@ -17943,10 +17521,7 @@ mod tests { expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable( HlsTerminalAssetIdentity::from_asset(&asset), ), - anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test( - &asset, - bundle_target_duration_ms, - ), + anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, bundle_target_duration_ms,), base_manifest: snapshot, asset, }), @@ -18532,9 +18107,8 @@ mod tests { } fn regression_origin_manifest(first_sequence: u64, segment_count: usize) -> Vec { - let mut manifest = format!( - "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n" - ); + let mut manifest = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); for offset in 0..segment_count { let sequence = first_sequence.saturating_add(u64::try_from(offset).unwrap_or(u64::MAX)); let _ = writeln!(&mut manifest, "#EXTINF:12.0,\n{sequence}.ts"); @@ -18613,14 +18187,7 @@ mod tests { proxy_seq: u64, logical_size: u64, ) { - let cache_key = session - .read() - .await - .segments - .get(&proxy_seq) - .expect("mapped sparse segment") - .cache_key - .clone(); + let cache_key = session.read().await.segments.get(&proxy_seq).expect("mapped sparse segment").cache_key.clone(); let metadata = app_state .hls_proxy .segment_cache() @@ -18628,24 +18195,22 @@ mod tests { .await .expect("sparse cache metadata read") .expect("READY sparse cache object"); - let file = tokio::fs::OpenOptions::new() - .write(true) - .open(&metadata.path) - .await - .expect("sparse cache object opens"); + let file = + tokio::fs::OpenOptions::new().write(true).open(&metadata.path).await.expect("sparse cache object opens"); file.set_len(logical_size).await.expect("sparse cache object extends"); let mut session = session.write().await; let segment = session.segments.get_mut(&proxy_seq).expect("sparse segment remains mapped"); - segment.status = SegmentCacheStatus::Ready { - content_length: logical_size, - ready_at_ms: super::current_time_millis(), - }; + segment.status = + SegmentCacheStatus::Ready { content_length: logical_size, ready_at_ms: super::current_time_millis() }; session.advance_media_readiness_generation(); session.render_and_store_manifest(super::current_time_millis()).expect("sparse timeline renders"); } fn terminal_test_asset() -> Arc { - let bytes = bytes::Bytes::from_static(include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts"))); + let bytes = bytes::Bytes::from_static(include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/channel_unavailable.ts" + ))); let buffer = TransportStreamBuffer::new(bytes.to_vec()); snapshot_terminal_media_asset(&buffer).expect("terminal test asset is valid") } @@ -18665,13 +18230,7 @@ mod tests { .expect("live lease accepts publication preparation"); assert!(app_state .hls_proxy - .commit_access_lease_manifest_publication( - lease_id, - proxy_session_id, - publication_guard, - snapshot, - now_ms, - ) + .commit_access_lease_manifest_publication(lease_id, proxy_session_id, publication_guard, snapshot, now_ms,) .await .is_committed()); @@ -18681,11 +18240,8 @@ mod tests { .and_then(|responses| responses.channel_unavailable.as_ref()) .and_then(|buffer| snapshot_terminal_media_asset(buffer).ok()) .expect("configured terminal test asset"); - let terminal_key = prepared_terminal_bundle_key( - &terminal_asset, - target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); + let terminal_key = + prepared_terminal_bundle_key(&terminal_asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); let state = app_state.hls_proxy.start_prepared_terminal_bundle( terminal_asset, target_duration_ms, @@ -18748,7 +18304,8 @@ mod tests { let proxy_session_id = ProxySessionId(proxy_session_id.to_string()); let lease_id = HlsAccessLeaseId(lease_id.to_string()); let buffer = TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/hls/channel_unavailable.ts")).to_vec(), + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .to_vec(), ); let asset = snapshot_terminal_media_asset(&buffer).expect("terminal test asset is valid"); let base_manifest = HlsLeaseManifestSnapshot { @@ -18775,8 +18332,7 @@ mod tests { container: HlsMediaContainer::MpegTs, }; let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); - let base_splice_evidence = - Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); let terminal_splice_evidence = base_splice_evidence.clone(); let plan = build_terminal_tail_plan(HlsTerminalTailBuildInput { generation: HlsTerminalTailGeneration(17), @@ -18804,7 +18360,7 @@ mod tests { asset, }) .expect("terminal test plan is compatible"); - let protection = crate::api::model::HlsTerminalTailProtection { + let protection = HlsTerminalTailProtection { generation: plan.generation, base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), key_bindings: plan.key_bindings(), @@ -18900,7 +18456,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); let cache_key = { @@ -18935,7 +18491,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); let cache_key = { @@ -19060,9 +18616,7 @@ mod tests { (bandwidth, uri) } - async fn single_variant_uri(response: Response) -> String { - single_variant_master_playlist(response).await.1 - } + async fn single_variant_uri(response: Response) -> String { single_variant_master_playlist(response).await.1 } fn access_lease_id_from_variant_uri(uri: &str) -> &str { uri.trim_end_matches("/manifest.m3u8").rsplit('/').next().expect("access lease id in variant URI") @@ -19192,7 +18746,7 @@ mod tests { async fn register_hls_cache_stream_for_stats_test( app_state: &Arc, - session: &crate::api::model::HlsSessionHandle, + session: &HlsSessionHandle, proxy_session_id: &ProxySessionId, session_token: &str, fingerprint: &Fingerprint, @@ -19365,7 +18919,8 @@ mod tests { let path = String::from_utf8_lossy(&request[..read]) .lines() .next() - .and_then(|line| line.split_whitespace().nth(1)).map_or_else(|| "/".to_string(), str::to_owned); + .and_then(|line| line.split_whitespace().nth(1)) + .map_or_else(|| "/".to_string(), str::to_owned); let current_key_bytes = Arc::clone(&*key_bytes.read().await); let body = if path_has_extension(&path, "m3u8") { manifest_requests.fetch_add(1, Ordering::SeqCst); @@ -19473,19 +19028,14 @@ mod tests { return; } let request = String::from_utf8_lossy(&buf[..read]); - let path = request - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .unwrap_or("/"); + let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); if path_has_extension(path, "m3u8") { manifest_requests.fetch_add(1, Ordering::SeqCst); } let TestBinaryOriginResponse { status, location, body } = handler(path); let reason = status.canonical_reason().unwrap_or("Status"); - let location_header = location.map_or_else(String::new, |location| { - format!("Location: {location}\r\n") - }); + let location_header = + location.map_or_else(String::new, |location| format!("Location: {location}\r\n")); let response = format!( "HTTP/1.1 {} {reason}\r\n{location_header}Content-Length: {}\r\nConnection: close\r\n\r\n", status.as_u16(), @@ -19508,10 +19058,7 @@ mod tests { async fn spawn_test_status_origin(status: StatusCode, body: &'static [u8]) -> TestSegmentOrigin { let body = Arc::<[u8]>::from(body); - spawn_test_binary_origin(Arc::new(move |_path| { - TestBinaryOriginResponse::new(status, Arc::clone(&body)) - })) - .await + spawn_test_binary_origin(Arc::new(move |_path| TestBinaryOriginResponse::new(status, Arc::clone(&body)))).await } struct TestEncodedManifestOrigin { @@ -19765,7 +19312,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; @@ -19819,7 +19366,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; @@ -19869,7 +19416,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); session.write().await.mark_for_gc_removal(); @@ -19898,7 +19445,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); { @@ -20125,7 +19672,7 @@ mod tests { let session = app_state .hls_proxy .sessions() - .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) .await .expect("session should exist"); { diff --git a/backend/src/api/endpoints/hls_terminal_response.rs b/backend/app/src/api/endpoints/hls_terminal_response.rs similarity index 87% rename from backend/src/api/endpoints/hls_terminal_response.rs rename to backend/app/src/api/endpoints/hls_terminal_response.rs index a95d77e90..103a1375b 100644 --- a/backend/src/api/endpoints/hls_terminal_response.rs +++ b/backend/app/src/api/endpoints/hls_terminal_response.rs @@ -1,14 +1,4 @@ -use crate::api::{ - api_utils::try_unwrap_body, - model::{ - commit_terminal_tail_if_lease_reserve_requires_cutover, finite_hls_immutable_media_response, - finite_hls_media_head_response, finite_hls_media_response, publication_late_after_ms, - retry_after_secs_from_ms, terminal_tail_manifest_body, - AppState, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsCacheResponseContext, - HlsLeasePlaybackMode, HlsSessionHandle, HlsTerminalFailedClosedReason, HlsTerminalResolution, - HlsTerminalSegmentPath, HlsTerminalTailPlan, ProxySessionId, - }, -}; +use crate::api::{api_utils::try_unwrap_body, model::AppState}; use axum::{ body::Body, http::{header, HeaderValue, StatusCode}, @@ -16,6 +6,13 @@ use axum::{ }; use log::warn; use std::sync::Arc; +use tuliprox_hls::api::{ + commit_terminal_tail_if_lease_reserve_requires_cutover, finite_hls_immutable_media_response, + finite_hls_media_head_response, finite_hls_media_response, publication_late_after_ms, retry_after_secs_from_ms, + terminal_tail_manifest_body, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsCacheResponseContext, + HlsLeasePlaybackMode, HlsSessionHandle, HlsTerminalFailedClosedReason, HlsTerminalResolution, + HlsTerminalSegmentPath, HlsTerminalTailPlan, ProxySessionId, +}; const HLS_TERMINAL_ENDPOINT_MAX_REEVALUATIONS: usize = 2; const HLS_TERMINAL_SEGMENT_CACHE_CONTROL: &str = "private, max-age=300, immutable"; @@ -55,17 +52,13 @@ pub(super) async fn hls_manifest_terminal_preflight( return if lease.state == HlsAccessLeaseState::Pending { HlsManifestTerminalPreflight::BootstrapPendingLease } else { - HlsManifestTerminalPreflight::FailClosed { - reason: HlsTerminalFailedClosedReason::LeaseStateUnavailable, - } + HlsManifestTerminalPreflight::FailClosed { reason: HlsTerminalFailedClosedReason::LeaseStateUnavailable } }; }; let (publication_late, capacity_recovery_blocks_ready_timeline) = { let session = session.read().await; - let target_duration_ms = session - .origin_control - .target_duration_snapshot_ms - .unwrap_or(manifest.target_duration_ms); + let target_duration_ms = + session.origin_control.target_duration_snapshot_ms.unwrap_or(manifest.target_duration_ms); let ready_timeline = session.ready_timeline_snapshot( lease.playback_cursor.ready_timeline_start_proxy_seq(manifest.first_proxy_seq), now_ms, @@ -91,9 +84,7 @@ pub(super) fn hls_terminal_endpoint_action(resolution: HlsTerminalResolution) -> HlsTerminalResolution::LiveAllowed => HlsTerminalEndpointAction::ServeLive, HlsTerminalResolution::Committed => HlsTerminalEndpointAction::ReloadTerminal, HlsTerminalResolution::Reevaluate => HlsTerminalEndpointAction::Reevaluate, - HlsTerminalResolution::Pending { retry_after_ms } => { - HlsTerminalEndpointAction::RetryAfter { retry_after_ms } - } + HlsTerminalResolution::Pending { retry_after_ms } => HlsTerminalEndpointAction::RetryAfter { retry_after_ms }, HlsTerminalResolution::FailedClosed { reason } => HlsTerminalEndpointAction::FailClosed { reason }, } } @@ -198,9 +189,7 @@ pub(super) fn hls_terminal_playback_response( } } -pub(super) fn hls_terminal_failed_closed_response( - reason: HlsTerminalFailedClosedReason, -) -> axum::response::Response { +pub(super) fn hls_terminal_failed_closed_response(reason: HlsTerminalFailedClosedReason) -> axum::response::Response { warn!("HLS terminal manifest failed closed: reason={}", reason.as_label()); StatusCode::SERVICE_UNAVAILABLE.into_response() } @@ -221,23 +210,18 @@ async fn current_hls_terminal_lease( access_lease_id: &HlsAccessLeaseId, now_ms: u64, ) -> Result> { - app_state - .hls_proxy - .access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms) - .await - .ok_or_else(|| Box::new(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable))) + app_state.hls_proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await.ok_or_else( + || Box::new(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable)), + ) } async fn current_hls_terminal_session( app_state: &Arc, proxy_session_id: &ProxySessionId, ) -> Result> { - app_state - .hls_proxy - .sessions() - .get_by_proxy_session_id(proxy_session_id) - .await - .ok_or_else(|| Box::new(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable))) + app_state.hls_proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.ok_or_else(|| { + Box::new(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable)) + }) } pub(super) async fn resolve_hls_terminal_manifest_state( @@ -258,7 +242,7 @@ pub(super) async fn resolve_hls_terminal_manifest_state( return Err(Box::new(response)); } let resolution = commit_terminal_tail_if_lease_reserve_requires_cutover( - app_state, + &app_state.hls_ctx(), &decision_session, proxy_session_id, &lease, diff --git a/backend/src/api/endpoints/library_api.rs b/backend/app/src/api/endpoints/library_api.rs similarity index 86% rename from backend/src/api/endpoints/library_api.rs rename to backend/app/src/api/endpoints/library_api.rs index 09a98d188..5d2b87fbf 100644 --- a/backend/src/api/endpoints/library_api.rs +++ b/backend/app/src/api/endpoints/library_api.rs @@ -1,16 +1,17 @@ -use crate::{api::{ - model::AppState, - tasks::{spawn_library_scan, LibraryScanTaskOptions}, -}, auth::permission_layer, library::{resolve_metadata_storage_path, LibraryProcessor, MetadataStorage}}; +use crate::{ + api::{ + auth_middleware::permission_layer, + model::AppState, + tasks::{spawn_library_scan, LibraryScanTaskOptions}, + }, + library::{resolve_metadata_storage_path, LibraryProcessor, MetadataStorage}, +}; use axum::response::IntoResponse; use log::{debug, warn}; use serde_json::json; -use shared::model::{ - permission::Permission, LibraryScanRequest, LibraryStatus, OperationRunAccepted, -}; +use shared::model::{permission::Permission, LibraryScanRequest, LibraryStatus, OperationRunAccepted}; use std::sync::Arc; - // Triggers a library scan async fn scan_library( axum::extract::State(app_state): axum::extract::State>, @@ -64,13 +65,12 @@ async fn get_library_status( if config.enabled { let client = app_state.http_client.load_full().as_ref().clone(); // Get statistics from processor - let processor = - LibraryProcessor::new( - config.clone(), - config_snapshot.metadata_update.as_ref(), - client, - &config_snapshot.storage_dir, - ); + let processor = LibraryProcessor::new( + config.clone(), + config_snapshot.metadata_update.as_ref(), + client, + &config_snapshot.storage_dir, + ); let entries = processor.get_all_entries().await; let movies = entries.iter().filter(|e| e.metadata.is_movie()).count(); @@ -82,9 +82,12 @@ async fn get_library_status( movies, series, path: Some( - resolve_metadata_storage_path(config_snapshot.metadata_update.as_ref(), &config_snapshot.storage_dir) - .to_string_lossy() - .to_string(), + resolve_metadata_storage_path( + config_snapshot.metadata_update.as_ref(), + &config_snapshot.storage_dir, + ) + .to_string_lossy() + .to_string(), ), }; @@ -114,10 +117,8 @@ async fn get_thumbnail( return axum::http::StatusCode::NOT_FOUND.into_response(); } - let storage_path = resolve_metadata_storage_path( - config_snapshot.metadata_update.as_ref(), - &config_snapshot.storage_dir, - ); + let storage_path = + resolve_metadata_storage_path(config_snapshot.metadata_update.as_ref(), &config_snapshot.storage_dir); let storage = MetadataStorage::new(storage_path); if let Some(entry) = storage.load_by_uuid(&id).await { diff --git a/backend/src/api/endpoints/log_ws_api.rs b/backend/app/src/api/endpoints/log_ws_api.rs similarity index 94% rename from backend/src/api/endpoints/log_ws_api.rs rename to backend/app/src/api/endpoints/log_ws_api.rs index 577f841aa..7e9c883a7 100644 --- a/backend/src/api/endpoints/log_ws_api.rs +++ b/backend/app/src/api/endpoints/log_ws_api.rs @@ -13,7 +13,7 @@ use axum::{ use log::{trace, warn}; use serde::Deserialize; use shared::{ - model::{LogLevel, LogWsMessage, Permission, ROLE_ADMIN, WsCloseCode}, + model::{LogLevel, LogWsMessage, Permission, WsCloseCode, ROLE_ADMIN}, utils::concat_path_leading_slash, }; use std::sync::Arc; @@ -91,12 +91,7 @@ async fn wait_for_socket_auth(socket: &mut WebSocket, secret_key: Option<&[u8]>) } } -async fn handle_socket( - mut socket: WebSocket, - app_state: Arc, - auth_required: bool, - query: LogWsQuery, -) { +async fn handle_socket(mut socket: WebSocket, app_state: Arc, auth_required: bool, query: LogWsQuery) { let secret_key = get_secret_key(&app_state, auth_required); let mut is_authorized = !auth_required; let mut min_level: Option = query.min_level.as_deref().and_then(|s| s.parse().ok()); @@ -115,10 +110,8 @@ async fn handle_socket( // Send history let history = get_log_history(); - let filtered_history: Vec<_> = history - .into_iter() - .filter(|e| min_level.is_none_or(|lvl| e.level.matches(lvl))) - .collect(); + let filtered_history: Vec<_> = + history.into_iter().filter(|e| min_level.is_none_or(|lvl| e.level.matches(lvl))).collect(); if let Ok(history_json) = serde_json::to_string(&LogWsMessage::History(filtered_history)) { if socket.send(Message::Text(history_json.into())).await.is_err() { @@ -202,9 +195,11 @@ pub fn log_ws_api_register(web_auth_enabled: bool, web_ui_path: &str) -> axum::R #[cfg(test)] mod tests { use super::*; + use crate::{ + auth::{create_jwt_admin, create_jwt_web_user}, + model::WebAuthConfig, + }; use shared::model::permission::PermissionSet; - use crate::model::WebAuthConfig; - use crate::auth::{create_jwt_admin, create_jwt_web_user}; #[test] fn test_log_ws_query_parsing() { diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/app/src/api/endpoints/m3u_api.rs similarity index 95% rename from backend/src/api/endpoints/m3u_api.rs rename to backend/app/src/api/endpoints/m3u_api.rs index bb5ff91d9..151608e89 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/app/src/api/endpoints/m3u_api.rs @@ -82,7 +82,7 @@ fn m3u_api_with_auth( api_req: &UserApiRequest, ) -> Result<(Arc, Arc), ApiUserAuthError> { let (user, target) = get_user_target(api_req, app_state).ok_or(ApiUserAuthError::AuthFailed)?; - check_network_access_only(&user, fingerprint, app_state)?; + check_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip)?; Ok((user, target)) } @@ -96,7 +96,7 @@ fn m3u_api_stream_network_auth( ) -> Result<(Arc, Arc), ApiUserAuthError> { let (user, target) = get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state) .ok_or(ApiUserAuthError::AuthFailed)?; - check_network_access_only(&user, fingerprint, app_state)?; + check_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip)?; Ok((user, target)) } @@ -172,7 +172,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( .await; } return crate::api::model::create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, crate::api::model::CustomVideoStreamType::ChannelUnavailable, ) @@ -184,7 +184,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( return axum::http::StatusCode::BAD_REQUEST.into_response(); } - if user.permission_denied(app_state) { + if user.permission_denied(&app_state.app_config) { if is_hls_manifest_request { return hls_admission_failure_manifest_response( app_state, @@ -213,7 +213,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( let user_session = app_state.active_users.get_and_update_user_session(&user.username, &playback_session_token).await; let (admission, _grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( - app_state, + &app_state.admission_ctx(), &user, fingerprint, pli.item_type, @@ -356,7 +356,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( }; let (connection_admission, grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( - app_state, + &app_state.admission_ctx(), &user, fingerprint, pli.item_type, @@ -443,10 +443,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH if is_session_request && extension == shared::defaults::HLS_EXT { let Some(stream_context) = HlsEntryStreamContext::from_playlist_item(&pli) else { - error!( - "HLS input stream identity missing for virtual_id={}; refresh target playlist", - pli.virtual_id - ); + error!("HLS input stream identity missing for virtual_id={}; refresh target playlist", pli.virtual_id); return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); }; let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, pli.virtual_id); @@ -623,7 +620,9 @@ async fn m3u_api_stream( } if let Some((req_virtual_id, archive)) = parse_flat_flussonic_archive(stream_req.stream_id) { - let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { + let pli = match m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target) + .await + { Ok(pli) => pli, Err(err) => { error!("Failed to read M3U item for native archive stream id {req_virtual_id}: {err}"); @@ -665,21 +664,22 @@ async fn m3u_api_stream( let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); - let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { - Ok(pli) => pli, - Err(err) => { - error!("Failed to read m3u item for stream id {req_virtual_id}: {err}"); - if stream_ext == Some(HLS_EXT) { - return axum::http::StatusCode::NOT_FOUND.into_response(); + let pli = + match m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await { + Ok(pli) => pli, + Err(err) => { + error!("Failed to read m3u item for stream id {req_virtual_id}: {err}"); + if stream_ext == Some(HLS_EXT) { + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + return crate::api::model::create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + crate::api::model::CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); } - return crate::api::model::create_custom_video_stream_response( - app_state, - &fingerprint.addr, - crate::api::model::CustomVideoStreamType::ChannelUnavailable, - ) - .into_response(); - } - }; + }; let input = try_option_bad_request!( app_state.app_config.get_input_by_name(&pli.input_name), @@ -728,13 +728,14 @@ async fn m3u_api_stream_nested( return axum::http::StatusCode::BAD_REQUEST.into_response(); } let req_virtual_id: u32 = try_result_bad_request!(stream_req.stream_id.trim().parse()); - let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { - Ok(pli) => pli, - Err(err) => { - error!("Failed to read M3U item for nested stream id {req_virtual_id}: {err}"); - return axum::http::StatusCode::NOT_FOUND.into_response(); - } - }; + let pli = + match m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await { + Ok(pli) => pli, + Err(err) => { + error!("Failed to read M3U item for nested stream id {req_virtual_id}: {err}"); + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + }; let input = try_option_bad_request!( app_state.app_config.get_input_by_name(&pli.input_name), true, @@ -809,16 +810,16 @@ async fn m3u_api_catchup( if target.id != target_id { return axum::http::StatusCode::FORBIDDEN.into_response(); } - if let Err(err) = check_network_access_only(&user, &fingerprint, &app_state) { + if let Err(err) = check_network_access_only(&user, &fingerprint, &app_state.app_config, &app_state.geoip) { return err.into_player_response(app_state.app_config.get_auth_error_status()); } - if user.permission_denied(&app_state) || !target.has_output(TargetType::M3u) { + if user.permission_denied(&app_state.app_config) || !target.has_output(TargetType::M3u) { return axum::http::StatusCode::FORBIDDEN.into_response(); } let _user_guard = app_state.app_config.file_locks.write_lock_str(&user.username).await; let pli = try_result_not_found!( - m3u_get_item_for_stream_id(virtual_id, &app_state, &target).await, + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, &target).await, true, format!("Failed to read m3u item for stream id {virtual_id}") ); @@ -862,7 +863,13 @@ fn m3u_api_resource_auth( ) -> Result<(Arc, Arc), ApiUserAuthError> { let (user, target) = get_user_target_by_credentials(username, password, api_req, app_state).ok_or(ApiUserAuthError::AuthFailed)?; - resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), app_state)?; + resolve_api_user_context( + user.clone(), + target.clone(), + fingerprint.clone(), + &app_state.app_config, + &app_state.geoip, + )?; Ok((user, target)) } @@ -892,13 +899,14 @@ async fn m3u_api_resource( debug!("Target has no m3u playlist {target_name}"); return axum::http::StatusCode::BAD_REQUEST.into_response(); } - let m3u_item = match m3u_get_item_for_stream_id(m3u_stream_id, &app_state, &target).await { - Ok(item) => item, - Err(err) => { - error!("Failed to get m3u url: {}", sanitize_sensitive_info(&err.to_string())); - return axum::http::StatusCode::NOT_FOUND.into_response(); - } - }; + let m3u_item = + match m3u_get_item_for_stream_id(m3u_stream_id, &app_state.app_config, &app_state.playlists, &target).await { + Ok(item) => item, + Err(err) => { + error!("Failed to get m3u url: {}", sanitize_sensitive_info(&err.to_string())); + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + }; if !user.allows_item_type(m3u_item.item_type) || !(user.t_filter.is_none() || user.allows_content(&shared::model::PlaylistItem::from(&m3u_item))) diff --git a/backend/src/api/endpoints/mod.rs b/backend/app/src/api/endpoints/mod.rs similarity index 100% rename from backend/src/api/endpoints/mod.rs rename to backend/app/src/api/endpoints/mod.rs diff --git a/backend/src/api/endpoints/provider_resolve_api.rs b/backend/app/src/api/endpoints/provider_resolve_api.rs similarity index 87% rename from backend/src/api/endpoints/provider_resolve_api.rs rename to backend/app/src/api/endpoints/provider_resolve_api.rs index 0a527e68a..5dce43198 100644 --- a/backend/src/api/endpoints/provider_resolve_api.rs +++ b/backend/app/src/api/endpoints/provider_resolve_api.rs @@ -1,11 +1,14 @@ use crate::{ api::{api_utils::redirect, model::AppState}, auth::{check_network_access_only, Fingerprint}, - model::{resolve_provider_scheme_url_with_provider_index, ApiProxyServerInfo, ConfigInput, ConfigTarget, ProxyUserCredentials}, + model::{ + resolve_provider_scheme_url_with_provider_index, ApiProxyServerInfo, ConfigInput, ConfigTarget, + ProxyUserCredentials, + }, repository::{m3u_get_item_for_stream_id, storage_const, xtream_get_item_for_stream_id}, utils::{ - decode_provider_resolve_token, extract_extension_from_url, sanitize_sensitive_info, - ProviderResolveToken, PROVIDER_RESOLVE_ROUTE_PREFIX, + decode_provider_resolve_token, extract_extension_from_url, sanitize_sensitive_info, ProviderResolveToken, + PROVIDER_RESOLVE_ROUTE_PREFIX, }, }; use axum::{extract, response::IntoResponse}; @@ -58,8 +61,11 @@ fn provider_resolve_output_kind(target: &ConfigTarget) -> Option Result { if let Some(provider) = input.get_resolve_provider(url) { - let (_, resolved) = - resolve_provider_scheme_url_with_provider_index(url, Some(Arc::clone(&provider)), provider.get_current_index())?; + let (_, resolved) = resolve_provider_scheme_url_with_provider_index( + url, + Some(Arc::clone(&provider)), + provider.get_current_index(), + )?; return Ok(resolved.into_owned()); } input.resolve_url(url).map(Cow::into_owned) @@ -82,10 +88,9 @@ fn resolve_provider_playlist_item_location( let ext = extract_extension_from_url(item.url).unwrap_or_default(); let base_url = server_info.get_base_url(); Ok(match output_kind { - ProviderResolveOutputKind::Xtream => format!( - "{base_url}/{stream_type}/{}/{}/{}{}", - user.username, user.password, item.virtual_id, ext - ), + ProviderResolveOutputKind::Xtream => { + format!("{base_url}/{stream_type}/{}/{}/{}{}", user.username, user.password, item.virtual_id, ext) + } ProviderResolveOutputKind::M3u => format!( "{base_url}/{}/{stream_type}/{}/{}/{}{}", storage_const::M3U_STREAM_PATH, @@ -118,7 +123,8 @@ async fn load_provider_resolve_item( match output_kind { ProviderResolveOutputKind::Xtream => xtream_get_item_for_stream_id( decoded_virtual_id, - app_state, + &app_state.app_config, + &app_state.playlists, target, Some(decoded_cluster), ) @@ -127,16 +133,16 @@ async fn load_provider_resolve_item( virtual_id: item.virtual_id, item_type: item.item_type, cluster: item.xtream_cluster, - content_allowed: user.t_filter.is_none() - || user.allows_content(&shared::model::PlaylistItem::from(&item)), + content_allowed: user.t_filter.is_none() || user.allows_content(&shared::model::PlaylistItem::from(&item)), url: item.url, input_name: item.input_name, }) .map_err(|err| TuliproxError::RepositoryXtream(err.to_string())), ProviderResolveOutputKind::M3u => { - let item = m3u_get_item_for_stream_id(decoded_virtual_id, app_state, target) - .await - .map_err(|err| TuliproxError::RepositoryM3u(err.to_string()))?; + let item = + m3u_get_item_for_stream_id(decoded_virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .map_err(|err| TuliproxError::RepositoryM3u(err.to_string()))?; if !item.item_type.is_cluster(decoded_cluster) { return Err(TuliproxError::RepositoryM3u(format!( "M3U item {} is not in requested cluster {}", @@ -177,10 +183,10 @@ async fn provider_resolve( if target.id != decoded.target_id { return axum::http::StatusCode::FORBIDDEN.into_response(); } - if let Err(err) = check_network_access_only(&user, &fingerprint, &app_state) { + if let Err(err) = check_network_access_only(&user, &fingerprint, &app_state.app_config, &app_state.geoip) { return err.into_player_response(app_state.app_config.get_auth_error_status()); } - if user.permission_denied(&app_state) || !user.allows_cluster(decoded.cluster) { + if user.permission_denied(&app_state.app_config) || !user.allows_cluster(decoded.cluster) { return axum::http::StatusCode::FORBIDDEN.into_response(); } let Some(output_kind) = provider_resolve_output_kind(&target) else { @@ -192,13 +198,16 @@ async fn provider_resolve( return axum::http::StatusCode::BAD_REQUEST.into_response(); }; - let item = match load_provider_resolve_item(output_kind, decoded.virtual_id, decoded.cluster, &app_state, &target, &user).await { - Ok(item) => item, - Err(err) => { - debug!("Provider resolve item lookup failed: {err}"); - return axum::http::StatusCode::NOT_FOUND.into_response(); - } - }; + let item = + match load_provider_resolve_item(output_kind, decoded.virtual_id, decoded.cluster, &app_state, &target, &user) + .await + { + Ok(item) => item, + Err(err) => { + debug!("Provider resolve item lookup failed: {err}"); + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + }; if !user.allows_item_type(item.item_type) || !item.content_allowed { return axum::http::StatusCode::FORBIDDEN.into_response(); } @@ -229,10 +238,8 @@ async fn provider_resolve( } pub fn provider_resolve_api_register() -> axum::Router> { - axum::Router::new().route( - &format!("{PROVIDER_RESOLVE_ROUTE_PREFIX}/{{token}}"), - axum::routing::get(provider_resolve), - ) + axum::Router::new() + .route(&format!("{PROVIDER_RESOLVE_ROUTE_PREFIX}/{{token}}"), axum::routing::get(provider_resolve)) } #[cfg(test)] @@ -264,9 +271,7 @@ mod tests { user } - fn input() -> ConfigInput { - input_with_provider_urls(vec!["http://provider.example.com".into()], None) - } + fn input() -> ConfigInput { input_with_provider_urls(vec!["http://provider.example.com".into()], None) } fn input_with_provider_urls(urls: Vec>, current_index: Option) -> ConfigInput { let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { diff --git a/backend/src/api/endpoints/rbac_api.rs b/backend/app/src/api/endpoints/rbac_api.rs similarity index 88% rename from backend/src/api/endpoints/rbac_api.rs rename to backend/app/src/api/endpoints/rbac_api.rs index 5b1e157fc..f617743da 100644 --- a/backend/src/api/endpoints/rbac_api.rs +++ b/backend/app/src/api/endpoints/rbac_api.rs @@ -3,6 +3,7 @@ use crate::{ auth::{generate_password_from_input, verify_token, AuthBearer}, model::{RbacGroup, WebAuthConfig, WebUiUser}, utils, + utils::{get_default_user_file_path, get_default_user_group_file_path}, }; use axum::{ extract::{Path, State}, @@ -14,17 +15,18 @@ use log::{info, warn}; use rand::Rng; use serde::Deserialize; use serde_json::json; -use shared::model::{ - permission::{permission_from_name, Permission, PermissionSet, PERMISSION_NAMES}, - RbacGroupDto, WebUiUserDto, +use shared::{ + defaults::{is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path}, + model::{ + permission::{permission_from_name, Permission, PermissionSet, PERMISSION_NAMES}, + RbacGroupDto, WebUiUserDto, + }, }; use std::{ collections::HashSet, path::{Path as FsPath, PathBuf}, sync::Arc, }; -use shared::defaults::{is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path}; -use crate::utils::{get_default_user_file_path, get_default_user_group_file_path}; const RBAC_MUTATION_LOCK: &str = "rbac:mutation"; @@ -96,7 +98,8 @@ async fn write_text_file_atomic(path: &FsPath, content: &str) -> Result<(), std: } fn serialize_users_file(users: &[WebUiUser]) -> String { - users.iter() + users + .iter() .map(|user| { if user.groups.is_empty() || (user.groups.len() == 1 && user.groups[0].eq_ignore_ascii_case("admin")) { format!("{}:{}", user.username, user.password_hash) @@ -109,7 +112,8 @@ fn serialize_users_file(users: &[WebUiUser]) -> String { } fn serialize_groups_file(groups: &[RbacGroup]) -> String { - groups.iter() + groups + .iter() .map(|group| { let permissions = PERMISSION_NAMES .iter() @@ -201,7 +205,9 @@ fn validate_permission_dependencies(permission_names: &[String]) -> Result<(), V } } -fn user_has_admin_group(user: &WebUiUser) -> bool { user.groups.iter().any(|group| group.eq_ignore_ascii_case("admin")) } +fn user_has_admin_group(user: &WebUiUser) -> bool { + user.groups.iter().any(|group| group.eq_ignore_ascii_case("admin")) +} fn count_admin_users(users: &[WebUiUser]) -> usize { users.iter().filter(|user| user_has_admin_group(user)).count() } @@ -356,11 +362,9 @@ async fn list_users(State(app_state): State>) -> impl IntoResponse .and_then(|web_ui| web_ui.auth.as_ref()) .and_then(|auth| auth.t_users.as_ref()) .map(|users| { - users.iter() - .map(|user| WebUiUserDto { - username: user.username.clone(), - groups: user.groups.clone(), - }) + users + .iter() + .map(|user| WebUiUserDto { username: user.username.clone(), groups: user.groups.clone() }) .collect() }) .unwrap_or_default(); @@ -374,28 +378,18 @@ async fn list_users_unprotected() -> impl IntoResponse { async fn list_groups(State(app_state): State>) -> impl IntoResponse { let config = app_state.app_config.config.load(); - let mut groups = vec![RbacGroupDto { - name: "admin".to_string(), - permissions: vec!["*".to_string()], - builtin: true, - }]; + let mut groups = + vec![RbacGroupDto { name: "admin".to_string(), permissions: vec!["*".to_string()], builtin: true }]; - if let Some(parsed_groups) = config - .web_ui - .as_ref() - .and_then(|web_ui| web_ui.auth.as_ref()) - .and_then(|auth| auth.t_groups.as_ref()) + if let Some(parsed_groups) = + config.web_ui.as_ref().and_then(|web_ui| web_ui.auth.as_ref()).and_then(|auth| auth.t_groups.as_ref()) { for group in parsed_groups { let permissions = PERMISSION_NAMES .iter() .filter_map(|(name, permission)| group.permissions.contains(*permission).then_some((*name).to_string())) .collect(); - groups.push(RbacGroupDto { - name: group.name.clone(), - permissions, - builtin: false, - }); + groups.push(RbacGroupDto { name: group.name.clone(), permissions, builtin: false }); } } @@ -403,22 +397,14 @@ async fn list_groups(State(app_state): State>) -> impl IntoRespons } async fn list_groups_unprotected() -> impl IntoResponse { - let groups = vec![RbacGroupDto { - name: "admin".to_string(), - permissions: vec!["*".to_string()], - builtin: true, - }]; + let groups = vec![RbacGroupDto { name: "admin".to_string(), permissions: vec!["*".to_string()], builtin: true }]; Json(groups) } - async fn list_permissions() -> impl IntoResponse { let permissions = PERMISSION_NAMES .iter() - .map(|(name, permission)| PermissionInfo { - name, - reserved: matches!(permission, Permission::EpgWrite), - }) + .map(|(name, permission)| PermissionInfo { name, reserved: matches!(permission, Permission::EpgWrite) }) .collect::>(); Json(permissions) } @@ -432,7 +418,8 @@ async fn create_user( return (StatusCode::BAD_REQUEST, Json(json!({"error": "Username cannot be empty"}))).into_response(); } if request.password.len() < 8 { - return (StatusCode::BAD_REQUEST, Json(json!({"error": "Password must be at least 8 characters"}))).into_response(); + return (StatusCode::BAD_REQUEST, Json(json!({"error": "Password must be at least 8 characters"}))) + .into_response(); } let groups = normalize_groups(&request.groups); if let Err((status, error)) = reject_empty_groups(&groups) { @@ -463,20 +450,18 @@ async fn create_user( let available_groups = web_auth.t_groups.clone().unwrap_or_default(); if users.iter().any(|user| user.username.eq_ignore_ascii_case(&username)) { - return (StatusCode::CONFLICT, Json(json!({"error": format!("User '{username}' already exists")}))).into_response(); + return (StatusCode::CONFLICT, Json(json!({"error": format!("User '{username}' already exists")}))) + .into_response(); } if let Err(err) = validate_group_names(&groups, &available_groups) { return (StatusCode::BAD_REQUEST, Json(json!({"error": err}))).into_response(); } - users.push(WebUiUser { - username: username.clone(), - password_hash: hash, - groups, - }); + users.push(WebUiUser { username: username.clone(), password_hash: hash, groups }); let (userfile_path, _) = resolve_auth_paths(&web_auth, &config_path); - if let Err(response) = save_and_reprepare_auth_file(&app_state, &userfile_path, &serialize_users_file(&users)).await { + if let Err(response) = save_and_reprepare_auth_file(&app_state, &userfile_path, &serialize_users_file(&users)).await + { return response.into_response(); } @@ -494,7 +479,8 @@ async fn update_user( return (StatusCode::BAD_REQUEST, Json(json!({"error": "Username cannot be empty"}))).into_response(); } if request.password.as_ref().is_some_and(|password| password.len() < 8) { - return (StatusCode::BAD_REQUEST, Json(json!({"error": "Password must be at least 8 characters"}))).into_response(); + return (StatusCode::BAD_REQUEST, Json(json!({"error": "Password must be at least 8 characters"}))) + .into_response(); } let groups = normalize_groups(&request.groups); @@ -535,12 +521,10 @@ async fn update_user( return (StatusCode::NOT_FOUND, Json(json!({"error": format!("User '{username}' not found")}))).into_response(); }; - let removing_admin = user_has_admin_group(&users[user_index]) && !groups.iter().any(|group| group.eq_ignore_ascii_case("admin")); + let removing_admin = + user_has_admin_group(&users[user_index]) && !groups.iter().any(|group| group.eq_ignore_ascii_case("admin")); if removing_admin && count_admin_users(&users) == 1 { - return ( - StatusCode::CONFLICT, - Json(json!({"error": "Cannot remove admin group from the last admin user"})), - ) + return (StatusCode::CONFLICT, Json(json!({"error": "Cannot remove admin group from the last admin user"}))) .into_response(); } @@ -550,7 +534,8 @@ async fn update_user( } let (userfile_path, _) = resolve_auth_paths(&web_auth, &config_path); - if let Err(response) = save_and_reprepare_auth_file(&app_state, &userfile_path, &serialize_users_file(&users)).await { + if let Err(response) = save_and_reprepare_auth_file(&app_state, &userfile_path, &serialize_users_file(&users)).await + { return response.into_response(); } @@ -568,7 +553,9 @@ async fn delete_user( return (StatusCode::BAD_REQUEST, Json(json!({"error": "Username cannot be empty"}))).into_response(); } - if current_username(&app_state, &token).is_some_and(|current_username| current_username.eq_ignore_ascii_case(&username)) { + if current_username(&app_state, &token) + .is_some_and(|current_username| current_username.eq_ignore_ascii_case(&username)) + { return (StatusCode::FORBIDDEN, Json(json!({"error": "Admin cannot delete themselves"}))).into_response(); } @@ -589,7 +576,8 @@ async fn delete_user( users.remove(user_index); let (userfile_path, _) = resolve_auth_paths(&web_auth, &config_path); - if let Err(response) = save_and_reprepare_auth_file(&app_state, &userfile_path, &serialize_users_file(&users)).await { + if let Err(response) = save_and_reprepare_auth_file(&app_state, &userfile_path, &serialize_users_file(&users)).await + { return response.into_response(); } @@ -623,16 +611,16 @@ async fn create_group( }; let mut groups = web_auth.t_groups.clone().unwrap_or_default(); if groups.iter().any(|group| group.name.eq_ignore_ascii_case(&name)) { - return (StatusCode::CONFLICT, Json(json!({"error": format!("Group '{name}' already exists")}))).into_response(); + return (StatusCode::CONFLICT, Json(json!({"error": format!("Group '{name}' already exists")}))) + .into_response(); } - groups.push(RbacGroup { - name: name.clone(), - permissions: normalize_permissions(&request.permissions), - }); + groups.push(RbacGroup { name: name.clone(), permissions: normalize_permissions(&request.permissions) }); let (_, groupfile_path) = resolve_auth_paths(&web_auth, &config_path); - if let Err(response) = save_and_reprepare_auth_file(&app_state, &groupfile_path, &serialize_groups_file(&groups)).await { + if let Err(response) = + save_and_reprepare_auth_file(&app_state, &groupfile_path, &serialize_groups_file(&groups)).await + { return response.into_response(); } @@ -673,7 +661,9 @@ async fn update_group( groups[group_index].permissions = normalize_permissions(&request.permissions); let (_, groupfile_path) = resolve_auth_paths(&web_auth, &config_path); - if let Err(response) = save_and_reprepare_auth_file(&app_state, &groupfile_path, &serialize_groups_file(&groups)).await { + if let Err(response) = + save_and_reprepare_auth_file(&app_state, &groupfile_path, &serialize_groups_file(&groups)).await + { return response.into_response(); } @@ -681,10 +671,7 @@ async fn update_group( StatusCode::OK.into_response() } -async fn delete_group( - State(app_state): State>, - Path(name): Path, -) -> impl IntoResponse { +async fn delete_group(State(app_state): State>, Path(name): Path) -> impl IntoResponse { let name = normalize_name(&name); if name.is_empty() { return (StatusCode::BAD_REQUEST, Json(json!({"error": "Group name cannot be empty"}))).into_response(); @@ -723,7 +710,9 @@ async fn delete_group( groups.remove(group_index); let (_, groupfile_path) = resolve_auth_paths(&web_auth, &config_path); - if let Err(response) = save_and_reprepare_auth_file(&app_state, &groupfile_path, &serialize_groups_file(&groups)).await { + if let Err(response) = + save_and_reprepare_auth_file(&app_state, &groupfile_path, &serialize_groups_file(&groups)).await + { return response.into_response(); } @@ -770,24 +759,16 @@ pub fn rbac_api_register(app_state: Arc) -> Router> { pub fn rbac_api_register_unprotected(_app_state: Arc) -> Router> { Router::new() - .route( - "/rbac/users", axum::routing::get(list_users_unprotected) - ) - .route( - "/rbac/groups", axum::routing::get(list_groups_unprotected) - ) - .route( - "/rbac/permissions", axum::routing::get(list_permissions) - ) + .route("/rbac/users", axum::routing::get(list_users_unprotected)) + .route("/rbac/groups", axum::routing::get(list_groups_unprotected)) + .route("/rbac/permissions", axum::routing::get(list_permissions)) } - #[cfg(test)] mod tests { use super::{reject_empty_groups, resolve_auth_paths, validate_permission_dependencies}; + use crate::{model::WebAuthConfig, utils}; use axum::http::StatusCode; - use crate::model::WebAuthConfig; - use crate::utils; #[test] fn rejects_write_permission_without_matching_read_permission() { @@ -798,10 +779,7 @@ mod tests { #[test] fn rejects_empty_group_assignments() { - assert_eq!( - reject_empty_groups(&[]), - Err((StatusCode::BAD_REQUEST, "At least one group must be assigned")), - ); + assert_eq!(reject_empty_groups(&[]), Err((StatusCode::BAD_REQUEST, "At least one group must be assigned")),); } #[test] diff --git a/backend/src/api/endpoints/recording_api.rs b/backend/app/src/api/endpoints/recording_api.rs similarity index 99% rename from backend/src/api/endpoints/recording_api.rs rename to backend/app/src/api/endpoints/recording_api.rs index 1595a1549..abb563d8f 100644 --- a/backend/src/api/endpoints/recording_api.rs +++ b/backend/app/src/api/endpoints/recording_api.rs @@ -294,7 +294,7 @@ pub async fn preview_recording_conflicts( post_roll_secs: body.candidate.post_roll_secs, priority: body.candidate.priority, }; - let service = crate::api::model::recording_service::RecordingService::from_app_state(&state); + let service = crate::api::model::recording_service::RecordingService::from_ctx(&state.recording_ctx()); let preview = match service.preview_conflicts(&claims, &request).await { Ok(preview) => preview, Err(err) => return service_error_response(&err), @@ -936,7 +936,7 @@ pub async fn recording_availability( { return error_response(StatusCode::FORBIDDEN, "recording_forbidden"); } - if !crate::api::model::recording::recording_supervisor::recording_enabled(app_state.as_ref()) { + if !crate::api::model::recording::recording_supervisor::recording_enabled(&app_state.app_config) { return error_response(StatusCode::NOT_IMPLEMENTED, "recording_disabled"); } StatusCode::NO_CONTENT.into_response() @@ -962,7 +962,7 @@ pub async fn require_recording_enabled( request: axum::extract::Request, next: axum::middleware::Next, ) -> axum::response::Response { - if crate::api::model::recording::recording_supervisor::recording_enabled(app_state.as_ref()) { + if crate::api::model::recording::recording_supervisor::recording_enabled(&app_state.app_config) { next.run(request).await } else { error_response(StatusCode::NOT_IMPLEMENTED, "recording_disabled") diff --git a/backend/src/api/endpoints/recording_media_api.rs b/backend/app/src/api/endpoints/recording_media_api.rs similarity index 98% rename from backend/src/api/endpoints/recording_media_api.rs rename to backend/app/src/api/endpoints/recording_media_api.rs index 036b17677..7d0f61fde 100644 --- a/backend/src/api/endpoints/recording_media_api.rs +++ b/backend/app/src/api/endpoints/recording_media_api.rs @@ -268,11 +268,7 @@ async fn resolve_for_open(app_state: &AppState, claims: &Claims, uuid: &str) -> let file_meta = no_follow_path_in_root(&recording_root, &abs_path) .await .ok_or_else(|| Box::new(access_error_to_response(&CatalogAccessError::NotFound)))?; - Ok(ResolvedMedia { - abs_path, - recording_root, - size: file_meta.len(), - }) + Ok(ResolvedMedia { abs_path, recording_root, size: file_meta.len() }) } /// `GET /library/recording/playback/{uuid}` — supports HTTP Range @@ -410,7 +406,9 @@ mod tests { use crate::{api::model::create_test_app_state, auth::create_jwt_admin, model::Config}; use axum::http::Request; use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; - use shared::model::{Permission, UserId, WebUiConfigDto, CURRENT_PERMISSION_SCHEMA_VERSION, PERM_ALL, ROLE_ADMIN, TOKEN_NO_AUTH}; + use shared::model::{ + Permission, UserId, WebUiConfigDto, CURRENT_PERMISSION_SCHEMA_VERSION, PERM_ALL, ROLE_ADMIN, TOKEN_NO_AUTH, + }; fn config_with_web_auth(enabled: bool, secret: &str) -> Config { let web_ui = WebUiConfigDto { @@ -435,9 +433,7 @@ mod tests { } let request = builder.body(()).expect("request"); let (mut parts, ()) = request.into_parts(); - AuthClaims::from_request_parts(&mut parts, state) - .await - .map_err(Box::new) + AuthClaims::from_request_parts(&mut parts, state).await.map_err(Box::new) } #[tokio::test] diff --git a/backend/src/api/endpoints/stream_history_api.rs b/backend/app/src/api/endpoints/stream_history_api.rs similarity index 85% rename from backend/src/api/endpoints/stream_history_api.rs rename to backend/app/src/api/endpoints/stream_history_api.rs index 9574e9065..1877a5d21 100644 --- a/backend/src/api/endpoints/stream_history_api.rs +++ b/backend/app/src/api/endpoints/stream_history_api.rs @@ -1,26 +1,30 @@ -use std::cmp::Reverse; -use std::collections::{BinaryHeap, HashMap}; -use std::io; -use std::path::Path; -use std::str::FromStr; -use std::sync::Arc; - -use crate::api::api_utils::json_or_bin_response; -use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader; -use crate::api::model::AppState; -use crate::model::StreamHistoryRecord; -use crate::repository::{QosSnapshotRecord, QosSnapshotRepository, StreamHistoryFileReader}; -use crate::utils::stream_history_viewer::{discover_files, resolve_time_range, CompiledFilter, StreamHistoryQuery, TimeRange}; - -use axum::extract::{Path as AxumPath, Query, State}; -use axum::http::StatusCode; -use axum::response::{IntoResponse, Response}; +use crate::{ + api::{api_utils::json_or_bin_response, endpoints::extract_accept_header::ExtractAcceptHeader, model::AppState}, + model::StreamHistoryRecord, + repository::{ + discover_files, resolve_time_range, CompiledFilter, QosSnapshotRecord, QosSnapshotRepository, + StreamHistoryFileReader, StreamHistoryQuery, TimeRange, + }, +}; +use axum::{ + extract::{Path as AxumPath, Query, State}, + http::StatusCode, + response::{IntoResponse, Response}, +}; use regex::{Regex, RegexBuilder}; use serde::{Deserialize, Serialize}; use shared::model::{ - PageRequestDto, PagedResponseDto, SearchMode, StreamHistoryEventType, StreamHistoryPageRequestDto, - StreamHistoryProviderSummaryDto, StreamHistorySearchField, QosSnapshotRecordDto, StreamHistoryQueryRequestDto, - StreamHistoryRecordDto, + PageRequestDto, PagedResponseDto, QosSnapshotRecordDto, SearchMode, StreamHistoryEventType, + StreamHistoryPageRequestDto, StreamHistoryProviderSummaryDto, StreamHistoryQueryRequestDto, StreamHistoryRecordDto, + StreamHistorySearchField, +}; +use std::{ + cmp::Reverse, + collections::{BinaryHeap, HashMap}, + io, + path::Path, + str::FromStr, + sync::Arc, }; use strum::IntoEnumIterator; @@ -30,9 +34,7 @@ fn compile_search_fields(fields: Option>) -> Result { record.bytes_sent.is_some_and(|value| matcher.is_match(&value.to_string())) } - StreamHistorySearchField::FirstByteLatencyMs => record - .first_byte_latency_ms - .is_some_and(|value| matcher.is_match(&value.to_string())), + StreamHistorySearchField::FirstByteLatencyMs => { + record.first_byte_latency_ms.is_some_and(|value| matcher.is_match(&value.to_string())) + } StreamHistorySearchField::UserAgent => { record.user_agent.as_deref().is_some_and(|value| matcher.is_match(value)) } StreamHistorySearchField::ItemType => { record.item_type.as_ref().is_some_and(|value| matcher.is_match(&value.to_string())) } - StreamHistorySearchField::Container => { - record.container.as_deref().is_some_and(|value| matcher.is_match(value)) + StreamHistorySearchField::Container => record.container.as_deref().is_some_and(|value| matcher.is_match(value)), + StreamHistorySearchField::DisconnectReason => { + record.disconnect_reason.as_ref().is_some_and(|value| matcher.is_match(&value.to_string())) } - StreamHistorySearchField::DisconnectReason => record - .disconnect_reason - .as_ref() - .is_some_and(|value| matcher.is_match(&value.to_string())), StreamHistorySearchField::SourceAddr => { record.source_addr.as_deref().is_some_and(|value| matcher.is_match(value)) } @@ -166,20 +165,16 @@ impl HlsSessionAccumulator { self.total_bytes = self.total_bytes.saturating_add(record.bytes_sent.unwrap_or(0)); match record.event_type { StreamHistoryEventType::Connect => { - let should_replace = self - .first_connect - .as_ref() - .is_none_or(|current| record.event_ts_utc < current.event_ts_utc); + let should_replace = + self.first_connect.as_ref().is_none_or(|current| record.event_ts_utc < current.event_ts_utc); if should_replace { self.first_connect = Some(record); } } StreamHistoryEventType::Disconnect => { self.disconnect_count = self.disconnect_count.saturating_add(1); - let should_replace = self - .last_disconnect - .as_ref() - .is_none_or(|current| record.event_ts_utc >= current.event_ts_utc); + let should_replace = + self.last_disconnect.as_ref().is_none_or(|current| record.event_ts_utc >= current.event_ts_utc); if should_replace { self.last_disconnect = Some(record); } @@ -304,13 +299,7 @@ impl TopHistoryPageCollector { "Stream history page window too large: page={page}, page_size={page_size}, start={start}, capacity={capacity}, max={MAX_STREAM_HISTORY_PAGE_HEAP_CAPACITY}" )); } - Ok(Self { - capacity, - start, - limit, - total_items: 0, - heap: BinaryHeap::with_capacity(capacity), - }) + Ok(Self { capacity, start, limit, total_items: 0, heap: BinaryHeap::with_capacity(capacity) }) } fn push(&mut self, record: StreamHistoryRecord, matcher: Option<&Regex>, fields: &[StreamHistorySearchField]) { @@ -323,9 +312,7 @@ impl TopHistoryPageCollector { return; } - let ranked = RankedHistoryRecord { - record, - }; + let ranked = RankedHistoryRecord { record }; if self.heap.len() < self.capacity { self.heap.push(Reverse(ranked)); @@ -360,10 +347,8 @@ fn push_hls_or_non_hls( fields: &[StreamHistorySearchField], ) { let is_hls = matches!(record.container.as_deref(), Some("mpegts" | "fmp4" | "hls")); - let is_hls_session_event = matches!( - record.event_type, - StreamHistoryEventType::Connect | StreamHistoryEventType::Disconnect - ); + let is_hls_session_event = + matches!(record.event_type, StreamHistoryEventType::Connect | StreamHistoryEventType::Disconnect); if is_hls && is_hls_session_event { let key: HlsKey = ( record.source_addr.clone(), @@ -435,7 +420,9 @@ fn paginate_aggregated_records( } filtered.sort_unstable_by_key(|record| Reverse(record.event_ts_utc)); - let Ok(mut collector) = TopHistoryPageCollector::new(page, page_size) else { return PagedResponseDto::new(Vec::new(), page, page_size, 0) }; + let Ok(mut collector) = TopHistoryPageCollector::new(page, page_size) else { + return PagedResponseDto::new(Vec::new(), page, page_size, 0); + }; for record in filtered { collector.push(record, None, &[]); } @@ -468,12 +455,7 @@ pub(crate) async fn stream_history_page_query( }; // Build time range - let query = StreamHistoryQuery { - from: params.from.clone(), - to: params.to.clone(), - path: None, - filter: None, - }; + let query = StreamHistoryQuery { from: params.from.clone(), to: params.to.clone(), path: None, filter: None }; let time_range = match resolve_time_range(&query) { Ok(tr) => tr, @@ -486,33 +468,42 @@ pub(crate) async fn stream_history_page_query( }; let page = page_req.page; let page_size = page_req.page_size; - let batch_records = if let Some(hw) = app_state - .connection_manager - .history_writer() - .load() - .as_ref() - { + let batch_records = if let Some(hw) = app_state.connection_manager.history_writer().load().as_ref() { hw.get_current_batch().await.unwrap_or_else(|_| Vec::new()) } else { Vec::new() }; let result = tokio::task::spawn_blocking(move || { - let empty_filter = CompiledFilter::compile(&HashMap::new()) - .map(Arc::new) - .map_err(|err| StreamHistoryPageQueryError::Internal(format!("Failed to compile stream history filter: {err}")))?; + let empty_filter = CompiledFilter::compile(&HashMap::new()).map(Arc::new).map_err(|err| { + StreamHistoryPageQueryError::Internal(format!("Failed to compile stream history filter: {err}")) + })?; let file_iter = futures::executor::block_on(collect_records_iter(&history_dir, time_range, empty_filter)) - .map_err(|err| StreamHistoryPageQueryError::Internal(format!("Failed to open stream history files: {err}")))?; + .map_err(|err| { + StreamHistoryPageQueryError::Internal(format!("Failed to open stream history files: {err}")) + })?; - paginate_stream_history_records(file_iter, batch_records, search_matcher.as_ref(), &search_fields, page, page_size) - .map_err(StreamHistoryPageQueryError::InvalidWindow) - }).await; + paginate_stream_history_records( + file_iter, + batch_records, + search_matcher.as_ref(), + &search_fields, + page, + page_size, + ) + .map_err(StreamHistoryPageQueryError::InvalidWindow) + }) + .await; match result { Ok(Ok(response)) => axum::Json(response).into_response(), - Ok(Err(StreamHistoryPageQueryError::InvalidWindow(message))) => error_response(StatusCode::BAD_REQUEST, message), - Ok(Err(StreamHistoryPageQueryError::Internal(message))) => error_response(StatusCode::INTERNAL_SERVER_ERROR, message), + Ok(Err(StreamHistoryPageQueryError::InvalidWindow(message))) => { + error_response(StatusCode::BAD_REQUEST, message) + } + Ok(Err(StreamHistoryPageQueryError::Internal(message))) => { + error_response(StatusCode::INTERNAL_SERVER_ERROR, message) + } Err(e) => error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Task failed: {e}")), } } @@ -550,12 +541,7 @@ pub(crate) async fn stream_history_summary_query( }; let filters_arc = Arc::new(filters); - let batch_records = if let Some(hw) = app_state - .connection_manager - .history_writer() - .load() - .as_ref() - { + let batch_records = if let Some(hw) = app_state.connection_manager.history_writer().load().as_ref() { hw.get_current_batch().await.unwrap_or_else(|_| Vec::new()) } else { Vec::new() @@ -570,14 +556,12 @@ pub(crate) async fn stream_history_summary_query( match result { Ok(Ok(summaries)) => json_or_bin_response(accept.as_deref(), &summaries).into_response(), - Ok(Err(err)) => error_response( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to discover history files: {err}"), - ), - Err(err) => error_response( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Stream history summary task failed: {err}"), - ), + Ok(Err(err)) => { + error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to discover history files: {err}")) + } + Err(err) => { + error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Stream history summary task failed: {err}")) + } } } @@ -596,21 +580,16 @@ pub(crate) async fn qos_snapshot_query( }; let limit = params.limit.unwrap_or(100).max(1); - let result = tokio::task::spawn_blocking(move || { - collect_filtered_qos_snapshots(Path::new(&storage_dir), &filters, limit) - }) - .await; + let result = + tokio::task::spawn_blocking(move || collect_filtered_qos_snapshots(Path::new(&storage_dir), &filters, limit)) + .await; match result { Ok(Ok(records)) => json_or_bin_response(accept.as_deref(), &records).into_response(), - Ok(Err(err)) => error_response( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read QoS snapshots: {err}"), - ), - Err(err) => error_response( - StatusCode::INTERNAL_SERVER_ERROR, - format!("QoS snapshot query task failed: {err}"), - ), + Ok(Err(err)) => { + error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read QoS snapshots: {err}")) + } + Err(err) => error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("QoS snapshot query task failed: {err}")), } } @@ -623,20 +602,17 @@ pub(crate) async fn qos_snapshot_detail_query( return error_response(StatusCode::SERVICE_UNAVAILABLE, "QoS aggregation is not enabled"); }; - let result = tokio::task::spawn_blocking(move || load_qos_snapshot(&storage_dir, &stream_identity_key)) - .await; + let result = tokio::task::spawn_blocking(move || load_qos_snapshot(&storage_dir, &stream_identity_key)).await; match result { Ok(Ok(Some(record))) => json_or_bin_response(accept.as_deref(), &record).into_response(), Ok(Ok(None)) => error_response(StatusCode::NOT_FOUND, "QoS snapshot not found"), - Ok(Err(err)) => error_response( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read QoS snapshot detail: {err}"), - ), - Err(err) => error_response( - StatusCode::INTERNAL_SERVER_ERROR, - format!("QoS snapshot detail task failed: {err}"), - ), + Ok(Err(err)) => { + error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read QoS snapshot detail: {err}")) + } + Err(err) => { + error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("QoS snapshot detail task failed: {err}")) + } } } @@ -644,8 +620,8 @@ pub(crate) async fn qos_snapshot_detail_query( /// Filters are applied lazily — records are never collected into a Vec. /// Errors opening files or reading records are logged but do not stop iteration. struct RecordFileIter { - files: std::vec::IntoIter, - current_iter: Option>>>, + files: std::vec::IntoIter, + current_iter: Option>>>, time_range: TimeRange, filters: Arc, } @@ -653,12 +629,7 @@ struct RecordFileIter { impl RecordFileIter { async fn new(dir: &str, time_range: TimeRange, filters: Arc) -> io::Result { let files = discover_files(Path::new(dir), &time_range).await?; - Ok(Self { - files: files.into_iter(), - current_iter: None, - time_range, - filters, - }) + Ok(Self { files: files.into_iter(), current_iter: None, time_range, filters }) } } @@ -671,7 +642,7 @@ impl Iterator for RecordFileIter { loop { if self.current_iter.is_none() { let file = self.files.next()?; - let iter: Box>> = if file.is_archive { + let iter: Box>> = if file.is_archive { match StreamHistoryFileReader::from_archive(&file.path, Some(self.time_range)) { Ok((r, _)) => Box::new(r), Err(e) => { @@ -712,7 +683,6 @@ impl Iterator for RecordFileIter { } } - /// Returns a lazy iterator over filtered records from history files. async fn collect_records_iter( dir: &str, @@ -725,9 +695,7 @@ async fn collect_records_iter( #[cfg(test)] // Test-only baseline implementation used to validate the iterator-based summary // path against a simpler whole-slice aggregation. -pub(crate) fn aggregate_provider_summaries( - records: &[StreamHistoryRecord], -) -> Vec { +pub(crate) fn aggregate_provider_summaries(records: &[StreamHistoryRecord]) -> Vec { #[derive(Default)] struct Acc { session_count: u64, @@ -775,7 +743,7 @@ pub(crate) fn aggregate_provider_summaries( /// Lazily aggregates provider summaries from a (possibly infinite) iterator of records. /// Unlike `aggregate_provider_summaries` which collects into a Vec first, this processes /// records one-by-one without any intermediate allocation. -pub fn aggregate_provider_summaries_from_iter>( +pub fn aggregate_provider_summaries_from_iter>( records: I, ) -> Vec { #[derive(Default)] @@ -847,9 +815,7 @@ impl CompiledQosSnapshotFilter { } fn matches(&self, snapshot: &QosSnapshotRecord) -> bool { - self.stream_identity_key - .as_ref() - .is_none_or(|value| snapshot.stream_identity_key == *value) + self.stream_identity_key.as_ref().is_none_or(|value| snapshot.stream_identity_key == *value) && self.input_name.as_ref().is_none_or(|value| snapshot.input_name.as_ref() == value) && self.provider_name.as_ref().is_none_or(|value| snapshot.provider_name.as_ref() == value) && self.item_type.as_ref().is_none_or(|value| snapshot.item_type.as_str() == value) @@ -893,12 +859,14 @@ fn load_qos_snapshot(storage_dir: &str, stream_identity_key: &str) -> io::Result #[cfg(test)] mod tests { use super::*; - use crate::model::{QosAggregationConfig, ResourceRetryConfig, ReverseProxyConfig}; - use crate::repository::{ - QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow, + use crate::{ + model::{QosAggregationConfig, ResourceRetryConfig, ReverseProxyConfig}, + repository::{QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow}, + }; + use shared::{ + model::{DisconnectReason, PlaylistItemType}, + utils::Internable, }; - use shared::model::{DisconnectReason, PlaylistItemType}; - use shared::utils::Internable; fn make_record( provider_name: &str, @@ -1011,10 +979,12 @@ mod tests { connect.event_ts_utc = 100; connect.bytes_sent = None; - let mut first_disconnect = make_record("acme", Some(10), Some(120), Some(50), Some(DisconnectReason::ProviderError)); + let mut first_disconnect = + make_record("acme", Some(10), Some(120), Some(50), Some(DisconnectReason::ProviderError)); first_disconnect.event_ts_utc = 110; - let mut final_disconnect = make_record("acme", Some(10), Some(80), Some(50), Some(DisconnectReason::ClientClosed)); + let mut final_disconnect = + make_record("acme", Some(10), Some(80), Some(50), Some(DisconnectReason::ClientClosed)); final_disconnect.event_ts_utc = 120; let rows = aggregate_hls_session(&[&connect, &first_disconnect, &final_disconnect]); @@ -1067,14 +1037,7 @@ mod tests { 1, 1, )?; - let second = paginate_stream_history_records( - vec![right, left].into_iter(), - Vec::new(), - None, - &[], - 1, - 1, - )?; + let second = paginate_stream_history_records(vec![right, left].into_iter(), Vec::new(), None, &[], 1, 1)?; assert_eq!(first.items.len(), 1); assert_eq!(second.items.len(), 1); @@ -1099,11 +1062,7 @@ mod tests { item_type: PlaylistItemType::Live, updated_at: 100, last_event_at: 99, - window_24h: QosSnapshotWindow { - score: score_24h, - confidence: 70, - ..QosSnapshotWindow::default() - }, + window_24h: QosSnapshotWindow { score: score_24h, confidence: 70, ..QosSnapshotWindow::default() }, window_7d: QosSnapshotWindow::default(), window_30d: QosSnapshotWindow::default(), daily_buckets: std::collections::BTreeMap::::new(), @@ -1122,10 +1081,7 @@ mod tests { raw.insert("target_name".to_string(), "target-b".to_string()); let filter = CompiledQosSnapshotFilter::compile(&raw).expect("qos snapshot filter should compile"); - let mut filtered = snapshots - .into_iter() - .filter(|snapshot| filter.matches(snapshot)) - .collect::>(); + let mut filtered = snapshots.into_iter().filter(|snapshot| filter.matches(snapshot)).collect::>(); filtered.sort_by(qos_snapshot_order); assert_eq!(filtered.len(), 1); assert_eq!(filtered[0].stream_identity_key, "stream-b"); @@ -1158,17 +1114,11 @@ mod tests { assert!(get_qos_storage_directory_from_config(&cfg).is_none()); if let Some(reverse_proxy) = cfg.reverse_proxy.as_mut() { - reverse_proxy.qos_aggregation = Some(QosAggregationConfig { - enabled: true, - interval_secs: 300, - compaction_interval_secs: 86_400, - }); + reverse_proxy.qos_aggregation = + Some(QosAggregationConfig { enabled: true, interval_secs: 300, compaction_interval_secs: 86_400 }); } - assert_eq!( - get_qos_storage_directory_from_config(&cfg).as_deref(), - Some("/var/lib/tuliprox") - ); + assert_eq!(get_qos_storage_directory_from_config(&cfg).as_deref(), Some("/var/lib/tuliprox")); } #[test] @@ -1183,10 +1133,7 @@ mod tests { raw.insert("provider_name".to_string(), "provider-a".to_string()); let filter = CompiledQosSnapshotFilter::compile(&raw).expect("qos snapshot filter should compile"); - let mut filtered = snapshots - .into_iter() - .filter(|snapshot| filter.matches(snapshot)) - .collect::>(); + let mut filtered = snapshots.into_iter().filter(|snapshot| filter.matches(snapshot)).collect::>(); filtered.sort_by(qos_snapshot_order); assert_eq!(filtered.len(), 2); assert_eq!(filtered[0].stream_identity_key, "stream-b"); diff --git a/backend/src/api/endpoints/user_api.rs b/backend/app/src/api/endpoints/user_api.rs similarity index 96% rename from backend/src/api/endpoints/user_api.rs rename to backend/app/src/api/endpoints/user_api.rs index 3627b2ffe..40ac43db9 100644 --- a/backend/src/api/endpoints/user_api.rs +++ b/backend/app/src/api/endpoints/user_api.rs @@ -1,11 +1,10 @@ use crate::{ api::{ - api_utils::{ - get_user_target_by_username, get_username_from_auth_header, try_unwrap_body, - }, + api_utils::{get_user_target_by_username, get_username_from_auth_header, try_unwrap_body}, + auth_middleware::validator_api_user, model::AppState, }, - auth::{validator_api_user, AuthBearer}, + auth::AuthBearer, model::{AppConfig, ConfigTarget, PlaylistXtreamCategory}, repository::{ iter_raw_m3u_target_playlist, load_user_bouquet_as_json, save_user_bouquet, xtream_get_playlist_categories, @@ -64,7 +63,7 @@ async fn playlist_categories( ) -> impl axum::response::IntoResponse + Send { if let Some(username) = get_username_from_auth_header(&token, &app_state) { if let Some((user, target)) = get_user_target_by_username(username.as_str(), &app_state) { - if user.permission_denied(&app_state) { + if user.permission_denied(&app_state.app_config) { return axum::http::StatusCode::FORBIDDEN.into_response(); } let target_name = &target.name; @@ -158,7 +157,7 @@ async fn save_playlist_bouquet( ) -> impl axum::response::IntoResponse + Send { if let Some(username) = get_username_from_auth_header(&token, &app_state) { if let Some((user, target)) = get_user_target_by_username(username.as_str(), &app_state) { - if user.permission_denied(&app_state) { + if user.permission_denied(&app_state.app_config) { return axum::http::StatusCode::FORBIDDEN.into_response(); } let config = &app_state.app_config.config.load(); @@ -182,7 +181,7 @@ async fn playlist_bouquet( ) -> impl axum::response::IntoResponse + Send { if let Some(username) = get_username_from_auth_header(&token, &app_state) { if let Some((user, _target)) = get_user_target_by_username(username.as_str(), &app_state) { - if user.permission_denied(&app_state) { + if user.permission_denied(&app_state.app_config) { return axum::http::StatusCode::FORBIDDEN.into_response(); } let config = &app_state.app_config.config.load(); diff --git a/backend/src/api/endpoints/user_visibility.rs b/backend/app/src/api/endpoints/user_visibility.rs similarity index 94% rename from backend/src/api/endpoints/user_visibility.rs rename to backend/app/src/api/endpoints/user_visibility.rs index d1bb4f2f9..02753392d 100644 --- a/backend/src/api/endpoints/user_visibility.rs +++ b/backend/app/src/api/endpoints/user_visibility.rs @@ -1,5 +1,7 @@ -use crate::model::{AppConfig, ConfigTarget, ProxyUserCredentials}; -use crate::repository::{iter_raw_m3u_target_playlist, iter_raw_xtream_target_playlist}; +use crate::{ + model::{AppConfig, ConfigTarget, ProxyUserCredentials}, + repository::{iter_raw_m3u_target_playlist, iter_raw_xtream_target_playlist}, +}; use shared::model::{PlaylistItem, TargetType, XtreamCluster}; use std::collections::HashSet; use tokio_stream::StreamExt; diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/app/src/api/endpoints/v1_api.rs similarity index 99% rename from backend/src/api/endpoints/v1_api.rs rename to backend/app/src/api/endpoints/v1_api.rs index 1c04f3e03..6901bcdb2 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/app/src/api/endpoints/v1_api.rs @@ -1,6 +1,7 @@ use crate::{ api::{ api_utils::{internal_server_error, json_or_bin_response, try_unwrap_body}, + auth_middleware::permission_layer, endpoints::{ download_api, extract_accept_header::ExtractAcceptHeader, @@ -18,7 +19,6 @@ use crate::{ }, model::AppState, }, - auth::permission_layer, processing::geoip::{update_geoip_db, GeoIpUpdateError}, utils::ip_checker::get_ips, VERSION, @@ -91,7 +91,7 @@ async fn streams( async fn geoip_update( axum::extract::State(app_state): axum::extract::State>, ) -> axum::response::Response { - match update_geoip_db(&app_state).await { + match update_geoip_db(&app_state.app_config, &app_state.http_client.load(), &app_state.geoip).await { Ok(()) => axum::http::StatusCode::OK.into_response(), Err(GeoIpUpdateError::Disabled | GeoIpUpdateError::DownloadFailed(_)) => { axum::http::StatusCode::BAD_REQUEST.into_response() diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs similarity index 88% rename from backend/src/api/endpoints/v1_api_config.rs rename to backend/app/src/api/endpoints/v1_api_config.rs index 18b989f5a..b174c070d 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -1,11 +1,19 @@ -use crate::{api::{ - api_utils::{internal_server_error, try_unwrap_body}, - config_file::ConfigFile, - model::AppState, -}, auth::{permission_layer, verify_token, AuthBearer}, iptv::xtream::{get_xtream_stream_url_base, xtream_login}, model::{validate_library_paths_from_dto, ApiProxyConfig, InputSource, UserPlan}, utils, utils::{ - persist_messaging_templates, plans_file_path, prepare_sources_batch, prepare_users, read_api_proxy_file, - read_plans_file, save_plans, request::download_text_content, -}}; +use crate::{ + api::{ + api_utils::{internal_server_error, try_unwrap_body}, + auth_middleware::permission_layer, + config_file::ConfigFile, + model::AppState, + }, + auth::{verify_token, AuthBearer}, + config_loader::{ + persist_messaging_templates, plans_file_path, prepare_sources_batch, prepare_users, read_api_proxy_file, + read_plans_file, save_plans, + }, + iptv::xtream::{get_xtream_stream_url_base, xtream_login}, + model::{validate_library_paths_from_dto, ApiProxyConfig, InputSource, UserPlan}, + utils::request::download_text_content, +}; use axum::{ http::{header::IF_MATCH, HeaderMap, HeaderName, HeaderValue, StatusCode}, response::IntoResponse, @@ -13,19 +21,18 @@ use axum::{ }; use log::error; use serde_json::json; -use shared::model::InputFetchMethod; use shared::{ error::TuliproxError, - model::permission::{Permission, PermissionSet}, - model::{ApiProxyConfigDto, ConfigDto, PlansConfigDto, SourcesConfigDto, XtreamLoginRequest}, + model::{ + permission::{Permission, PermissionSet}, + ApiProxyConfigDto, ConfigDto, InputFetchMethod, PlansConfigDto, SourcesConfigDto, XtreamLoginRequest, + }, utils::{ parse_provider_scheme_url_parts, HEADER_CONFIG_API_PROXY_REVISION, HEADER_CONFIG_MAIN_REVISION, HEADER_CONFIG_SOURCES_REVISION, HEADER_IF_MATCH, PROVIDER_SCHEME_PREFIX, }, }; -use std::collections::HashMap; -use std::path::Path; -use std::sync::Arc; +use std::{collections::HashMap, path::Path, sync::Arc}; fn file_revision_from_bytes(bytes: &[u8]) -> String { blake3::hash(bytes).to_hex().to_string() } @@ -133,7 +140,7 @@ pub(in crate::api::endpoints) async fn intern_save_config_api_proxy( api_proxy: &ApiProxyConfigDto, file_path: &str, ) -> Option { - match utils::save_api_proxy(file_path, backup_dir, api_proxy).await { + match crate::config_loader::save_api_proxy(file_path, backup_dir, api_proxy).await { Ok(()) => {} Err(err) => { error!("Failed to save api-proxy.yml {err}"); @@ -144,7 +151,7 @@ pub(in crate::api::endpoints) async fn intern_save_config_api_proxy( } async fn intern_save_config_main(file_path: &str, backup_dir: &str, cfg: &ConfigDto) -> Option { - match utils::save_main_config(file_path, backup_dir, cfg).await { + match crate::config_loader::save_main_config(file_path, backup_dir, cfg).await { Ok(()) => {} Err(err) => { error!("Failed to save config.yml {err}"); @@ -187,7 +194,7 @@ async fn save_config_main( } else if let Err(err) = validate_library_paths_from_dto(&cfg) { (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response() } else { - if let Err(err) = persist_messaging_templates(&app_state, &mut cfg).await { + if let Err(err) = persist_messaging_templates(&app_state.app_config, &mut cfg).await { error!("Failed to persist messaging templates: {err}"); return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))) .into_response(); @@ -232,24 +239,27 @@ async fn save_config_sources( return response; } - let templates_to_persist = match utils::validate_source_config_for_persist(&app_state, &sources).await { - Ok(value) => value, - Err(err) => { - error!("Failed to validate source.yml {err}"); - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))) - .into_response(); - } - }; + let templates_to_persist = + match crate::config_loader::validate_source_config_for_persist(&app_state.app_config, &sources).await { + Ok(value) => value, + Err(err) => { + error!("Failed to validate source.yml {err}"); + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))) + .into_response(); + } + }; if let Some(template_definition) = templates_to_persist.as_ref() { - if let Err(err) = utils::persist_templates_config(&app_state, template_definition).await { + if let Err(err) = + crate::config_loader::persist_templates_config(&app_state.app_config, template_definition).await + { error!("Failed to save template config {err}"); return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))) .into_response(); } } - match utils::persist_source_config(&app_state, None, sources).await { + match crate::config_loader::persist_source_config(&app_state.app_config, None, sources).await { Ok(_) => {} Err(err) => { error!("Failed to persist source.yml {err}"); @@ -323,12 +333,9 @@ async fn save_config_api_proxy_config( return internal_server_error!(); } }; - if let Some(response) = require_matching_revision( - &headers, - ¤t_revision, - HEADER_CONFIG_API_PROXY_REVISION, - "api-proxy.yml", - ) { + if let Some(response) = + require_matching_revision(&headers, ¤t_revision, HEADER_CONFIG_API_PROXY_REVISION, "api-proxy.yml") + { return response; } @@ -354,17 +361,10 @@ async fn save_config_api_proxy_config( let stored_plans = updated_api_proxy.plans.clone(); let mut updated_api_proxy_dto = ApiProxyConfigDto::from(&updated_api_proxy); if let Err(err) = updated_api_proxy_dto.prepare() { - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))) - .into_response(); + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response(); } - if let Some(err) = intern_save_config_api_proxy( - &backup_dir, - &updated_api_proxy_dto, - &api_proxy_file_path, - ) - .await - { + if let Some(err) = intern_save_config_api_proxy(&backup_dir, &updated_api_proxy_dto, &api_proxy_file_path).await { return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({"error": err.to_string()}))) .into_response(); } @@ -387,11 +387,7 @@ async fn save_config_api_proxy_config( async fn get_config_common(app_state: &Arc, permissions: Option) -> axum::response::Response { let (config_file_path, sources_file_path, api_proxy_file_path) = { let paths = app_state.app_config.paths.load(); - ( - paths.config_file_path.clone(), - paths.sources_file_path.clone(), - paths.api_proxy_file_path.clone(), - ) + (paths.config_file_path.clone(), paths.sources_file_path.clone(), paths.api_proxy_file_path.clone()) }; let main_revision = match read_file_revision(&config_file_path).await { @@ -418,7 +414,7 @@ async fn get_config_common(app_state: &Arc, permissions: Option { @@ -432,7 +428,8 @@ async fn get_config_common(app_state: &Arc, permissions: Option, permissions: Option>) -> impl IntoResponse + Send { +async fn config_unprotected( + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { get_config_common(&app_state, None).await } @@ -454,10 +453,7 @@ async fn config( let Some(permissions) = decode_permissions(&app_state, &token) else { return axum::http::StatusCode::UNAUTHORIZED.into_response(); }; - if !has_any_permission( - permissions, - &[Permission::ConfigRead, Permission::SourceRead, Permission::UserRead], - ) { + if !has_any_permission(permissions, &[Permission::ConfigRead, Permission::SourceRead, Permission::UserRead]) { return axum::http::StatusCode::FORBIDDEN.into_response(); } @@ -517,7 +513,7 @@ async fn config_batch_content( None, false, ) - .await + .await { Ok((content, _path)) => { // Return CSV with explicit content-type @@ -567,11 +563,7 @@ async fn get_xtream_login_info( Ok(login_info) => axum::Json(login_info.unwrap_or_default()).into_response(), Err(err) => { error!("Failed to get xtream login info: {err}"); - ( - StatusCode::BAD_GATEWAY, - axum::Json(json!({"error": "Failed to get Xtream login info"})), - ) - .into_response() + (StatusCode::BAD_GATEWAY, axum::Json(json!({"error": "Failed to get Xtream login info"}))).into_response() } } } @@ -583,22 +575,21 @@ fn build_xtream_login_input_source( let url = request.url.trim(); let provider = if url.starts_with(PROVIDER_SCHEME_PREFIX) { let (provider_name, _) = parse_provider_scheme_url_parts(url)?; - let request_provider = request - .providers - .as_ref() - .and_then(|request_providers| { - request_providers - .iter() - .find(|provider| provider.name.as_ref() == provider_name) - .map(crate::model::ConfigProvider::from) - .map(Arc::new) - }); + let request_provider = request.providers.as_ref().and_then(|request_providers| { + request_providers + .iter() + .find(|provider| provider.name.as_ref() == provider_name) + .map(crate::model::ConfigProvider::from) + .map(Arc::new) + }); Some( request_provider .into_iter() .chain(providers.iter().cloned()) .find(|provider| provider.name.as_ref() == provider_name) - .ok_or_else(|| TuliproxError::ConfigInput(format!("Provider config for '{provider_name}' not found")))?, + .ok_or_else(|| { + TuliproxError::ConfigInput(format!("Provider config for '{provider_name}' not found")) + })?, ) } else { None @@ -669,7 +660,10 @@ pub fn v1_api_config_register(router: Router>) -> axum::Router) -> Router> { let base_read = Router::new() @@ -696,12 +690,7 @@ pub fn v1_api_config_register_with_permissions(app_state: &Arc) -> Rou .route("/config/plans", axum::routing::put(save_config_plans)) .layer(permission_layer!(app_state, Permission::ConfigWrite)); - Router::new() - .merge(base_read) - .merge(config_read) - .merge(source_read) - .merge(source_write) - .merge(config_write) + Router::new().merge(base_read).merge(config_read).merge(source_read).merge(source_write).merge(config_write) } #[cfg(test)] @@ -712,11 +701,10 @@ mod tests { }; use crate::model::ConfigProvider; use axum::http::{HeaderMap, HeaderValue, StatusCode}; - use shared::model::TemplateDefinitionDto; use shared::{ model::{ ApiProxyConfigDto, ApiProxyServerInfoDto, AppConfigDto, ConfigDto, ConfigProviderDto, Permission, - PermissionSet, SourcesConfigDto, TargetUserDto, XtreamLoginRequest, + PermissionSet, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, XtreamLoginRequest, }, utils::{HEADER_CONFIG_SOURCES_REVISION, HEADER_IF_MATCH}, }; @@ -725,12 +713,7 @@ mod tests { #[test] fn require_matching_revision_rejects_missing_if_match_header() { let headers = HeaderMap::new(); - let response = require_matching_revision( - &headers, - "rev-a", - HEADER_CONFIG_SOURCES_REVISION, - "source.yml", - ) + let response = require_matching_revision(&headers, "rev-a", HEADER_CONFIG_SOURCES_REVISION, "source.yml") .expect("missing if-match header must fail"); assert_eq!(response.status(), StatusCode::PRECONDITION_REQUIRED); } @@ -738,16 +721,8 @@ mod tests { #[test] fn require_matching_revision_accepts_exact_match() { let mut headers = HeaderMap::new(); - headers.insert( - HEADER_IF_MATCH, - HeaderValue::from_str("rev-a").expect("header value should be valid"), - ); - let result = require_matching_revision( - &headers, - "rev-a", - HEADER_CONFIG_SOURCES_REVISION, - "source.yml", - ); + headers.insert(HEADER_IF_MATCH, HeaderValue::from_str("rev-a").expect("header value should be valid")); + let result = require_matching_revision(&headers, "rev-a", HEADER_CONFIG_SOURCES_REVISION, "source.yml"); assert!(result.is_none(), "exact revision match should be accepted"); } @@ -762,10 +737,7 @@ mod tests { message: String::from("hello"), path: None, }], - user: vec![TargetUserDto { - target: String::from("target-a"), - credentials: vec![], - }], + user: vec![TargetUserDto { target: String::from("target-a"), credentials: vec![] }], use_user_db: true, auth_error_status: 401, } @@ -775,10 +747,7 @@ mod tests { fn filter_app_config_clears_unauthorized_sections() { let permissions: PermissionSet = Permission::ConfigRead.into(); let mut app_config = AppConfigDto { - config: ConfigDto { - storage_dir: Some(String::from("storage")), - ..ConfigDto::default() - }, + config: ConfigDto { storage_dir: Some(String::from("storage")), ..ConfigDto::default() }, sources: SourcesConfigDto { inputs: vec![], sources: vec![], diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/app/src/api/endpoints/v1_api_playlist.rs similarity index 95% rename from backend/src/api/endpoints/v1_api_playlist.rs rename to backend/app/src/api/endpoints/v1_api_playlist.rs index 55d2d5c58..38c54b309 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist.rs @@ -1,9 +1,12 @@ +// Recording-source resolution moved into the recording subsystem so it no +// longer has to call back into `endpoints`. use crate::{ api::{ api_utils::{ create_api_proxy_user, json_or_bin_response, resource_response, try_option_bad_request, try_result_bad_request, try_unwrap_body, }, + auth_middleware::permission_layer, endpoints::{ api_playlist_utils::{ get_playlist_for_custom_provider, get_playlist_for_input, get_playlist_for_target, @@ -17,18 +20,22 @@ use crate::{ xtream_player_api_stream_with_token, ApiStreamContext, ApiStreamRequest, }, }, - model::AppState, + model::{ + recording::recording_source_resolution::{resolve_recording_config, resolve_recording_target}, + AppState, + }, }, - auth::{create_access_token, permission_layer, verify_access_token}, + auth::{create_access_token, verify_access_token}, iptv::{stalker::client::validate_public_playable_url, xtream}, model::{ parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, AppConfig, ConfigInput, ConfigInputFlags, ConfigInputOptions, EpgSource, EpgSourceType, IcsDummyPolicy, InputSource, }, processing::{ + epg::get_input_raw_epg_file_path, parser::{ - ics::parse_ics_file_to_channel, - xmltv::{merge_epg_channels_by_priority_with_dummy_policies, EpgDummyPolicySource}, + ics::parse_ics_file_to_channel, + xmltv::{merge_epg_channels_by_priority_with_dummy_policies, EpgDummyPolicySource}, }, processor::re_resolve_stalker_url, }, @@ -36,11 +43,7 @@ use crate::{ iter_raw_m3u_target_playlist, iter_raw_xtream_target_playlist, m3u_get_item_for_stream_id, xtream_get_item_for_stream_id, }, - utils::{ - epg::get_input_raw_epg_file_path, - file_exists_async, - request, - }, + utils::{file_exists_async, request}, }; use axum::{response::IntoResponse, Router}; use log::{debug, error}; @@ -175,21 +178,6 @@ fn build_recording_stream_url( Some(url.into()) } -pub(in crate::api) fn build_recording_source_descriptor( - target_name: &str, - input_name: &str, - virtual_id: u32, - cluster: XtreamCluster, -) -> Option { - let mut url = Url::parse("tuliprox-recording://source").ok()?; - url.query_pairs_mut() - .append_pair("target_name", target_name) - .append_pair("input_name", input_name) - .append_pair("virtual_id", &virtual_id.to_string()) - .append_pair("cluster", cluster.as_stream_type()); - Some(url.into()) -} - pub(in crate::api) fn build_webplayer_recording_url( app_config: &crate::model::AppConfig, target_id: u16, @@ -204,13 +192,7 @@ pub(in crate::api) fn build_webplayer_recording_url( .and_then(|web_ui| web_ui.player_server.as_ref()) .map_or("default", |server_name| server_name.as_str()); let server_info = app_config.get_server_info(server_name)?; - Some(build_playlist_webplayer_url( - &server_info.get_base_url(), - &access_token, - target_id, - virtual_id, - cluster, - )) + Some(build_playlist_webplayer_url(&server_info.get_base_url(), &access_token, target_id, virtual_id, cluster)) } pub(in crate::api) fn build_stable_recording_url( @@ -228,14 +210,7 @@ pub(in crate::api) fn build_stable_recording_url( .and_then(|web_ui| web_ui.player_server.as_ref()) .map_or("default", |server_name| server_name.as_str()); let server_info = app_config.get_server_info(server_name)?; - build_recording_stream_url( - &server_info.get_base_url(), - &access_token, - target_name, - input_name, - virtual_id, - cluster, - ) + build_recording_stream_url(&server_info.get_base_url(), &access_token, target_name, input_name, virtual_id, cluster) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -244,35 +219,6 @@ pub(in crate::api) struct ResolvedRecordingSource { pub input_name: String, } -#[derive(Debug, Clone)] -pub(in crate::api) struct ResolvedRecordingConfig { - pub target: Arc, - pub input: Arc, -} - -pub(in crate::api) fn resolve_recording_config( - sources: &crate::model::SourcesConfig, - target_name: &str, - input_name: &str, -) -> Option { - let source = sources.sources.iter().find(|source| { - source.inputs.iter().any(|configured_input| configured_input.as_ref() == input_name) - && source.targets.iter().any(|target| target.name == target_name) - })?; - let target = source.targets.iter().find(|target| target.name == target_name)?.clone(); - let input = sources.inputs.iter().find(|input| input.name.as_ref() == input_name)?.clone(); - Some(ResolvedRecordingConfig { target, input }) -} - -pub(in crate::api) fn resolve_recording_target( - app_config: &crate::model::AppConfig, - target_name: &str, - input_name: &str, -) -> Option> { - resolve_recording_config(app_config.sources.load().as_ref(), target_name, input_name) - .map(|resolved| resolved.target) -} - pub(in crate::api) async fn resolve_target_recording_source( app_config: &crate::model::AppConfig, target_name: &str, @@ -389,7 +335,7 @@ async fn load_xmltv_epg_source_channels( } } - parse_xmltv_for_web_ui_from_url(app_state, resolved_url).await + parse_xmltv_for_web_ui_from_url(&app_state.app_config, &app_state.http_client.load(), resolved_url).await } async fn load_ics_epg_source_channels( @@ -762,7 +708,7 @@ async fn playlist_webplayer_stream( let req_virtual_id: u32 = try_result_bad_request!(stream_id.trim().parse()); let pli = try_result_bad_request!( - m3u_get_item_for_stream_id(req_virtual_id, &app_state, &target).await, + m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await, true, format!("Failed to read m3u item for stream id {req_virtual_id}") ); @@ -821,7 +767,7 @@ async fn playlist_recording_stream( } let pli = try_result_bad_request!( - m3u_get_item_for_stream_id(virtual_id, &app_state, &resolved.target).await, + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, &resolved.target).await, true, format!("Failed to read m3u item for stream id {virtual_id}") ); @@ -906,7 +852,7 @@ async fn playlist_epg( ) .into_response(); } - match parse_xmltv_for_web_ui_from_url(&app_state, &url).await { + match parse_xmltv_for_web_ui_from_url(&app_state.app_config, &app_state.http_client.load(), &url).await { Ok(epg) => { let config = app_state.app_config.config.load(); let web_ui_path = config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); @@ -962,7 +908,8 @@ async fn stalker_resource_response( cluster: XtreamCluster, provider_id: u32, ) -> axum::response::Response { - let Some(input) = app_state.app_config.get_input_by_id(input_id).filter(|input| input.input_type.is_stalker()) else { + let Some(input) = app_state.app_config.get_input_by_id(input_id).filter(|input| input.input_type.is_stalker()) + else { return axum::http::StatusCode::NOT_FOUND.into_response(); }; let kind = match cluster { @@ -1105,8 +1052,7 @@ async fn playlist_filter_preview( if target.has_output(TargetType::Xtream) { let mut any_cluster_read = false; for cluster in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { - if let Some(mut iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, &target, cluster).await - { + if let Some(mut iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, &target, cluster).await { any_cluster_read = true; while let Some(entry) = iterator.next().await { match entry { @@ -1189,10 +1135,7 @@ pub fn v1_api_playlist_register_public(router: Router>) -> axum::R "/playlist/webplayer/{token}/{target_id}/{cluster}/{stream_id}", axum::routing::get(playlist_webplayer_stream), ) - .route( - "/playlist/recording/{token}/{cluster}/{virtual_id}", - axum::routing::get(playlist_recording_stream), - ) + .route("/playlist/recording/{token}/{cluster}/{virtual_id}", axum::routing::get(playlist_recording_stream)) } pub fn v1_api_playlist_register_with_permissions( @@ -1230,8 +1173,14 @@ async fn playlist_episode_item( if let Some(target) = app_state.app_config.get_target_by_id(target_id) { if target.has_output(TargetType::Xtream) { if let Ok(vid) = virtual_id.parse::() { - if let Ok(pli) = - xtream_get_item_for_stream_id(vid, &app_state, &target, Some(XtreamCluster::Series)).await + if let Ok(pli) = xtream_get_item_for_stream_id( + vid, + &app_state.app_config, + &app_state.playlists, + &target, + Some(XtreamCluster::Series), + ) + .await { return axum::Json(json!(UiPlaylistItem::from(pli))).into_response(); } @@ -1244,31 +1193,29 @@ async fn playlist_episode_item( #[cfg(test)] mod tests { - use super::{resolve_provider_url_for_request, resolve_recording_config}; + use super::resolve_provider_url_for_request; use crate::{ api::model::{ - ActiveProviderManager, ActiveUserManager, AppState, ConnectionManager, DownloadQueue, EventManager, - MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, + recording::recording_source_resolution::resolve_recording_config, ActiveProviderManager, ActiveUserManager, + AppState, ConnectionManager, DownloadQueue, EventManager, MetadataUpdateManager, PlaylistStorageState, + SharedStreamManager, }, model::{ - AppConfig, Config, ConfigInput, ConfigProvider, ConfigSource, ConfigTarget, SourcesConfig, - StreamHistoryConfig, VideoDownloadConfig, - }, - utils::{ - epg::{get_input_raw_epg_file_path, get_input_raw_xmltv_file_path}, - GeoIp, + AppConfig, Config, ConfigInput, ConfigInputOptions, ConfigProvider, ConfigSource, ConfigTarget, + SourcesConfig, StreamHistoryConfig, VideoDownloadConfig, }, + processing::epg::{get_input_raw_epg_file_path, get_input_raw_xmltv_file_path}, + repository::GeoIp, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::{ - extract::{Path as AxumPath, Query, State}, body::Body, + extract::{Path as AxumPath, Query, State}, http::{Request, StatusCode}, response::IntoResponse, Json, Router, }; use chrono::Utc; - use crate::model::ConfigInputOptions; use serde_json::json; use shared::{ foundation::Filter, @@ -1398,22 +1345,24 @@ mod tests { fn recording_target_resolution_uses_stable_name_and_input_across_runtime_ids() { let input_a = Arc::new(ConfigInput { id: 7, name: "input-a".intern(), ..Default::default() }); let input_b = Arc::new(ConfigInput { id: 8, name: "input-b".intern(), ..Default::default() }); - let target = |id, name: &str| Arc::new(ConfigTarget { - id, - enabled: true, - name: name.to_string(), - options: None, - sort: None, - filter: Filter::default(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - watch: None, - use_memory_cache: false, - }); + let target = |id, name: &str| { + Arc::new(ConfigTarget { + id, + enabled: true, + name: name.to_string(), + options: None, + sort: None, + filter: Filter::default(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + watch: None, + use_memory_cache: false, + }) + }; let sources = |later_target_id| { let inputs = vec![Arc::clone(&input_a), Arc::clone(&input_b)]; SourcesConfig { @@ -1809,7 +1758,7 @@ mod tests { #[test] fn recording_source_descriptor_is_token_free_and_percent_encodes_names() { - let url = super::build_recording_source_descriptor( + let url = crate::api::model::recording::recording_source_resolution::build_recording_source_descriptor( "News/HD &+", "input/name ?+", 42, @@ -1830,7 +1779,7 @@ mod tests { #[test] fn future_scheduled_recording_descriptor_round_trips_without_token() { - let url = super::build_recording_source_descriptor( + let url = crate::api::model::recording::recording_source_resolution::build_recording_source_descriptor( "stable-target", "input-a", 42, @@ -2273,10 +2222,7 @@ mod tests { }), ..Default::default() }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![], - }; + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_config = test_app_config(Arc::clone(&input), source); let mut sources = app_config.sources.load().as_ref().clone(); sources.inputs.push(Arc::new(ConfigInput { @@ -2330,7 +2276,10 @@ mod tests { .await .expect("resource response"); assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT, "{resource_path}"); - assert_eq!(response.headers().get("location").and_then(|value| value.to_str().ok()), Some("http://8.8.8.8/live/101")); + assert_eq!( + response.headers().get("location").and_then(|value| value.to_str().ok()), + Some("http://8.8.8.8/live/101") + ); let response = router .clone() diff --git a/backend/src/api/endpoints/v1_api_user.rs b/backend/app/src/api/endpoints/v1_api_user.rs similarity index 87% rename from backend/src/api/endpoints/v1_api_user.rs rename to backend/app/src/api/endpoints/v1_api_user.rs index aeeda3982..0d01aa23a 100644 --- a/backend/src/api/endpoints/v1_api_user.rs +++ b/backend/app/src/api/endpoints/v1_api_user.rs @@ -1,7 +1,12 @@ -use crate::{api::{ - model::AppState, - panel_api::{sync_panel_api_alias_pool_for_target, target_has_alias_pool_min}, -}, auth::permission_layer, model::{ApiProxyConfig, ProxyUserCredentials, TargetUser}, repository::store_api_user}; +use crate::{ + api::{ + auth_middleware::permission_layer, + model::AppState, + panel_api::{sync_panel_api_alias_pool_for_target, target_has_alias_pool_min}, + }, + model::{ApiProxyConfig, ProxyUserCredentials, TargetUser}, + repository::store_api_user, +}; use axum::{response::IntoResponse, Router}; use serde_json::json; use shared::{ @@ -49,12 +54,11 @@ async fn save_config_api_proxy_user( } // Trial plans: new users without an explicit expiry get the trial window. if !is_update { - if let Some(plan) = - credential.plan.as_ref().and_then(|name| api_proxy.plans.iter().find(|p| p.name == *name)) - { + if let Some(plan) = credential.plan.as_ref().and_then(|name| api_proxy.plans.iter().find(|p| p.name == *name)) { if let Some(trial_secs) = plan.t_trial_duration_secs { if credential.exp_date.is_none() { - let expires = chrono::Utc::now().timestamp().saturating_add(i64::try_from(trial_secs).unwrap_or(i64::MAX)); + let expires = + chrono::Utc::now().timestamp().saturating_add(i64::try_from(trial_secs).unwrap_or(i64::MAX)); credential.exp_date = Some(expires); } if credential.status.is_none() { @@ -105,14 +109,18 @@ async fn save_config_api_proxy_user( } } - // ---------- if update but no user found -> Error ---------- - if is_update && existing_user_index.is_none() { - return ( - axum::http::StatusCode::BAD_REQUEST, - axum::Json(json!({"error": format!("User {} not found", credential.username)})), - ) - .into_response(); - } + let existing_user_position = if is_update { + let (Some(user_idx), Some(user_target_idx)) = (existing_user_index, existing_user_target_index) else { + return ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"error": format!("User {} not found", credential.username)})), + ) + .into_response(); + }; + Some((user_idx, user_target_idx)) + } else { + None + }; // ---------- create target if new target does not exist ---------- let target_idx = if let Some(idx) = existing_target_index { @@ -122,11 +130,8 @@ async fn save_config_api_proxy_user( api_proxy.user.len() - 1 }; - if is_update { + if let Some((user_idx, user_target_idx)) = existing_user_position { let mut remove_empty_target = false; - // existing_user_index and existing_user_target_index exists at this point - let user_idx = existing_user_index.unwrap(); - let user_target_idx = existing_user_target_index.unwrap(); if user_target_idx == target_idx { // Update @@ -267,19 +272,9 @@ pub fn v1_api_user_register_with_permissions( app_state: &Arc, ) -> axum::Router> { let user_write_routes = Router::new() - .route( - "/{target}", - axum::routing::post(save_config_api_proxy_user) - .put(save_config_api_proxy_user) - ) - .route( - "/{target}/{username}", - axum::routing::delete(delete_config_api_proxy_user) - ) - .route( - "/{username}/session/{session_token}", - axum::routing::delete(terminate_user_session) - ) + .route("/{target}", axum::routing::post(save_config_api_proxy_user).put(save_config_api_proxy_user)) + .route("/{target}/{username}", axum::routing::delete(delete_config_api_proxy_user)) + .route("/{username}/session/{session_token}", axum::routing::delete(terminate_user_session)) .layer(permission_layer!(app_state, Permission::UserWrite)); router.nest("/user", user_write_routes) diff --git a/backend/src/api/endpoints/web_index.rs b/backend/app/src/api/endpoints/web_index.rs similarity index 95% rename from backend/src/api/endpoints/web_index.rs rename to backend/app/src/api/endpoints/web_index.rs index ec5d958dd..d8c7af41c 100644 --- a/backend/src/api/endpoints/web_index.rs +++ b/backend/app/src/api/endpoints/web_index.rs @@ -52,8 +52,8 @@ async fn token( .t_users .as_ref() .and_then(|users| users.iter().find(|user| user.username.eq_ignore_ascii_case(username))); - let is_admin = - user_entry.is_some_and(|user| user.groups.iter().any(|group| group.eq_ignore_ascii_case("admin"))); + let is_admin = user_entry + .is_some_and(|user| user.groups.iter().any(|group| group.eq_ignore_ascii_case("admin"))); let user_groups = user_entry.map(|user| user.groups.clone()).unwrap_or_default(); debug!( "Web login success candidate: username='{username}', groups={user_groups:?}, is_admin={is_admin}, permissions={permissions}", @@ -106,11 +106,7 @@ async fn token_refresh( let claims = token_data.claims; let username = claims.username.as_str(); - if claims - .roles - .iter() - .any(|role| role.eq_ignore_ascii_case(shared::model::ROLE_API_USER)) - { + if claims.roles.iter().any(|role| role.eq_ignore_ascii_case(shared::model::ROLE_API_USER)) { let Some(user) = app_state.app_config.get_user_credentials(username) else { return axum::http::StatusCode::UNAUTHORIZED.into_response(); }; @@ -127,7 +123,8 @@ async fn token_refresh( let Some(users) = web_auth.t_users.as_ref() else { return axum::http::StatusCode::UNAUTHORIZED.into_response(); }; - let Some(user) = users.iter().find(|candidate| candidate.username.eq_ignore_ascii_case(username)) else { + let Some(user) = users.iter().find(|candidate| candidate.username.eq_ignore_ascii_case(username)) + else { return axum::http::StatusCode::UNAUTHORIZED.into_response(); }; @@ -140,10 +137,7 @@ async fn token_refresh( let resolved_permissions = web_auth.resolve_permissions(username); debug!( "Web token refresh: username='{}', groups={:?}, is_admin={}, permissions={}", - username, - user.groups, - is_admin, - resolved_permissions + username, user.groups, is_admin, resolved_permissions ); let new_token = if is_admin { create_jwt_admin(web_auth, username, current_pwd_version) @@ -416,7 +410,12 @@ pub fn index_register_with_path(web_dir_path: &Path, web_ui_path: &str) -> axum: #[cfg(test)] mod tests { use super::api_user_can_access_web_ui; - use axum::{body::Body, http::{Method, Request, StatusCode}, routing::{get, post}, Router}; + use axum::{ + body::Body, + http::{Method, Request, StatusCode}, + routing::{get, post}, + Router, + }; use shared::utils::concat_path_leading_slash; use tower::ServiceExt; @@ -440,13 +439,7 @@ mod tests { .nest(&format!("/{web_ui_path}/"), web_ui_router); let response = router - .oneshot( - Request::builder() - .method(Method::POST) - .uri("/tuli/auth/token") - .body(Body::empty()) - .unwrap(), - ) + .oneshot(Request::builder().method(Method::POST).uri("/tuli/auth/token").body(Body::empty()).unwrap()) .await .unwrap(); diff --git a/backend/src/api/endpoints/websocket_api.rs b/backend/app/src/api/endpoints/websocket_api.rs similarity index 99% rename from backend/src/api/endpoints/websocket_api.rs rename to backend/app/src/api/endpoints/websocket_api.rs index 6cd3dd918..3942f1ce1 100644 --- a/backend/src/api/endpoints/websocket_api.rs +++ b/backend/app/src/api/endpoints/websocket_api.rs @@ -399,7 +399,7 @@ async fn recording_frame_for_session(app_state: &AppState, claims: &Claims) -> P // schedulers; the socket has to agree, or a client would keep // receiving live recording data while every REST call answered // `501 recording_disabled`. - if !crate::api::model::recording::recording_supervisor::recording_enabled(app_state) { + if !crate::api::model::recording::recording_supervisor::recording_enabled(&app_state.app_config) { return ProtocolMessage::RecordingWsError { code: "recording_disabled".to_string() }; } if let Some(RecordingViewDenial::TokenRefreshRequired) = recording_view_denial(claims) { diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/app/src/api/endpoints/xmltv_api.rs similarity index 86% rename from backend/src/api/endpoints/xmltv_api.rs rename to backend/app/src/api/endpoints/xmltv_api.rs index 8f38a39cf..aa5aace8a 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/app/src/api/endpoints/xmltv_api.rs @@ -2,18 +2,21 @@ use crate::{ api::{ api_utils::{ coalesce_byte_stream, create_api_proxy_user, empty_json_response_as_array, get_user_target, - get_user_target_by_credentials, internal_server_error, - resource_response, + get_user_target_by_credentials, internal_server_error, resource_response, stream_json_or_bin_response_try_stream, try_unwrap_body, }, model::{AppState, UserApiRequest, UserApiRequestQueryOrBody}, + static_headers::CT_XML, + }, + auth::{resolve_api_user_context, Fingerprint}, + model::{ + ApiProxyServerInfo, Config, ConfigTarget, ProxyUserCredentials, EPG_ATTRIB_ID, EPG_ATTRIB_LANG, + EPG_TAG_CATEGORY, EPG_TAG_CHANNEL, EPG_TAG_LIVE, EPG_TAG_NEW, }, - auth::Fingerprint, - model::{Config, ConfigTarget, ProxyUserCredentials, EPG_ATTRIB_ID, EPG_TAG_CHANNEL}, repository::{ - epg_query_channels_by_storage_key, get_target_storage_path, m3u_get_epg_file_path_for_target, - storage_const, xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTreeQuery, - LockedReceiverStream, XML_PREAMBLE, + epg_query_channels_by_storage_key, get_target_storage_path, m3u_get_epg_file_path_for_target, storage_const, + xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTreeQuery, LockedReceiverStream, + XML_PREAMBLE, }, utils, utils::{ @@ -34,13 +37,18 @@ use shared::{ }, utils::{concat_path, concat_path_leading_slash, obfuscate_text, Internable}, }; -use std::{collections::HashMap, path::{Path, PathBuf}, sync::Arc}; -use tokio::{io::{AsyncWrite, AsyncWriteExt}, sync::mpsc, task}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::{ + io::{AsyncWrite, AsyncWriteExt}, + sync::mpsc, + task, +}; use tokio_stream::StreamExt; use tokio_util::io::ReaderStream; -use crate::api::static_headers::CT_XML; -use crate::auth::resolve_api_user_context; -use crate::model::{ApiProxyServerInfo, EPG_ATTRIB_LANG, EPG_TAG_CATEGORY, EPG_TAG_LIVE, EPG_TAG_NEW}; pub fn get_empty_epg_response() -> axum::response::Response { try_unwrap_body!(axum::response::Response::builder() @@ -90,10 +98,7 @@ pub(in crate::api) fn get_epg_path_for_target_by_type( } TargetType::M3u => { if let Some(target_path) = get_target_storage_path(config, &target.name) { - return get_epg_path_for_target_of_type( - &target.name, - m3u_get_epg_file_path_for_target(&target_path), - ); + return get_epg_path_for_target_of_type(&target.name, m3u_get_epg_file_path_for_target(&target_path)); } None } @@ -140,7 +145,8 @@ pub async fn serve_epg_web_ui( let query = match BPlusTreeQuery::, EpgChannel>::try_new(&epg_path) { Ok(query) => query, Err(error) => { - let message = format!("Failed to open epg db for target {target_name} {}: {error}", epg_path.display()); + let message = + format!("Failed to open epg db for target {target_name} {}: {error}", epg_path.display()); error!("{message}"); let _ = tx.blocking_send(Err(message)); return; @@ -169,19 +175,28 @@ pub async fn serve_epg_web_ui( epg_path_for_log.display() ); let _ = join_error_tx - .send(Err(format!("EPG web UI producer task failed for target {} {}: {err}", target_name_for_log, epg_path_for_log.display()))) + .send(Err(format!( + "EPG web UI producer task failed for target {} {}: {err}", + target_name_for_log, + epg_path_for_log.display() + ))) .await; } }); - let stream = LockedReceiverStream::new(rx, iter_lock) - .map(move |result| result.map(|channel| rewrite_epg_channel_resource_url(&encrypt_secret, &resource_url, channel))); + let stream = LockedReceiverStream::new(rx, iter_lock).map(move |result| { + result.map(|channel| rewrite_epg_channel_resource_url(&encrypt_secret, &resource_url, channel)) + }); return stream_json_or_bin_response_try_stream(accept, stream); } try_unwrap_body!(empty_json_response_as_array()) } -pub fn rewrite_epg_channel_resource_url(encrypt_secret: &[u8; 16], resource_url: &str, mut channel: EpgChannel) -> EpgChannel { +pub fn rewrite_epg_channel_resource_url( + encrypt_secret: &[u8; 16], + resource_url: &str, + mut channel: EpgChannel, +) -> EpgChannel { let Some(icon) = channel.icon.as_ref() else { return channel; }; @@ -210,9 +225,7 @@ async fn write_programme_classification_tags( elem.push_attribute((EPG_ATTRIB_LANG, lang.as_ref())); } writer.write_event_async(Event::Start(elem)).await?; - writer - .write_event_async(Event::Text(BytesText::new(category.value.as_ref()))) - .await?; + writer.write_event_async(Event::Text(BytesText::new(category.value.as_ref()))).await?; writer.write_event_async(Event::End(BytesEnd::new(EPG_TAG_CATEGORY))).await?; } @@ -237,24 +250,24 @@ async fn serve_epg_with_rewrites( return get_empty_epg_response(); } - let epg_processing_options = get_epg_processing_options(app_state, user, target); - let lowercase_display_names = target - .options - .as_ref() - .is_some_and(ConfigTargetOptions::lowercase_xmltv_display_names); + let epg_processing_options = get_epg_processing_options(&app_state.app_config, user, target); + let lowercase_display_names = + target.options.as_ref().is_some_and(ConfigTargetOptions::lowercase_xmltv_display_names); let server_info = app_state.app_config.get_user_server_info(user); let base_url = if !matches!(epg_processing_options.time_shift, EpgTimeShift::None) || epg_processing_options.rewrite_urls { - server_info.as_ref().map(|si| concat_string!( - &si.get_base_url(), - "/", - storage_const::EPG_RESOURCE_PATH, - "/", - &user.username, - "/", - &user.password - )) + server_info.as_ref().map(|si| { + concat_string!( + &si.get_base_url(), + "/", + storage_const::EPG_RESOURCE_PATH, + "/", + &user.username, + "/", + &user.password + ) + }) } else { None }; @@ -317,7 +330,9 @@ async fn serve_epg_with_rewrites( error!("EPG: Failed to write xml header {err}"); return; } - if let Err(err) = tx.write_all(format!(r#""#).as_bytes()).await + if let Err(err) = tx + .write_all(format!(r#""#).as_bytes()) + .await { error!("EPG: Failed to write xml tv header {err}"); return; @@ -352,9 +367,7 @@ async fn serve_epg_with_rewrites( continue_on_err!(writer.write_event_async(Event::Start(elem)).await); let title: &str = channel.title.as_deref().unwrap_or(""); let display_name = lowercase_xmltv_text(title, lowercase_display_names); - continue_on_err!(writer - .write_event_async(Event::Text(BytesText::new(display_name.as_ref()))) - .await); + continue_on_err!(writer.write_event_async(Event::Text(BytesText::new(display_name.as_ref()))).await); let elem = BytesEnd::new("display-name"); continue_on_err!(writer.write_event_async(Event::End(elem)).await); @@ -490,10 +503,7 @@ fn get_applied_epg_timeshift( (start_str, stop_str, start_ts, stop_ts) } -fn get_applied_epg_timestamps( - programme: &EpgProgramme, - epg_processing_options: &EpgProcessingOptions, -) -> (i64, i64) { +fn get_applied_epg_timestamps(programme: &EpgProgramme, epg_processing_options: &EpgProcessingOptions) -> (i64, i64) { match &epg_processing_options.time_shift { EpgTimeShift::None | EpgTimeShift::TimeZone(_) => (programme.start, programme.stop), EpgTimeShift::Fixed(m) => { @@ -503,9 +513,7 @@ fn get_applied_epg_timestamps( } } -fn get_source_epg_timestamps(programme: &EpgProgramme) -> (i64, i64) { - (programme.start, programme.stop) -} +fn get_source_epg_timestamps(programme: &EpgProgramme) -> (i64, i64) { (programme.start, programme.stop) } fn format_epg_timeshift_strings( programme: &EpgProgramme, @@ -569,10 +577,7 @@ pub async fn serve_short_epg( limit: u32, has_archive: bool, ) -> axum::response::Response { - let lowercase_ids = target - .options - .as_ref() - .is_some_and(ConfigTargetOptions::lowercase_epg_ids); + let lowercase_ids = target.options.as_ref().is_some_and(ConfigTargetOptions::lowercase_epg_ids); let output_case = EpgIdOutputCase::from_lowercase(lowercase_ids); let storage_key = canonicalize_output_epg_id(channel_id, output_case); let response_channel_id = Arc::clone(&storage_key); @@ -581,7 +586,7 @@ pub async fn serve_short_epg( let limit = if limit > 0 { limit } else { DEFAULT_SHORT_EPG_LIMIT }; if file_exists_async(epg_path).await { if let Some(epg_channel) = get_epg_channel_by_storage_key(app_state, &storage_key, epg_path).await { - let epg_processing_options = get_epg_processing_options(app_state, user, target); + let epg_processing_options = get_epg_processing_options(&app_state.app_config, user, target); ShortEpgResultDto { epg_listings: epg_channel .get_programme_with_limit(limit) @@ -654,28 +659,20 @@ async fn serve_stream_epg( epg_path: &Path, prepared: PreparedStreamEpgRequest, ) -> StreamEpgResponse { - let epg_processing_options = get_epg_processing_options(app_state, user, target); + let epg_processing_options = get_epg_processing_options(&app_state.app_config, user, target); let live_now = chrono::Utc::now().timestamp(); let PreparedStreamEpgRequest { entries: prepared_entries } = prepared; - let storage_keys = prepared_entries - .iter() - .map(|entry| Arc::clone(&entry.storage_key)) - .collect(); + let storage_keys = prepared_entries.iter().map(|entry| Arc::clone(&entry.storage_key)).collect(); - let channels: Vec> = match epg_query_channels_by_storage_key( - &app_state.app_config.file_locks, - epg_path, - storage_keys, - ) - .await - { - Ok(results) => results.into_iter().map(|(_, channel)| channel).collect(), - Err(err) => { - error!("{err}"); - std::iter::repeat_with(|| None).take(prepared_entries.len()).collect() - } - }; + let channels: Vec> = + match epg_query_channels_by_storage_key(&app_state.app_config.file_locks, epg_path, storage_keys).await { + Ok(results) => results.into_iter().map(|(_, channel)| channel).collect(), + Err(err) => { + error!("{err}"); + std::iter::repeat_with(|| None).take(prepared_entries.len()).collect() + } + }; let mut entries = Vec::with_capacity(channels.len()); @@ -684,7 +681,9 @@ async fn serve_stream_epg( let window_start = reference_ts.saturating_sub(STREAM_EPG_ARCHIVE_WINDOW_BACK_SECS); let window_end = reference_ts.saturating_add(STREAM_EPG_ARCHIVE_WINDOW_FWD_SECS); let programmes = match channel { - Some(ch) => stream_epg_programmes_for_channel(&ch.programmes, &epg_processing_options, window_start, window_end), + Some(ch) => { + stream_epg_programmes_for_channel(&ch.programmes, &epg_processing_options, window_start, window_end) + } None => Vec::new(), }; @@ -724,9 +723,7 @@ fn stream_epg_programmes_for_channel( .collect() } -fn group_stream_epg_items( - items: Vec, -) -> Result)>, String> { +fn group_stream_epg_items(items: Vec) -> Result)>, String> { if items.is_empty() { return Err("items must not be empty".to_string()); } @@ -740,9 +737,7 @@ fn group_stream_epg_items( return Err("epg_channel_id must not be empty".to_string()); } if item.epg_channel_id.len() > MAX_STREAM_EPG_CHANNEL_ID_BYTES { - return Err(format!( - "epg_channel_id must not exceed {MAX_STREAM_EPG_CHANNEL_ID_BYTES} bytes" - )); + return Err(format!("epg_channel_id must not exceed {MAX_STREAM_EPG_CHANNEL_ID_BYTES} bytes")); } let Some(target_id) = item.target_id else { return Err("all items must include target_id".to_string()); @@ -792,10 +787,9 @@ pub(crate) async fn stream_epg_api( let Some(target) = app_state.app_config.get_target_by_id(target_id) else { return stream_epg_bad_request(&format!("unknown target_id: {target_id}")); }; - let output_case = EpgIdOutputCase::from_lowercase(target - .options - .as_ref() - .is_some_and(ConfigTargetOptions::lowercase_epg_ids)); + let output_case = EpgIdOutputCase::from_lowercase( + target.options.as_ref().is_some_and(ConfigTargetOptions::lowercase_epg_ids), + ); let prepared = prepare_stream_epg_request(&items, output_case); let Some(epg_path) = get_epg_path_for_target(config.as_ref(), &target) else { @@ -810,11 +804,10 @@ pub(crate) async fn stream_epg_api( let result = StreamEpgResponse { entries }; match serde_json::to_string(&result) { - Ok(json) => ( - axum::http::StatusCode::OK, - [(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string())], - json, - ).into_response(), + Ok(json) => { + (axum::http::StatusCode::OK, [(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string())], json) + .into_response() + } Err(_) => internal_server_error!(), } } @@ -830,13 +823,23 @@ pub(crate) async fn stream_epg_api( /// let router = xmltv_api_register(); /// // A GET request to /xmltv.php with valid query parameters will invoke this handler. /// ``` -async fn xmltv_api(fingerprint: &Fingerprint, api_req: UserApiRequest, app_state: &Arc) -> impl IntoResponse + Send { +async fn xmltv_api( + fingerprint: &Fingerprint, + api_req: UserApiRequest, + app_state: &Arc, +) -> impl IntoResponse + Send { api_req.log_sanitized("xmltv_api"); let auth_status = app_state.app_config.get_auth_error_status(); let Some((user, target)) = get_user_target(&api_req, app_state) else { return auth_status.into_response(); }; - if let Err(e) = resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), app_state) { + if let Err(e) = resolve_api_user_context( + user.clone(), + target.clone(), + fingerprint.clone(), + &app_state.app_config, + &app_state.geoip, + ) { return e.into_player_response(auth_status); } @@ -875,7 +878,13 @@ async fn epg_api_resource( let Some((user, target)) = get_user_target_by_credentials(&username, &password, &api_req, &app_state) else { return auth_status.into_response(); }; - if let Err(e) = resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), &app_state) { + if let Err(e) = resolve_api_user_context( + user.clone(), + target.clone(), + fingerprint.clone(), + &app_state.app_config, + &app_state.geoip, + ) { return e.into_player_response(auth_status); } @@ -911,27 +920,16 @@ pub fn xmltv_api_register() -> axum::Router> { #[cfg(test)] mod tests { use super::{ - empty_stream_epg_entries, - from_programme, - get_epg_path_for_target, - get_epg_path_for_target_by_type, - group_stream_epg_items, - prepare_stream_epg_request, - rewrite_epg_channel_resource_url, - serve_epg, - serve_short_epg, - serve_stream_epg, - stream_epg_api, - stream_epg_programmes_for_channel, - write_programme_classification_tags, - MAX_STREAM_EPG_CHANNEL_ID_BYTES, - MAX_STREAM_EPG_ITEMS, + empty_stream_epg_entries, from_programme, get_epg_path_for_target, get_epg_path_for_target_by_type, + group_stream_epg_items, prepare_stream_epg_request, rewrite_epg_channel_resource_url, serve_epg, + serve_short_epg, serve_stream_epg, stream_epg_api, stream_epg_programmes_for_channel, + write_programme_classification_tags, MAX_STREAM_EPG_CHANNEL_ID_BYTES, MAX_STREAM_EPG_ITEMS, }; use crate::{ api::model::{create_test_app_state, AppState}, model::{ - Config, ConfigTarget, Epg, IcsEpgSourceConfig, M3uTargetOutput, ProxyUserCredentials, - TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, + Config, ConfigTarget, Epg, IcsEpgSourceConfig, M3uTargetOutput, ProxyUserCredentials, TargetOutput, + XtreamTargetFlagsSet, XtreamTargetOutput, }, processing::parser::ics::parse_ics_file_to_channel, repository::{epg_write_file, BPlusTree}, @@ -950,8 +948,7 @@ mod tests { }; use std::{ collections::HashMap, - fs, - io, + fs, io, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -962,11 +959,7 @@ mod tests { struct ErroringWriter; impl AsyncWrite for ErroringWriter { - fn poll_write( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - _buf: &[u8], - ) -> Poll> { + fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll> { Poll::Ready(Err(io::Error::new(io::ErrorKind::ConnectionReset, "synthetic write failure"))) } @@ -1034,36 +1027,23 @@ mod tests { } fn test_config_with_storage(storage_dir: &str) -> Config { - Config { - storage_dir: storage_dir.to_string(), - ..Default::default() - } + Config { storage_dir: storage_dir.to_string(), ..Default::default() } } - fn test_app_state() -> Arc { - create_test_app_state(Config::default()) - } + fn test_app_state() -> Arc { create_test_app_state(Config::default()) } fn test_target_with_epg_options(lowercase_ids: bool, lowercase_display_names: bool) -> Arc { let mut target = test_target_with_xtream_only(); target.options = Some(ConfigTargetOptions { - epg_output: EpgOutputOptions { - lowercase_ids, - lowercase_xmltv_display_names: lowercase_display_names, - }, + epg_output: EpgOutputOptions { lowercase_ids, lowercase_xmltv_display_names: lowercase_display_names }, ..ConfigTargetOptions::default() }); Arc::new(target) } - fn write_test_epg_db(path: &std::path::Path, channel: EpgChannel) { - write_test_epg_channels(path, [channel]); - } + fn write_test_epg_db(path: &std::path::Path, channel: EpgChannel) { write_test_epg_channels(path, [channel]); } - fn write_test_epg_channels( - path: &std::path::Path, - channels: impl IntoIterator, - ) { + fn write_test_epg_channels(path: &std::path::Path, channels: impl IntoIterator) { let mut tree = BPlusTree::, EpgChannel>::new(); for channel in channels { tree.insert(Arc::clone(&channel.id), channel); @@ -1071,14 +1051,17 @@ mod tests { tree.store(path).expect("test EPG DB should be written"); } - fn stream_epg_item(id: impl Into, target_id: Option, reference_ts: Option) -> StreamEpgItemRequest { + fn stream_epg_item( + id: impl Into, + target_id: Option, + reference_ts: Option, + ) -> StreamEpgItemRequest { StreamEpgItemRequest { epg_channel_id: id.into(), target_id, reference_ts } } async fn response_body_text(response: axum::response::Response) -> String { - let body = axum::body::to_bytes(response.into_body(), 64 * 1024) - .await - .expect("response body should be readable"); + let body = + axum::body::to_bytes(response.into_body(), 64 * 1024).await.expect("response body should be readable"); String::from_utf8(body.to_vec()).expect("response body should be UTF-8") } @@ -1104,23 +1087,14 @@ mod tests { ) .expect("write ICS fixture"); - let channel = parse_ics_file_to_channel( - &ics_path, - "f1.calendar".intern(), - None, - &IcsEpgSourceConfig::default(), - ) - .await - .expect("parse ICS fixture"); + let channel = + parse_ics_file_to_channel(&ics_path, "f1.calendar".intern(), None, &IcsEpgSourceConfig::default()) + .await + .expect("parse ICS fixture"); let epg_path = dir.path().join("epg.db"); epg_write_file( "ics-target", - &Epg { - priority: 0, - logo_override: false, - attributes: None, - children: vec![Arc::new(channel)], - }, + &Epg { priority: 0, logo_override: false, attributes: None, children: vec![Arc::new(channel)] }, &epg_path, &HashMap::, Arc>::new(), &EpgOutputOptions::default(), @@ -1130,17 +1104,8 @@ mod tests { let config = test_config_with_storage(dir.path().to_string_lossy().as_ref()); let app_state = create_test_app_state(config); let target = Arc::new(test_target_with_xtream_and_m3u()); - let response = serve_epg( - &app_state, - &epg_path, - &ProxyUserCredentials::default(), - &target, - None, - ) - .await; - let body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("read XMLTV response body"); + let response = serve_epg(&app_state, &epg_path, &ProxyUserCredentials::default(), &target, None).await; + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("read XMLTV response body"); let xml = String::from_utf8(body.to_vec()).expect("XMLTV response is UTF-8"); assert!(xml.contains(r#""#)); @@ -1245,17 +1210,13 @@ mod tests { )]) .unwrap_err(); - assert_eq!( - err, - format!("epg_channel_id must not exceed {MAX_STREAM_EPG_CHANNEL_ID_BYTES} bytes") - ); + assert_eq!(err, format!("epg_channel_id must not exceed {MAX_STREAM_EPG_CHANNEL_ID_BYTES} bytes")); } #[test] fn test_group_stream_epg_items_rejects_excessive_item_count() { - let items = (0..=MAX_STREAM_EPG_ITEMS) - .map(|index| stream_epg_item(format!("epg-{index}"), Some(1), None)) - .collect(); + let items = + (0..=MAX_STREAM_EPG_ITEMS).map(|index| stream_epg_item(format!("epg-{index}"), Some(1), None)).collect(); let err = group_stream_epg_items(items).unwrap_err(); @@ -1279,17 +1240,14 @@ mod tests { async fn stream_epg_api_returns_bad_request_for_invalid_item() { let response = stream_epg_api( axum::extract::State(test_app_state()), - axum::extract::Json(StreamEpgRequest { - items: vec![stream_epg_item("", Some(1), None)], - }), + axum::extract::Json(StreamEpgRequest { items: vec![stream_epg_item("", Some(1), None)] }), ) .await .into_response(); assert_eq!(response.status(), axum::http::StatusCode::BAD_REQUEST); let body = response_body_text(response).await; - let error = serde_json::from_str::(&body) - .expect("bad-request response should contain JSON"); + let error = serde_json::from_str::(&body).expect("bad-request response should contain JSON"); assert_eq!(error["error"], "epg_channel_id must not be empty"); } @@ -1347,11 +1305,7 @@ mod tests { ]; let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); - let storage_keys = prepared - .entries - .iter() - .map(|entry| entry.storage_key.as_ref()) - .collect::>(); + let storage_keys = prepared.entries.iter().map(|entry| entry.storage_key.as_ref()).collect::>(); assert_eq!(storage_keys, vec!["example.channel", "Äbc.id"]); assert_eq!(prepared.entries[0].reference_ts, Some(100)); @@ -1392,10 +1346,8 @@ mod tests { #[test] fn empty_stream_epg_entries_use_prepared_canonical_ids() { - let items = [ - stream_epg_item("Example.Channel", Some(7), None), - stream_epg_item("example.CHANNEL", Some(7), None), - ]; + let items = + [stream_epg_item("Example.Channel", Some(7), None), stream_epg_item("example.CHANNEL", Some(7), None)]; let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); let entries = empty_stream_epg_entries(7, &prepared); @@ -1416,19 +1368,9 @@ mod tests { fn short_epg_response_uses_canonical_id_for_both_id_fields() { let stream_id = "42".intern(); let epg_id = "example.channel".intern(); - let programme = EpgProgramme::new_all( - 100, - 200, - Arc::clone(&epg_id), - Some("News".intern()), - None, - None, - ); - let options = EpgProcessingOptions { - rewrite_urls: false, - time_shift: EpgTimeShift::None, - encrypt_secret: [0; 16], - }; + let programme = EpgProgramme::new_all(100, 200, Arc::clone(&epg_id), Some("News".intern()), None, None); + let options = + EpgProcessingOptions { rewrite_urls: false, time_shift: EpgTimeShift::None, encrypt_secret: [0; 16] }; let result = from_programme(&stream_id, &epg_id, &programme, &options, false); @@ -1440,19 +1382,9 @@ mod tests { fn from_programme_sets_has_archive_only_when_capability_true() { let stream_id = "42".intern(); let epg_id = "example.channel".intern(); - let programme = EpgProgramme::new_all( - 100, - 200, - Arc::clone(&epg_id), - Some("News".intern()), - None, - None, - ); - let options = EpgProcessingOptions { - rewrite_urls: false, - time_shift: EpgTimeShift::None, - encrypt_secret: [0; 16], - }; + let programme = EpgProgramme::new_all(100, 200, Arc::clone(&epg_id), Some("News".intern()), None, None); + let options = + EpgProcessingOptions { rewrite_urls: false, time_shift: EpgTimeShift::None, encrypt_secret: [0; 16] }; let archived = from_programme(&stream_id, &epg_id, &programme, &options, true); let live_only = from_programme(&stream_id, &epg_id, &programme, &options, false); @@ -1465,15 +1397,11 @@ mod tests { fn lowercased_xmltv_display_name_remains_xml_escaped() { let display_name = lowercase_xmltv_text("CAFÉ NETWORK & ", true); let mut writer = quick_xml::Writer::new(Vec::new()); - writer - .write_event(Event::Start(BytesStart::new("display-name"))) - .expect("display-name start should serialize"); + writer.write_event(Event::Start(BytesStart::new("display-name"))).expect("display-name start should serialize"); writer .write_event(Event::Text(BytesText::new(display_name.as_ref()))) .expect("display-name text should serialize"); - writer - .write_event(Event::End(BytesEnd::new("display-name"))) - .expect("display-name end should serialize"); + writer.write_event(Event::End(BytesEnd::new("display-name"))).expect("display-name end should serialize"); let xml = String::from_utf8(writer.into_inner()).expect("serialized XML should be UTF-8"); @@ -1510,18 +1438,13 @@ mod tests { let app_state = test_app_state(); let target = test_target_with_epg_options(true, true); - let response = serve_epg( - &app_state, - &epg_path, - &ProxyUserCredentials::default(), - &target, - None, - ) - .await; + let response = serve_epg(&app_state, &epg_path, &ProxyUserCredentials::default(), &target, None).await; let xml = response_body_text(response).await; assert!(xml.contains(r#""#), "XMLTV output: {xml}"); - assert!(xml.contains(r#""#)); + assert!(xml.contains( + r#""# + )); assert!(xml.contains("café network & <hd>")); assert!(xml.contains("News & Updates")); assert!(xml.contains("Keep <Case>")); @@ -1569,14 +1492,7 @@ mod tests { let app_state = test_app_state(); let target = Arc::new(test_target_with_xtream_only()); - let response = serve_epg( - &app_state, - &epg_path, - &ProxyUserCredentials::default(), - &target, - None, - ) - .await; + let response = serve_epg(&app_state, &epg_path, &ProxyUserCredentials::default(), &target, None).await; let xml = response_body_text(response).await; assert!(xml.contains(r#""#)); @@ -1608,14 +1524,7 @@ mod tests { let app_state = test_app_state(); let target = Arc::new(test_target_with_xtream_only()); - let response = serve_epg( - &app_state, - &epg_path, - &ProxyUserCredentials::default(), - &target, - None, - ) - .await; + let response = serve_epg(&app_state, &epg_path, &ProxyUserCredentials::default(), &target, None).await; let xml = response_body_text(response).await; let first_position = xml.find(r#""#).expect("first channel should exist"); let second_position = xml.find(r#""#).expect("second channel should exist"); @@ -1646,14 +1555,7 @@ mod tests { let app_state = test_app_state(); let target = test_target_with_epg_options(false, true); - let response = serve_epg( - &app_state, - &epg_path, - &ProxyUserCredentials::default(), - &target, - None, - ) - .await; + let response = serve_epg(&app_state, &epg_path, &ProxyUserCredentials::default(), &target, None).await; let xml = response_body_text(response).await; assert!(xml.contains(r#""#)); @@ -1686,14 +1588,7 @@ mod tests { let app_state = test_app_state(); let target = test_target_with_epg_options(true, false); - let response = serve_epg( - &app_state, - &epg_path, - &ProxyUserCredentials::default(), - &target, - None, - ) - .await; + let response = serve_epg(&app_state, &epg_path, &ProxyUserCredentials::default(), &target, None).await; let xml = response_body_text(response).await; assert!(xml.contains(r#""#)); @@ -1815,22 +1710,12 @@ mod tests { let target = test_target_with_epg_options(true, false); let items = [ stream_epg_item("Example.Channel", Some(target.id), Some(first_reference)), - stream_epg_item( - "example.CHANNEL", - Some(target.id), - Some(first_reference + 7 * 24 * 3600), - ), + stream_epg_item("example.CHANNEL", Some(target.id), Some(first_reference + 7 * 24 * 3600)), ]; let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); - let response = serve_stream_epg( - &app_state, - &ProxyUserCredentials::default(), - &target, - &epg_path, - prepared, - ) - .await; + let response = + serve_stream_epg(&app_state, &ProxyUserCredentials::default(), &target, &epg_path, prepared).await; assert_eq!(response.entries.len(), 1); assert_eq!(response.entries[0].epg_channel_id, "example.channel"); @@ -1883,14 +1768,8 @@ mod tests { ]; let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::Preserve); - let response = serve_stream_epg( - &app_state, - &ProxyUserCredentials::default(), - &target, - &epg_path, - prepared, - ) - .await; + let response = + serve_stream_epg(&app_state, &ProxyUserCredentials::default(), &target, &epg_path, prepared).await; assert_eq!(response.entries.len(), 2); assert_eq!(response.entries[0].epg_channel_id, "Example.Channel"); @@ -1939,11 +1818,8 @@ mod tests { fn stream_epg_programmes_for_channel_filters_using_shifted_fixed_times() { let window_start = 10_000; let window_end = window_start + 8 * 3600; - let epg_processing_options = EpgProcessingOptions { - rewrite_urls: false, - time_shift: EpgTimeShift::Fixed(120), - encrypt_secret: [0; 16], - }; + let epg_processing_options = + EpgProcessingOptions { rewrite_urls: false, time_shift: EpgTimeShift::Fixed(120), encrypt_secret: [0; 16] }; let programmes = vec![ EpgProgramme::new_all( window_start - 7_300, @@ -1981,5 +1857,4 @@ mod tests { assert_eq!(filtered[0].start_timestamp, window_start + 7_260); assert_eq!(filtered[0].stop_timestamp, window_start + 7_800); } - } diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/app/src/api/endpoints/xtream_api.rs similarity index 94% rename from backend/src/api/endpoints/xtream_api.rs rename to backend/app/src/api/endpoints/xtream_api.rs index 811f85ff9..40d64fd5c 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/app/src/api/endpoints/xtream_api.rs @@ -8,11 +8,11 @@ use crate::{ admission_failure_response, coalesce_byte_stream, create_api_proxy_user, create_catchup_session_key, create_m3u_catchup_session_key, create_playback_session_fingerprint, create_session_fingerprint, empty_json_response_as_array, empty_json_response_as_object, force_provider_stream_response, - get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, - internal_server_error, is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, - local_stream_response, redirect, redirect_response, resource_response, separate_number_and_remainder, - should_allow_exhausted_shared_reconnect, stream_response, try_option_bad_request, try_result_bad_request, - try_unwrap_body, RedirectParams, resolve_initial_stalker_playback_url, + get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, internal_server_error, + is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, local_stream_response, + redirect, redirect_response, resolve_initial_stalker_playback_url, resource_response, + separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, + try_option_bad_request, try_result_bad_request, try_unwrap_body, RedirectParams, }, endpoints::{ hls_api::{ @@ -40,9 +40,7 @@ use crate::{ get_target_id_mapping, get_target_storage_path, storage_const, user_get_bouquet_filter, xtream_get_collection_path, xtream_get_item_for_stream_id, xtream_load_rewrite_playlist, VirtualIdRecord, }, - utils::{ - apply_timeshift, debug_if_enabled, file_exists_async, parse_timeshift, request, trace_if_enabled, - }, + utils::{apply_timeshift, debug_if_enabled, file_exists_async, parse_timeshift, request, trace_if_enabled}, }; use axum::{http::HeaderMap, response::IntoResponse}; use bytes::Bytes; @@ -55,6 +53,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; use shared::{ concat_string, + defaults::{HLS_EXT, TS_EXT}, error::TuliproxError, model::{ create_stream_channel_with_type, ConnectFailureReason, PlaylistEntry, PlaylistItemType, ProxyType, @@ -64,7 +63,6 @@ use shared::{ deserialize_as_string, extract_extension_from_url, generate_provider_playlist_uuid, sanitize_sensitive_info, trim_slash, Internable, }, - defaults::{HLS_EXT, TS_EXT}, }; use std::{ fmt::{Display, Formatter, Write}, @@ -264,7 +262,7 @@ async fn xtream_player_api_stream( Some((user, target)) => (user, target), }; // Network access check only - permission check is done later with full stream info - if let Err(e) = check_network_access_only(&user, fingerprint, app_state) { + if let Err(e) = check_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip) { return e.into_player_response(auth_status); } @@ -282,7 +280,7 @@ async fn xtream_player_api_stream( return auth_status.into_response(); } return create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, CustomVideoStreamType::ChannelUnavailable, ) @@ -290,7 +288,9 @@ async fn xtream_player_api_stream( } let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); - let Ok(mut pli) = xtream_get_item_for_stream_id(req_virtual_id, app_state, &target, None).await else { + let Ok(mut pli) = + xtream_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target, None).await + else { error!("Failed to read xtream item for stream id {req_virtual_id}"); if is_hls_manifest_request { return hls_custom_video_manifest_response( @@ -302,7 +302,7 @@ async fn xtream_player_api_stream( .await; } return create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, CustomVideoStreamType::ChannelUnavailable, ) @@ -319,7 +319,8 @@ async fn xtream_player_api_stream( user.allows_cluster(XtreamCluster::Live) } else { user.allows_item_type(pli.item_type) - }) && (user.t_filter.is_none() || user.allows_content(&shared::model::PlaylistItem::from(&pli))); + }) && (user.t_filter.is_none() + || user.allows_content(&shared::model::PlaylistItem::from(&pli))); if !output_allowed { if is_hls_manifest_request { return hls_custom_video_manifest_response( @@ -331,7 +332,7 @@ async fn xtream_player_api_stream( .await; } return create_custom_video_stream_response( - app_state, + &app_state.provider_stream_ctx(), &fingerprint.addr, CustomVideoStreamType::ChannelUnavailable, ) @@ -352,7 +353,7 @@ async fn xtream_player_api_stream( ) ); - if user.permission_denied(app_state) { + if user.permission_denied(&app_state.app_config) { let stream_channel = create_stream_channel_with_type(target.id, &pli, pli.item_type); if is_hls_playback_request(stream_ext, &pli) { return hls_admission_failure_manifest_response( @@ -395,7 +396,7 @@ async fn xtream_player_api_stream( let user_session = app_state.active_users.get_and_update_user_session(&user.username, &playback_session_token).await; let (admission, _grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( - app_state, + &app_state.admission_ctx(), &user, fingerprint, pli.item_type, @@ -562,7 +563,7 @@ async fn xtream_player_api_stream( }; let (connection_admission, grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( - app_state, + &app_state.admission_ctx(), &user, fingerprint, item_type, @@ -644,10 +645,7 @@ async fn xtream_player_api_stream( // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH if is_session_request && playback_ext == shared::defaults::HLS_EXT { let Some(stream_context) = HlsEntryStreamContext::from_playlist_item(&pli) else { - error!( - "HLS input stream identity missing for virtual_id={}; refresh target playlist", - pli.virtual_id - ); + error!("HLS input stream identity missing for virtual_id={}; refresh target playlist", pli.virtual_id); return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); }; let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, pli.virtual_id); @@ -677,8 +675,8 @@ async fn xtream_player_api_stream( let archive_reference = m3u_archive_epg_reference_ts(stream_url.as_ref()) .or_else(|| m3u_archive_epg_reference_ts(pli.url.as_ref())) .or_else(|| m3u_catchup_epg_reference_from_session_token(&session_key)); - let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type) - .with_epg_reference_ts(archive_reference); + let stream_channel = + create_stream_channel_with_type(target.id, &pli, item_type).with_epg_reference_ts(archive_reference); let pinned_provider = user_session.as_ref().filter(|_| item_type.requires_provider_affinity()).map(|session| &session.provider); @@ -766,23 +764,19 @@ fn resolve_m3u_xtream_timeshift( return Ok(None); } let (duration, start) = action_path.split_once('/').ok_or_else(|| { - TuliproxError::ApiXtream( - "M3U Xtream timeshift requires action_path in 'duration/start' form".to_string(), - ) - })?; - let props = item.additional_properties.as_ref().ok_or_else(|| { - TuliproxError::ApiXtream("M3U Xtream timeshift item has no stream properties".to_string()) + TuliproxError::ApiXtream("M3U Xtream timeshift requires action_path in 'duration/start' form".to_string()) })?; + let props = item + .additional_properties + .as_ref() + .ok_or_else(|| TuliproxError::ApiXtream("M3U Xtream timeshift item has no stream properties".to_string()))?; let StreamProperties::Live(live) = props else { - return Err(TuliproxError::ApiXtream( - "M3U Xtream timeshift requires a live stream".to_string(), - )); + return Err(TuliproxError::ApiXtream("M3U Xtream timeshift requires a live stream".to_string())); }; - let catchup = live.catchup.as_ref().ok_or_else(|| { - TuliproxError::ApiXtream( - "M3U Xtream timeshift item has no catch-up metadata".to_string(), - ) - })?; + let catchup = live + .catchup + .as_ref() + .ok_or_else(|| TuliproxError::ApiXtream("M3U Xtream timeshift item has no catch-up metadata".to_string()))?; let resolved_source = input.resolve_url(&item.url)?; let resolved = resolve_xtream_m3u_catchup_url(resolved_source.as_ref(), catchup, start, duration)?; Ok(Some(resolved)) @@ -878,7 +872,14 @@ pub(in crate::api) async fn xtream_player_api_stream_with_resolved_target( let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); let mut pli = try_result_bad_request!( - xtream_get_item_for_stream_id(req_virtual_id, app_state, &target, Some(stream_req.context.cluster())).await, + xtream_get_item_for_stream_id( + req_virtual_id, + &app_state.app_config, + &app_state.playlists, + &target, + Some(stream_req.context.cluster()) + ) + .await, true, format!("Failed to read xtream item for stream id {req_virtual_id}") ); @@ -926,11 +927,8 @@ pub(in crate::api) async fn xtream_player_api_stream_with_resolved_target( .into_response(); } - let resolution_item_type = if stream_req.context == ApiStreamContext::Timeshift { - PlaylistItemType::Catchup - } else { - pli.item_type - }; + let resolution_item_type = + if stream_req.context == ApiStreamContext::Timeshift { PlaylistItemType::Catchup } else { pli.item_type }; pli.url = match resolve_initial_stalker_playback_url( app_state, &input, @@ -1033,7 +1031,9 @@ async fn xtream_player_api_resource( else { return auth_status.into_response(); }; - if let Err(e) = check_permission_and_network_access_only(&user, fingerprint, app_state) { + if let Err(e) = + check_permission_and_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip) + { return e.into_player_response(auth_status); } let target_name = &target.name; @@ -1044,7 +1044,7 @@ async fn xtream_player_api_resource( let req_virtual_id: u32 = try_result_bad_request!(resource_req.stream_id.trim().parse()); let resource = resource_req.action_path.trim(); let pli = try_result_bad_request!( - xtream_get_item_for_stream_id(req_virtual_id, app_state, &target, None).await, + xtream_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target, None).await, true, format!("Failed to read xtream item for stream id {req_virtual_id}") ); @@ -1277,7 +1277,10 @@ pub async fn xtream_get_stream_info_response( Err(_) => return try_unwrap_body!(empty_json_response_as_object()), }; - let Ok(pli) = xtream_get_item_for_stream_id(virtual_id, app_state, target, Some(cluster)).await else { + let Ok(pli) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, Some(cluster)) + .await + else { return empty_stream_info_response(cluster); }; @@ -1330,7 +1333,8 @@ pub async fn xtream_get_stream_info_response( // fetch info from the upstream provider if let Ok(content) = xtream::get_xtream_stream_info( &app_state.http_client.load(), - app_state, + &app_state.app_config, + &app_state.playlists, user, &input, target, @@ -1382,17 +1386,18 @@ async fn xtream_get_short_epg( Err(_) => return get_empty_epg_response().into_response(), }; - if let Ok(pli) = xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await { + if let Ok(pli) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await + { // Content filter: hidden items expose no EPG either if !(user.t_filter.is_none() || user.allows_content(&shared::model::PlaylistItem::from(&pli))) { return axum::Json(json!(ShortEpgResultDto::default())).into_response(); } let config = &app_state.app_config.config.load(); let has_archive = pli_supports_archive(app_state, &pli); - if let (Some(epg_path), Some(channel_id)) = ( - get_epg_path_for_target_by_type(config, target, TargetType::Xtream), - &pli.epg_channel_id, - ) { + if let (Some(epg_path), Some(channel_id)) = + (get_epg_path_for_target_by_type(config, target, TargetType::Xtream), &pli.epg_channel_id) + { if file_exists_async(&epg_path).await { return serve_short_epg( app_state, @@ -1423,7 +1428,10 @@ async fn xtream_get_short_epg( return match api_utils::resolve_redirect_location(Some(&input), &info_url) { Ok(redirect_url) => redirect(redirect_url.as_ref()).into_response(), Err(err) => { - error!("Failed to resolve redirect url: {}", sanitize_sensitive_info(&err.to_string())); + error!( + "Failed to resolve redirect url: {}", + sanitize_sensitive_info(&err.to_string()) + ); axum::http::StatusCode::BAD_REQUEST.into_response() } }; @@ -1499,9 +1507,7 @@ async fn xtream_player_api_handle_content_action( ) .await { - categories.retain(|c| { - c.category_id.parse::().is_ok_and(|id| visible.contains(&id)) - }); + categories.retain(|c| c.category_id.parse::().is_ok_and(|id| visible.contains(&id))); } return Some(axum::Json(categories).into_response()); } @@ -1535,7 +1541,14 @@ async fn xtream_get_catchup_response( }; let pli = try_result_bad_request!( - xtream_get_item_for_stream_id(req_virtual_id, app_state, target, Some(XtreamCluster::Live)).await + xtream_get_item_for_stream_id( + req_virtual_id, + &app_state.app_config, + &app_state.playlists, + target, + Some(XtreamCluster::Live) + ) + .await ); // Content filter: hidden items expose no catch-up table either, so a @@ -1686,7 +1699,7 @@ async fn xtream_player_api( let Some((user, target)) = get_user_target(&api_req, app_state) else { return auth_status.into_response(); }; - if let Err(e) = check_network_access_only(&user, fingerprint, app_state) { + if let Err(e) = check_network_access_only(&user, fingerprint, &app_state.app_config, &app_state.geoip) { return e.into_player_response(auth_status); } if !target.has_output(TargetType::Xtream) { @@ -1704,7 +1717,7 @@ async fn xtream_player_api( ); } - if user.permission_denied(app_state) { + if user.permission_denied(&app_state.app_config) { return axum::http::StatusCode::FORBIDDEN.into_response(); } @@ -1777,14 +1790,8 @@ async fn xtream_player_api( let category_id = api_req.category_id.trim().parse::().ok(); // Handle general content actions - if let Some(response) = xtream_player_api_handle_content_action( - app_state, - &target, - action, - category_id, - &user, - ) - .await + if let Some(response) = + xtream_player_api_handle_content_action(app_state, &target, action, category_id, &user).await { return response.into_response(); } @@ -1792,15 +1799,17 @@ async fn xtream_player_api( let result = match action { crate::model::XC_ACTION_GET_LIVE_STREAMS => skip_flag_optional!( skip_live, - xtream_load_rewrite_playlist(XtreamCluster::Live, app_state, &target, category_id, &user).await + xtream_load_rewrite_playlist(XtreamCluster::Live, &app_state.app_config, &target, category_id, &user).await ), crate::model::XC_ACTION_GET_VOD_STREAMS => skip_flag_optional!( skip_vod, - xtream_load_rewrite_playlist(XtreamCluster::Video, app_state, &target, category_id, &user).await + xtream_load_rewrite_playlist(XtreamCluster::Video, &app_state.app_config, &target, category_id, &user) + .await ), crate::model::XC_ACTION_GET_SERIES => skip_flag_optional!( skip_series, - xtream_load_rewrite_playlist(XtreamCluster::Series, app_state, &target, category_id, &user).await + xtream_load_rewrite_playlist(XtreamCluster::Series, &app_state.app_config, &target, category_id, &user) + .await ), _ => Some(Err(TuliproxError::ApiXtream(format!("Unknown api call: {action} for target: {}", target.name)))), }; @@ -1844,9 +1853,11 @@ where Bytes::from(line) }) }); - coalesce_byte_stream(stream::once(async { Ok::(Bytes::from("[")) }) - .chain(mapped) - .chain(stream::once(async { Ok::(Bytes::from("]")) }))) + coalesce_byte_stream( + stream::once(async { Ok::(Bytes::from("[")) }) + .chain(mapped) + .chain(stream::once(async { Ok::(Bytes::from("]")) })), + ) } async fn xtream_player_api_get( @@ -1955,31 +1966,29 @@ pub fn xtream_api_register() -> axum::Router> { #[cfg(test)] mod tests { use super::{ - empty_stream_info_response, get_xtream_player_api_stream_url, resolve_m3u_xtream_timeshift, - is_hls_playback_request, override_live_hls_extension, recording_input_matches, resolve_xtream_playback_extension, - xtream_get_short_epg, xtream_player_api_stream, xtream_player_api_stream_with_token, ApiStreamContext, - ApiStreamRequest, XtreamApiTimeShiftRequest, + empty_stream_info_response, get_xtream_player_api_stream_url, is_hls_playback_request, + override_live_hls_extension, recording_input_matches, resolve_m3u_xtream_timeshift, + resolve_xtream_playback_extension, xtream_get_short_epg, xtream_player_api_stream, + xtream_player_api_stream_with_token, ApiStreamContext, ApiStreamRequest, XtreamApiTimeShiftRequest, }; - use crate::auth::Fingerprint; use crate::{ - api::model::{ - create_test_app_state, AppState, PlaylistStorage, PlaylistXtreamStorage, UserApiRequest, - }, + api::model::{create_test_app_state, AppState, PlaylistStorage, PlaylistXtreamStorage, UserApiRequest}, + auth::Fingerprint, model::{ Config, ConfigInput, ConfigTarget, Epg, IcsEpgSourceConfig, ProxyUserCredentials, SourcesConfig, TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, }, processing::parser::ics::parse_ics_file_to_channel, repository::{ - epg_write_file, xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTree, - VirtualIdRecord, + epg_write_file, xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTree, VirtualIdRecord, }, }; use arc_swap::ArcSwapOption; use axum::{http::HeaderMap, response::IntoResponse}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine}; - use shared::{error::TuliproxError, foundation::Filter}; use shared::{ + error::TuliproxError, + foundation::Filter, model::{ ClusterFlags, InputType, PlaylistItemType, ProcessingOrder, ProxyUserStatus, StreamProperties, UUIDType, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, @@ -2142,22 +2151,14 @@ mod tests { .await .expect("parse ICS fixture"); - let config = Config { - storage_dir: dir.path().to_string_lossy().into_owned(), - ..Config::default() - }; + let config = Config { storage_dir: dir.path().to_string_lossy().into_owned(), ..Config::default() }; let target = Arc::new(short_epg_target()); let xtream_storage = xtream_get_storage_path(&config, &target.name).expect("xtream storage path"); std::fs::create_dir_all(&xtream_storage).expect("create xtream storage"); let epg_path = xtream_get_epg_file_path_for_target(&xtream_storage); epg_write_file( &target.name, - &Epg { - priority: 0, - logo_override: false, - attributes: None, - children: vec![Arc::new(channel)], - }, + &Epg { priority: 0, logo_override: false, attributes: None, children: vec![Arc::new(channel)] }, &epg_path, &std::collections::HashMap::, Arc>::new(), &shared::model::EpgOutputOptions::default(), @@ -2194,9 +2195,7 @@ mod tests { let mut user = ProxyUserCredentials::default(); user.output_clusters = ClusterFlags::all(); - let response = xtream_get_short_epg(&app_state, &user, &target, "100", 4) - .await - .into_response(); + let response = xtream_get_short_epg(&app_state, &user, &target, "100", 4).await.into_response(); let body = response_body_text(response).await.expect("read short EPG response"); let json: serde_json::Value = serde_json::from_str(&body).expect("parse short EPG JSON"); let decode_listing_text = |field: &str| { @@ -2210,12 +2209,7 @@ mod tests { assert_eq!(decode_listing_text("description"), "Imported from ICS"); } - fn m3u_catchup_item( - name: &str, - input_name: &str, - url: &str, - catchup_source: Option<&str>, - ) -> XtreamPlaylistItem { + fn m3u_catchup_item(name: &str, input_name: &str, url: &str, catchup_source: Option<&str>) -> XtreamPlaylistItem { XtreamPlaylistItem { virtual_id: 100, provider_id: 0, @@ -2326,22 +2320,14 @@ mod tests { .await .expect("parse ICS fixture"); - let config = Config { - storage_dir: dir.path().to_string_lossy().into_owned(), - ..Config::default() - }; + let config = Config { storage_dir: dir.path().to_string_lossy().into_owned(), ..Config::default() }; let target = Arc::new(short_epg_target()); let xtream_storage = xtream_get_storage_path(&config, &target.name).expect("xtream storage path"); std::fs::create_dir_all(&xtream_storage).expect("create xtream storage"); let epg_path = xtream_get_epg_file_path_for_target(&xtream_storage); epg_write_file( &target.name, - &Epg { - priority: 0, - logo_override: false, - attributes: None, - children: vec![Arc::new(channel)], - }, + &Epg { priority: 0, logo_override: false, attributes: None, children: vec![Arc::new(channel)] }, &epg_path, &std::collections::HashMap::, Arc>::new(), &shared::model::EpgOutputOptions::default(), @@ -2718,10 +2704,7 @@ mod tests { .expect("resolver ok") .expect("M3U input handled by bridge"); - assert_eq!( - resolved.url, - "http://provider.example/channel/video-1704067200-3600.m3u8" - ); + assert_eq!(resolved.url, "http://provider.example/channel/video-1704067200-3600.m3u8"); assert!(!resolved.discriminator.is_empty()); } @@ -2737,11 +2720,7 @@ mod tests { assert_ne!(first.url, second.url); assert_ne!(first.discriminator, second.discriminator); - let fp = Fingerprint::new( - "fp".to_string(), - "127.0.0.1".to_string(), - "127.0.0.1:0".parse().unwrap(), - ); + let fp = Fingerprint::new("fp".to_string(), "127.0.0.1".to_string(), "127.0.0.1:0".parse().unwrap()); assert_ne!( crate::api::api_utils::create_m3u_catchup_session_key(&fp, "alice", 100, &first.discriminator), crate::api::api_utils::create_m3u_catchup_session_key(&fp, "alice", 100, &second.discriminator), @@ -2756,8 +2735,8 @@ mod tests { } let input = m3u_timeshift_input(); - let err = resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:00-00") - .expect_err("missing catchup must error"); + let err = + resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:00-00").expect_err("missing catchup must error"); assert!(matches!(err, TuliproxError::ApiXtream(_))); } @@ -2775,8 +2754,7 @@ mod tests { #[test] fn m3u_timeshift_stream_url_uses_resolved_archive_for_credential_bearing_input() { let input = m3u_timeshift_input(); - let resolved_url: Arc = - Arc::from("http://provider.example/channel/video-1704067200-3600.m3u8"); + let resolved_url: Arc = Arc::from("http://provider.example/channel/video-1704067200-3600.m3u8"); let url = get_xtream_player_api_stream_url(&input, ApiStreamContext::Timeshift, "60/2024-01-01:00-00", &resolved_url) @@ -2829,10 +2807,7 @@ mod tests { use crate::auth::create_access_token; let dir = tempdir().expect("temp dir"); - let config = Config { - storage_dir: dir.path().to_string_lossy().into_owned(), - ..Config::default() - }; + let config = Config { storage_dir: dir.path().to_string_lossy().into_owned(), ..Config::default() }; let app_state = create_test_app_state(config); let target = Arc::new(short_epg_target()); let input = m3u_timeshift_input(); @@ -2846,12 +2821,7 @@ mod tests { &HeaderMap::new(), &app_state, target.id, - ApiStreamRequest::from_access_token( - ApiStreamContext::Timeshift, - &token, - "100.ts", - "", - ), + ApiStreamRequest::from_access_token(ApiStreamContext::Timeshift, &token, "100.ts", ""), ) .await .into_response(); diff --git a/backend/src/api/endpoints/xtream_url.rs b/backend/app/src/api/endpoints/xtream_url.rs similarity index 97% rename from backend/src/api/endpoints/xtream_url.rs rename to backend/app/src/api/endpoints/xtream_url.rs index 6cc8a6fdc..ac6cb5666 100644 --- a/backend/src/api/endpoints/xtream_url.rs +++ b/backend/app/src/api/endpoints/xtream_url.rs @@ -25,4 +25,4 @@ //! re-export module is the smallest step that unblocks the dependency //! direction without that risk. -pub(in crate::api) use super::xtream_api::{get_query_path, get_xtream_player_api_stream_url, ApiStreamContext}; \ No newline at end of file +pub(in crate::api) use super::xtream_api::{get_query_path, get_xtream_player_api_stream_url, ApiStreamContext}; diff --git a/backend/src/api/hdhomerun_proprietary.rs b/backend/app/src/api/hdhomerun_proprietary.rs similarity index 100% rename from backend/src/api/hdhomerun_proprietary.rs rename to backend/app/src/api/hdhomerun_proprietary.rs diff --git a/backend/src/api/http_layers.rs b/backend/app/src/api/http_layers.rs similarity index 64% rename from backend/src/api/http_layers.rs rename to backend/app/src/api/http_layers.rs index 00bf32892..01595bcb4 100644 --- a/backend/src/api/http_layers.rs +++ b/backend/app/src/api/http_layers.rs @@ -1,6 +1,4 @@ -pub(crate) fn create_cors_layer( - methods: [axum::http::Method; N], -) -> tower_http::cors::CorsLayer { +pub(crate) fn create_cors_layer(methods: [axum::http::Method; N]) -> tower_http::cors::CorsLayer { tower_http::cors::CorsLayer::new() .allow_origin(tower_http::cors::Any) .allow_methods(methods) diff --git a/backend/src/api/main_api.rs b/backend/app/src/api/main_api.rs similarity index 89% rename from backend/src/api/main_api.rs rename to backend/app/src/api/main_api.rs index a4f4ccd9e..3f652d3b0 100644 --- a/backend/src/api/main_api.rs +++ b/backend/app/src/api/main_api.rs @@ -1,43 +1,42 @@ use crate::{ api::{ api_utils::{get_build_time, get_server_time}, - tasks::exec_config_watch, endpoints::{ custom_video_stream_api::cvs_api_register, download_api::{resume_download_worker_if_needed, spawn_download_services}, hdhomerun_api::hdhr_api_register, hls_api::hls_api_register, + log_ws_api::log_ws_api_register, m3u_api::m3u_api_register, provider_resolve_api::provider_resolve_api_register, v1_api::v1_api_register, web_index::{index_register_with_path, index_register_without_path}, websocket_api::ws_api_register, - log_ws_api::log_ws_api_register, xmltv_api::xmltv_api_register, xtream_api::xtream_api_register, }, hdhomerun_proprietary::spawn_proprietary_tasks, - tasks::spawn_ssdp_discover_task, http_layers::create_cors_layer, model::{ create_cache, create_http_client, create_http_client_no_redirect, create_public_http_client_no_redirect, - exec_provider_dns, - exec_hls_cache_gc, exec_hls_lifecycle, ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, - ConnectionManager, DownloadQueue, EventManager, EventMessage, HdHomerunAppState, HlsProvisioningState, - HlsProxyManager, ManualPlaylistUpdateRequest, MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, - UpdateGuard, exec_qos_aggregation, + exec_provider_dns, exec_qos_aggregation, load_playlists_into_memory_cache, recording_rule_scheduler::spawn_recording_rule_scheduler, - recording_supervisor::start_recording_supervisors, + recording_supervisor::start_recording_supervisors, ActiveProviderManager, ActiveUserManager, AppState, + CancelTokens, ConnectionManager, DownloadQueue, EventManager, EventMessage, HdHomerunAppState, + HlsProvisioningState, ManualPlaylistUpdateRequest, MetadataUpdateManager, PlaylistStorageState, + SharedStreamManager, UpdateGuard, }, - panel_api::sync_panel_api_exp_dates_on_boot, - tasks::{exec_interner_prune, exec_scheduler, exec_xtream_expiry_sync}, + panel_api::{sync_panel_api_exp_dates, sync_panel_api_exp_dates_on_boot}, serve::serve, sys_usage::exec_system_usage, + tasks::{ + exec_config_watch, exec_interner_prune, exec_scheduler, exec_xtream_expiry_sync, spawn_ssdp_discover_task, + }, }, model::{AppConfig, Config, HdHomeRunFlags, Healthcheck, ProcessTargets, RateLimitConfig}, processing::processor::exec_processing, - repository::{get_geoip_path, load_playlists_into_memory_cache}, - utils::{exec_file_lock_prune, get_default_web_root_path, GeoIp}, + repository::{get_geoip_path, GeoIp}, + utils::{exec_file_lock_prune, get_default_web_root_path}, VERSION, }; use arc_swap::{ArcSwap, ArcSwapOption}; @@ -56,13 +55,19 @@ use std::{ collections::{HashMap, HashSet}, net::SocketAddr, path::PathBuf, - sync::{atomic::{AtomicI8, AtomicU64, Ordering}, Arc}, + sync::{ + atomic::{AtomicI8, AtomicU64, Ordering}, + Arc, + }, }; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use tower_governor::key_extractor::SmartIpKeyExtractor; -use tower_http::compression::predicate::{DefaultPredicate, Predicate, SizeAbove}; -use tower_http::services::ServeDir; +use tower_http::{ + compression::predicate::{DefaultPredicate, Predicate, SizeAbove}, + services::ServeDir, +}; +use tuliprox_hls::api::{exec_hls_cache_gc, exec_hls_lifecycle, HlsProxyManager}; const METADATA_TRIGGER_WAIT_CYCLE_LIMIT: u32 = 900; static CORRUPT_DOWNLOADS_STATE_SUFFIX: AtomicU64 = AtomicU64::new(1); @@ -102,9 +107,7 @@ async fn recover_persisted_downloads_state(downloads: &DownloadQueue) -> Result< } Ok(()) } - Err(err) => Err(TuliproxError::Io( - format!("Failed to load persisted downloads: {err}"), - )), + Err(err) => Err(TuliproxError::Io(format!("Failed to load persisted downloads: {err}"))), } } @@ -125,8 +128,8 @@ async fn resume_downloads_after_bind(app_state: &Arc, download_cfg: &c // Cloned out of the `ArcSwap` guard first: the guard must not be held // across the await below. let downloads_cancel = app_state.cancel_tokens.load().downloads.clone(); - start_recording_supervisors(app_state, &downloads_cancel).await; - spawn_recording_rule_scheduler(app_state, &downloads_cancel); + start_recording_supervisors(&app_state.recording_ctx(), &downloads_cancel).await; + spawn_recording_rule_scheduler(&app_state.recording_ctx(), &downloads_cancel); if let Err(err) = resume_download_worker_if_needed(app_state.as_ref(), download_cfg).await { error!("Failed to resume persisted downloads during startup; continuing with downloads paused: {err}"); } @@ -209,7 +212,14 @@ fn spawn_metadata_trigger_update( Arc::clone(&app_config), proc_targets, Some(Arc::clone(&event_manager)), - Some(app_state_clone.clone()), + Some({ + let state = Arc::clone(&app_state_clone); + std::sync::Arc::new(move || { + let state = Arc::clone(&state); + Box::pin(async move { sync_panel_api_exp_dates(&state).await }) + as std::pin::Pin + Send>> + }) + }), Some(Arc::clone(&playlist_state)), Some(update_guard.clone()), disabled_headers.clone(), @@ -255,7 +265,10 @@ fn spawn_metadata_trigger_update( fn get_web_dir_path(web_ui_enabled: bool, web_root: &str) -> Result { let web_dir_path = if web_root.is_empty() { get_default_web_root_path() } else { PathBuf::from(web_root) }; if web_ui_enabled && (!&web_dir_path.exists() || !&web_dir_path.is_dir()) { - return Err(TuliproxError::Server(format!("web_root does not exist or is not a directory: {}", web_dir_path.display()))); + return Err(TuliproxError::Server(format!( + "web_root does not exist or is not a directory: {}", + web_dir_path.display() + ))); } Ok(web_dir_path) } @@ -292,11 +305,7 @@ async fn ready( // No enabled input or alias left means no capacity can ever be served. let exhausted = slots.is_empty() || is_exhausted(slots, &sources.group_lookup); let status_label = if exhausted { "exhausted" } else { "ready" }; - let status_code = if exhausted { - axum::http::StatusCode::SERVICE_UNAVAILABLE - } else { - axum::http::StatusCode::OK - }; + let status_code = if exhausted { axum::http::StatusCode::SERVICE_UNAVAILABLE } else { axum::http::StatusCode::OK }; (status_code, axum::Json(ReadyResponse { status: status_label })) } @@ -329,10 +338,8 @@ async fn create_shared_data( let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(app_config, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - let rewrite_secret = config - .reverse_proxy - .as_ref() - .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); + let rewrite_secret = + config.reverse_proxy.as_ref().map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); let hls_proxy = Arc::new(HlsProxyManager::from_hls_cache_config_and_secret( config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()), &rewrite_secret, @@ -361,27 +368,27 @@ async fn create_shared_data( let (manual_update_sender, manual_update_rx) = mpsc::channel::(1); let app_state = AppState { - forced_targets: Arc::new(ArcSwap::new(Arc::clone(forced_targets))), - app_config: Arc::clone(app_config), - http_client: Arc::new(ArcSwap::from_pointee(client)), - http_client_no_redirect: Arc::new(ArcSwap::from_pointee(client_no_redirect)), - public_http_client_no_redirect: Arc::new(ArcSwap::from_pointee(public_client_no_redirect)), - downloads: Arc::new(DownloadQueue::new_with_state_file(Some(downloads_state_file))), - cache: Arc::new(ArcSwapOption::from(cache)), - shared_stream_manager, - hls_proxy, - hls_provisioning: Arc::new(HlsProvisioningState::new()), - active_users, - active_provider, - connection_manager, - event_manager, - cancel_tokens, - playlists: Arc::new(PlaylistStorageState::new()), - geoip, - update_guard: UpdateGuard::new(), - metadata_manager, - manual_update_sender, - }; + forced_targets: Arc::new(ArcSwap::new(Arc::clone(forced_targets))), + app_config: Arc::clone(app_config), + http_client: Arc::new(ArcSwap::from_pointee(client)), + http_client_no_redirect: Arc::new(ArcSwap::from_pointee(client_no_redirect)), + public_http_client_no_redirect: Arc::new(ArcSwap::from_pointee(public_client_no_redirect)), + downloads: Arc::new(DownloadQueue::new_with_state_file(Some(downloads_state_file))), + cache: Arc::new(ArcSwapOption::from(cache)), + shared_stream_manager, + hls_proxy, + hls_provisioning: Arc::new(HlsProvisioningState::new()), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens, + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + update_guard: UpdateGuard::new(), + metadata_manager, + manual_update_sender, + }; recover_persisted_downloads_state_for_startup(&app_state.downloads).await; @@ -402,12 +409,20 @@ async fn run_manual_update_worker( Arc::clone(&app_state.app_config), request.targets, Some(Arc::clone(&app_state.event_manager)), - Some(Arc::clone(&app_state)), + Some({ + let state = Arc::clone(&app_state); + std::sync::Arc::new(move || { + let state = Arc::clone(&state); + Box::pin(async move { sync_panel_api_exp_dates(&state).await }) + as std::pin::Pin + Send>> + }) + }), Some(Arc::clone(&app_state.playlists)), Some(app_state.update_guard.clone()), app_state.get_disabled_headers(), Some(Arc::clone(&app_state.active_provider)), - Some(Arc::clone(&app_state.metadata_manager)), + Some(Arc::clone(&app_state.metadata_manager) + as std::sync::Arc), None, Some(permit), ) @@ -428,9 +443,7 @@ async fn cancel_all_service_tokens(app_state: &Arc) { // Use the manager's shutdown() rather than cancelling the token directly so // the is_shutdown flag is set and workers do not attempt to restart after cancellation. app_state.metadata_manager.shutdown(); - if tokio::time::timeout(std::time::Duration::from_secs(30), app_state.connection_manager.shutdown()) - .await - .is_err() + if tokio::time::timeout(std::time::Duration::from_secs(30), app_state.connection_manager.shutdown()).await.is_err() { warn!("Connection manager shutdown timed out after 30s, forcing exit"); } @@ -438,7 +451,7 @@ async fn cancel_all_service_tokens(app_state: &Arc) { #[cfg(unix)] async fn wait_for_shutdown_signal() -> Result<&'static str, std::io::Error> { - use tokio::signal::unix::{SignalKind, signal}; + use tokio::signal::unix::{signal, SignalKind}; let mut terminate = signal(SignalKind::terminate())?; let mut quit = signal(SignalKind::quit())?; @@ -476,7 +489,14 @@ fn exec_update_on_boot(client: &reqwest::Client, app_state: &Arc, targ app_config_clone, targets_clone, event_manager, - Some(app_state_clone), + Some({ + let state = Arc::clone(&app_state_clone); + std::sync::Arc::new(move || { + let state = Arc::clone(&state); + Box::pin(async move { sync_panel_api_exp_dates(&state).await }) + as std::pin::Pin + Send>> + }) + }), Some(playlist_state), update_guard, disabled_headers, @@ -527,9 +547,7 @@ fn allow_response_compression( } fn create_compression_layer() -> tower_http::compression::CompressionLayer { - let predicate = DefaultPredicate::new() - .and(SizeAbove::new(1024)) - .and(allow_response_compression); + let predicate = DefaultPredicate::new().and(SizeAbove::new(1024)).and(allow_response_compression); tower_http::compression::CompressionLayer::new() .br(false) .deflate(true) @@ -622,7 +640,7 @@ pub async fn start_server(app_config: Arc, targets: Arc, targets: Arc, targets: Arc, targets: &Arc Result<&'static str, std::io::Error> { mod tests { use super::{corrupt_downloads_state_path, recover_persisted_downloads_state}; use crate::api::model::DownloadQueue; - use std::{path::PathBuf, time::{SystemTime, UNIX_EPOCH}}; + use std::{ + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, + }; fn temp_state_file(name: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("time") - .as_nanos(); + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).expect("time").as_nanos(); std::env::temp_dir().join(format!("tuliprox_{name}_{nanos}.json")) } @@ -946,10 +961,8 @@ mod tests { let state_file = temp_state_file("corrupt_downloads_state"); std::fs::write(&state_file, "{ not valid json").expect("write corrupt state"); let corrupt_dir = state_file.parent().expect("state dir").to_path_buf(); - let expected_prefix = format!( - "{}_corrupt.", - state_file.file_stem().and_then(|stem| stem.to_str()).expect("state stem") - ); + let expected_prefix = + format!("{}_corrupt.", state_file.file_stem().and_then(|stem| stem.to_str()).expect("state stem")); let expected_extension = state_file.extension().and_then(|ext| ext.to_str()).expect("state ext"); let downloads = DownloadQueue::new_with_state_file(Some(state_file.clone())); @@ -986,7 +999,8 @@ mod tests { MetadataUpdateManager, PlaylistStorageState, ProviderHandle, SharedStreamManager, UpdateGuard, }, model::{AppConfig, Config, ConfigInput, MediaToolCapabilities, ProcessTargets, SourcesConfig}, - utils::{FileLockManager, GeoIp}, + repository::GeoIp, + utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::response::IntoResponse; diff --git a/backend/src/api/mod.rs b/backend/app/src/api/mod.rs similarity index 92% rename from backend/src/api/mod.rs rename to backend/app/src/api/mod.rs index 402fcf533..94d427d15 100644 --- a/backend/src/api/mod.rs +++ b/backend/app/src/api/mod.rs @@ -1,4 +1,5 @@ pub mod api_utils; +pub mod auth_middleware; mod http_layers; pub mod model; mod panel_api; @@ -9,6 +10,6 @@ mod endpoints; pub(crate) mod hdhomerun_proprietary; pub mod main_api; mod serve; +pub(crate) mod static_headers; mod sys_usage; mod tasks; -pub(crate) mod static_headers; diff --git a/backend/src/api/model/app_state.rs b/backend/app/src/api/model/app_state.rs similarity index 90% rename from backend/src/api/model/app_state.rs rename to backend/app/src/api/model/app_state.rs index f168e7c0f..bb85a31fd 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/app/src/api/model/app_state.rs @@ -2,11 +2,9 @@ use crate::{ api::{ endpoints::download_api::{resume_download_worker_if_needed, spawn_download_services}, model::{ - active_user_manager::ConnectionAdmission, metadata_update_manager::MetadataUpdateManager, - provider_dns_manager::exec_provider_dns, qos_aggregation_manager::exec_qos_aggregation, - recording_rule_scheduler::spawn_recording_rule_scheduler, ActiveProviderManager, ActiveUserManager, - ConnectionManager, DownloadQueue, EventManager, HlsProvisioningState, HlsProxyManager, PlaylistStorage, - PlaylistStorageState, SharedStreamManager, UpdateGuard, + load_target_into_memory_cache, recording_rule_scheduler::spawn_recording_rule_scheduler, + ActiveProviderManager, ActiveUserManager, ConnectionManager, DownloadQueue, EventManager, + HlsProvisioningState, PlaylistStorage, PlaylistStorageState, SharedStreamManager, UpdateGuard, }, tasks::{exec_config_watch, exec_scheduler}, }, @@ -14,11 +12,11 @@ use crate::{ AppConfig, Config, ConfigProvider, ConfigTarget, GracePeriodOptions, HdHomeRunConfig, HdHomeRunDeviceConfig, ProcessTargets, ReverseProxyDisabledHeaderConfig, ScheduleConfig, SourcesConfig, }, - repository::{get_geoip_path, load_target_into_memory_cache}, + repository::{get_geoip_path, GeoIp}, utils::{ reload_logger, request::{create_client, create_client_with_redirect, PublicIpResolver}, - GeoIp, LRUResourceCache, + LRUResourceCache, }, }; use arc_swap::{ArcSwap, ArcSwapOption}; @@ -32,7 +30,6 @@ use shared::{ }; use std::{ collections::HashMap, - ffi::OsStr, sync::{atomic::AtomicI8, Arc}, time::Duration, }; @@ -41,7 +38,9 @@ use tokio::{ task, }; use tokio_util::sync::CancellationToken; -use url::Url; +use tuliprox_hls::api::HlsProxyManager; +use tuliprox_metadata::manager::MetadataUpdateManager; +use tuliprox_session::{provider_dns_manager::exec_provider_dns, qos_aggregation_manager::exec_qos_aggregation}; macro_rules! cancel_service { ($field: ident, $flag:expr, $changes:expr, $cancel_tokens:expr) => { @@ -117,7 +116,7 @@ async fn update_target_caches(app_state: &Arc, target_changes: Option< match target.cache_status { TargetCacheState::UnchangedFalse | TargetCacheState::UnchangedTrue => {} // skip this TargetCacheState::ChangedToTrue => { - load_target_into_memory_cache(app_state, &target.target).await; + load_target_into_memory_cache(&app_state.app_config, &app_state.playlists, &target.target).await; } TargetCacheState::ChangedToFalse => { to_remove.push(target.name.clone()); @@ -231,10 +230,10 @@ fn start_services(app_state: &Arc, changes: &UpdateChanges) { } if changes.flags.contains(UpdateChangesFlags::ProviderDns) { - exec_provider_dns(app_state, &app_state.cancel_tokens.load().provider_dns); + exec_provider_dns(&app_state.app_config, &app_state.cancel_tokens.load().provider_dns); } if changes.flags.contains(UpdateChangesFlags::QosAggregation) { - exec_qos_aggregation(app_state, &app_state.cancel_tokens.load().qos_aggregation); + exec_qos_aggregation(&app_state.app_config, &app_state.cancel_tokens.load().qos_aggregation); let history_cfg = app_state.app_config.config.load().reverse_proxy.as_ref().and_then(|rp| rp.stream_history.clone()); let connection_manager = Arc::clone(&app_state.connection_manager); @@ -244,7 +243,7 @@ fn start_services(app_state: &Arc, changes: &UpdateChanges) { } if changes.flags.contains(UpdateChangesFlags::Downloads) { spawn_download_services(app_state, &app_state.cancel_tokens.load().downloads); - spawn_recording_rule_scheduler(app_state, &app_state.cancel_tokens.load().downloads); + spawn_recording_rule_scheduler(&app_state.recording_ctx(), &app_state.cancel_tokens.load().downloads); let config = app_state.app_config.config.load(); if let Some(download_cfg) = config.video.as_ref().and_then(|video| video.download.as_ref()).cloned() { let app_state = Arc::clone(app_state); @@ -622,39 +621,6 @@ impl AppState { self.active_users.user_connections(username).await } - pub(crate) async fn get_connection_admission( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - ) -> ConnectionAdmission { - self.active_users.connection_admission(username, max_connections, soft_connections).await - } - - pub(crate) async fn get_connection_admission_for_session( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - session_token: &str, - ) -> ConnectionAdmission { - self.active_users - .connection_admission_for_session(username, max_connections, soft_connections, session_token) - .await - } - - pub(crate) async fn get_connection_admission_for_session_activation( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - session_token: &str, - ) -> ConnectionAdmission { - self.active_users - .connection_admission_for_session_activation(username, max_connections, soft_connections, session_token) - .await - } - pub async fn get_connection_permission( &self, username: &str, @@ -774,70 +740,9 @@ impl AppState { pub fn get_grace_options(&self) -> GracePeriodOptions { self.app_config.get_grace_options() } - pub fn should_use_manual_redirects(&self) -> bool { - let config = self.app_config.config.load(); - config.proxy.as_ref().is_some_and(|proxy| should_use_manual_redirect_for_proxy(proxy.url.as_str())) - || proxy_env_present() - } + pub fn should_use_manual_redirects(&self) -> bool { crate::model::should_use_manual_redirects(&self.app_config) } - pub fn get_encrypt_secret(&self) -> [u8; 16] { - self.app_config.get_reverse_proxy_rewrite_secret().unwrap_or(self.app_config.encrypt_secret) - } -} - -fn proxy_env_present() -> bool { should_use_manual_redirects_for_env_vars(std::env::vars_os()) } - -fn parse_proxy_url_with_http_fallback(proxy_url: &str) -> Option { - let trimmed = proxy_url.trim(); - if trimmed.is_empty() { - return None; - } - - if let Ok(url) = Url::parse(trimmed) { - if matches!(url.scheme().to_ascii_lowercase().as_str(), "http" | "https") { - return Some(url); - } - if trimmed.contains("://") { - return None; - } - } - - if trimmed.contains("://") { - return None; - } - if trimmed.starts_with('/') || trimmed.starts_with('\\') { - return None; - } - - Url::parse(format!("http://{trimmed}").as_str()).ok() -} - -fn should_use_manual_redirect_for_proxy(proxy_url: &str) -> bool { - parse_proxy_url_with_http_fallback(proxy_url).is_some_and(|url| { - matches!(url.scheme().to_ascii_lowercase().as_str(), "http" | "https") && url.host_str().is_some() - }) -} - -fn should_use_manual_redirects_for_env_vars(vars: I) -> bool -where - I: IntoIterator, - K: AsRef, - V: AsRef, -{ - const ENV_KEYS: [&str; 3] = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; - - vars.into_iter().any(|(key, value)| { - let Some(key) = key.as_ref().to_str() else { - return false; - }; - let Some(value) = value.as_ref().to_str() else { - return false; - }; - let value = value.trim(); - ENV_KEYS.iter().any(|candidate| candidate.eq_ignore_ascii_case(key)) - && !value.is_empty() - && should_use_manual_redirect_for_proxy(value) - }) + pub fn get_encrypt_secret(&self) -> [u8; 16] { self.app_config.get_encrypt_secret() } } fn schedules_changed(a: &[ScheduleConfig], b: &[ScheduleConfig]) -> bool { @@ -932,11 +837,11 @@ pub struct HdHomerunAppState { #[cfg(test)] mod tests { - use super::{ - qos_aggregation_changed, schedules_changed, should_use_manual_redirect_for_proxy, - should_use_manual_redirects_for_env_vars, video_download_changed, + use super::{qos_aggregation_changed, schedules_changed, video_download_changed}; + use crate::model::{ + should_use_manual_redirect_for_proxy, should_use_manual_redirects_for_env_vars, Config, ScheduleConfig, + VideoDownloadConfig, }; - use crate::model::{Config, ScheduleConfig, VideoDownloadConfig}; use shared::model::{ QosAggregationConfigDto, ReverseProxyConfigDto, ScheduleTaskType, StreamHistoryConfigDto, WebAuthConfigDto, WebUiConfigDto, diff --git a/backend/app/src/api/model/app_state_view.rs b/backend/app/src/api/model/app_state_view.rs new file mode 100644 index 000000000..14ec592df --- /dev/null +++ b/backend/app/src/api/model/app_state_view.rs @@ -0,0 +1,99 @@ +//! Views onto [`AppState`]: the subsets each subsystem actually reads. +//! +//! Every extracted subsystem takes a small context struct - the handles it +//! needs - rather than the server's root state. Those structs live in their own +//! crates and cannot name `AppState`, so something has to build them, and this +//! is it. +//! +//! # Why the trait is defined here +//! +//! The obvious shape - a `FromAppState` trait in `tuliprox-core`, implemented by +//! each context beside its own definition - does not compile. The orphan rule +//! allows `impl ForeignTrait for LocalType` or `impl LocalTrait for ForeignType`, +//! and that arrangement is neither: the trait would be core's, the context would +//! be (say) `tuliprox-hls`'s, and the impl would have to live in this binary, +//! which owns neither. It also could not be written in `tuliprox-hls`, because +//! nothing there can name `AppState`. +//! +//! So the trait is local to the binary and the contexts are the foreign types. +//! That is the one arrangement the orphan rule permits, and it puts every view +//! in one file - which is the property worth having anyway: "what does this +//! subsystem read from the server?" is answerable without opening its crate. +//! +//! # What is not here +//! +//! `WeakHlsCtx` is derived from `HlsCtx`, not from `AppState`. `SharedStreamCtx` +//! borrows rather than owning its handles, so it cannot be built by value. +//! Neither is a view in this sense. + +use super::AppState; + +/// A subset of [`AppState`] that one subsystem reads. +pub trait AppStateView { + /// Build the view by cloning the handles it names. + fn from_app_state(state: &AppState) -> Self; +} + +impl AppState { + /// The subset that `V` reads. + /// + /// Prefer the named accessors below at call sites; this is what they call, + /// and what generic code can use. + pub fn view(&self) -> V { V::from_app_state(self) } +} + +/// Declare a view: the accessor name, the context type, and the `AppState` +/// fields it clones. Field names are the same on both sides by construction - +/// a view renaming a handle would be a view lying about what it reads. +macro_rules! app_state_views { + ($( + $(#[$meta:meta])* + $accessor:ident => $ctx:ty { $($field:ident),+ $(,)? } + )+) => { + $( + impl AppStateView for $ctx { + fn from_app_state(state: &AppState) -> Self { + Self { $($field: ::core::clone::Clone::clone(&state.$field)),+ } + } + } + + impl AppState { + $(#[$meta])* + #[must_use] + pub fn $accessor(&self) -> $ctx { + self.view() + } + } + )+ + }; +} + +app_state_views! { + /// The handles the DVR needs: the recording queue and what feeds it. + recording_ctx => crate::api::model::recording::recording_ctx::RecordingCtx { + app_config, downloads, event_manager, http_client, + } + + /// The handles the HLS proxy needs: itself, plus provider allocation and + /// session accounting. + hls_ctx => crate::api::model::hls_cache::HlsCtx { + app_config, hls_proxy, active_provider, connection_manager, active_users, + } + + /// The handles admission reads: it decides over connections and users. + admission_ctx => tuliprox_session::admission::AdmissionCtx { + app_config, active_users, connection_manager, + } + + /// The handles the provider side of a stream needs: the redirect-aware + /// HTTP clients. + provider_stream_ctx => tuliprox_session::stream_ctx::ProviderStreamCtx { + app_config, connection_manager, http_client_no_redirect, public_http_client_no_redirect, + } + + /// The handles the background metadata worker reads. + metadata_update_ctx => tuliprox_metadata::ctx::MetadataUpdateCtx { + app_config, active_provider, connection_manager, event_manager, playlists, update_guard, + http_client, http_client_no_redirect, + } +} diff --git a/backend/src/api/model/hls_provisioning.rs b/backend/app/src/api/model/hls_provisioning.rs similarity index 87% rename from backend/src/api/model/hls_provisioning.rs rename to backend/app/src/api/model/hls_provisioning.rs index 6821c5ef9..f955a32cb 100644 --- a/backend/src/api/model/hls_provisioning.rs +++ b/backend/app/src/api/model/hls_provisioning.rs @@ -1,10 +1,6 @@ use crate::{ api::{ - model::{ - build_hls_standalone_custom_plan, is_custom_video_stream_enabled, AppState, CustomVideoStreamType, - HlsAccessLease, HlsAccessLeaseId, HlsRuntimeCustomTailReason, HlsStandaloneCustomAccess, - ProxySessionId, TransportStreamBuffer, - }, + model::{is_custom_video_stream_enabled, AppState, CustomVideoStreamType, TransportStreamBuffer}, panel_api::{can_provision_on_exhausted, try_provision_account_on_exhausted}, }, model::{ConfigInput, CustomStreamResponse, ProxyUserCredentials}, @@ -16,11 +12,12 @@ use axum::{ }; use dashmap::DashMap; use log::{debug, error}; -use shared::{ - utils::{sanitize_sensitive_info}, - defaults::{CUSTOM_VIDEO_PREFIX} -}; +use shared::{defaults::CUSTOM_VIDEO_PREFIX, utils::sanitize_sensitive_info}; use std::sync::Arc; +use tuliprox_hls::api::{ + build_hls_standalone_custom_plan, HlsAccessLease, HlsAccessLeaseId, HlsRuntimeCustomTailReason, + HlsStandaloneCustomAccess, ProxySessionId, +}; const PROVISIONING_HLS_TARGET_DURATION_SECS: u64 = 2; const PROVISIONING_HLS_EXTINF: &str = "2.000000"; @@ -121,10 +118,8 @@ impl HlsProvisioningState { return 0; } if let Some(mut group) = self.jobs.get_mut(&input_name) { - let counted_jobs = group - .running_jobs - .saturating_add(group.ready_slots) - .saturating_add(group.recent_failed_jobs); + let counted_jobs = + group.running_jobs.saturating_add(group.ready_slots).saturating_add(group.recent_failed_jobs); if counted_jobs >= desired_jobs { return 0; } @@ -232,8 +227,7 @@ impl HlsProvisioningState { } pub(in crate::api) fn clear_consumer(&self, input_name: &Arc, virtual_id: u32) { - self.consumers - .remove(&HlsProvisioningConsumerKey::new(Arc::clone(input_name), virtual_id)); + self.consumers.remove(&HlsProvisioningConsumerKey::new(Arc::clone(input_name), virtual_id)); } pub(in crate::api) fn mark_handoff_once( @@ -290,25 +284,18 @@ impl HlsProvisioningState { group.last_failed_at_ms = None; } } - let latest_reference_ms = [ - Some(group.last_started_at_ms), - group.last_ready_at_ms, - group.last_failed_at_ms, - ] - .into_iter() - .flatten() - .max() - .unwrap_or_default(); + let latest_reference_ms = [Some(group.last_started_at_ms), group.last_ready_at_ms, group.last_failed_at_ms] + .into_iter() + .flatten() + .max() + .unwrap_or_default(); group.running_jobs > 0 || group.ready_slots > 0 || group.recent_failed_jobs > 0 || latest_reference_ms.saturating_add(HLS_PROVISIONING_STALE_MARKER_MS) >= now_ms }); self.consumers.retain(|_, consumer| { - consumer - .last_seen_at_ms - .max(consumer.created_at_ms) - .saturating_add(HLS_PROVISIONING_STALE_MARKER_MS) + consumer.last_seen_at_ms.max(consumer.created_at_ms).saturating_add(HLS_PROVISIONING_STALE_MARKER_MS) >= now_ms }); self.handoffs.retain(|_, marked_at_ms| { @@ -339,10 +326,7 @@ pub(crate) fn hls_custom_video_type_configured(app_state: &Arc, video_ return false; } let custom_stream_response = app_state.app_config.custom_stream_response.load(); - custom_stream_response - .as_ref() - .and_then(|response| custom_video_asset(response, video_type)) - .is_some() + custom_stream_response.as_ref().and_then(|response| custom_video_asset(response, video_type)).is_some() } fn custom_video_asset( @@ -387,9 +371,7 @@ pub(crate) fn hls_panel_provisioning_manifest_path(user: &ProxyUserCredentials, ) } -pub(crate) fn hls_provisioning_discontinuity_sequence(_now_ms: u64) -> u64 { - 0 -} +pub(crate) fn hls_provisioning_discontinuity_sequence(_now_ms: u64) -> u64 { 0 } fn build_hls_panel_provisioning_manifest_body(mut segment_url: impl FnMut(usize) -> String) -> String { let media_sequence = 0; @@ -421,9 +403,7 @@ pub(crate) fn build_hls_custom_video_manifest_body( return None; } - Some(build_hls_panel_provisioning_manifest_body(|index| { - hls_panel_provisioning_segment_url(base_url, user, index) - })) + Some(build_hls_panel_provisioning_manifest_body(|index| hls_panel_provisioning_segment_url(base_url, user, index))) } pub(crate) async fn hls_custom_video_manifest_response_with_virtual_id( @@ -433,14 +413,7 @@ pub(crate) async fn hls_custom_video_manifest_response_with_virtual_id( fallback_status: StatusCode, _virtual_id: Option, ) -> axum::response::Response { - hls_custom_video_manifest_response_with_access( - app_state, - user, - video_type, - fallback_status, - None, - ) - .await + hls_custom_video_manifest_response_with_access(app_state, user, video_type, fallback_status, None).await } pub(crate) async fn hls_custom_video_manifest_response_for_access_lease( @@ -457,14 +430,7 @@ pub(crate) async fn hls_custom_video_manifest_response_for_access_lease( lease.issued_at_ms, lease.valid_until_ms, ); - hls_custom_video_manifest_response_with_access( - app_state, - user, - video_type, - fallback_status, - Some(access), - ) - .await + hls_custom_video_manifest_response_with_access(app_state, user, video_type, fallback_status, Some(access)).await } async fn hls_custom_video_manifest_response_with_access( @@ -483,16 +449,15 @@ async fn hls_custom_video_manifest_response_with_access( let base_url = server_info.get_base_url(); let manifest = if video_type == CustomVideoStreamType::Provisioning { - build_hls_panel_provisioning_manifest_body(|index| { - hls_panel_provisioning_segment_url(&base_url, user, index) - }) + build_hls_panel_provisioning_manifest_body(|index| hls_panel_provisioning_segment_url(&base_url, user, index)) } else { let Some(reason) = HlsRuntimeCustomTailReason::from_video_type(video_type) else { return fallback_status.into_response(); }; let access = access.unwrap_or_else(|| HlsStandaloneCustomAccess::for_user(user.username.clone())); let Ok(plan) = - build_hls_standalone_custom_plan(app_state, &base_url, access, reason, current_time_millis()).await + build_hls_standalone_custom_plan(&app_state.hls_ctx(), &base_url, access, reason, current_time_millis()) + .await else { return fallback_status.into_response(); }; @@ -516,17 +481,12 @@ pub(crate) async fn try_hls_panel_provisioning_manifest_response( fallback_status: StatusCode, ) -> Option { let now_ms = current_time_millis(); - app_state - .hls_provisioning - .touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + app_state.hls_provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); if provisioning_enabled { start_hls_panel_provisioning_once(app_state, input); } - let status = if let Some(status) = app_state - .hls_provisioning - .consumer_status(&input.name, virtual_id, now_ms) - { + let status = if let Some(status) = app_state.hls_provisioning.consumer_status(&input.name, virtual_id, now_ms) { status } else if provisioning_enabled { HlsProvisioningStatus::InProgress @@ -541,27 +501,24 @@ pub(crate) async fn try_hls_panel_provisioning_manifest_response( if !hls_custom_video_type_configured(app_state, CustomVideoStreamType::Provisioning) { return Some(fallback_status.into_response()); } - let server_path = - server_path.map(str::to_string).or_else(|| app_state.app_config.get_user_server_info(user).and_then(|server| server.path)); - let manifest_path = - redirect_paths.waiting_manifest_path.map_or_else(|| hls_panel_provisioning_manifest_path(user, virtual_id), str::to_string); - Some(hls_virtual_entry_redirect_response( - &manifest_path, - server_path.as_deref(), - )) + let server_path = server_path + .map(str::to_string) + .or_else(|| app_state.app_config.get_user_server_info(user).and_then(|server| server.path)); + let manifest_path = redirect_paths + .waiting_manifest_path + .map_or_else(|| hls_panel_provisioning_manifest_path(user, virtual_id), str::to_string); + Some(hls_virtual_entry_redirect_response(&manifest_path, server_path.as_deref())) } - HlsProvisioningStatus::ProviderExhausted => { - Some( - hls_custom_video_manifest_response_with_virtual_id( - app_state, - user, - CustomVideoStreamType::ProviderConnectionsExhausted, - fallback_status, - Some(virtual_id), - ) - .await, + HlsProvisioningStatus::ProviderExhausted => Some( + hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::ProviderConnectionsExhausted, + fallback_status, + Some(virtual_id), ) - } + .await, + ), } } @@ -571,9 +528,7 @@ pub(crate) fn start_hls_panel_provisioning_once(app_state: &Arc, input } let key = Arc::clone(&input.name); let now_ms = current_time_millis(); - let jobs_to_start = app_state - .hls_provisioning - .start_jobs_for_waiting_consumers(&key, now_ms); + let jobs_to_start = app_state.hls_provisioning.start_jobs_for_waiting_consumers(&key, now_ms); if jobs_to_start == 0 { return false; } @@ -592,9 +547,7 @@ pub(crate) fn start_hls_panel_provisioning_once(app_state: &Arc, input if ready { app_state.hls_provisioning.mark_job_ready(Arc::clone(&key), finished_at_ms); } else { - app_state - .hls_provisioning - .mark_job_provider_exhausted(Arc::clone(&key), finished_at_ms); + app_state.hls_provisioning.mark_job_provider_exhausted(Arc::clone(&key), finished_at_ms); } debug!( "HLS panel provisioning completed: input={} outcome={} ready={}", @@ -674,10 +627,7 @@ mod tests { state.job_status(&input, ready_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS), Some(HlsProvisioningStatus::Ready) ); - assert_eq!( - state.job_status(&input, ready_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS + 1), - None - ); + assert_eq!(state.job_status(&input, ready_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS + 1), None); } #[test] @@ -691,13 +641,7 @@ mod tests { assert!(!state.mark_handoff_once(&input, 57, Some(&proxy_session_id), Some(&access_lease_id), 1_100)); let next_access_lease_id = HlsAccessLeaseId("next-access-lease".to_string()); - assert!(state.mark_handoff_once( - &input, - 57, - Some(&proxy_session_id), - Some(&next_access_lease_id), - 1_200 - )); + assert!(state.mark_handoff_once(&input, 57, Some(&proxy_session_id), Some(&next_access_lease_id), 1_200)); } #[test] @@ -708,13 +652,7 @@ mod tests { let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); let marked_at_ms = 10_000; - assert!(state.mark_handoff_once( - &input, - 57, - Some(&proxy_session_id), - Some(&access_lease_id), - marked_at_ms - )); + assert!(state.mark_handoff_once(&input, 57, Some(&proxy_session_id), Some(&access_lease_id), marked_at_ms)); assert!(!state.mark_handoff_once( &input, 57, diff --git a/backend/app/src/api/model/mod.rs b/backend/app/src/api/model/mod.rs new file mode 100644 index 000000000..df4c3a590 --- /dev/null +++ b/backend/app/src/api/model/mod.rs @@ -0,0 +1,96 @@ +mod app_state; +mod app_state_view; +mod hls_provisioning; +mod proxy; +mod streams; + +#[cfg(test)] +pub(in crate::api) use self::hls_provisioning::{ + build_hls_custom_video_manifest_body, hls_panel_provisioning_manifest_path, +}; +pub(in crate::api) use self::hls_provisioning::{ + hls_custom_video_manifest_response_for_access_lease, hls_custom_video_manifest_response_with_virtual_id, + hls_provisioning_discontinuity_sequence, hls_virtual_entry_redirect_response, + parse_hls_panel_provisioning_segment_route_name, start_hls_panel_provisioning_once, + try_hls_panel_provisioning_manifest_response, HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, +}; +pub(crate) use self::streams::*; +pub use self::{app_state::*, app_state_view::*, hls_provisioning::HlsProvisioningState, proxy::*}; +// Provider value types moved to `model`; re-exported so `api` keeps its names. +pub use crate::model::provider::*; +// Update semaphores moved to `model`; re-exported so `api` keeps its names. +pub use crate::model::update_guard::*; +pub use crate::model::{stream_error::*, update_task::*}; +// In-memory playlist storage moved to `repository`; re-exported so `api` call +// sites keep their names. +pub use crate::repository::playlist_mem_cache::*; +// Dependency-free model types moved to `tuliprox-core`, and the provider +// response-header helpers to `tuliprox-session` beside the header types they +// operate on. Re-exported so `api` call sites keep their names. +pub use tuliprox_core::model::{batch_result_collector::*, user_api_request::*, xtream_response::*}; +// HTTP range parsing moved to `tuliprox_core::utils`; re-exported so `api` call +// sites keep their names. +pub use tuliprox_core::utils::byte_range::{resolve_single_byte_range, SingleByteRange}; +// The recording queue and the DVR moved to `tuliprox-dvr`; re-exported so `api` +// call sites keep their names, module paths included. +pub use tuliprox_dvr::{download, recording}; +pub use tuliprox_dvr::{download::*, recording::*}; +// Keep the crate alias while call sites migrate to its explicit `api` facade. +pub use tuliprox_hls as hls_cache; +// The HLS proxy symbols this crate's own tests reach through `api::model`. +// Explicit rather than a glob so the list stays a statement of what the tests +// actually need, and so pruning `tuliprox_hls::api` shows up here as a +// compile error instead of silently widening. +#[cfg(test)] +pub use tuliprox_hls::api::{ + begin_hls_origin_account_io, build_hls_origin_session_owner, build_hls_standalone_custom_plan, + build_proxy_session_id, build_transient_resource_id, commit_terminal_tail_if_lease_reserve_requires_cutover, + finish_hls_origin_account_io, hls_manifest_acceptance_directive_for_session, is_hls_provisioning_segment, + is_hls_transient_full_object_cacheable_request, scrub_hls_origin_headers, trigger_origin_refresh_sync, + CacheAccessState, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAvailabilityReevaluationRegistration, + HlsEffectiveOriginAcquirePolicy, HlsFreshManifestRequiredReason, HlsLeasePlaybackMode, HlsLifecycleEvent, + HlsLifecycleEventKey, HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, + HlsManifestCommitRequirement, HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, + HlsOriginIoContext, HlsOriginSource, HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsProxyManager, + HlsRuntimeCustomTailReason, HlsSegmentFile, HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionMode, + HlsSessionStoreOutcome, HlsStandaloneCustomAccess, HlsTerminalFailedClosedReason, HlsTerminalResolution, + HlsTerminalSegmentPath, HlsTerminalTailPlan, HlsTerminalTailProtection, LiveHlsOriginEntry, MapCacheStatus, + MapEntry, OriginMapKey, OriginRefreshRequest, OriginSegmentFetchRef, OriginSegmentKey, ProxyMapId, ProxySessionId, + RenderedManifest, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, SegmentFetchPriority, + TransientObjectCacheKey, TransientObjectCacheStatus, TransientPassthroughReason, TransientResourceId, + TransientResourceKind, TransientResourceRef, HLS_ACCESS_LEASE_ID_PLACEHOLDER, +}; +#[cfg(test)] +pub use tuliprox_hls::{ + build_terminal_tail_plan, prepare_terminal_base_evidence, prepared_terminal_bundle_key, + snapshot_terminal_media_asset, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, + HlsAvailabilityReevaluationFinishReason, HlsAvailabilityReevaluationMode, HlsBandwidthPersistenceState, + HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestAcceptanceExhaustionReason, + HlsManifestAcceptanceTrigger, HlsManifestDeliveryMode, HlsManifestSourceRenderMarker, HlsMapSignature, + HlsMediaContainer, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsOriginPathCondition, + HlsPreparedTerminalBundleState, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, HlsRecoveryWorkload, + HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, + HlsTerminalBaseSegmentAvailability, HlsTerminalMediaAsset, HlsTerminalMediaPreparationState, + HlsTerminalTailBuildInput, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, HlsTransitionMarginMs, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, +}; +// Background metadata resolution moved to `tuliprox-metadata`; re-exported so +// `api` call sites keep their names. +pub use tuliprox_metadata::{ctx::MetadataUpdateCtx, manager::*}; +// Background workers moved to the layers they read: provider DNS and QoS +// aggregation to `tuliprox-session`, the playlist cache loader to +// `tuliprox-repository`. Re-exported so `api` call sites keep their names. +pub use tuliprox_repository::playlist_cache_loader::*; +// Provider allocation and the streaming-session runtime moved to +// `tuliprox-session`; re-exported so `api` call sites keep their names, module +// paths included. +pub use tuliprox_session::{ + active_provider_manager, active_user_manager, admission_strategy, connection_manager, event_manager, meter, + provider_lineup_manager, stream, +}; +pub use tuliprox_session::{ + active_provider_manager::*, active_user_manager::*, admission_strategy::*, connection_manager::*, event_manager::*, + meter::*, provider_dns_manager::*, provider_lineup_manager::*, qos_aggregation_manager::*, response_headers::*, + stream::*, streams::*, +}; diff --git a/backend/src/api/model/proxy/mod.rs b/backend/app/src/api/model/proxy/mod.rs similarity index 74% rename from backend/src/api/model/proxy/mod.rs rename to backend/app/src/api/model/proxy/mod.rs index cae1a98e0..bb3b3139a 100644 --- a/backend/src/api/model/proxy/mod.rs +++ b/backend/app/src/api/model/proxy/mod.rs @@ -4,4 +4,5 @@ //! `header_policy` is the canonical home for hop-by-hop header rules used by both //! the HLS cache proxy and the MPEG-TS reverse proxy — see its module docs. -pub mod header_policy; +// Hop-by-hop header policy moved to `tuliprox-hls`, its only consumer. +pub use tuliprox_hls::header_policy; diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/app/src/api/model/streams/active_client_stream.rs similarity index 97% rename from backend/src/api/model/streams/active_client_stream.rs rename to backend/app/src/api/model/streams/active_client_stream.rs index e24caddb6..ba95acb1e 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/app/src/api/model/streams/active_client_stream.rs @@ -1,6 +1,5 @@ use crate::{ api::{ - api_utils::get_stream_options, model::{ connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, create_provider_stream, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, CleanupEvent, @@ -33,6 +32,7 @@ use std::{ }; use tokio::sync::Notify; use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use tuliprox_session::stream_options::get_stream_options; const DIRECT_BODY_IDLE_TIMEOUT_SECS: u64 = 90; const BODY_IDLE_TIMEOUT_ERROR_CLASS: &str = "body_idle_timeout"; @@ -135,7 +135,7 @@ struct GracePeriodParams { grace_active_version: Option, // Set when the stream was admitted via a user-grace strategy. // On user-grace failure, remaining strategies are evaluated before final deny. - grace_resolution_context: Option, + grace_resolution_context: Option, // The `ConnectionKind` from the original admission decision. // Preserved in `GraceResolutionContext.kind` and also passed directly here // so the grace task can use the runtime value when evaluating remaining strategies. @@ -262,8 +262,15 @@ impl ActiveClientStreamState { stream }; if let Some(ctx) = self.timed_stream_context.as_ref() { - TimedClientStream::new(&ctx.app_state, stream, ctx.duration_secs, self.fingerprint.addr, ctx.virtual_id) - .boxed() + TimedClientStream::new( + &ctx.app_state.app_config, + &ctx.app_state.connection_manager, + stream, + ctx.duration_secs, + self.fingerprint.addr, + ctx.virtual_id, + ) + .boxed() } else { stream } @@ -531,7 +538,15 @@ fn wrap_timed_client_stream_if_needed( Some(mins) => { let secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); if secs > 0 { - TimedClientStream::new(app_state, stream, secs, addr, virtual_id).boxed() + TimedClientStream::new( + &app_state.app_config, + &app_state.connection_manager, + stream, + secs, + addr, + virtual_id, + ) + .boxed() } else { stream } @@ -554,7 +569,7 @@ fn create_deferred_provider_open_future( let request_url = stream_details.request_url.as_deref()?; let input = find_input_by_provider_name(app_state.as_ref(), provider_name)?; let stream_url = url::Url::parse(request_url).ok()?; - let stream_options = get_stream_options(app_state); + let stream_options = get_stream_options(&app_state.app_config); let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); let disabled_headers = app_state.get_disabled_headers(); let mut provider_stream_factory_options = @@ -647,17 +662,13 @@ impl Stream for ActiveClientStream { }; let future = Box::pin(async move { match create_provider_stream( - &app_state, + &app_state.provider_stream_ctx(), &client, context.provider_stream_factory_options, ) .await { - Some(response) - if matches!( - response.info, - Some((_, _, _, Some(_))) - ) => { + Some(response) if matches!(response.info, Some((_, _, _, Some(_)))) => { let Some((_headers, _status, _response_url, Some(custom_video_type))) = response.info else { @@ -1150,28 +1161,29 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op // User-grace failed. Evaluate remaining strategies before final deny. if let Some(ref ctx) = grace_resolution_context { let eviction_guard = if socket_bound { - crate::api::api_utils::EvictionReentryGuard::SocketPlayback { virtual_id } + tuliprox_session::admission::EvictionReentryGuard::SocketPlayback { virtual_id } } else { - crate::api::api_utils::EvictionReentryGuard::Session( + tuliprox_session::admission::EvictionReentryGuard::Session( // Defensive fallback: an empty token will not match any real session. session_token.as_deref().unwrap_or_default(), ) }; - let remaining_result = crate::api::api_utils::evaluate_remaining_strategies_after_grace( - &app_state, - &username, - max_connections, - user.soft_connections, - &fingerprint.client_ip, - &fingerprint.addr, - true, - session_token.as_deref(), - true, - eviction_guard, - ctx, - grace_kind, - ) - .await; + let remaining_result = + tuliprox_session::admission::evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + &username, + max_connections, + user.soft_connections, + &fingerprint.client_ip, + &fingerprint.addr, + true, + session_token.as_deref(), + true, + eviction_guard, + ctx, + grace_kind, + ) + .await; match remaining_result.admission.permission { shared::model::UserConnectionPermission::Allowed | shared::model::UserConnectionPermission::GracePeriod => { @@ -1347,30 +1359,27 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op #[cfg(test)] mod tests { - use super::super::buffered_stream::BufferedStream; use super::{ - create_active_client_stream, create_deferred_provider_open_future, should_use_direct_body_idle_timeout, - stream_grace_period, ActiveClientStream, ActiveClientStreamParams, ActiveClientStreamState, CustomVideoBuffers, - DeferredProviderOpenOutcome, DeferredProviderOpenState, DirectBodyIdleTimeout, GracePeriodParams, StreamMode, - TimedStreamContext, DIRECT_BODY_IDLE_TIMEOUT_SECS, + super::buffered_stream::BufferedStream, create_active_client_stream, create_deferred_provider_open_future, + should_use_direct_body_idle_timeout, stream_grace_period, ActiveClientStream, ActiveClientStreamParams, + ActiveClientStreamState, CustomVideoBuffers, DeferredProviderOpenOutcome, DeferredProviderOpenState, + DirectBodyIdleTimeout, GracePeriodParams, StreamMode, TimedStreamContext, DIRECT_BODY_IDLE_TIMEOUT_SECS, }; use crate::{ - api::{ - api_utils::GraceResolutionContext, - model::{ - connection_manager::PROVIDER_END_NOT_SET, ActiveProviderManager, ActiveUserManager, AppState, - BoxedProviderStream, CancelTokens, ConnectionManager, CreateUserSessionParams, CustomVideoStreamType, - DownloadQueue, EventManager, MetadataUpdateManager, PlaylistStorageState, - ProviderContentRepresentationMode, ProviderHandle, SharedStreamManager, StreamDetails, StreamError, - UpdateGuard, - }, + api::model::{ + connection_manager::PROVIDER_END_NOT_SET, ActiveProviderManager, ActiveUserManager, AppState, + BoxedProviderStream, CancelTokens, ConnectionManager, CreateUserSessionParams, CustomVideoStreamType, + DownloadQueue, EventManager, GraceResolutionContext, MetadataUpdateManager, PlaylistStorageState, + ProviderContentRepresentationMode, ProviderHandle, SharedStreamManager, StreamDetails, StreamError, + UpdateGuard, }, auth::Fingerprint, model::{ AppConfig, Config, ConfigInput, GracePeriodOptions, MediaToolCapabilities, ProcessTargets, ProxyUserCredentials, SourcesConfig, StreamConfig, }, - utils::{FileLockManager, GeoIp}, + repository::GeoIp, + utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::{body::Body, http::HeaderMap}; diff --git a/backend/src/api/model/streams/client_stream.rs b/backend/app/src/api/model/streams/client_stream.rs similarity index 100% rename from backend/src/api/model/streams/client_stream.rs rename to backend/app/src/api/model/streams/client_stream.rs diff --git a/backend/src/api/model/streams/custom_video_stream.rs b/backend/app/src/api/model/streams/custom_video_stream.rs similarity index 100% rename from backend/src/api/model/streams/custom_video_stream.rs rename to backend/app/src/api/model/streams/custom_video_stream.rs diff --git a/backend/app/src/api/model/streams/metering_stream.rs b/backend/app/src/api/model/streams/metering_stream.rs new file mode 100644 index 000000000..ea3d6c387 --- /dev/null +++ b/backend/app/src/api/model/streams/metering_stream.rs @@ -0,0 +1,97 @@ +use crate::api::model::{BoxedProviderStream, EventManager, StreamError, StreamMeterHandle}; +use bytes::Bytes; +use futures::{ + stream::Stream, + task::{Context, Poll}, +}; +use std::{pin::Pin, sync::Arc}; + +/// Thin stream wrapper that only counts bytes in a shared meter handle. +pub struct MeteringStream { + inner: BoxedProviderStream, + meter: Arc, + event_manager: Arc, +} + +impl MeteringStream { + pub fn new(inner: BoxedProviderStream, meter: Arc, event_manager: Arc) -> Self { + meter.mark_attached(); + Self { inner, meter, event_manager } + } +} + +impl Stream for MeteringStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + match Pin::new(&mut this.inner).poll_next(cx) { + Poll::Ready(Some(Ok(bytes))) => { + this.meter.record_bytes(bytes.len() as u64); + Poll::Ready(Some(Ok(bytes))) + } + other => other, + } + } +} + +impl Drop for MeteringStream { + fn drop(&mut self) { + let event_manager = Arc::clone(&self.event_manager); + let meter_uid = self.meter.meter_uid(); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { + event_manager.flush_and_unregister_meter(meter_uid).await; + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::MeteringStream; + use crate::api::model::{BoxedProviderStream, EventManager, MeterReading, StreamError, StreamMeterHandle}; + use bytes::Bytes; + use futures::{stream, StreamExt}; + use std::sync::Arc; + use tokio::{ + task::yield_now, + time::{advance, Duration}, + }; + + fn boxed_stream(chunks: usize) -> BoxedProviderStream { + stream::iter((0..chunks).map(|_| Ok::(Bytes::from_static(b"abc")))).boxed() + } + + #[tokio::test] + async fn metering_stream_counts_bytes_without_snapshot_logic() { + let event_manager = Arc::new(EventManager::new()); + let meter = Arc::new(StreamMeterHandle::new(11, Arc::downgrade(&event_manager))); + let mut stream = MeteringStream::new(boxed_stream(4), Arc::clone(&meter), event_manager); + + while stream.next().await.is_some() {} + + let reading = meter.snapshot(); + assert_eq!(reading, MeterReading { meter_uid: 11, bytes_total: 12, bytes_window: 12 }); + } + + #[tokio::test(start_paused = true)] + async fn unattached_meter_handle_unregisters_on_drop() { + let event_manager = Arc::new(EventManager::new()); + let meter = Arc::new(StreamMeterHandle::new(17, Arc::downgrade(&event_manager))); + event_manager.register_meter(Arc::clone(&meter)).await; + event_manager.register_meter_client(71, 17).await; + event_manager.stream_meter_subscriber_connected(); + let mut meter_events = event_manager.get_meter_channel(); + meter.record_bytes(1024); + drop(meter); + yield_now().await; + let entries = meter_events.recv().await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].meter_uid, 17); + assert_eq!(entries[0].uids, vec![71]); + assert_eq!(entries[0].total_kb, 1); + advance(Duration::from_secs(3)).await; + assert!(meter_events.try_recv().is_err(), "meter must be unregistered after final flush"); + } +} diff --git a/backend/app/src/api/model/streams/mod.rs b/backend/app/src/api/model/streams/mod.rs new file mode 100644 index 000000000..e7d52e732 --- /dev/null +++ b/backend/app/src/api/model/streams/mod.rs @@ -0,0 +1,48 @@ +//! The streaming layer. +//! +//! This stays in `api` rather than joining `tuliprox-session`, and not because +//! of `AppState`. Two independent things pin it here: +//! +//! - `active_client_stream` and `provider_stream` call panel provisioning +//! (`can_provision_on_exhausted`, `run_panel_api_provisioning_probe`, +//! `find_input_by_provider_name`), which takes the whole root state and does +//! HTTP against an external panel. +//! - `provider_stream_factory` logs origin content-coding through +//! `tuliprox-hls`, and `tuliprox-hls` already depends on `tuliprox-session`, +//! so moving it down would close a cycle. +//! +//! Together those say something real: the streaming layer is where `session`, +//! `hls`, `iptv` and panel provisioning are composed. That is what `api` is. +//! The provider half no longer names `AppState` - it takes a +//! `ProviderStreamCtx` - but not naming the root state is not the same as being +//! able to leave. + +mod client_stream; +mod custom_video_stream; +mod provisioning_stream; +mod timed_client_stream; +// mod chunked_buffer; +mod active_client_stream; +mod metering_stream; +mod provider_stream; +mod provider_stream_factory; +mod throttled_stream; + +// Moved to `utils::network::persist_pipe`; re-exported so api call sites keep +// their existing names. +pub(in crate::api) use self::{ + active_client_stream::*, custom_video_stream::*, metering_stream::*, provider_stream::*, + provider_stream_factory::*, provisioning_stream::*, throttled_stream::*, timed_client_stream::*, +}; +pub(crate) use crate::mpegts::transport_stream_buffer::*; +pub use crate::utils::network::persist_pipe::*; +// Defined by the HTTP client that implements the timeout; re-exported here +// because the streaming layer applies the same default. +pub use crate::utils::network::request::STREAM_IDLE_TIMEOUT; +// Moved to `shared::model`: it is vocabulary shared between the code that +// decides a substitution is needed and the code that performs it. Re-exported +// so api call sites keep their existing names. +pub(crate) use shared::model::CustomVideoStreamType; +// Shared-stream fan-out and its buffer moved to `tuliprox-session`; +// re-exported so api call sites keep their names, module paths included. +pub(crate) use tuliprox_session::streams::buffered_stream; diff --git a/backend/src/api/model/streams/provider_stream.rs b/backend/app/src/api/model/streams/provider_stream.rs similarity index 86% rename from backend/src/api/model/streams/provider_stream.rs rename to backend/app/src/api/model/streams/provider_stream.rs index 1517c2bff..0beed4d70 100644 --- a/backend/src/api/model/streams/provider_stream.rs +++ b/backend/app/src/api/model/streams/provider_stream.rs @@ -1,82 +1,21 @@ use crate::{ - api::{ - api_utils::{mark_response_as_uncompressed, try_unwrap_body, HeaderFilter}, - model::{ - stream::{BoxedProviderStream, ProviderStreamResponse}, - AppState, CleanupEvent, CustomVideoStream, ProvisioningStream, ThrottledStream, TimedClientStream, - TransportStreamBuffer, - }, + api::model::{ + stream::{BoxedProviderStream, ProviderStreamResponse}, + CleanupEvent, CustomVideoStream, ProvisioningStream, ThrottledStream, TimedClientStream, TransportStreamBuffer, }, model::AppConfig, }; use axum::response::IntoResponse; use log::trace; use reqwest::StatusCode; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use shared::{error::TuliproxError, model::PlaylistItemType}; -use std::{fmt, net::SocketAddr, str::FromStr, sync::Arc}; +use shared::model::{CustomVideoStreamType, PlaylistItemType}; +use std::net::SocketAddr; use tokio_util::sync::CancellationToken; - -#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] -pub enum CustomVideoStreamType { - ChannelUnavailable, - UserConnectionsExhausted, - ProviderConnectionsExhausted, - LowPriorityPreempted, - UserAccountExpired, - Provisioning, - HlsSessionOrLeaseExpired, -} - -impl fmt::Display for CustomVideoStreamType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - CustomVideoStreamType::ChannelUnavailable => "channel_unavailable", - CustomVideoStreamType::UserConnectionsExhausted => "user_connections_exhausted", - CustomVideoStreamType::ProviderConnectionsExhausted => "provider_connections_exhausted", - CustomVideoStreamType::LowPriorityPreempted => "low_priority_preempted", - CustomVideoStreamType::UserAccountExpired => "user_account_expired", - CustomVideoStreamType::Provisioning => "provisioning", - CustomVideoStreamType::HlsSessionOrLeaseExpired => "hls_session_or_lease_expired", - }; - write!(f, "{s}") - } -} - -impl FromStr for CustomVideoStreamType { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "channel_unavailable" => Ok(Self::ChannelUnavailable), - "user_connections_exhausted" => Ok(Self::UserConnectionsExhausted), - "provider_connections_exhausted" => Ok(Self::ProviderConnectionsExhausted), - "low_priority_preempted" => Ok(Self::LowPriorityPreempted), - "user_account_expired" => Ok(Self::UserAccountExpired), - "provisioning" => Ok(Self::Provisioning), - "hls_session_or_lease_expired" => Ok(Self::HlsSessionOrLeaseExpired), - _ => Err(TuliproxError::Config(format!("Unknown stream type: {s}"))), - } - } -} - -impl Serialize for CustomVideoStreamType { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} -impl<'de> Deserialize<'de> for CustomVideoStreamType { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - Self::from_str(&s).map_err(serde::de::Error::custom) - } -} +use tuliprox_core::{ + try_unwrap_body, + utils::{request_headers::HeaderFilter, response_compression::mark_response_as_uncompressed}, +}; +use tuliprox_session::stream_ctx::ProviderStreamCtx; fn prepare_video_headers(headers: &[(String, String)]) -> Vec<(String, String)> { let mut h: Vec<(String, String)> = headers @@ -116,9 +55,8 @@ fn apply_custom_stream_timeout(cfg: &AppConfig, stream: BoxedProviderStream) -> /// `custom_stream_response_error_status` instead. This allows a downstream Nginx /// with `proxy_intercept_errors on;` to sever the socket instead of seeing an /// infinite 200 OK loop. -pub(crate) fn is_custom_video_stream_enabled(cfg: &AppConfig) -> bool { - cfg.config.load().custom_stream_response_enabled -} +// Moved to `tuliprox_core::model`: the HLS proxy reads it too. +pub(crate) use tuliprox_core::model::is_custom_video_stream_enabled; /// Returns the HTTP status code the custom-video factories should produce when /// `custom_stream_response_enabled` is `false`. The configured value is validated @@ -263,11 +201,11 @@ pub fn create_panel_api_provisioning_stream_with_stop( } pub fn create_custom_video_stream_response( - app_state: &Arc, + ctx: &ProviderStreamCtx, addr: &SocketAddr, video_response: CustomVideoStreamType, ) -> impl axum::response::IntoResponse + Send { - let config = &app_state.app_config; + let config = &ctx.app_config; if let (Some(stream), Some((headers, status_code, _, _))) = match video_response { CustomVideoStreamType::ChannelUnavailable => create_channel_unavailable_stream(config, &[], StatusCode::OK), CustomVideoStreamType::UserConnectionsExhausted => create_user_connections_exhausted_stream(config, &[]), @@ -279,7 +217,7 @@ pub fn create_custom_video_stream_response( CustomVideoStreamType::Provisioning => create_panel_api_provisioning_stream(config, &[]), CustomVideoStreamType::HlsSessionOrLeaseExpired => create_hls_session_or_lease_expired_stream(config, &[]), } { - app_state.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProviderConnection { + ctx.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProviderConnection { addr: *addr, video_type: video_response, }); diff --git a/backend/src/api/model/streams/provider_stream_factory.rs b/backend/app/src/api/model/streams/provider_stream_factory.rs similarity index 95% rename from backend/src/api/model/streams/provider_stream_factory.rs rename to backend/app/src/api/model/streams/provider_stream_factory.rs index cb812ee42..496db11c9 100644 --- a/backend/src/api/model/streams/provider_stream_factory.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory.rs @@ -1,15 +1,9 @@ use crate::{ - api::{ - api_utils::{get_headers_from_request, StreamOptions}, - model::{ - create_channel_unavailable_stream, extract_hls_provider_session_headers, get_header_filter_for_item_type, - get_response_headers, - log_hls_origin_content_coding, - model_utils::{provider_response_headers, ProviderResponseHeaderError}, - streams::{buffered_stream::BufferedStream, client_stream::ClientStream}, - AppState, CustomVideoStreamType, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, - ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT, - }, + api::model::{ + create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, + streams::{buffered_stream::BufferedStream, client_stream::ClientStream}, + CustomVideoStreamType, ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, + STREAM_IDLE_TIMEOUT, }, iptv::stalker::client::validate_public_playable_url, model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig}, @@ -46,6 +40,16 @@ use std::{ time::{Duration, Instant}, }; use tokio_util::{io::ReaderStream, sync::CancellationToken}; +use tuliprox_core::utils::request_headers::get_headers_from_request; +use tuliprox_hls::api::{ + extract_hls_provider_session_headers, log_hls_origin_content_coding, HlsOriginContentCodingObjectKind, + HlsOriginContentCodingSource, +}; +use tuliprox_session::{ + response_headers::{provider_response_headers, ProviderResponseHeaderError}, + stream_ctx::ProviderStreamCtx, + stream_options::StreamOptions, +}; use url::Url; const RETRY_SECONDS: u64 = 5; @@ -74,8 +78,8 @@ create_bitset!( BufferEnabled, ShareStream, PipeStream, - RangeRequested - ,PublicDestinationRequired + RangeRequested, + PublicDestinationRequired ); #[derive(Debug, Clone)] @@ -146,8 +150,7 @@ impl ProviderStreamFactoryOptions { .map(ToString::to_string); let filter_header = get_header_filter_for_item_type(*item_type); let mut req_headers = get_headers_from_request(req_headers, &filter_header); - let requested_range = - req_headers.remove(RANGE.as_str()).and_then(|value| HeaderValue::from_bytes(&value).ok()); + let requested_range = req_headers.remove(RANGE.as_str()).and_then(|value| HeaderValue::from_bytes(&value).ok()); let merged_input_headers = merge_provider_request_headers(*input_headers, *session_headers); @@ -345,7 +348,7 @@ fn merge_provider_request_headers( } fn record_provider_open_failure( - app_state: &Arc, + ctx: &ProviderStreamCtx, stream_options: &ProviderStreamFactoryOptions, reason: ConnectFailureReason, provider_http_status: Option, @@ -356,9 +359,8 @@ fn record_provider_open_failure( stream_options.get_provider().map_or_else(|| "unknown".intern(), |provider| provider.name.clone()); let Some(info) = stream_options.build_connect_failed_stream_info(provider_name) else { return }; // Resolve target_name from target_id using the stable target config name. - let target_name = - app_state.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); - app_state.connection_manager.record_connect_failed_with_provider_failure( + let target_name = ctx.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); + ctx.connection_manager.record_connect_failed_with_provider_failure( &info, reason, failure_stage, @@ -546,11 +548,8 @@ fn prepare_client( } fn remove_sensitive_headers(headers: &mut axum::http::HeaderMap) { - let names_to_remove = headers - .keys() - .filter(|name| !is_safe_cross_origin_redirect_header(name.as_str())) - .cloned() - .collect::>(); + let names_to_remove = + headers.keys().filter(|name| !is_safe_cross_origin_redirect_header(name.as_str())).cloned().collect::>(); for name in names_to_remove { headers.remove(name); } @@ -806,12 +805,12 @@ fn provider_decoded_body_error(error: &io::Error) -> StreamError { #[allow(clippy::too_many_lines)] async fn provider_stream_request( - app_state: &Arc, + ctx: &ProviderStreamCtx, request_client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, ) -> Result, ProviderStreamRequestFailure> { let use_manual_redirects = stream_options.requires_public_destination() - || app_state.should_use_manual_redirects() + || tuliprox_core::model::should_use_manual_redirects(&ctx.app_config) || provider_headers_require_manual_redirects(stream_options.get_headers()); if log_enabled!(log::Level::Debug) { let diagnostics = @@ -824,18 +823,18 @@ async fn provider_stream_request( } let response_result = if use_manual_redirects { let client_no_redirect = if stream_options.requires_public_destination() { - app_state.public_http_client_no_redirect.load() + ctx.public_http_client_no_redirect.load() } else { - app_state.http_client_no_redirect.load() + ctx.http_client_no_redirect.load() }; - send_with_manual_redirects(&client_no_redirect, stream_options, &app_state.app_config).await + send_with_manual_redirects(&client_no_redirect, stream_options, &ctx.app_config).await } else { // Use send_with_retry_and_provider for automatic failover support let url = stream_options.get_url(); let provider = stream_options.get_provider().cloned(); send_with_retry_and_provider_policy( - &app_state.app_config, + &ctx.app_config, url, provider.as_ref(), false, @@ -961,7 +960,7 @@ async fn provider_stream_request( } async fn get_provider_stream( - app_state: &Arc, + ctx: &ProviderStreamCtx, client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, ) -> Result, ProviderStreamRequestFailure> { @@ -971,7 +970,7 @@ async fn get_provider_stream( let mut connect_err: u32 = 1; while stream_options.should_continue() { - match provider_stream_request(app_state, client, stream_options).await { + match provider_stream_request(ctx, client, stream_options).await { Ok(Some(stream_response)) => { return Ok(Some(stream_response)); } @@ -1032,7 +1031,7 @@ async fn get_provider_stream( } debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); stream_options.cancel_reconnect(); - app_state.connection_manager.release_provider_connection(&stream_options.addr).await; + ctx.connection_manager.release_provider_connection(&stream_options.addr).await; Err(ProviderStreamRequestFailure::Status { status: StatusCode::SERVICE_UNAVAILABLE, provider_error_class: "service_unavailable", @@ -1042,11 +1041,11 @@ async fn get_provider_stream( #[allow(clippy::too_many_lines)] pub async fn create_provider_stream( - app_state: &Arc, + ctx: &ProviderStreamCtx, client: &reqwest::Client, stream_options: ProviderStreamFactoryOptions, ) -> Option { - match get_provider_stream(app_state, client, &stream_options).await { + match get_provider_stream(ctx, client, &stream_options).await { Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers })) => { if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info { let reason = match custom_video_type { @@ -1058,7 +1057,7 @@ pub async fn create_provider_stream( _ => None, }; if let Some(reason) = reason { - record_provider_open_failure(app_state, &stream_options, reason, None, None); + record_provider_open_failure(ctx, &stream_options, reason, None, None); } } let continue_signal = stream_options.get_reconnect_flag_clone(); @@ -1076,7 +1075,8 @@ pub async fn create_provider_stream( stream }; Some(ProviderStreamFactoryResponse { - stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()).boxed(), + stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()) + .boxed(), info, provider_session_headers, }) @@ -1084,16 +1084,16 @@ pub async fn create_provider_stream( Ok(None) => None, Err(failure) => { let status = failure.status(); - app_state.connection_manager.release_provider_connection(&stream_options.addr).await; + ctx.connection_manager.release_provider_connection(&stream_options.addr).await; record_provider_open_failure( - app_state, + ctx, &stream_options, ConnectFailureReason::ChannelUnavailable, Some(status), Some(failure.provider_error_class()), ); if let (Some(boxed_provider_stream), response_info) = create_channel_unavailable_stream( - &app_state.app_config, + &ctx.app_config, &get_response_headers(stream_options.get_headers()), StatusCode::OK, ) { @@ -1277,8 +1277,13 @@ mod tests { input_headers: Option<&HashMap>, session_headers: Option<&HashMap>, ) -> ProviderStreamFactoryOptions { - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: false, buffer_size: 0, buffer_max_bytes: 0, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: false, + buffer_size: 0, + buffer_max_bytes: 0, + pipe_provider_stream: false, + }; ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { addr: "127.0.0.1:8080".parse().unwrap(), item_type: PlaylistItemType::Catchup, @@ -1423,22 +1428,11 @@ mod tests { let stream_url = Url::parse("http://provider.example/movie").unwrap(); let client = reqwest::Client::new(); - for requested_range in [ - "bytes=100-199", - "bytes=100-", - "bytes=-100", - "bytes=0-0", - "bytes=0-0,200-299", - ] { + for requested_range in ["bytes=100-199", "bytes=100-", "bytes=-100", "bytes=0-0", "bytes=0-0,200-299"] { let mut req_headers = HeaderMap::new(); req_headers.insert(reqwest::header::RANGE, requested_range.parse().unwrap()); - let options = test_options( - ProviderContentRepresentationMode::PreserveOrigin, - &stream_url, - &req_headers, - None, - None, - ); + let options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None); let (request, partial) = prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin); @@ -1466,15 +1460,11 @@ mod tests { None, ); - let request = prepare_client( - &reqwest::Client::new(), - &options, - None, - ProviderRequestCredentialState::OriginalOrigin, - ) - .0 - .build() - .unwrap(); + let request = + prepare_client(&reqwest::Client::new(), &options, None, ProviderRequestCredentialState::OriginalOrigin) + .0 + .build() + .unwrap(); assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=100-199"); } @@ -1483,8 +1473,13 @@ mod tests { fn test_provider_stream_factory_options_propagate_buffer_max_bytes() { let addr = "127.0.0.1:8080".parse().unwrap(); let stream_url = Url::parse("http://example.com/stream").unwrap(); - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: true, buffer_size: 1024, buffer_max_bytes: 4096, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 4096, + pipe_provider_stream: false, + }; let req_headers = HeaderMap::new(); let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { addr, @@ -1511,8 +1506,13 @@ mod tests { fn test_provider_stream_factory_options_range_logic() { let addr = "127.0.0.1:8080".parse().unwrap(); let stream_url = Url::parse("http://example.com/stream").unwrap(); - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: true, buffer_size: 1024, buffer_max_bytes: 0, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + }; let disabled_headers = None; // Case 1: VOD, no initial range requested @@ -1610,8 +1610,13 @@ mod tests { let addr = "127.0.0.1:8080".parse().unwrap(); let stream_url = Url::parse("http://example.com/segment.ts").unwrap(); let req_headers = HeaderMap::new(); - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: true, buffer_size: 1024, buffer_max_bytes: 0, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + }; let hls_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { addr, @@ -1660,8 +1665,13 @@ mod tests { let addr = "127.0.0.1:8080".parse().unwrap(); let stream_url = Url::parse("http://example.com/shared.ts").unwrap(); let req_headers = HeaderMap::new(); - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: true, buffer_size: 1024, buffer_max_bytes: 0, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + }; let shared_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { addr, @@ -1692,8 +1702,13 @@ mod tests { let addr = "127.0.0.1:8080".parse().unwrap(); let stream_url = Url::parse("http://example.com/stream").unwrap(); let req_headers = HeaderMap::new(); - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: true, buffer_size: 1024, buffer_max_bytes: 0, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + }; let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { addr, @@ -1737,7 +1752,10 @@ mod tests { assert_eq!(info.provider.as_ref(), "provider-a"); assert_eq!(info.channel.input_name.as_ref(), "input-a"); assert_eq!(info.channel.virtual_id, 77); - assert_eq!(options.headers.get(reqwest::header::USER_AGENT).and_then(|value| value.to_str().ok()), Some("Channel-UA")); + assert_eq!( + options.headers.get(reqwest::header::USER_AGENT).and_then(|value| value.to_str().ok()), + Some("Channel-UA") + ); } #[test] @@ -1766,8 +1784,13 @@ mod tests { let req_headers = HeaderMap::new(); let mut session_headers = HashMap::new(); session_headers.insert(String::from("cookie"), String::from("sid=abc; pref=1")); - let stream_options = - StreamOptions { stream_retry: true, buffer_enabled: true, buffer_size: 1024, buffer_max_bytes: 0, pipe_provider_stream: false }; + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + }; let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { addr, @@ -1812,10 +1835,9 @@ mod tests { assert!(result.is_ok()); if let Ok(response) = result { assert_eq!(response.provider_session_headers.get("cookie").map(String::as_str), Some("sid=abc")); - assert!(response - .info - .as_ref() - .is_some_and(|(headers, _, _, _)| headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("set-cookie")))); + assert!(response.info.as_ref().is_some_and(|(headers, _, _, _)| headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("set-cookie")))); } } @@ -1853,15 +1875,11 @@ mod tests { let sensitive_headers = ["authorization", "cookie", "proxy-authorization", "x-api-key", "x-provider-token"]; let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); - let same_origin_request = prepare_client( - &client, - &options, - Some(&same_origin), - ProviderRequestCredentialState::OriginalOrigin, - ) - .0 - .build() - .unwrap(); + let same_origin_request = + prepare_client(&client, &options, Some(&same_origin), ProviderRequestCredentialState::OriginalOrigin) + .0 + .build() + .unwrap(); for name in sensitive_headers { assert!(same_origin_request.headers().contains_key(name), "same-origin request lost {name}"); } @@ -1935,10 +1953,7 @@ mod tests { } else { ProviderRequestCredentialState::OriginalOrigin }; - let request = prepare_client(&client, &options, target, credential_state) - .0 - .build() - .unwrap(); + let request = prepare_client(&client, &options, target, credential_state).0.build().unwrap(); assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); if target == Some(&cross_origin) { diff --git a/backend/src/api/model/streams/provisioning_stream.rs b/backend/app/src/api/model/streams/provisioning_stream.rs similarity index 92% rename from backend/src/api/model/streams/provisioning_stream.rs rename to backend/app/src/api/model/streams/provisioning_stream.rs index 1629cf6e5..ea16c14b6 100644 --- a/backend/src/api/model/streams/provisioning_stream.rs +++ b/backend/app/src/api/model/streams/provisioning_stream.rs @@ -39,9 +39,7 @@ impl Stream for ProvisioningStream { if let Some(chunk) = self.buffer.next_chunk() { // Reset idle deadline on each delivered chunk. - self.idle_deadline - .as_mut() - .reset(Instant::now() + Duration::from_secs(STREAM_IDLE_TIMEOUT)); + self.idle_deadline.as_mut().reset(Instant::now() + Duration::from_secs(STREAM_IDLE_TIMEOUT)); Poll::Ready(Some(Ok(chunk))) } else { self.buffer.register_waker(cx.waker()); diff --git a/backend/src/api/model/streams/throttled_stream.rs b/backend/app/src/api/model/streams/throttled_stream.rs similarity index 94% rename from backend/src/api/model/streams/throttled_stream.rs rename to backend/app/src/api/model/streams/throttled_stream.rs index f89823693..c660c208f 100644 --- a/backend/src/api/model/streams/throttled_stream.rs +++ b/backend/app/src/api/model/streams/throttled_stream.rs @@ -21,12 +21,7 @@ impl ThrottledStream { pub fn new(inner: S, throttle_kbps: usize) -> Self { assert!(throttle_kbps > 0, "Rate must be greater than 0"); let rate_bytes_per_sec = (throttle_kbps as f64) * 1000.0 / 8.0; - Self { - inner, - rate_bytes_per_sec, - delay: Box::pin(sleep(Duration::ZERO)), - delay_active: false, - } + Self { inner, rate_bytes_per_sec, delay: Box::pin(sleep(Duration::ZERO)), delay_active: false } } } diff --git a/backend/src/api/model/streams/timed_client_stream.rs b/backend/app/src/api/model/streams/timed_client_stream.rs similarity index 68% rename from backend/src/api/model/streams/timed_client_stream.rs rename to backend/app/src/api/model/streams/timed_client_stream.rs index 9e322dfb1..d6358945e 100644 --- a/backend/src/api/model/streams/timed_client_stream.rs +++ b/backend/app/src/api/model/streams/timed_client_stream.rs @@ -1,28 +1,22 @@ use crate::{ - api::model::{stream_error::StreamError, AppState, BoxedProviderStream}, + api::model::{BoxedProviderStream, ConnectionManager}, + model::{AppConfig, StreamError}, utils::debug_if_enabled, }; use bytes::Bytes; use futures::Stream; use shared::{ - model::VirtualId, - utils::{sanitize_sensitive_info}, - defaults::{default_kick_secs}, -}; -use std::{ - future::Future, - net::SocketAddr, - pin::Pin, - sync::Arc, - task::Poll, - time::Duration, + defaults::default_kick_secs, + model::{DisconnectReason, VirtualId}, + utils::sanitize_sensitive_info, }; +use std::{future::Future, net::SocketAddr, pin::Pin, sync::Arc, task::Poll, time::Duration}; use tokio::time::{sleep_until, Instant, Sleep}; -use shared::model::DisconnectReason; enum TimeoutAction { Kick { - app_state: Arc, + app_config: Arc, + connection_manager: Arc, addr: SocketAddr, virtual_id: VirtualId, }, @@ -41,7 +35,8 @@ pub struct TimedClientStream { impl TimedClientStream { pub(crate) fn new( - app_state: &Arc, + app_config: &Arc, + connection_manager: &Arc, inner: BoxedProviderStream, duration: u32, addr: SocketAddr, @@ -51,7 +46,12 @@ impl TimedClientStream { Self { inner, deadline, - timeout_action: TimeoutAction::Kick { app_state: Arc::clone(app_state), addr, virtual_id }, + timeout_action: TimeoutAction::Kick { + app_config: Arc::clone(app_config), + connection_manager: Arc::clone(connection_manager), + addr, + virtual_id, + }, } } @@ -69,15 +69,10 @@ impl Stream for TimedClientStream { // wakes this task exactly when the deadline fires — even if the upstream // provider is stalled and emitting no data. if self.deadline.as_mut().poll(cx).is_ready() { - if let TimeoutAction::Kick { app_state, addr, virtual_id } = &self.timeout_action { - let kick_secs = app_state - .app_config - .config - .load() - .web_ui - .as_ref() - .map_or_else(default_kick_secs, |wc| wc.kick_secs); - let connection_manager = Arc::clone(&app_state.connection_manager); + if let TimeoutAction::Kick { app_config, connection_manager, addr, virtual_id } = &self.timeout_action { + let kick_secs = + app_config.config.load().web_ui.as_ref().map_or_else(default_kick_secs, |wc| wc.kick_secs); + let connection_manager = Arc::clone(connection_manager); let addr = *addr; let virtual_id = *virtual_id; debug_if_enabled!( @@ -86,12 +81,7 @@ impl Stream for TimedClientStream { ); tokio::spawn(async move { let _ = connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - kick_secs, - DisconnectReason::Timeout, - ) + .close_connection_with_reason_and_block(&addr, virtual_id, kick_secs, DisconnectReason::Timeout) .await; }); } diff --git a/backend/src/api/panel_api.rs b/backend/app/src/api/panel_api.rs similarity index 94% rename from backend/src/api/panel_api.rs rename to backend/app/src/api/panel_api.rs index f4ea2e202..99c2164a4 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/app/src/api/panel_api.rs @@ -1,3 +1,24 @@ +//! Provisioning accounts against an upstream Xtream panel. +//! +//! This stays in `api`, and the reason is a cycle rather than `AppState`. It +//! reads only two fields off the root state - the configuration and the HTTP +//! client - so the state is not what pins it. What pins it is +//! `api::model::streams`: +//! +//! - this module calls `provider_stream::create_panel_api_provisioning_stream_with_stop` +//! and `create_provider_connections_exhausted_stream` to answer an exhausted +//! request with a provisioning clip; +//! - `provider_stream` and `active_client_stream` call back here for +//! `can_provision_on_exhausted`, `run_panel_api_provisioning_probe` and +//! `find_input_by_provider_name`. +//! +//! So provisioning and streaming are one mutually recursive cluster, roughly +//! 11.5k lines, and neither half moves without the other. Extracting them would +//! mean moving both together into a package that also depends on `hls`, +//! `session`, `iptv`, `repository` and `config-loader` - and `ConfigFile`, which +//! mutates the root state directly, would have to move too. That is a +//! deliberate not-yet, not an oversight. + use crate::{ api::{ config_file::ConfigFile, @@ -6,18 +27,18 @@ use crate::{ StreamDetails, }, }, + config_loader::{persist_source_config, read_sources_file_from_path}, model::{ - is_input_expired, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, PanelApiConfig, PanelApiQueryParam, - ProxyUserCredentials, + is_input_expired, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, PanelApiConfig, + PanelApiQueryParam, ProxyUserCredentials, }, repository::{ compare_alias_exp_date_with_order, csv_patch_batch_append, csv_patch_batch_remove_expired, csv_patch_batch_sort_by_exp_date, csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, get_csv_file_path, AliasExpDateSortOrder, }, - utils::{debug_if_enabled, format_http_status, persist_source_config, read_sources_file_from_path, request}, + utils::{debug_if_enabled, format_http_status, request}, }; -use smallvec::SmallVec; use axum::http::{header, HeaderMap, Method, StatusCode}; use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use chrono_tz::Tz; @@ -29,15 +50,15 @@ use shared::{ concat_string, create_bitset, error::{string_to_io_error, TuliproxError}, model::{ - ConfigInputAliasDto, ConfigInputDto, InputType, PanelApiAliasPoolSizeValue, PanelApiProvisioningMethod, - ProxyUserStatus, SourcesConfigDto, VirtualId, + ConfigInputAliasDto, ConfigInputDto, DisconnectReason, InputType, PanelApiAliasPoolSizeValue, + PanelApiProvisioningMethod, ProxyUserStatus, SourcesConfigDto, VirtualId, }, utils::{ get_base_url_from_str, get_credentials_from_url, get_credentials_from_url_str, get_i64_from_serde_value, - get_string_from_serde_value, parse_timestamp, sanitize_sensitive_info, Internable, - PROVIDER_SCHEME_PREFIX, + get_string_from_serde_value, parse_timestamp, sanitize_sensitive_info, Internable, PROVIDER_SCHEME_PREFIX, }, }; +use smallvec::SmallVec; use std::{ cmp::Ordering, collections::{HashMap, HashSet}, @@ -49,7 +70,6 @@ use std::{ }; use tokio_util::sync::CancellationToken; use url::Url; -use shared::model::DisconnectReason; #[derive(Debug, Clone)] struct AccountCredentials { @@ -234,10 +254,14 @@ fn validate_type_is_m3u(params: &[PanelApiQueryParam]) -> Result<(), TuliproxErr fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); let Some(api_key) = api_key else { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: {section} must contain query param 'api_key' (use value 'auto')"))); + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: {section} must contain query param 'api_key' (use value 'auto')" + ))); }; if api_key.value.trim().is_empty() { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: {section} query param 'api_key' must not be empty (use value 'auto')"))); + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" + ))); } Ok(()) } @@ -264,7 +288,9 @@ fn validate_client_new_params(params: &[PanelApiQueryParam]) -> Result<(), Tulip require_api_key_param(params, "query_parameter.client_new")?; validate_type_is_m3u(params)?; if params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("user")) { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_new must not contain query param 'user'".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_new must not contain query param 'user'".to_string(), + )); } Ok(()) } @@ -361,9 +387,12 @@ fn parse_panel_api_provisioning_offset_secs(offset: &str) -> Result Result<(), TuliproxError> { @@ -377,7 +406,9 @@ fn validate_panel_api_config(cfg: &PanelApiConfig) -> Result<(), TuliproxError> return Err(TuliproxError::ConfigPanelApi("panel_api: api_key is missing".to_string())); } if cfg.query_parameter.client_info.is_empty() { - return Err(TuliproxError::ConfigPanelApi("panel_api: query_parameter.client_info must be configured".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: query_parameter.client_info must be configured".to_string(), + )); } validate_client_info_params(&cfg.query_parameter.client_info)?; if !cfg.query_parameter.client_new.is_empty() { @@ -395,23 +426,31 @@ fn validate_panel_api_config(cfg: &PanelApiConfig) -> Result<(), TuliproxError> let (min_val, max_val) = alias_pool_limit_values(cfg); if let Some(PanelApiAliasPoolSizeValue::Number(value)) = min_val { if *value == 0 { - return Err(TuliproxError::ConfigPanelApi("panel_api.alias_pool.size.min must be greater than 0".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.min must be greater than 0".to_string(), + )); } } if let Some(PanelApiAliasPoolSizeValue::Number(value)) = max_val { if *value == 0 { - return Err(TuliproxError::ConfigPanelApi("panel_api.alias_pool.size.max must be greater than 0".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.max must be greater than 0".to_string(), + )); } } let min = min_val.and_then(PanelApiAliasPoolSizeValue::as_number); let max = max_val.and_then(PanelApiAliasPoolSizeValue::as_number); if let (Some(min), Some(max)) = (min, max) { if min > max { - return Err(TuliproxError::ConfigPanelApi("panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string(), + )); } } if cfg.provisioning.probe_interval_sec == 0 { - return Err(TuliproxError::ConfigPanelApi("panel_api.provisioning.probe_interval_sec must be greater than 0".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api.provisioning.probe_interval_sec must be greater than 0".to_string(), + )); } if let Some(offset) = cfg.provisioning.offset.as_deref() { let _secs = parse_panel_api_provisioning_offset_secs(offset)?; @@ -434,7 +473,9 @@ fn resolve_query_params( if value.eq_ignore_ascii_case("auto") { if key.eq_ignore_ascii_case("api_key") { let Some(k) = api_key.filter(|s| !s.trim().is_empty()) else { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: query param {key} uses 'auto' but panel_api.api_key is missing"))); + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: query param {key} uses 'auto' but panel_api.api_key is missing" + ))); }; value = k.to_string(); } else if key.eq_ignore_ascii_case("username") { @@ -459,7 +500,8 @@ fn resolve_query_params( } fn build_panel_url(base_url: &str, query_params: &[(String, String)]) -> Result { - let mut url = Url::parse(base_url).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: invalid url {base_url}: {e}")))?; + let mut url = Url::parse(base_url) + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: invalid url {base_url}: {e}")))?; { let mut pairs = url.query_pairs_mut(); for (k, v) in query_params { @@ -506,10 +548,7 @@ fn sanitize_panel_api_json_for_log(value: &Value, sanitize_sensitive: bool) -> V fn panel_api_retryable_status(status: StatusCode) -> bool { status.is_server_error() - || matches!( - status, - StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY - ) + || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY) } fn panel_api_retry_after_from_header_value(raw: &str) -> Option { @@ -530,9 +569,7 @@ fn panel_api_retry_after_from_header_value(raw: &str) -> Option { } else { u64::try_from(secs).unwrap_or(PANEL_API_DEFAULT_RETRY_AFTER_SECS) }; - Some(Duration::from_secs( - secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS), - )) + Some(Duration::from_secs(secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS))) } fn panel_api_retry_after_from_headers(headers: &HeaderMap) -> Duration { @@ -547,12 +584,7 @@ async fn panel_get_json(app_state: &AppState, url: Url) -> Result Result input_source.get_provider(), false, |resolved_url| { - client - .get(resolved_url.clone()) - .headers(headers.clone()) - .timeout(std::time::Duration::from_secs(30)) + client.get(resolved_url.clone()).headers(headers.clone()).timeout(std::time::Duration::from_secs(30)) }, ) .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api request failed: {e}")))?; + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api request failed: {e}")))?; let status = resp.status(); - let body = resp.text().await.map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api read response failed: {e}")))?; - let json: Value = - serde_json::from_str(&body).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api invalid json (http {status}): {e}")))?; + let body = resp + .text() + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api read response failed: {e}")))?; + let json: Value = serde_json::from_str(&body) + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api invalid json (http {status}): {e}")))?; let sanitize_sensitive = app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); if let Ok(json_str) = serde_json::to_string(&json_for_log) { @@ -676,7 +709,9 @@ async fn panel_client_new( let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_new response is not a JSON object/array".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_new response is not a JSON object/array".to_string(), + )); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { @@ -691,7 +726,9 @@ async fn panel_client_new( return Ok((u, p, base)); } } - Err(TuliproxError::ConfigPanelApi("panel_api: client_new response missing username/password (and no parsable url)".to_string())) + Err(TuliproxError::ConfigPanelApi( + "panel_api: client_new response missing username/password (and no parsable url)".to_string(), + )) } async fn panel_client_renew( @@ -706,7 +743,9 @@ async fn panel_client_renew( let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_renew response is not a JSON object/array".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_renew response is not a JSON object/array".to_string(), + )); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { @@ -727,7 +766,9 @@ async fn panel_client_info_raw( let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_info response is not a JSON object/array".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_info response is not a JSON object/array".to_string(), + )); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { @@ -855,9 +896,12 @@ async fn persist_panel_api_time_cache( cache: &PanelApiTimeCache, ) -> Result<(), TuliproxError> { if let Some(parent) = cache_path.parent() { - tokio::fs::create_dir_all(parent).await.map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to create cache dir: {e}")))?; + tokio::fs::create_dir_all(parent) + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to create cache dir: {e}")))?; } - let content = serde_json::to_string_pretty(cache).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: {e}")))?; + let content = + serde_json::to_string_pretty(cache).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: {e}")))?; let _lock = app_state.app_config.file_locks.write_lock(cache_path).await; tokio::fs::write(cache_path, content) .await @@ -880,7 +924,9 @@ async fn panel_account_info( let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi("panel_api: account_info response is not a JSON object/array".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: account_info response is not a JSON object/array".to_string(), + )); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { @@ -905,7 +951,9 @@ async fn panel_client_adult_content( let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; let json = panel_get_json(app_state, url).await?; let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_adult_content response is not a JSON object/array".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_adult_content response is not a JSON object/array".to_string(), + )); }; let status_ok = obj.get("status").is_some_and(parse_boolish); if !status_ok { @@ -1038,7 +1086,9 @@ fn resolve_alias_pool_limits( }; if let (Some(min), Some(max)) = (min, max) { if min > max { - return Err(TuliproxError::ConfigPanelApi("panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string())); + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string(), + )); } } Ok((min, max)) @@ -1388,9 +1438,11 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa for patch in patches { match patch { SourcesYmlPatch::UpdatePanelApiCredits { input_name, credits } => { - let idx = *inputs_by_name - .get(input_name.as_ref()) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let Some(panel_api) = doc.inputs[idx].panel_api.as_mut() else { return Err(TuliproxError::ConfigPanelApi(format!( "panel_api: could not find panel_api for input '{input_name}' in source.yml" @@ -1402,9 +1454,11 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa } } SourcesYmlPatch::SortAliases { input_name, order } => { - let idx = *inputs_by_name - .get(input_name.as_ref()) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { continue; }; @@ -1415,9 +1469,11 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa } } SourcesYmlPatch::UpdateExpDate { input_name, account_name, exp_date } => { - let idx = *inputs_by_name - .get(input_name.as_ref()) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; if account_name == input_name { if doc.inputs[idx].exp_date != Some(*exp_date) || !doc.inputs[idx].enabled @@ -1435,25 +1491,21 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa "panel_api: could not find alias '{account_name}' under input '{input_name}' in source.yml" ))); }; - let aliases = doc.inputs[idx] - .aliases - .as_mut() - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")))?; + let aliases = doc.inputs[idx].aliases.as_mut().ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")) + })?; if aliases[alias_idx].exp_date != Some(*exp_date) || aliases[alias_idx].max_connections != 1 { aliases[alias_idx].exp_date = Some(*exp_date); aliases[alias_idx].max_connections = 1; changed = true; } } - SourcesYmlPatch::UpdateRootCredentials { - input_name, - username, - password, - exp_date, - } => { - let idx = *inputs_by_name - .get(input_name.as_ref()) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + SourcesYmlPatch::UpdateRootCredentials { input_name, username, password, exp_date } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let input = &mut doc.inputs[idx]; let exp_date_changed = exp_date.is_some() && input.exp_date != *exp_date; if input.username.as_deref() != Some(username.as_str()) @@ -1472,9 +1524,11 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa } } SourcesYmlPatch::PersistProvisionedAccount { input_name, username, password, exp_date } => { - let idx = *inputs_by_name - .get(input_name.as_ref()) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let input = &mut doc.inputs[idx]; let current_root_is_usable = input.exp_date.is_some() && !is_input_expired_at(input.exp_date, get_current_timestamp()); @@ -1530,18 +1584,19 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa } } SourcesYmlPatch::UpdateAliasCredentials { input_name, alias_name, username, password, exp_date } => { - let idx = *inputs_by_name - .get(input_name.as_ref()) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let Some(alias_idx) = alias_indices[idx].get(alias_name).copied() else { return Err(TuliproxError::ConfigPanelApi(format!( "panel_api: could not find alias '{alias_name}' under input '{input_name}' in source.yml" ))); }; - let aliases = doc.inputs[idx] - .aliases - .as_mut() - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")))?; + let aliases = doc.inputs[idx].aliases.as_mut().ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")) + })?; let alias = &mut aliases[alias_idx]; let exp_date_changed = exp_date.is_some() && alias.exp_date != *exp_date; if alias.username.as_deref() != Some(username.as_str()) @@ -1559,9 +1614,11 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa } } SourcesYmlPatch::AddAlias { input_name, alias_name, base_url, username, password, exp_date } => { - let idx = *inputs_by_name - .get(input_name) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let alias_idx = append_sources_yml_alias( input_name, &mut doc.inputs[idx], @@ -1575,9 +1632,11 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa changed = true; } SourcesYmlPatch::RemoveExpiredAliases { input_name } => { - let idx = *inputs_by_name - .get(input_name) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let idx = *inputs_by_name.get(input_name).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: could not find input '{input_name}' in source.yml" + )) + })?; let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { continue; }; @@ -1612,7 +1671,7 @@ async fn persist_sources_yml_with_patches( return Ok(false); } - persist_source_config(app_state, Some(sources_path), sources).await?; + persist_source_config(&app_state.app_config, Some(sources_path), sources).await?; Ok(true) } @@ -1743,9 +1802,12 @@ async fn try_renew_expired_account( { debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); } - if let Err(err) = - csv_patch_batch_sort_by_exp_date(input.input_type, &csv_path, AliasExpDateSortOrder::NewestFirst) - .await + if let Err(err) = csv_patch_batch_sort_by_exp_date( + input.input_type, + &csv_path, + AliasExpDateSortOrder::NewestFirst, + ) + .await { debug_if_enabled!("panel_api failed to sort csv accounts after renew: {}", err); } @@ -1824,9 +1886,7 @@ async fn try_refresh_root_account_on_exhausted( let (outcome, active_username, active_password, credentials_changed) = if renew_enabled && !old_username.is_empty() && !old_password.is_empty() { match panel_client_renew(app_state, panel_cfg, old_username.as_str(), old_password.as_str()).await { - Ok(()) => { - (PanelApiProvisionOutcome::Renewed, old_username.clone(), old_password.clone(), false) - } + Ok(()) => (PanelApiProvisionOutcome::Renewed, old_username.clone(), old_password.clone(), false), Err(err) => { debug_if_enabled!( "panel_api client_renew failed for root {}: {}", @@ -1899,15 +1959,12 @@ async fn try_refresh_root_account_on_exhausted( return None; } - let refreshed_exp_date = panel_client_info(app_state, panel_cfg, active_username.as_str(), active_password.as_str(), None) - .await - .ok() - .flatten(); - let exp_date = if credentials_changed { - refreshed_exp_date - } else { - refreshed_exp_date.or(input.exp_date) - }; + let refreshed_exp_date = + panel_client_info(app_state, panel_cfg, active_username.as_str(), active_password.as_str(), None) + .await + .ok() + .flatten(); + let exp_date = if credentials_changed { refreshed_exp_date } else { refreshed_exp_date.or(input.exp_date) }; if is_batch { let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); @@ -1944,7 +2001,9 @@ async fn try_refresh_root_account_on_exhausted( debug_if_enabled!("panel_api failed to persist root provisioning to csv: {}", err); return None; } - if let Err(err) = csv_patch_batch_sort_by_exp_date(input.input_type, &csv_path, AliasExpDateSortOrder::NewestFirst).await { + if let Err(err) = + csv_patch_batch_sort_by_exp_date(input.input_type, &csv_path, AliasExpDateSortOrder::NewestFirst).await + { debug_if_enabled!("panel_api failed to sort csv accounts after root provisioning: {}", err); return None; } @@ -2111,11 +2170,7 @@ pub async fn try_provision_account_on_exhausted( app_state: &Arc, input_name: &Arc, ) -> Option { - let _input_lock = app_state - .app_config - .file_locks - .write_lock_str(format!("panel_api:{input_name}").as_str()) - .await; + let _input_lock = app_state.app_config.file_locks.write_lock_str(format!("panel_api:{input_name}").as_str()).await; let current_input = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()); let Some(input) = current_input.as_deref() else { debug_if_enabled!( @@ -2163,7 +2218,8 @@ pub async fn try_provision_account_on_exhausted( let sources_path = PathBuf::from(&sources_file_path); if let Some(outcome) = - try_refresh_root_account_on_exhausted(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional).await + try_refresh_root_account_on_exhausted(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional) + .await { debug_if_enabled!( "panel_api: provisioning succeeded via root {} for input {}", @@ -3564,7 +3620,8 @@ async fn sync_panel_api_for_input_on_boot( if panel_cfg.alias_pool.is_some() { if let Some(csv_path) = csv_path.as_ref() { let _csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - match csv_patch_batch_sort_by_exp_date(input.input_type, csv_path, AliasExpDateSortOrder::NewestFirst).await { + match csv_patch_batch_sort_by_exp_date(input.input_type, csv_path, AliasExpDateSortOrder::NewestFirst).await + { Ok(true) => any_change = true, Ok(false) => {} Err(err) => debug_if_enabled!( @@ -4015,9 +4072,9 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; + DisconnectReason::Provisioning, + ) + .await; return Ok(()); } @@ -4052,11 +4109,8 @@ pub fn create_panel_api_provisioning_stream_details( ) -> StreamDetails { let stop_signal = CancellationToken::new(); let headers = [("connection".to_string(), "close".to_string())]; - let (stream, stream_info) = create_panel_api_provisioning_stream_with_stop( - &app_state.app_config, - &headers, - stop_signal.clone(), - ); + let (stream, stream_info) = + create_panel_api_provisioning_stream_with_stop(&app_state.app_config, &headers, stop_signal.clone()); if stream.is_none() { debug_if_enabled!( @@ -4117,11 +4171,12 @@ mod tests { AliasExpDateSortOrder, PanelApiProbeTarget, SourcesYmlPatch, PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS, }; - use axum::http::StatusCode; use crate::model::{ConfigInput, ConfigProvider}; - use shared::model::{ConfigInputAliasDto, ConfigInputDto, ConfigProviderDto, InputType, ProviderUrlSelectionPolicy, SourcesConfigDto}; - use std::time::Duration; - use std::sync::Arc; + use axum::http::StatusCode; + use shared::model::{ + ConfigInputAliasDto, ConfigInputDto, ConfigProviderDto, InputType, ProviderUrlSelectionPolicy, SourcesConfigDto, + }; + use std::{sync::Arc, time::Duration}; use url::Url; fn source_alias(name: &str, exp_date: Option) -> ConfigInputAliasDto { @@ -4347,10 +4402,7 @@ mod tests { #[test] fn panel_api_retry_after_header_is_short_and_bounded() { - assert_eq!( - panel_api_retry_after_from_header_value("2"), - Some(Duration::from_secs(2)) - ); + assert_eq!(panel_api_retry_after_from_header_value("2"), Some(Duration::from_secs(2))); assert_eq!( panel_api_retry_after_from_header_value("0"), Some(Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) @@ -4384,8 +4436,7 @@ mod tests { ); let parsed = Url::parse(result.as_str()).expect("expected valid provider url"); - let pairs: Vec<(String, String)> = - parsed.query_pairs().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + let pairs: Vec<(String, String)> = parsed.query_pairs().map(|(k, v)| (k.to_string(), v.to_string())).collect(); assert!(pairs.contains(&("foo".to_string(), "bar".to_string()))); assert!(pairs.contains(&("username".to_string(), "new-user".to_string()))); assert!(pairs.contains(&("password".to_string(), "new-pass".to_string()))); @@ -4405,12 +4456,8 @@ mod tests { #[test] fn resolve_base_url_falls_back_when_panel_response_is_literal_null() { - let result = resolve_provisioned_account_base_url( - "http://input.example.org/path?x=1", - Some("null"), - "new", - "secret", - ); + let result = + resolve_provisioned_account_base_url("http://input.example.org/path?x=1", Some("null"), "new", "secret"); assert_eq!(result, "http://input.example.org/path?x=1"); } diff --git a/backend/src/api/serve.rs b/backend/app/src/api/serve.rs similarity index 94% rename from backend/src/api/serve.rs rename to backend/app/src/api/serve.rs index 0dbacba59..75b0f8e4c 100644 --- a/backend/src/api/serve.rs +++ b/backend/app/src/api/serve.rs @@ -1,5 +1,4 @@ use crate::api::model::{CloseConnectionSignal, ConnectionManager}; -use shared::model::DisconnectReason; use axum::{body::Body, extract::Request, response::Response}; use futures::FutureExt; use hyper::body::Incoming; @@ -9,6 +8,7 @@ use hyper_util::{ service::TowerToHyperService, }; use log::{debug, error, trace}; +use shared::model::DisconnectReason; use socket2::{SockRef, TcpKeepalive}; use std::{convert::Infallible, fmt::Debug, net::SocketAddr, pin::pin, sync::Arc, time::Duration}; use tokio::sync::watch; @@ -267,11 +267,8 @@ mod tests { epg_reference_ts: None, upstream_user_agent: None, }; - let upstream = PendingDropProbeStream { - dropped: Arc::clone(&state.upstream_dropped), - }; - let stream_details = - StreamDetails::from_stream(Box::pin(upstream), GracePeriodOptions::default()); + let upstream = PendingDropProbeStream { dropped: Arc::clone(&state.upstream_dropped) }; + let stream_details = StreamDetails::from_stream(Box::pin(upstream), GracePeriodOptions::default()); let stream = create_active_client_stream(ActiveClientStreamParams { stream_details, app_state: &state.app_state, @@ -299,9 +296,8 @@ mod tests { async fn assert_socket_disconnect_cleans_direct_series(disconnect: ClientDisconnect) { let app_state = create_test_app_state(Config::default()); let upstream_dropped = Arc::new(AtomicBool::new(false)); - let router = Router::new() - .route("/series", get(pending_direct_series_response)) - .with_state(DisconnectTestState { + let router = + Router::new().route("/series", get(pending_direct_series_response)).with_state(DisconnectTestState { app_state: Arc::clone(&app_state), upstream_dropped: Arc::clone(&upstream_dropped), }); @@ -335,9 +331,7 @@ mod tests { } ClientDisconnect::Reset => { let client = client.into_std().expect("convert test client to std socket"); - SockRef::from(&client) - .set_linger(Some(Duration::ZERO)) - .expect("configure reset-on-close"); + SockRef::from(&client).set_linger(Some(Duration::ZERO)).expect("configure reset-on-close"); drop(client); } } @@ -355,16 +349,7 @@ mod tests { }) .await .expect("socket disconnect should release registry state and drop the upstream"); - assert_eq!( - app_state - .active_provider - .active_connections() - .await - .unwrap_or_default() - .values() - .sum::(), - 0 - ); + assert_eq!(app_state.active_provider.active_connections().await.unwrap_or_default().values().sum::(), 0); server_cancel.cancel(); tokio::time::timeout(Duration::from_secs(1), server) diff --git a/backend/src/api/setup_api.rs b/backend/app/src/api/setup_api.rs similarity index 92% rename from backend/src/api/setup_api.rs rename to backend/app/src/api/setup_api.rs index 922ab23fa..2fb61ffef 100644 --- a/backend/src/api/setup_api.rs +++ b/backend/app/src/api/setup_api.rs @@ -1,11 +1,11 @@ use crate::{ api::{api_utils::serve_file, http_layers::create_cors_layer}, auth::generate_password_from_input, - model::validate_library_paths_from_dto, - utils::{ - file_exists, get_default_web_root_path_for_home, read_api_proxy_file, read_config_file_with_options, read_sources_file, - read_templates_file, resolve_template_persist_file_path, sanitize_sources_for_persist, + config_loader::{ + read_api_proxy_file, read_config_file_with_options, read_sources_file, sanitize_sources_for_persist, }, + model::validate_library_paths_from_dto, + utils::{file_exists, get_default_web_root_path_for_home, read_templates_file, resolve_template_persist_file_path}, }; use axum::{ extract::{Path, State}, @@ -19,14 +19,14 @@ use rand::Rng; use serde::{Deserialize, Serialize}; use serde_json::json; use shared::{ + defaults::{default_kick_secs, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH, DEFAULT_PORT, DEFAULT_STORAGE_DIR, USER_FILE}, error::TuliproxError, foundation::prepare_templates, model::{ ApiProxyConfigDto, ApiProxyServerInfoDto, AppConfigDto, ConfigApiDto, ConfigDto, ConfigPaths, PatternTemplate, SourcesConfigDto, TemplateDefinitionDto, TokenResponse, WebAuthConfigDto, WebUiConfigDto, TOKEN_NO_AUTH, }, - utils::{hex_encode}, - defaults::{default_kick_secs, DEFAULT_PORT, DEFAULT_STORAGE_DIR, USER_FILE, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH}, + utils::hex_encode, }; use std::{ collections::HashSet, @@ -39,7 +39,10 @@ use std::{ }, }; use tokio::sync::{oneshot, Mutex, RwLock}; -use tower_http::{compression::predicate::{DefaultPredicate, Predicate, SizeAbove}, services::ServeDir}; +use tower_http::{ + compression::predicate::{DefaultPredicate, Predicate, SizeAbove}, + services::ServeDir, +}; const DEFAULT_SETUP_HOST: &str = "0.0.0.0"; @@ -51,10 +54,7 @@ pub struct SetupWebUserCredentialDto { impl fmt::Debug for SetupWebUserCredentialDto { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SetupWebUserCredentialDto") - .field("username", &self.username) - .field("password", &"***") - .finish() + f.debug_struct("SetupWebUserCredentialDto").field("username", &self.username).field("password", &"***").finish() } } @@ -179,9 +179,7 @@ fn create_default_config_dto(home_path: &str) -> ConfigDto { api: ConfigApiDto { host: DEFAULT_SETUP_HOST.to_string(), port: DEFAULT_PORT, - web_root: get_default_web_root_path_for_home(FsPath::new(home_path)) - .display() - .to_string(), + web_root: get_default_web_root_path_for_home(FsPath::new(home_path)).display().to_string(), }, storage_dir: Some(DEFAULT_STORAGE_DIR.to_string()), custom_stream_response_path: Some(DEFAULT_CUSTOM_STREAM_RESPONSE_PATH.to_string()), @@ -204,7 +202,10 @@ async fn build_initial_draft(paths: &ConfigPaths) -> AppConfigDto { let mut draft = create_default_draft(paths.home_path.as_str()); if file_exists(&paths.config_file_path) { - match read_config_file_with_options(paths.config_file_path.as_str(), crate::utils::ReadConfigOptions { resolve_env: true, include_computed: false }) { + match read_config_file_with_options( + paths.config_file_path.as_str(), + crate::config_loader::ReadConfigOptions { resolve_env: true, include_computed: false }, + ) { Ok(cfg) => draft.config = cfg, Err(err) => warn!("Setup mode: failed to load existing config.yml: {err}"), } @@ -216,7 +217,7 @@ async fn build_initial_draft(paths: &ConfigPaths) -> AppConfigDto { false, false, draft.config.get_hdhr_device_overview().as_ref(), - None + None, ) .await { @@ -252,9 +253,7 @@ fn ensure_setup_defaults(config: &mut ConfigDto, home_path: &str) { config.api.port = DEFAULT_PORT; } if config.api.web_root.trim().is_empty() { - config.api.web_root = get_default_web_root_path_for_home(FsPath::new(home_path)) - .display() - .to_string(); + config.api.web_root = get_default_web_root_path_for_home(FsPath::new(home_path)).display().to_string(); } if config.storage_dir.as_ref().is_none_or(|dir| dir.trim().is_empty()) { config.storage_dir = Some(DEFAULT_STORAGE_DIR.to_string()); @@ -310,10 +309,7 @@ fn resolve_setup_web_dir(web_root: &FsPath, home_path: &str) -> Option } fn default_api_proxy_config() -> ApiProxyConfigDto { - ApiProxyConfigDto { - server: vec![create_default_api_proxy_server()], - ..ApiProxyConfigDto::default() - } + ApiProxyConfigDto { server: vec![create_default_api_proxy_server()], ..ApiProxyConfigDto::default() } } fn api_proxy_or_default(draft: &AppConfigDto) -> ApiProxyConfigDto { @@ -495,10 +491,7 @@ enum SetupPersistWriteError { Io(std::io::Error), } -async fn persist_yaml_file( - file_path: &FsPath, - payload: &T, -) -> Result<(), SetupPersistWriteError> { +async fn persist_yaml_file(file_path: &FsPath, payload: &T) -> Result<(), SetupPersistWriteError> { let mut content = String::new(); let options = serde_saphyr::ser_options! {prefer_block_scalars: false}; serde_saphyr::to_fmt_writer_with_options(&mut content, payload, options) @@ -515,19 +508,10 @@ async fn ensure_parent_dir(file_path: &FsPath) -> std::io::Result<()> { fn setup_io_error_response(file_path: &FsPath, operation: &str, err: &std::io::Error) -> axum::response::Response { let (status, message) = if err.kind() == ErrorKind::PermissionDenied { - error!( - "Setup mode: permission denied while trying to {operation} '{}': {err}", - file_path.display() - ); - ( - StatusCode::FORBIDDEN, - "Permission denied while accessing requested resource".to_string(), - ) + error!("Setup mode: permission denied while trying to {operation} '{}': {err}", file_path.display()); + (StatusCode::FORBIDDEN, "Permission denied while accessing requested resource".to_string()) } else { - error!( - "Setup mode: failed to {operation} '{}': {err}", - file_path.display() - ); + error!("Setup mode: failed to {operation} '{}': {err}", file_path.display()); (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".to_string()) }; @@ -792,10 +776,7 @@ async fn setup_complete_inner( "Setup mode: failed to serialize setup content for '{}': {serialize_err}", target_path.display() ); - ( - StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({ "error": "Internal server error" })), - ) + (StatusCode::INTERNAL_SERVER_ERROR, axum::Json(json!({ "error": "Internal server error" }))) .into_response() } }; @@ -858,30 +839,28 @@ fn create_compression_layer() -> tower_http::compression::CompressionLayer tower_http::compression::CompressionLayer Result<(), TuliproxError> { let draft = build_initial_draft(paths).await; let (host, port, web_root) = setup_bind_values(&draft, paths.home_path.as_str()); - let web_dir = resolve_setup_web_dir(&web_root, paths.home_path.as_str()) - .ok_or_else(|| TuliproxError::Server(format!("Setup mode requires a web directory. Tried '{}'", web_root.display())))?; + let web_dir = resolve_setup_web_dir(&web_root, paths.home_path.as_str()).ok_or_else(|| { + TuliproxError::Server(format!("Setup mode requires a web directory. Tried '{}'", web_root.display())) + })?; let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); let state = Arc::new(SetupModeState { @@ -965,12 +945,11 @@ pub async fn start_setup_server(paths: &ConfigPaths, missing_files: &[String]) - #[cfg(test)] mod tests { use super::api_proxy_or_default; - use shared::model::{ApiProxyConfigDto, AppConfigDto, TargetUserDto}; - #[cfg(unix)] use super::{setup_complete_inner, SetupCompleteRequestDto, SetupModeState, SetupWebUserCredentialDto}; #[cfg(unix)] use axum::{body::to_bytes, http::StatusCode}; + use shared::model::{ApiProxyConfigDto, AppConfigDto, TargetUserDto}; #[cfg(unix)] use std::{ path::PathBuf, @@ -1011,8 +990,8 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn setup_complete_reports_permission_denied_for_unwritable_target_dir() - -> Result<(), Box> { + async fn setup_complete_reports_permission_denied_for_unwritable_target_dir( + ) -> Result<(), Box> { use std::os::unix::fs::PermissionsExt; struct PermissionResetGuard(PathBuf); impl Drop for PermissionResetGuard { diff --git a/backend/src/api/static_headers.rs b/backend/app/src/api/static_headers.rs similarity index 100% rename from backend/src/api/static_headers.rs rename to backend/app/src/api/static_headers.rs diff --git a/backend/src/api/sys_usage.rs b/backend/app/src/api/sys_usage.rs similarity index 93% rename from backend/src/api/sys_usage.rs rename to backend/app/src/api/sys_usage.rs index 639e145ce..6efb82800 100644 --- a/backend/src/api/sys_usage.rs +++ b/backend/app/src/api/sys_usage.rs @@ -1,8 +1,10 @@ -use crate::api::model::AppState; -use crate::messaging::send_message as send_messaging; -use crate::model::{DiskAlertConfig, MessageContent}; #[cfg(target_os = "linux")] use crate::utils::parse_ascii_u64_bytes; +use crate::{ + api::model::AppState, + messaging::send_message as send_messaging, + model::{DiskAlertConfig, MessageContent}, +}; use shared::model::{DiskAlert, DiskAlertLevel, MsgKind, SystemInfo}; use std::{ sync::Arc, @@ -26,9 +28,7 @@ struct DiskAlertMonitor { } impl DiskAlertMonitor { - fn new() -> Self { - Self { last_level: None, last_notified_at: None } - } + fn new() -> Self { Self { last_level: None, last_notified_at: None } } #[allow(clippy::cast_precision_loss)] fn inspect(&mut self, cfg: &DiskAlertConfig, total_bytes: u64, free_bytes: u64) -> Option { @@ -51,9 +51,8 @@ impl DiskAlertMonitor { let now = Instant::now(); let state_changed = new_level != self.last_level; - let rearm_elapsed = self - .last_notified_at - .is_none_or(|t| now.duration_since(t).as_secs() >= cfg.repeat_interval_secs); + let rearm_elapsed = + self.last_notified_at.is_none_or(|t| now.duration_since(t).as_secs() >= cfg.repeat_interval_secs); let should_notify = new_level.is_some() && (state_changed || rearm_elapsed); @@ -62,13 +61,7 @@ impl DiskAlertMonitor { self.last_level = new_level; if should_notify { self.last_notified_at = Some(now); - Some(DiskAlert { - level: new_level.expect("new_level is Some when should_notify is true"), - total_bytes, - free_bytes, - used_bytes, - percent, - }) + new_level.map(|level| DiskAlert { level, total_bytes, free_bytes, used_bytes, percent }) } else { None } @@ -95,7 +88,7 @@ impl DiskProbe { /// `None` indicates the platform has no disk probe (e.g. fallback path). fn for_cwd() -> Option { let cwd = std::env::current_dir().ok()?; - Some(Self { path: DiskPath::from_path(&cwd) }) + Some(Self { path: DiskPath::from_path(&cwd)? }) } /// Return `(total_bytes, free_bytes_available_to_caller)`. @@ -121,13 +114,14 @@ impl DiskProbe { let mut total_bytes: u64 = 0; let mut total_free_bytes: u64 = 0; // SAFETY: `self.path.0` is a NUL-terminated UTF-16 string built from - // the CWD; the three output pointers alias ULARGE_INTEGER (= u64). + // the CWD; the three output pointers alias writable `u64` slots + // (windows-sys types them as `*mut u64`). let ok = unsafe { - winapi::um::fileapi::GetDiskFreeSpaceExW( + windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW( self.path.0.as_ptr(), - (&raw mut free_bytes_available).cast(), - (&raw mut total_bytes).cast(), - (&raw mut total_free_bytes).cast(), + &mut free_bytes_available, + &mut total_bytes, + &mut total_free_bytes, ) }; if ok == 0 { @@ -148,21 +142,19 @@ fn statvfs_counter_to_u64>(value: T) -> u64 { value.into() } #[cfg(unix)] impl DiskPath { - fn from_path(path: &std::path::Path) -> Self { + fn from_path(path: &std::path::Path) -> Option { use std::os::unix::ffi::OsStrExt; - // CString::new only fails if the path contains interior NULs, which - // `std::env::current_dir` cannot produce on a supported platform. - Self(std::ffi::CString::new(path.as_os_str().as_bytes()).expect("CWD contains NUL byte")) + std::ffi::CString::new(path.as_os_str().as_bytes()).ok().map(Self) } } #[cfg(windows)] impl DiskPath { - fn from_path(path: &std::path::Path) -> Self { + fn from_path(path: &std::path::Path) -> Option { use std::os::windows::ffi::OsStrExt; let mut wide: Vec = path.as_os_str().encode_wide().collect(); wide.push(0); - Self(wide) + Some(Self(wide)) } } @@ -203,7 +195,8 @@ pub fn exec_system_usage(app_state: &Arc) -> tokio::task::JoinHandle<( let Some(alert_cfg) = messaging.disk_alert.as_ref() else { continue }; alert_cfg.clone() }; - let Some(alert) = disk_alert_monitor.inspect(&alert_cfg, info.disk_total_bytes, info.disk_free_bytes) else { + let Some(alert) = disk_alert_monitor.inspect(&alert_cfg, info.disk_total_bytes, info.disk_free_bytes) + else { continue; }; let http_client = state.http_client.load(); @@ -562,18 +555,18 @@ mod platform { mod platform { use super::{CpuTracker, DiskProbe, SystemInfo}; use std::mem::size_of; - use winapi::{ - shared::minwindef::FILETIME, - um::{ - processthreadsapi::{GetCurrentProcess, GetProcessTimes}, - psapi::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, - sysinfoapi::{GlobalMemoryStatusEx, MEMORYSTATUSEX}, + use windows_sys::Win32::{ + Foundation::FILETIME, + System::{ + ProcessStatus::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, + SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX}, + Threading::{GetCurrentProcess, GetProcessTimes}, }, }; /// Pseudo-handle from `GetCurrentProcess` is process-wide and safe across threads. #[derive(Clone, Copy)] - struct ProcessHandle(winapi::um::winnt::HANDLE); + struct ProcessHandle(windows_sys::Win32::Foundation::HANDLE); // SAFETY: only stores the current-process pseudo-handle (-1), not an owned HANDLE. unsafe impl Send for ProcessHandle {} @@ -632,7 +625,7 @@ mod platform { (ok != 0).then_some(status.ullTotalPhys) } - fn query_process_memory_usage(process: winapi::um::winnt::HANDLE) -> Option { + fn query_process_memory_usage(process: windows_sys::Win32::Foundation::HANDLE) -> Option { let mut counters = PROCESS_MEMORY_COUNTERS { cb: u32::try_from(size_of::()).ok()?, ..unsafe { std::mem::zeroed() } @@ -644,14 +637,12 @@ mod platform { } #[allow(clippy::cast_precision_loss)] - fn query_process_cpu_time_secs(process: winapi::um::winnt::HANDLE) -> Option { + fn query_process_cpu_time_secs(process: windows_sys::Win32::Foundation::HANDLE) -> Option { let mut created = unsafe { std::mem::zeroed::() }; let mut exited = unsafe { std::mem::zeroed::() }; let mut kernel = unsafe { std::mem::zeroed::() }; let mut user = unsafe { std::mem::zeroed::() }; - let ok = unsafe { - GetProcessTimes(process, &raw mut created, &raw mut exited, &raw mut kernel, &raw mut user) - }; + let ok = unsafe { GetProcessTimes(process, &raw mut created, &raw mut exited, &raw mut kernel, &raw mut user) }; if ok == 0 { return None; } @@ -772,12 +763,7 @@ mod platform { let mut info = unsafe { zeroed::() }; let mut count = u32::try_from(size_of::() / size_of::()).ok()?; let rc = unsafe { - task_info( - mach_task_self(), - MACH_TASK_BASIC_INFO, - (&raw mut info).cast::(), - &raw mut count, - ) + task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (&raw mut info).cast::(), &raw mut count) }; (rc == KERN_SUCCESS).then_some(info.resident_size) } @@ -813,6 +799,14 @@ mod disk_probe_tests { use super::DiskProbe; use std::path::Path; + #[test] + fn disk_path_rejects_interior_nul() { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let path = Path::new(OsStr::from_bytes(b"invalid\0path")); + assert!(super::DiskPath::from_path(path).is_none()); + } + #[test] fn disk_probe_for_cwd_returns_nonzero_on_linux() { let probe = DiskProbe::for_cwd().expect("CWD should be available on Linux"); @@ -827,7 +821,10 @@ mod disk_probe_tests { fn disk_probe_with_relative_path_resolves_via_kernel() { // Relative paths are resolved against the process CWD by the kernel; // the sample call must not allocate or panic. - let probe = DiskProbe { path: super::DiskPath::from_path(Path::new(".")) }; + let Some(path) = super::DiskPath::from_path(Path::new(".")) else { + unreachable!("a relative path without NUL bytes must be accepted"); + }; + let probe = DiskProbe { path }; let _ = probe.sample(); } } @@ -907,7 +904,8 @@ mod tests { fn test_net_tracker_reports_bytes_per_second() { let mut tracker = super::NetTracker::new(); let _ = tracker.sample(1000, 500); - tracker.last_sample_at = tracker.last_sample_at.map(|instant| instant.checked_sub(Duration::from_secs(2)).unwrap()); + tracker.last_sample_at = + tracker.last_sample_at.map(|instant| instant.checked_sub(Duration::from_secs(2)).unwrap()); let sample = tracker.sample(3000, 1500); assert!((999.0..=1001.0).contains(&sample.rx_bytes_per_sec)); assert!((499.0..=501.0).contains(&sample.tx_bytes_per_sec)); diff --git a/backend/src/api/tasks/config_watch.rs b/backend/app/src/api/tasks/config_watch.rs similarity index 95% rename from backend/src/api/tasks/config_watch.rs rename to backend/app/src/api/tasks/config_watch.rs index ef21d88ec..82e2768cb 100644 --- a/backend/src/api/tasks/config_watch.rs +++ b/backend/app/src/api/tasks/config_watch.rs @@ -13,10 +13,7 @@ use notify::{ event::{AccessKind, AccessMode}, recommended_watcher, EventKind, RecursiveMode, Watcher, }; -use shared::{ - error::TuliproxError, - model::ConfigPaths, -}; +use shared::{error::TuliproxError, model::ConfigPaths}; use std::{ collections::{HashMap, HashSet}, path::{Path, PathBuf}, @@ -56,12 +53,11 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke } }); - let mut watcher = recommended_watcher(std_tx).map_err(|err| { - TuliproxError::Io(format!("Failed to init config file watcher {err}")) - })?; - watcher.watch(path, recursive_mode).map_err(|err| { - TuliproxError::Io(format!("Failed to start config file watcher {err}")) - })?; + let mut watcher = recommended_watcher(std_tx) + .map_err(|err| TuliproxError::Io(format!("Failed to init config file watcher {err}")))?; + watcher + .watch(path, recursive_mode) + .map_err(|err| TuliproxError::Io(format!("Failed to start config file watcher {err}")))?; info!("Watching config file changes {}", path.display()); let event_manager = Arc::clone(&app_state.event_manager); @@ -164,10 +160,7 @@ fn start_config_watch(app_state: &Arc, cancel_token: &CancellationToke Ok(()) } -async fn has_external_source_write( - file_locks: &crate::utils::FileLockManager, - paths: &HashSet, -) -> bool { +async fn has_external_source_write(file_locks: &crate::utils::FileLockManager, paths: &HashSet) -> bool { for path in paths { if !file_locks.is_internal_write_revision(path).await { return true; diff --git a/backend/src/api/tasks/hdhomerun_ssdp.rs b/backend/app/src/api/tasks/hdhomerun_ssdp.rs similarity index 100% rename from backend/src/api/tasks/hdhomerun_ssdp.rs rename to backend/app/src/api/tasks/hdhomerun_ssdp.rs diff --git a/backend/src/api/tasks/library_scan.rs b/backend/app/src/api/tasks/library_scan.rs similarity index 100% rename from backend/src/api/tasks/library_scan.rs rename to backend/app/src/api/tasks/library_scan.rs diff --git a/backend/app/src/api/tasks/mod.rs b/backend/app/src/api/tasks/mod.rs new file mode 100644 index 000000000..e75a9767d --- /dev/null +++ b/backend/app/src/api/tasks/mod.rs @@ -0,0 +1,28 @@ +//! Background tasks the server runs on a schedule or a watch. +//! +//! Measured 2026-08-26: this module is pinned to `api`, mostly by one type. +//! +//! - `config_watch` and `xtream_expiry` read only `app_config` (and one HTTP +//! client), but both call `ConfigFile::load_sources`, and `ConfigFile` exists +//! to reload configuration *into* `AppState` - thirteen references to it. It +//! is the root state's own reload path, so it does not move, and neither do +//! they. +//! - `scheduler` reads ten fields off `AppState`, including `forced_targets` +//! which lives nowhere else, and calls `panel_api::sync_panel_api_exp_dates`. +//! - `hdhomerun_ssdp` (139 lines, `core` only) and `library_scan` (55 lines) are +//! genuinely free. They stay because their natural homes would have to take a +//! dependency to accept them - `library` is a clean `shared`+`core` leaf and +//! would gain `session` for an event publish - and 194 lines is not worth +//! degrading a leaf crate or minting a package to hold. + +mod config_watch; +mod hdhomerun_ssdp; +mod library_scan; +mod scheduler; +mod xtream_expiry; + +pub(in crate::api) use config_watch::*; +pub(in crate::api) use hdhomerun_ssdp::*; +pub(in crate::api) use library_scan::*; +pub(in crate::api) use scheduler::*; +pub(in crate::api) use xtream_expiry::*; diff --git a/backend/src/api/tasks/scheduler.rs b/backend/app/src/api/tasks/scheduler.rs similarity index 92% rename from backend/src/api/tasks/scheduler.rs rename to backend/app/src/api/tasks/scheduler.rs index 5dec40620..51c766d25 100644 --- a/backend/src/api/tasks/scheduler.rs +++ b/backend/app/src/api/tasks/scheduler.rs @@ -1,16 +1,21 @@ use crate::{ api::{ - tasks::{spawn_library_scan, LibraryScanTaskOptions}, model::AppState, + tasks::{spawn_library_scan, LibraryScanTaskOptions}, }, model::{AppConfig, ProcessTargets, ScheduleConfig}, - processing::geoip::{update_geoip_db, GeoIpUpdateError}, - processing::processor::exec_processing, + processing::{ + geoip::{update_geoip_db, GeoIpUpdateError}, + processor::exec_processing, + }, utils::exit, }; use chrono::{DateTime, FixedOffset, Local}; use cron::Schedule; -use shared::{model::ScheduleTaskType, utils::{interner_gc, interner_len}}; +use shared::{ + model::ScheduleTaskType, + utils::{interner_gc, interner_len}, +}; use std::{ str::FromStr, sync::Arc, @@ -33,11 +38,7 @@ pub fn datetime_to_instant(datetime: DateTime) -> Instant { Instant::now() + duration_until } -pub fn exec_scheduler( - client: &reqwest::Client, - app_state: &Arc, - cancel: &CancellationToken, -) { +pub fn exec_scheduler(client: &reqwest::Client, app_state: &Arc, cancel: &CancellationToken) { let cfg = &app_state.app_config; let config = cfg.config.load(); let schedules: Vec = @@ -53,11 +54,7 @@ pub fn exec_scheduler( .is_some_and(|geoip| geoip.enabled), }; if !task_enabled { - log::info!( - "Skipping disabled scheduled task {:?} ({})", - schedule.task_type, - schedule.schedule - ); + log::info!("Skipping disabled scheduled task {:?} ({})", schedule.task_type, schedule.schedule); continue; } @@ -101,14 +98,7 @@ pub fn exec_scheduler( } ScheduleTaskType::LibraryScan | ScheduleTaskType::GeoIpUpdate => { tokio::spawn(async move { - start_scheduler( - http_client, - expression.as_str(), - task_type, - app_state_clone, - cancel_token, - ) - .await; + start_scheduler(http_client, expression.as_str(), task_type, app_state_clone, cancel_token).await; }); } } @@ -192,22 +182,26 @@ async fn run_playlist_update_inner( }; // Re-resolve targets from the CURRENT sources and forced_targets each time, // so that input/target ID changes from hot-reloads are picked up. - let targets = get_process_targets( - &app_state.app_config, - &app_state.forced_targets.load(), - schedule_target_names, - ); + let targets = get_process_targets(&app_state.app_config, &app_state.forced_targets.load(), schedule_target_names); exec_processing( client, Arc::clone(&app_state.app_config), targets, Some(Arc::clone(&app_state.event_manager)), - Some(Arc::clone(app_state)), + Some({ + let state = Arc::clone(app_state); + std::sync::Arc::new(move || { + let state = Arc::clone(&state); + Box::pin(async move { crate::api::sync_panel_api_exp_dates(&state).await }) + as std::pin::Pin + Send>> + }) + }), Some(app_state.playlists.clone()), Some(app_state.update_guard.clone()), app_state.get_disabled_headers(), Some(Arc::clone(&app_state.active_provider)), - Some(Arc::clone(&app_state.metadata_manager)), + Some(Arc::clone(&app_state.metadata_manager) + as std::sync::Arc), None, Some(permit), ) @@ -276,7 +270,8 @@ fn run_library_scan(client: &reqwest::Client, app_state: &Arc) { fn run_geoip_update(app_state: &Arc) { let app_state = Arc::clone(app_state); tokio::spawn(async move { - if let Err(err) = update_geoip_db(&app_state).await { + if let Err(err) = update_geoip_db(&app_state.app_config, &app_state.http_client.load(), &app_state.geoip).await + { if !matches!(err, GeoIpUpdateError::Disabled) { log::error!("Scheduled GeoIp update failed: {err}"); } diff --git a/backend/src/api/tasks/xtream_expiry.rs b/backend/app/src/api/tasks/xtream_expiry.rs similarity index 90% rename from backend/src/api/tasks/xtream_expiry.rs rename to backend/app/src/api/tasks/xtream_expiry.rs index 76e86e095..4703f17db 100644 --- a/backend/src/api/tasks/xtream_expiry.rs +++ b/backend/app/src/api/tasks/xtream_expiry.rs @@ -1,13 +1,18 @@ use crate::{ api::{config_file::ConfigFile, model::AppState}, + config_loader::{persist_source_config, read_sources_file_from_path}, iptv::xtream::get_xtream_stream_url_base, repository::{csv_patch_batch_update_exp_dates, get_csv_file_path, BatchExpDateUpdate}, - utils::{persist_source_config, read_sources_file_from_path, request}, + utils::request, }; use chrono::Utc; use log::{debug, warn}; use shared::{error::TuliproxError, model::InputType}; -use std::{collections::{HashMap, HashSet}, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + time::Duration, +}; use tokio_util::sync::CancellationToken; const REFRESH_INTERVAL_SECS: i64 = 24 * 60 * 60; @@ -247,14 +252,14 @@ fn prune_state(state: &mut ExpiryState, accounts: &[Account], now: i64) -> bool fn collect_accounts(app_state: &AppState) -> Vec { let sources = app_state.app_config.sources.load(); - sources.inputs.iter().filter(|input| input.enabled && input.input_type.is_xtream()).flat_map(|input| { - let batch_url = input.t_batch_url.clone(); - let panel = Arc::::from(panel_identity(&input.url, &input.name)); - let root = input - .username - .as_ref() - .zip(input.password.as_ref()) - .and_then(|(username, password)| { + sources + .inputs + .iter() + .filter(|input| input.enabled && input.input_type.is_xtream()) + .flat_map(|input| { + let batch_url = input.t_batch_url.clone(); + let panel = Arc::::from(panel_identity(&input.url, &input.name)); + let root = input.username.as_ref().zip(input.password.as_ref()).and_then(|(username, password)| { input.resolve_url(&input.url).ok().map(|url| Account { input_name: Arc::clone(&input.name), name: Arc::clone(&input.name), @@ -268,26 +273,31 @@ fn collect_accounts(app_state: &AppState) -> Vec { panel: Arc::clone(&panel), }) }); - root.into_iter().chain(input.aliases.iter().flatten().filter(|alias| alias.enabled).filter_map(move |alias| { - alias.username.as_ref().zip(alias.password.as_ref()).and_then(|(username, password)| { - input.resolve_url(&alias.url).ok().map(|url| Account { - input_name: Arc::clone(&input.name), - name: Arc::clone(&alias.name), - batch_url: batch_url.clone(), - url: url.into_owned(), - source_url: alias.url.clone(), - headers: input.headers.clone(), - username: username.clone(), - password: password.clone(), - exp_date: alias.exp_date, - panel: Arc::clone(&panel), - }) - }) - })) - }).collect() + root.into_iter().chain(input.aliases.iter().flatten().filter(|alias| alias.enabled).filter_map( + move |alias| { + alias.username.as_ref().zip(alias.password.as_ref()).and_then(|(username, password)| { + input.resolve_url(&alias.url).ok().map(|url| Account { + input_name: Arc::clone(&input.name), + name: Arc::clone(&alias.name), + batch_url: batch_url.clone(), + url: url.into_owned(), + source_url: alias.url.clone(), + headers: input.headers.clone(), + username: username.clone(), + password: password.clone(), + exp_date: alias.exp_date, + panel: Arc::clone(&panel), + }) + }) + }, + )) + }) + .collect() } -fn account_url(account: &Account) -> String { get_xtream_stream_url_base(&account.url, &account.username, &account.password) } +fn account_url(account: &Account) -> String { + get_xtream_stream_url_base(&account.url, &account.username, &account.password) +} fn panel_identity(source_url: &str, fallback: &str) -> String { url::Url::parse(source_url).map_or_else( @@ -367,18 +377,14 @@ fn account_is_current(account: &Account, current_keys: &HashSet) -> bool async fn persist_updates(app_state: &Arc, updates: &[(&Account, i64)]) -> Result, TuliproxError> { let current_keys = collect_accounts(app_state).iter().map(account_key).collect::>(); - let updates = updates - .iter() - .copied() - .filter(|(account, _)| account_is_current(account, ¤t_keys)) - .collect::>(); + let updates = + updates.iter().copied().filter(|(account, _)| account_is_current(account, ¤t_keys)).collect::>(); if updates.is_empty() { return Ok(Vec::new()); } let now = Utc::now().timestamp(); - let requires_reload = updates - .iter() - .any(|(account, exp_date)| account.exp_date != Some(*exp_date) || is_expired_at(*exp_date, now)); + let requires_reload = + updates.iter().any(|(account, exp_date)| account.exp_date != Some(*exp_date) || is_expired_at(*exp_date, now)); let mut source_updates = Vec::new(); let mut batch_updates = HashMap::>::new(); let mut updated_accounts = Vec::new(); @@ -412,12 +418,13 @@ async fn persist_updates(app_state: &Arc, updates: &[(&Account, i64)]) let mut source_changed = false; for (key, input_name, account_name, exp_date) in source_updates { if let Some(input) = sources.inputs.iter_mut().find(|input| input.name == input_name) { - source_changed |= input.update_account_expiration_date(&account_name, exp_date, is_expired_at(exp_date, now))?; + source_changed |= + input.update_account_expiration_date(&account_name, exp_date, is_expired_at(exp_date, now))?; updated_accounts.push(key); } } if source_changed { - persist_source_config(app_state, Some(sources_path), sources).await?; + persist_source_config(&app_state.app_config, Some(sources_path), sources).await?; app_state .app_config .file_locks @@ -440,12 +447,9 @@ async fn persist_updates(app_state: &Arc, updates: &[(&Account, i64)]) match csv_patch_batch_update_exp_dates(InputType::XtreamBatch, &csv_path, &updates, &backup_dir).await { Ok((batch_changed, matched_keys)) => { if batch_changed { - app_state - .app_config - .file_locks - .mark_internal_write_revision(&csv_path) - .await - .map_err(|err| TuliproxError::Io(format!("Failed to track internal alias CSV update: {err}")))?; + app_state.app_config.file_locks.mark_internal_write_revision(&csv_path).await.map_err(|err| { + TuliproxError::Io(format!("Failed to track internal alias CSV update: {err}")) + })?; } updated_accounts.extend(matched_keys); } @@ -523,7 +527,10 @@ mod tests { }; use reqwest::StatusCode; use shared::model::{ConfigInputAliasDto, ConfigInputDto}; - use std::{collections::{HashMap, HashSet}, sync::Arc}; + use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + }; fn account(password: &str) -> Account { Account { @@ -545,12 +552,7 @@ mod tests { let now = 1_000_000; assert!(super::is_expiry_refresh_due(None, None, None, now)); assert!(!super::is_expiry_refresh_due(Some(now + 2 * 24 * 60 * 60), Some(now - 1), None, now)); - assert!(super::is_expiry_refresh_due( - Some(now + 2 * 24 * 60 * 60), - Some(now - 24 * 60 * 60), - None, - now - )); + assert!(super::is_expiry_refresh_due(Some(now + 2 * 24 * 60 * 60), Some(now - 24 * 60 * 60), None, now)); assert!(!super::is_expiry_refresh_due(Some(now + 4 * 24 * 60 * 60), None, None, now)); assert!(!super::is_expiry_refresh_due(None, None, Some(now - 1), now)); } @@ -649,7 +651,10 @@ mod tests { #[test] fn expiry_response_accepts_xtream_string_and_number_formats() { assert_eq!(parse_expiry_date(&serde_json::json!({"user_info": {"exp_date": "2000000000"}})), Ok(2_000_000_000)); - assert_eq!(parse_expiry_date(&serde_json::json!({"user_info": {"exp_date": 2_000_000_000}})), Ok(2_000_000_000)); + assert_eq!( + parse_expiry_date(&serde_json::json!({"user_info": {"exp_date": 2_000_000_000}})), + Ok(2_000_000_000) + ); assert_eq!(parse_expiry_date(&serde_json::json!({"user_info": {"exp_date": null}})), Err(FetchError::Account)); assert_eq!(parse_expiry_date(&serde_json::json!({"user_info": {"exp_date": "0"}})), Err(FetchError::Account)); assert_eq!(parse_expiry_date(&serde_json::json!({"user_info": {"exp_date": -1}})), Err(FetchError::Account)); diff --git a/backend/src/main.rs b/backend/app/src/main.rs similarity index 77% rename from backend/src/main.rs rename to backend/app/src/main.rs index b7f0621e8..95685920d 100644 --- a/backend/src/main.rs +++ b/backend/app/src/main.rs @@ -1,9 +1,12 @@ -#![warn(clippy::all, clippy::pedantic)] -#![allow(clippy::module_name_repetitions)] -#![allow(clippy::must_use_candidate)] -#![allow(clippy::return_self_not_must_use)] -#![allow(clippy::missing_errors_doc)] -#![allow(clippy::large_futures)] +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::dbg_macro))] +// Clippy policy lives in [workspace.lints.clippy] in the root Cargo.toml and is +// opted into by backend/app/Cargo.toml's [lints] workspace = true. + +// Auto-trait resolution for this crate's deeply nested async call chains +// exceeds the default 128-step recursion limit. Without this, rustc emits +// `recursion_depth_exceeding_limit`, which is on its way to becoming a hard +// error (rust-lang/rust#159228). +#![recursion_limit = "256"] // #[cfg(target_os = "linux")] // #[global_allocator] @@ -12,18 +15,32 @@ // #[global_allocator] // static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; +// `debug_if_enabled!`, `trace_if_enabled!`, `exit!` and the config `from_impl!` +// family are exported by tuliprox-core; make them available unqualified as they +// were when they lived in this crate. #[macro_use] -mod modules; +extern crate tuliprox_core; -include_modules!(); +extern crate core; +extern crate env_logger; +extern crate pest; +// `api` used to be declared by an `include_modules!()` macro. rustfmt does not +// expand macros, so it never walked into this module and silently skipped every +// file under `api/` - `cargo fmt --all -- --check` passed on unformatted code. +// Declaring the module directly is what puts that subtree back under the gate. +pub mod api; + +// The media-server anti-corruption layer is its own package; aliased under its +// historical module name so `crate::media_server::X` paths keep resolving. use crate::{ + api::model::create_http_client, auth::generate_password, - library::LibraryProcessor, + library::{LibraryProcessor, MediaToolProbes}, model::{AppConfig, Config, Healthcheck, HealthcheckConfig, ProcessTargets, SourcesConfig}, processing::processor::exec_processing, - repository::run_startup_migrations, - utils::{config_file_reader, db_viewer, init_logger, request::create_client, resolve_env_var}, + repository::{db_viewer, run_startup_migrations}, + utils::{config_file_reader, init_logger, request::create_client, resolve_env_var}, }; use arc_swap::{access::Access, ArcSwap}; use chrono::{DateTime, Utc}; @@ -38,6 +55,19 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; +use tuliprox_auth as auth; +use tuliprox_config_loader as config_loader; +// The configuration model and shared utilities are their own package. Aliased +// under their historical module names so `crate::model::X` and +// `crate::utils::X` keep resolving across the backend. +use tuliprox_core::{model, utils}; +use tuliprox_iptv as iptv; +use tuliprox_library::library; +use tuliprox_media_server as media_server; +use tuliprox_messaging as messaging; +use tuliprox_mpegts as mpegts; +use tuliprox_processing as processing; +use tuliprox_repository as repository; #[allow(clippy::struct_excessive_bools)] #[derive(Parser)] @@ -124,8 +154,8 @@ struct Args { } impl Args { - fn db_viewer_args(&self) -> utils::DbViewerArgs<'_> { - utils::DbViewerArgs::new( + fn db_viewer_args(&self) -> repository::DbViewerArgs<'_> { + repository::DbViewerArgs::new( self.db_xtream_file_name.as_deref(), self.db_m3u_file_name.as_deref(), self.db_epg_file_name.as_deref(), @@ -155,7 +185,7 @@ async fn main() { db_viewer(&args.db_viewer_args()); if let Some(ref sh_input) = args.stream_history { - std::process::exit(utils::stream_history_viewer(sh_input).await); + std::process::exit(repository::stream_history_viewer(sh_input).await); } if args.genpwd { @@ -181,7 +211,7 @@ async fn main() { if args.scan_library || args.force_library_rescan { info!("Library scan mode requested"); let app_config = Arc::new( - utils::read_initial_app_config(&mut config_paths, true, true, false) + crate::config_loader::read_initial_app_config(&mut config_paths, true, true, false) .await .unwrap_or_else(|err| exit!("{}", err)), ); @@ -213,7 +243,7 @@ async fn main() { } } - let app_config = utils::read_initial_app_config(&mut config_paths, true, true, args.server) + let app_config = crate::config_loader::read_initial_app_config(&mut config_paths, true, true, args.server) .await .unwrap_or_else(|err| exit!("{err}")); print_info(&app_config).await; @@ -261,7 +291,7 @@ async fn print_info(app_config: &AppConfig) { if let Some(metadata_update) = config.metadata_update.as_ref() { info!("Metadata path: {}", metadata_update.cache_path); } - runtime_config_report::log_runtime_config_report(app_config).await; + config_loader::runtime_config_report::log_runtime_config_report(app_config).await; } fn get_file_paths(args: &Args) -> ConfigPaths { @@ -319,9 +349,9 @@ fn get_file_paths(args: &Args) -> ConfigPaths { utils::resolve_template_file_path(config_path.as_str(), Some(path.as_str())) }); let storage_path = if Path::new(&config_file).exists() { - match utils::read_config_file_with_options( + match crate::config_loader::read_config_file_with_options( &config_file, - crate::utils::ReadConfigOptions { resolve_env: true, include_computed: false }, + crate::config_loader::ReadConfigOptions { resolve_env: true, include_computed: false }, ) { Ok(cfg) => resolve_storage_path(&home_path, cfg.storage_dir.as_deref()), Err(err) => { @@ -379,10 +409,43 @@ async fn start_in_server_mode(cfg: Arc, targets: Arc) } } +/// Builds a library processor from the application configuration. +/// +/// This was `LibraryProcessor::from_app_config`. It moved here because it was +/// the only thing in `library` that knew what an `AppConfig` is, and because +/// building the HTTP client is the composition root's job, not the library's. +fn library_processor_from_app_config(app_config: &Arc) -> Option { + let Ok(client) = create_http_client(app_config) else { + error!("Failed to create HTTP client for LibraryProcessor, skipping library scan. Please check your configuration."); + return None; + }; + // `Access` is in scope here, so `.load()` alone is ambiguous. + let config = ArcSwap::load(&app_config.config); + let library_config = config.library.as_ref()?; + let probes = { + let for_ffprobe = Arc::clone(app_config); + let for_ffmpeg = Arc::clone(app_config); + MediaToolProbes::new( + Arc::new(move || { + let app_config = Arc::clone(&for_ffprobe); + Box::pin(async move { app_config.is_ffprobe_enabled().await }) + }), + Arc::new(move || { + let app_config = Arc::clone(&for_ffmpeg); + Box::pin(async move { app_config.is_ffmpeg_available().await }) + }), + ) + }; + Some( + LibraryProcessor::new(library_config.clone(), config.metadata_update.as_ref(), client, &config.storage_dir) + .with_tool_probes(probes), + ) +} + async fn scan_library_cli(app_config: &Arc, force_rescan: bool) { info!("Starting Library scan from CLI (force_rescan: {force_rescan})"); - let Some(processor) = LibraryProcessor::from_app_config(app_config) else { + let Some(processor) = library_processor_from_app_config(app_config) else { error!("Library is not enabled in configuration"); std::process::exit(1); }; diff --git a/backend/src/tools/flags_builder.rs b/backend/app/src/tools/flags_builder.rs similarity index 96% rename from backend/src/tools/flags_builder.rs rename to backend/app/src/tools/flags_builder.rs index 1f592fc14..52583b80f 100644 --- a/backend/src/tools/flags_builder.rs +++ b/backend/app/src/tools/flags_builder.rs @@ -1,3 +1,8 @@ +// Build-time asset packer for the flag atlas. Every `as` narrowing below is on a +// length or index bounded by the checked-in SVG set and by the on-disk format's +// own field widths, so truncation cannot occur for any input this tool accepts. +#![allow(clippy::cast_possible_truncation)] + /////////// // Flags Builder creates a flags.dat file from a directory with svg files named with .svg // This file is used by the frontend to display the flags. diff --git a/backend/tests/provider_dns_https_sni.rs b/backend/app/tests/provider_dns_https_sni.rs similarity index 98% rename from backend/tests/provider_dns_https_sni.rs rename to backend/app/tests/provider_dns_https_sni.rs index 57174bbea..9507a11bc 100644 --- a/backend/tests/provider_dns_https_sni.rs +++ b/backend/app/tests/provider_dns_https_sni.rs @@ -20,7 +20,7 @@ struct StrictSniResolver { impl fmt::Debug for StrictSniResolver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StrictSniResolver").field("expected_host", &self.expected_host).finish() + f.debug_struct("StrictSniResolver").field("expected_host", &self.expected_host).finish_non_exhaustive() } } diff --git a/backend/tests/proxy_redirect_auth.rs b/backend/app/tests/proxy_redirect_auth.rs similarity index 93% rename from backend/tests/proxy_redirect_auth.rs rename to backend/app/tests/proxy_redirect_auth.rs index 1a431f149..b2073fa06 100644 --- a/backend/tests/proxy_redirect_auth.rs +++ b/backend/app/tests/proxy_redirect_auth.rs @@ -162,26 +162,15 @@ async fn proxy_auth_lost_on_redirect_repro() { assert_eq!( status, StatusCode::PROXY_AUTHENTICATION_REQUIRED, - "Expected 407 when proxy auth is lost on redirect. missing_auth={}, uris={:?}", - missing_auth, - uris - ); - assert!( - missing_auth > 0, - "Expected missing proxy auth on redirect. missing_auth={}, uris={:?}", - missing_auth, - uris + "Expected 407 when proxy auth is lost on redirect. missing_auth={missing_auth}, uris={uris:?}" ); + assert!(missing_auth > 0, "Expected missing proxy auth on redirect. missing_auth={missing_auth}, uris={uris:?}"); } #[cfg(feature = "proxy-auth-regression")] #[tokio::test] async fn proxy_auth_survives_redirect_regression() { let (status, missing_auth, uris) = run_proxy_flow(true).await; - assert_eq!(status, StatusCode::OK, "Expected 200 OK after fix. missing_auth={}, uris={:?}", missing_auth, uris); - assert_eq!( - missing_auth, 0, - "Expected no missing proxy auth after fix. missing_auth={}, uris={:?}", - missing_auth, uris - ); + assert_eq!(status, StatusCode::OK, "Expected 200 OK after fix. missing_auth={missing_auth}, uris={uris:?}"); + assert_eq!(missing_auth, 0, "Expected no missing proxy auth after fix. missing_auth={missing_auth}, uris={uris:?}"); } diff --git a/backend/auth/Cargo.toml b/backend/auth/Cargo.toml new file mode 100644 index 000000000..7eb5fc265 --- /dev/null +++ b/backend/auth/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "tuliprox-auth" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Authentication primitives and access control for tuliprox" + +[lints] +workspace = true + +[dependencies] +shared = { workspace = true } +tuliprox-core = { workspace = true } +tuliprox-repository = { workspace = true } +arc-swap.workspace = true +axum = { version = "0.8.9", features = ["macros", "default", "ws"] } +base64.workspace = true +blake3.workspace = true +chrono.workspace = true +jsonwebtoken = { version = "10.4.0", default-features = false, features = ["aws_lc_rs"] } +log.workspace = true +rand = "0.9.5" +rust-argon2 = "3.0.0" +serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } +rpassword = "7.5.4" +zeroize.workspace = true diff --git a/backend/src/auth/access_token.rs b/backend/auth/src/access_token.rs similarity index 93% rename from backend/src/auth/access_token.rs rename to backend/auth/src/access_token.rs index cba60ae57..8a82f6942 100644 --- a/backend/src/auth/access_token.rs +++ b/backend/auth/src/access_token.rs @@ -1,9 +1,8 @@ -use shared::utils::{hex_decode, hex_encode}; use chrono::Utc; - -pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { - a.len() == b.len() && a.iter().zip(b.iter()).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 -} +use shared::utils::{hex_decode, hex_encode}; +// `constant_time_eq` lives in `utils::crypto_utils`; re-exported here for the +// authentication call sites that have always used it under this path. +pub use tuliprox_core::utils::constant_time_eq; // #[derive(Serialize, Deserialize, Debug)] // struct AccessToken { @@ -77,7 +76,7 @@ fn verify_access_token_at(token_str: &str, secret: &[u8; 32], current_timestamp: #[cfg(test)] mod tests { - use crate::auth::access_token::{create_access_token_at, verify_access_token, verify_access_token_at}; + use crate::access_token::{create_access_token_at, verify_access_token, verify_access_token_at}; use std::panic::catch_unwind; #[test] diff --git a/backend/src/auth/api_user_context.rs b/backend/auth/src/api_user_context.rs similarity index 77% rename from backend/src/auth/api_user_context.rs rename to backend/auth/src/api_user_context.rs index d485b2957..095c4b02b 100644 --- a/backend/src/auth/api_user_context.rs +++ b/backend/auth/src/api_user_context.rs @@ -1,17 +1,13 @@ -use crate::{ - api::{ - api_utils::{ - evaluate_network_access, log_network_access_allowed_geoip_unavailable, log_network_access_denied, - NetworkAccessDecision, NetworkAccessDenyReason, - }, - model::AppState, - }, - model::ProxyUserPermissionDenyReason, -}; +use arc_swap::ArcSwapOption; use axum::response::IntoResponse; use log::debug; use shared::utils::sanitize_sensitive_info; use std::sync::Arc; +use tuliprox_core::model::{AppConfig, ProxyUserPermissionDenyReason}; +use tuliprox_repository::{ + evaluate_network_access, log_network_access_allowed_geoip_unavailable, log_network_access_denied, GeoIp, + NetworkAccessDecision, NetworkAccessDenyReason, +}; #[derive(Debug, Clone)] pub enum PermissionDenyReason { @@ -31,9 +27,7 @@ impl From for PermissionDenyReason { ProxyUserPermissionDenyReason::Banned => PermissionDenyReason::Banned, ProxyUserPermissionDenyReason::Inactive | ProxyUserPermissionDenyReason::UnresolvedPlan - | ProxyUserPermissionDenyReason::InvalidFilter => { - PermissionDenyReason::Inactive - } + | ProxyUserPermissionDenyReason::InvalidFilter => PermissionDenyReason::Inactive, } } } @@ -83,20 +77,21 @@ impl ApiUserAuthError { #[derive(Debug)] pub struct ApiUserContext { - pub user: Arc, - pub target: Arc, - pub fingerprint: crate::auth::Fingerprint, + pub user: Arc, + pub target: Arc, + pub fingerprint: crate::Fingerprint, } /// Checks only network access (no permission check). Used by stream endpoints /// where permission check must happen later with full stream info for `admission_failure_response`. pub fn check_network_access_only( - user: &Arc, - fingerprint: &crate::auth::Fingerprint, - app_state: &Arc, + user: &Arc, + fingerprint: &crate::Fingerprint, + app_config: &Arc, + geoip: &Arc>, ) -> Result<(), ApiUserAuthError> { - let geoip_unavailable_policy = app_state.app_config.get_geoip_unavailable_policy(); - match evaluate_network_access(user, &fingerprint.client_ip, &app_state.geoip, geoip_unavailable_policy) { + let geoip_unavailable_policy = app_config.get_geoip_unavailable_policy(); + match evaluate_network_access(user, &fingerprint.client_ip, geoip, geoip_unavailable_policy) { NetworkAccessDecision::Allowed => Ok(()), NetworkAccessDecision::AllowedGeoIpUnavailable => { log_network_access_allowed_geoip_unavailable(&user.username, &fingerprint.client_ip); @@ -112,32 +107,34 @@ pub fn check_network_access_only( /// Checks network access without logging. Used for high-frequency polling endpoints /// (e.g., `HDHomeRun` `lineup_status`) where repeated log output would be noisy. pub fn try_check_network_access_only( - user: &Arc, - fingerprint: &crate::auth::Fingerprint, - app_state: &Arc, + user: &Arc, + fingerprint: &crate::Fingerprint, + app_config: &Arc, + geoip: &Arc>, ) -> Result<(), ApiUserAuthError> { - let geoip_unavailable_policy = app_state.app_config.get_geoip_unavailable_policy(); - match evaluate_network_access(user, &fingerprint.client_ip, &app_state.geoip, geoip_unavailable_policy) { + let geoip_unavailable_policy = app_config.get_geoip_unavailable_policy(); + match evaluate_network_access(user, &fingerprint.client_ip, geoip, geoip_unavailable_policy) { NetworkAccessDecision::Allowed | NetworkAccessDecision::AllowedGeoIpUnavailable => Ok(()), NetworkAccessDecision::Denied(reason) => Err(ApiUserAuthError::NetworkDenied(reason)), } } pub fn resolve_api_user_context( - user: Arc, - target: Arc, - fingerprint: crate::auth::Fingerprint, - app_state: &Arc, + user: Arc, + target: Arc, + fingerprint: crate::Fingerprint, + app_config: &Arc, + geoip: &Arc>, ) -> Result { // Permission check - if let Some(reason) = user.permission_denied_reason(app_state) { + if let Some(reason) = user.permission_denied_reason(app_config) { debug!("User access denied for {}: {:?}", sanitize_sensitive_info(&user.username), reason); return Err(ApiUserAuthError::PermissionDenied(reason.into())); } // Network access check with policy - let geoip_unavailable_policy = app_state.app_config.get_geoip_unavailable_policy(); - match evaluate_network_access(&user, &fingerprint.client_ip, &app_state.geoip, geoip_unavailable_policy) { + let geoip_unavailable_policy = app_config.get_geoip_unavailable_policy(); + match evaluate_network_access(&user, &fingerprint.client_ip, geoip, geoip_unavailable_policy) { NetworkAccessDecision::Allowed => Ok(ApiUserContext { user, target, fingerprint }), NetworkAccessDecision::AllowedGeoIpUnavailable => { log_network_access_allowed_geoip_unavailable(&user.username, &fingerprint.client_ip); @@ -153,19 +150,20 @@ pub fn resolve_api_user_context( /// Checks permission and network access without taking ownership. Used when /// user/target are needed after the auth check (avoids unnecessary Arc clones). pub fn check_permission_and_network_access_only( - user: &Arc, - fingerprint: &crate::auth::Fingerprint, - app_state: &Arc, + user: &Arc, + fingerprint: &crate::Fingerprint, + app_config: &Arc, + geoip: &Arc>, ) -> Result<(), ApiUserAuthError> { // Permission check - if let Some(reason) = user.permission_denied_reason(app_state) { + if let Some(reason) = user.permission_denied_reason(app_config) { debug!("User access denied for {}: {:?}", sanitize_sensitive_info(&user.username), reason); return Err(ApiUserAuthError::PermissionDenied(reason.into())); } // Network access check - let geoip_unavailable_policy = app_state.app_config.get_geoip_unavailable_policy(); - match evaluate_network_access(user, &fingerprint.client_ip, &app_state.geoip, geoip_unavailable_policy) { + let geoip_unavailable_policy = app_config.get_geoip_unavailable_policy(); + match evaluate_network_access(user, &fingerprint.client_ip, geoip, geoip_unavailable_policy) { NetworkAccessDecision::Allowed => Ok(()), NetworkAccessDecision::AllowedGeoIpUnavailable => { log_network_access_allowed_geoip_unavailable(&user.username, &fingerprint.client_ip); diff --git a/backend/src/auth/auth_basic.rs b/backend/auth/src/auth_basic.rs similarity index 74% rename from backend/src/auth/auth_basic.rs rename to backend/auth/src/auth_basic.rs index 79b06c7cd..dc43bf576 100644 --- a/backend/src/auth/auth_basic.rs +++ b/backend/auth/src/auth_basic.rs @@ -1,9 +1,9 @@ -use axum::extract::FromRequestParts; -use axum::http::request::Parts; -use axum::http::StatusCode; -use base64::Engine; -use base64::engine::general_purpose; -use crate::auth::Rejection; +use crate::Rejection; +use axum::{ + extract::FromRequestParts, + http::{request::Parts, StatusCode}, +}; +use base64::{engine::general_purpose, Engine}; #[derive(Debug, PartialEq, Eq, Clone)] pub struct AuthBasic(pub (String, String)); @@ -23,9 +23,7 @@ where } impl AuthBasic { - fn from_header(contents: (String, String)) -> Self { - Self(contents) - } + fn from_header(contents: (String, String)) -> Self { Self(contents) } fn decode_request_parts(req: &mut Parts) -> Result { let authorization = req @@ -40,7 +38,7 @@ impl AuthBasic { Some((scheme, contents)) if scheme.eq_ignore_ascii_case("Basic") => { let decoded = decode(contents)?; Ok(Self::from_header(decoded)) - }, + } _ => Err((StatusCode::FORBIDDEN, "`Authorization` header must be a basic auth")), } } @@ -49,9 +47,11 @@ impl AuthBasic { /// Decodes the two parts of basic auth using the colon fn decode(input: &str) -> Result<(String, String), Rejection> { // Decode from base64 into a string - let decoded = general_purpose::STANDARD.decode(input).map_err(|_| (StatusCode::FORBIDDEN, "Authorization header contains invalid characters"))?; - let decoded = String::from_utf8(decoded).map_err(|_| (StatusCode::FORBIDDEN, "Authorization header contains invalid characters"))?; - + let decoded = general_purpose::STANDARD + .decode(input) + .map_err(|_| (StatusCode::FORBIDDEN, "Authorization header contains invalid characters"))?; + let decoded = String::from_utf8(decoded) + .map_err(|_| (StatusCode::FORBIDDEN, "Authorization header contains invalid characters"))?; // Return depending on if password is present if let Some((username, password)) = decoded.split_once(':') { diff --git a/backend/src/auth/auth_bearer.rs b/backend/auth/src/auth_bearer.rs similarity index 74% rename from backend/src/auth/auth_bearer.rs rename to backend/auth/src/auth_bearer.rs index 8ce04938f..c9b472845 100644 --- a/backend/src/auth/auth_bearer.rs +++ b/backend/auth/src/auth_bearer.rs @@ -1,8 +1,8 @@ -use axum::extract::FromRequestParts; -use axum::http::HeaderMap; -use axum::http::request::Parts; -use axum::http::StatusCode; -use crate::auth::Rejection; +use crate::Rejection; +use axum::{ + extract::FromRequestParts, + http::{request::Parts, HeaderMap, StatusCode}, +}; #[derive(Debug, PartialEq, Eq, Clone)] pub struct AuthBearer(pub String); @@ -22,11 +22,9 @@ where } impl AuthBearer { - fn from_header(contents: &str) -> Self { - Self(contents.to_string()) - } + fn from_header(contents: &str) -> Self { Self(contents.to_string()) } - pub(crate) fn from_headers(headers: &HeaderMap) -> Result { + pub fn from_headers(headers: &HeaderMap) -> Result { let authorization = headers .get(axum::http::header::AUTHORIZATION) .ok_or((StatusCode::FORBIDDEN, "Authorization header is missing"))? @@ -40,7 +38,5 @@ impl AuthBearer { } } - fn decode_request_parts(req: &mut Parts) -> Result { - Self::from_headers(&req.headers) - } + fn decode_request_parts(req: &mut Parts) -> Result { Self::from_headers(&req.headers) } } diff --git a/backend/src/auth/authenticator.rs b/backend/auth/src/authenticator.rs similarity index 73% rename from backend/src/auth/authenticator.rs rename to backend/auth/src/authenticator.rs index d6adf1053..1d424d730 100644 --- a/backend/src/auth/authenticator.rs +++ b/backend/auth/src/authenticator.rs @@ -1,19 +1,13 @@ -use crate::{ - api::{api_utils::get_username_from_auth_header, model::AppState}, - auth::AuthBearer, - model::WebAuthConfig, -}; use chrono::{Duration, Local}; use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation}; -use log::warn; use shared::{ error::to_io_error, model::{ - permission::{permission_to_name, Permission, PermissionSet, PERM_ALL}, + permission::{PermissionSet, PERM_ALL}, Claims, UserId, CURRENT_PERMISSION_SCHEMA_VERSION, ROLE_ADMIN, ROLE_API_USER, }, }; -use std::sync::Arc; +use tuliprox_core::model::WebAuthConfig; pub fn create_jwt_admin( web_auth_config: &WebAuthConfig, @@ -98,7 +92,7 @@ fn create_jwt( } } -pub(crate) fn verify_token(token: &str, secret_key: &[u8]) -> Option> { +pub fn verify_token(token: &str, secret_key: &[u8]) -> Option> { if let Ok(token_data) = decode::(token, &DecodingKey::from_secret(secret_key), &Validation::new(Algorithm::HS256)) { @@ -172,7 +166,7 @@ impl std::fmt::Display for AuthError { /// version, and [`AuthError::MissingSubject`] when the /// `subject_id` is `None`. A token that passes both checks is fit /// for downstream permission and authorization checks. -pub(crate) fn validate_token_claims(claims: &Claims) -> Result<(), AuthError> { +pub fn validate_token_claims(claims: &Claims) -> Result<(), AuthError> { if claims.permission_schema_version < CURRENT_PERMISSION_SCHEMA_VERSION { return Err(AuthError::StaleSchema); } @@ -182,113 +176,13 @@ pub(crate) fn validate_token_claims(claims: &Claims) -> Result<(), AuthError> { Ok(()) } -fn validate_request( - app_state: &Arc, - token: &str, - verify_fn: fn(&str, &[u8]) -> bool, -) -> Result<(), AuthError> { - let config = app_state.app_config.config.load(); - let Some(web_auth_config) = config.web_ui.as_ref().and_then(|c| c.auth.as_ref()) else { - return Err(AuthError::InvalidToken); - }; - let secret_key = web_auth_config.secret.as_ref(); - let token_data = verify_token(token, secret_key).ok_or(AuthError::InvalidToken)?; - validate_token_claims(&token_data.claims)?; - if !verify_fn(token, secret_key) { - return Err(AuthError::Forbidden); - } - Ok(()) -} - -/// Build a stable, recognizable rejection response. Refresh-required -/// cases carry the `X-Token-Refresh: required` header so the -/// frontend can branch on it. `Forbidden` (a successful authentication -/// that nonetheless cannot perform the action) maps to 403 so it does -/// not collapse into the "you're not authenticated" 401 path. -fn rejection_for(err: AuthError) -> axum::response::Response { - use axum::http::StatusCode; - let status = match &err { - AuthError::Forbidden => StatusCode::FORBIDDEN, - _ => StatusCode::UNAUTHORIZED, - }; - let mut builder = axum::http::Response::builder().status(status); - if err.is_token_refresh_required() { - builder = builder.header("X-Token-Refresh", "required"); - } - builder - .body(axum::body::Body::from(err.to_string())) - .unwrap_or_else(|_| axum::http::Response::new(axum::body::Body::empty())) -} - -pub async fn validator_admin( - axum::extract::State(app_state): axum::extract::State>, - AuthBearer(token): AuthBearer, - request: axum::extract::Request, - next: axum::middleware::Next, -) -> axum::response::Response { - match validate_request(&app_state, &token, verify_token_admin) { - Ok(()) => next.run(request).await, - Err(err) => rejection_for(err), - } -} - -pub async fn validator_api_user( - axum::extract::State(app_state): axum::extract::State>, - AuthBearer(token): AuthBearer, - request: axum::extract::Request, - next: axum::middleware::Next, -) -> axum::response::Response { - if let Some(username) = get_username_from_auth_header(&token, &app_state) { - if let Some(user) = app_state.app_config.get_user_credentials(&username) { - if !user.ui_enabled { - return axum::http::Response::builder() - .status(axum::http::StatusCode::FORBIDDEN) - .body(axum::body::Body::from("principal does not have the required role".to_string())) - .unwrap_or_else(|_| axum::http::Response::new(axum::body::Body::empty())); - } - } - } - match validate_request(&app_state, &token, verify_token_api_user) { - Ok(()) => next.run(request).await, - Err(err) => rejection_for(err), - } -} - -pub async fn require_permission_inner( - permission: Permission, - axum::extract::State(app_state): axum::extract::State>, - AuthBearer(token): AuthBearer, - request: axum::extract::Request, - next: axum::middleware::Next, -) -> axum::response::Response { - let config = app_state.app_config.config.load(); - let Some(web_auth_config) = config.web_ui.as_ref().and_then(|c| c.auth.as_ref()) else { - return rejection_for(AuthError::InvalidToken); - }; - - let Some(token_data) = verify_token(&token, web_auth_config.secret.as_bytes()) else { - return rejection_for(AuthError::InvalidToken); - }; - if let Err(err) = validate_token_claims(&token_data.claims) { - return rejection_for(err); - } - - if !token_data.claims.permissions.contains(permission) { - let denied_permission = permission_to_name(permission).unwrap_or("unknown"); - warn!("User '{}' denied permission '{denied_permission}'", token_data.claims.username); - return rejection_for(AuthError::Forbidden); - } - - next.run(request).await -} - #[cfg(test)] mod tests { use super::*; - use crate::model::WebAuthConfig; use shared::model::{ permission::Permission, Claims, UserId, CURRENT_PERMISSION_SCHEMA_VERSION, ROLE_ADMIN, ROLE_API_USER, }; + use tuliprox_core::model::WebAuthConfig; fn test_web_auth_config() -> WebAuthConfig { WebAuthConfig { diff --git a/backend/auth/src/lib.rs b/backend/auth/src/lib.rs new file mode 100644 index 000000000..95a83e65a --- /dev/null +++ b/backend/auth/src/lib.rs @@ -0,0 +1,19 @@ +use axum::http::StatusCode; + +mod access_token; +mod api_user_context; +mod auth_basic; +mod auth_bearer; +mod authenticator; +// Client identity moved to `tuliprox-core`: the connection layer keys on it +// too. Re-exported so auth call sites keep their name. +pub use tuliprox_core::model::Fingerprint; +mod password; +mod recording_auth; + +type Rejection = (StatusCode, &'static str); + +pub use self::{ + access_token::*, api_user_context::*, auth_basic::*, auth_bearer::*, authenticator::*, password::*, + recording_auth::*, +}; diff --git a/backend/src/auth/password.rs b/backend/auth/src/password.rs similarity index 63% rename from backend/src/auth/password.rs rename to backend/auth/src/password.rs index 6f71b0f8d..2c31f7eb7 100644 --- a/backend/src/auth/password.rs +++ b/backend/auth/src/password.rs @@ -1,13 +1,8 @@ -use rand::{Rng}; -use rand::distr::Alphanumeric; +use rand::{distr::Alphanumeric, Rng}; use shared::error::str_to_io_error; fn generate_salt(length: usize) -> String { - let salt: String = rand::rng() - .sample_iter(&Alphanumeric) - .take(length) - .map(char::from) - .collect(); + let salt: String = rand::rng().sample_iter(&Alphanumeric).take(length).map(char::from).collect(); salt } @@ -38,18 +33,16 @@ pub fn generate_password_from_input(password: &str) -> std::io::Result { pub fn generate_password() -> std::io::Result { match rpassword::prompt_password("password> ") { - Ok(pwd1) => { - match rpassword::prompt_password("retype password> ") { - Ok(pwd2) => { - if pwd1.eq(&pwd2) { - generate_password_from_input(&pwd1) - } else { - Err(str_to_io_error("Passwords don't match")) - } + Ok(pwd1) => match rpassword::prompt_password("retype password> ") { + Ok(pwd2) => { + if pwd1.eq(&pwd2) { + generate_password_from_input(&pwd1) + } else { + Err(str_to_io_error("Passwords don't match")) } - Err(err) => Err(err) } + Err(err) => Err(err), }, - Err(err) => Err(err) + Err(err) => Err(err), } } diff --git a/backend/src/auth/recording_auth.rs b/backend/auth/src/recording_auth.rs similarity index 84% rename from backend/src/auth/recording_auth.rs rename to backend/auth/src/recording_auth.rs index 0e822177c..044e08bb1 100644 --- a/backend/src/auth/recording_auth.rs +++ b/backend/auth/src/recording_auth.rs @@ -17,9 +17,11 @@ //! checks": system retention can act on behalf of any user, but only on //! eligible terminal states and only through the retention worker. -use shared::model::permission::Permission; -use shared::model::recording::{RecordingMetadata, RecordingVisibility}; -use shared::model::{Claims, ROLE_ADMIN, UserId}; +use shared::model::{ + permission::Permission, + recording::{RecordingMetadata, RecordingVisibility}, + Claims, UserId, ROLE_ADMIN, +}; /// Actions the recording system distinguishes. Each action maps to one /// or more HTTP routes and one or more WebSocket messages. New @@ -151,9 +153,7 @@ impl TerminalState { /// `true` when the recording is in a terminal state that is /// eligible for retention delete. Only the completed, failed /// and cancelled states qualify. - pub fn is_eligible_for_retention(&self) -> bool { - matches!(self, Self::Completed | Self::Failed | Self::Cancelled) - } + pub fn is_eligible_for_retention(&self) -> bool { matches!(self, Self::Completed | Self::Failed | Self::Cancelled) } } impl<'a> RecordingSubject<'a> { @@ -175,9 +175,7 @@ impl RecordingDecision { pub fn is_allow(&self) -> bool { matches!(self, Self::Allow) } } -fn is_admin(claims: &Claims) -> bool { - claims.roles.iter().any(|r| r == ROLE_ADMIN) -} +fn is_admin(claims: &Claims) -> bool { claims.roles.iter().any(|r| r == ROLE_ADMIN) } /// The `username` of the synthetic `Claims` the retention worker /// builds to act on shared / orphan / legacy-owned recordings. @@ -206,8 +204,7 @@ pub fn is_system_principal(claims: &Claims) -> bool { // alone would not survive a future auth refactor that issues the // builtin admin subject_id to a real account. The supervisor is // the only place that mints a Claims with both sentinel values. - claims.username == SYSTEM_PRINCIPAL_USERNAME - && claims.subject_id.as_ref().is_some_and(UserId::is_builtin_admin) + claims.username == SYSTEM_PRINCIPAL_USERNAME && claims.subject_id.as_ref().is_some_and(UserId::is_builtin_admin) } fn check_create(claims: &Claims, action: RecordingAction) -> RecordingDecision { @@ -239,17 +236,11 @@ fn owner_of(meta: &RecordingMetadata) -> Option { } } -fn is_visibility(meta: &RecordingMetadata, want: RecordingVisibility) -> bool { - meta.visibility == want -} +fn is_visibility(meta: &RecordingMetadata, want: RecordingVisibility) -> bool { meta.visibility == want } -fn has_recording_read(claims: &Claims) -> bool { - claims.permissions.contains(Permission::RecordingRead) -} +fn has_recording_read(claims: &Claims) -> bool { claims.permissions.contains(Permission::RecordingRead) } -fn has_recording_write(claims: &Claims) -> bool { - claims.permissions.contains(Permission::RecordingWrite) -} +fn has_recording_write(claims: &Claims) -> bool { claims.permissions.contains(Permission::RecordingWrite) } /// The principal decision. Pure function — does not touch the /// filesystem, the network, or the queue. @@ -382,11 +373,7 @@ fn check_read_action( RecordingDecision::Allow } -fn check_mutate_action( - claims: &Claims, - subject_id: &UserId, - meta: &RecordingMetadata, -) -> RecordingDecision { +fn check_mutate_action(claims: &Claims, subject_id: &UserId, meta: &RecordingMetadata) -> RecordingDecision { if !has_recording_write(claims) { return RecordingDecision::Deny(DenyReason::MissingPermission(Permission::RecordingWrite)); } @@ -407,10 +394,7 @@ fn check_mutate_action( fn action_requires_owner(action: RecordingAction) -> bool { matches!( action, - RecordingAction::Edit - | RecordingAction::Cancel - | RecordingAction::Delete - | RecordingAction::ManageRule + RecordingAction::Edit | RecordingAction::Cancel | RecordingAction::Delete | RecordingAction::ManageRule ) } @@ -442,8 +426,10 @@ pub fn authorize_orphan(claims: &Claims) -> RecordingDecision { #[cfg(test)] mod tests { use super::*; - use shared::model::recording::{AiringStatus, EpgEpisodeMetadata, RecordingOwner}; - use shared::model::{Claims, CURRENT_PERMISSION_SCHEMA_VERSION, UserId}; + use shared::model::{ + recording::{AiringStatus, EpgEpisodeMetadata, RecordingOwner}, + Claims, UserId, CURRENT_PERMISSION_SCHEMA_VERSION, + }; fn make_claims( username: &str, @@ -502,24 +488,16 @@ mod tests { make_meta(RecordingOwner::User(UserId::from(uid)), visibility) } - fn legacy_meta() -> RecordingMetadata { - make_meta(RecordingOwner::LegacyAdmin, RecordingVisibility::Private) - } + fn legacy_meta() -> RecordingMetadata { make_meta(RecordingOwner::LegacyAdmin, RecordingVisibility::Private) } fn subject(uid: &str) -> UserId { UserId::from(uid) } - fn read_perms() -> shared::model::permission::PermissionSet { - Permission::RecordingRead.into() - } - fn write_perms() -> shared::model::permission::PermissionSet { - Permission::RecordingWrite.into() - } + fn read_perms() -> shared::model::permission::PermissionSet { Permission::RecordingRead.into() } + fn write_perms() -> shared::model::permission::PermissionSet { Permission::RecordingWrite.into() } fn read_write_perms() -> shared::model::permission::PermissionSet { Permission::RecordingRead | Permission::RecordingWrite } - fn config_read_perms() -> shared::model::permission::PermissionSet { - Permission::ConfigRead.into() - } + fn config_read_perms() -> shared::model::permission::PermissionSet { Permission::ConfigRead.into() } // --- read / playback / download --- @@ -528,7 +506,12 @@ mod tests { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], read_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Read, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Read, + &sub, + ); assert!(d.is_allow(), "expected allow, got {d:?}"); } @@ -537,7 +520,12 @@ mod tests { let claims = make_claims("bob", Some(subject("web:bob")), vec!["WEB"], read_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Read, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Read, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::NotOwner))); } @@ -546,7 +534,12 @@ mod tests { let claims = make_claims("bob", Some(subject("web:bob")), vec!["WEB"], read_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Shared); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Read, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Read, + &sub, + ); assert!(d.is_allow()); } @@ -555,7 +548,12 @@ mod tests { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], config_read_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Read, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Read, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::MissingPermission(Permission::RecordingRead)))); } @@ -564,7 +562,12 @@ mod tests { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], read_perms()); let meta = legacy_meta(); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Read, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Read, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::LegacyAdminReserved))); } @@ -574,7 +577,12 @@ mod tests { fn create_private_with_recording_write_is_allowed() { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], write_perms()); let sub = RecordingSubject::new(None, TerminalState::Active, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::CreatePrivate, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::CreatePrivate, + &sub, + ); assert!(d.is_allow()); } @@ -582,7 +590,12 @@ mod tests { fn create_shared_requires_administrator_role() { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], read_write_perms()); let sub = RecordingSubject::new(None, TerminalState::Active, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::CreateShared, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::CreateShared, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::NotAdministrator))); } @@ -590,7 +603,12 @@ mod tests { fn create_shared_with_admin_role_and_recording_write_is_allowed() { let claims = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], read_write_perms()); let sub = RecordingSubject::new(None, TerminalState::Active, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::CreateShared, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::CreateShared, + &sub, + ); assert!(d.is_allow()); } @@ -601,7 +619,12 @@ mod tests { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Scheduled, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Edit, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Edit, + &sub, + ); assert!(d.is_allow()); } @@ -610,7 +633,12 @@ mod tests { let claims = make_claims("bob", Some(subject("web:bob")), vec!["WEB"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Scheduled, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Edit, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Edit, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::NotOwner))); } @@ -619,7 +647,12 @@ mod tests { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Shared); let sub = RecordingSubject::new(Some(&meta), TerminalState::Scheduled, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Edit, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Edit, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::NotAdministrator))); } @@ -628,7 +661,12 @@ mod tests { let claims = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], write_perms()); let meta = make_meta(RecordingOwner::User(UserId::from("web:alice")), RecordingVisibility::Shared); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::Delete, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::Delete, + &sub, + ); assert!(d.is_allow()); } @@ -638,11 +676,21 @@ mod tests { fn manage_rule_requires_administrator_and_recording_write() { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], write_perms()); let sub = RecordingSubject::new(None, TerminalState::Active, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::ManageRule, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::ManageRule, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::NotAdministrator))); let admin = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], write_perms()); - let d = authorize(&admin, admin.subject_id.as_ref().expect("test subject_id present"), RecordingAction::ManageRule, &sub); + let d = authorize( + &admin, + admin.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::ManageRule, + &sub, + ); assert!(d.is_allow()); } @@ -653,10 +701,20 @@ mod tests { let claims = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub_active = RecordingSubject::new(Some(&meta), TerminalState::Active, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::SystemRetentionDelete, &sub_active); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::SystemRetentionDelete, + &sub_active, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::IneligibleState))); let sub_deleting = RecordingSubject::new(Some(&meta), TerminalState::Deleting, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::SystemRetentionDelete, &sub_deleting); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::SystemRetentionDelete, + &sub_deleting, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::IneligibleState))); } @@ -669,7 +727,12 @@ mod tests { let claims = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::SystemRetentionDelete, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::SystemRetentionDelete, + &sub, + ); assert!(d.is_allow(), "retention delete must bypass owner for eligible Completed; got {d:?}"); } @@ -678,7 +741,12 @@ mod tests { let claims = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], read_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::SystemRetentionDelete, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::SystemRetentionDelete, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::MissingPermission(Permission::RecordingWrite)))); } @@ -687,7 +755,12 @@ mod tests { let claims = make_claims("alice", Some(subject("web:alice")), vec!["WEB"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, true); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::SystemRetentionDelete, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::SystemRetentionDelete, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::NotAdministrator))); } @@ -696,7 +769,12 @@ mod tests { let claims = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], write_perms()); let meta = owner_meta("web:alice", RecordingVisibility::Private); let sub = RecordingSubject::new(Some(&meta), TerminalState::Completed, false); - let d = authorize(&claims, claims.subject_id.as_ref().expect("test subject_id present"), RecordingAction::SystemRetentionDelete, &sub); + let d = authorize( + &claims, + claims.subject_id.as_ref().expect("test subject_id present"), + RecordingAction::SystemRetentionDelete, + &sub, + ); assert!(matches!(d, RecordingDecision::Deny(DenyReason::InvalidPath))); } @@ -725,7 +803,10 @@ mod tests { assert!(matches!(authorize_orphan(&web), RecordingDecision::Deny(DenyReason::NotAdministrator))); let admin = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], config_read_perms()); - assert!(matches!(authorize_orphan(&admin), RecordingDecision::Deny(DenyReason::MissingPermission(Permission::RecordingRead)))); + assert!(matches!( + authorize_orphan(&admin), + RecordingDecision::Deny(DenyReason::MissingPermission(Permission::RecordingRead)) + )); let admin_ok = make_claims("admin", Some(UserId::builtin_admin()), vec!["ADMIN"], read_perms()); assert!(authorize_orphan(&admin_ok).is_allow()); @@ -752,12 +833,7 @@ mod tests { let mut perms = shared::model::permission::PermissionSet::new(); perms.set(Permission::RecordingWrite); perms.set(Permission::RecordingRead); - let claims = make_claims( - SYSTEM_PRINCIPAL_USERNAME, - Some(UserId::builtin_admin()), - vec![ROLE_ADMIN], - perms, - ); + let claims = make_claims(SYSTEM_PRINCIPAL_USERNAME, Some(UserId::builtin_admin()), vec![ROLE_ADMIN], perms); assert!(is_system_principal(&claims)); } @@ -769,12 +845,7 @@ mod tests { let mut perms = shared::model::permission::PermissionSet::new(); perms.set(Permission::RecordingWrite); perms.set(Permission::RecordingRead); - let claims = make_claims( - "alice", - Some(UserId::builtin_admin()), - vec![ROLE_ADMIN], - perms, - ); + let claims = make_claims("alice", Some(UserId::builtin_admin()), vec![ROLE_ADMIN], perms); assert!(!is_system_principal(&claims)); } @@ -788,12 +859,7 @@ mod tests { let mut perms = shared::model::permission::PermissionSet::new(); perms.set(Permission::RecordingWrite); perms.set(Permission::RecordingRead); - let claims = make_claims( - SYSTEM_PRINCIPAL_USERNAME, - Some(UserId::builtin_admin()), - vec![ROLE_ADMIN], - perms, - ); + let claims = make_claims(SYSTEM_PRINCIPAL_USERNAME, Some(UserId::builtin_admin()), vec![ROLE_ADMIN], perms); let meta = make_meta(RecordingOwner::User(UserId::from("web:alice")), RecordingVisibility::Private); let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, false); assert_eq!( diff --git a/backend/src/auth/user.rs b/backend/auth/src/user.rs similarity index 100% rename from backend/src/auth/user.rs rename to backend/auth/src/user.rs diff --git a/backend/btree/Cargo.toml b/backend/btree/Cargo.toml new file mode 100644 index 000000000..621e10622 --- /dev/null +++ b/backend/btree/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "tuliprox-btree" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Persistent B+Tree storage engine used by tuliprox" + +[lints] +workspace = true + +[features] +# Publishes the fixture builders that construct legacy v2 databases and +# sorted-index sidecars by hand. They are `#[cfg(test)]` inside this crate; the +# feature exposes the same items to a dependent crate's tests, which need to +# build the same fixtures to exercise startup migration and index fallback. +# Enable it from `[dev-dependencies]` only - it is off for a normal build. +test-support = [] + +[dependencies] +crc32fast = "1.5.1" +fs2 = "0.4.3" +log.workspace = true +lz4_flex.workspace = true +memmap2 = "0.9.11" +parking_lot = "0.12.5" +rmp-serde = "1.3.1" +serde.workspace = true +smallvec = "1.15.2" +# `tempfile` and `uuid` are production dependencies here, not test-only: the v3 +# migration path stages its output through a uniquely named temporary file. +tempfile = "3.27.0" +uuid = { version = "1.26.0", features = ["v4"] } + +[dev-dependencies] +rand = "0.9.5" diff --git a/backend/btree/src/codec.rs b/backend/btree/src/codec.rs new file mode 100644 index 000000000..0f734f497 --- /dev/null +++ b/backend/btree/src/codec.rs @@ -0,0 +1,40 @@ +//! `MessagePack` codec for persisted B+Tree keys and values. +//! +//! The engine owns its own codec so that the on-disk encoding is a property of +//! the storage engine rather than of whatever application happens to link it. +//! The encoding is byte-for-byte the encoding this database has always written: +//! `rmp_serde` in its default (compact, positional) form. + +use log::error; +use std::io; + +#[inline] +fn encode_error(error: E) -> io::Error { io::Error::other(error.to_string()) } + +#[inline] +pub(crate) fn binary_serialize(value: &T) -> io::Result> +where + T: ?Sized + serde::Serialize, +{ + rmp_serde::to_vec(value).map_err(encode_error) +} + +#[inline] +pub(crate) fn binary_deserialize(value: &[u8]) -> io::Result +where + T: for<'a> serde::Deserialize<'a>, +{ + rmp_serde::from_slice(value).map_err(|error| { + error!("Failed to decode {error}"); + encode_error(error) + }) +} + +#[inline] +pub(crate) fn binary_serialize_into(writer: &mut W, value: &T) -> io::Result<()> +where + W: io::Write, + T: ?Sized + serde::Serialize, +{ + rmp_serde::encode::write(writer, value).map_err(encode_error) +} diff --git a/backend/src/repository/bplustree/common.rs b/backend/btree/src/common.rs similarity index 74% rename from backend/src/repository/bplustree/common.rs rename to backend/btree/src/common.rs index 869dfc1ed..b331db6da 100644 --- a/backend/src/repository/bplustree/common.rs +++ b/backend/btree/src/common.rs @@ -1,7 +1,11 @@ use log::warn; -use memmap2::Mmap; #[cfg(unix)] pub(crate) use memmap2::Advice; +use memmap2::Mmap; +#[cfg(unix)] +use std::os::unix::fs::FileExt; +#[cfg(windows)] +use std::os::windows::fs::FileExt; use std::{ ffi::OsString, fs::{self, File}, @@ -9,11 +13,6 @@ use std::{ path::{Path, PathBuf}, }; -#[cfg(unix)] -use std::os::unix::fs::FileExt; -#[cfg(windows)] -use std::os::windows::fs::FileExt; - /// Windows/memmap2 has no madvise-style Advice API; keep a stub so callers stay portable. #[cfg(not(unix))] #[derive(Debug, Clone, Copy)] @@ -106,7 +105,7 @@ pub(crate) fn write_all_at_offset(file: &File, buf: &[u8], offset: u64) -> io::R Ok(()) } -pub(crate) fn sidecar_lock_path(filepath: &Path) -> PathBuf { +pub fn sidecar_lock_path(filepath: &Path) -> PathBuf { if let Some(stem) = filepath.file_stem() { let mut name = OsString::from("."); name.push(stem); @@ -132,30 +131,22 @@ pub(crate) fn resolved_path_identity(path: &Path) -> io::Result { format!("path has no file name for identity resolution: {}", path.display()), ) })?; - let parent = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - fs::canonicalize(parent) - .map(|resolved_parent| resolved_parent.join(leaf)) - .map_err(|parent_error| { - io::Error::new( - parent_error.kind(), - format!( - "failed to resolve parent directory for path identity {}: {parent_error}", - path.display() - ), - ) - }) + let parent = + path.parent().filter(|parent| !parent.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")); + fs::canonicalize(parent).map(|resolved_parent| resolved_parent.join(leaf)).map_err(|parent_error| { + io::Error::new( + parent_error.kind(), + format!("failed to resolve parent directory for path identity {}: {parent_error}", path.display()), + ) + }) + } + Err(error) => { + Err(io::Error::new(error.kind(), format!("failed to resolve path identity {}: {error}", path.display()))) } - Err(error) => Err(io::Error::new( - error.kind(), - format!("failed to resolve path identity {}: {error}", path.display()), - )), } } -pub(crate) fn ensure_distinct_sidecar_lock_domains(published: &Path, staging: &Path) -> io::Result<()> { +pub fn ensure_distinct_sidecar_lock_domains(published: &Path, staging: &Path) -> io::Result<()> { let published_lock = sidecar_lock_path(published); let staging_lock = sidecar_lock_path(staging); let published_identity = resolved_path_identity(&published_lock)?; @@ -178,23 +169,47 @@ pub(crate) fn remove_file_if_exists(path: &Path) -> io::Result<()> { match fs::remove_file(path) { Ok(()) => Ok(()), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(io::Error::new( - error.kind(), - format!("failed to remove file {}: {error}", path.display()), - )), + Err(error) => Err(io::Error::new(error.kind(), format!("failed to remove file {}: {error}", path.display()))), } } -pub(crate) fn parent_or_dot(path: &Path) -> &Path { - path.parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")) +/// Extension of the sorted-index sidecar that accompanies a published database. +/// +/// The sidecar name is derived from the database name by the storage engine, not +/// by the application: a reader that opens `x.db` has to find `x.idx` beside it. +pub(crate) const FILE_SUFFIX_INDEX: &str = "idx"; + +/// Path of the sorted-index sidecar belonging to `db_path`. +pub fn get_file_path_for_db_index(db_path: &Path) -> PathBuf { db_path.with_extension(FILE_SUFFIX_INDEX) } + +/// Buffer size for the engine's own buffered readers and writers. +pub(crate) const IO_BUFFER_SIZE: usize = 256 * 1024; + +/// Only the v2 write path buffers writes, and that path is a fixture builder. +#[cfg(any(test, feature = "test-support"))] +pub(crate) fn file_writer(w: W) -> io::BufWriter { io::BufWriter::with_capacity(IO_BUFFER_SIZE, w) } + +pub(crate) fn file_reader(r: R) -> io::BufReader { io::BufReader::with_capacity(IO_BUFFER_SIZE, r) } + +/// Move `src` onto `dest`, degrading to a copy when the rename fails. +/// +/// This is the semantics the v2 writer has always relied on: a rename that +/// cannot be performed (for example across a filesystem boundary) falls back to +/// a copy, and the source file is left in place for the caller to clean up. +#[cfg(any(test, feature = "test-support"))] +pub(crate) fn rename_or_copy(src: &Path, dest: &Path) -> io::Result<()> { + if fs::rename(src, dest).is_err() { + fs::copy(src, dest)?; + } + Ok(()) } -pub(crate) fn same_parent_directory(left: &Path, right: &Path) -> bool { - parent_or_dot(left) == parent_or_dot(right) +pub(crate) fn parent_or_dot(path: &Path) -> &Path { + path.parent().filter(|parent| !parent.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")) } +pub(crate) fn same_parent_directory(left: &Path, right: &Path) -> bool { parent_or_dot(left) == parent_or_dot(right) } + pub(crate) fn require_same_parent_directory(staging: &Path, published: &Path) -> io::Result<()> { if same_parent_directory(staging, published) { Ok(()) diff --git a/backend/btree/src/lib.rs b/backend/btree/src/lib.rs new file mode 100644 index 000000000..328e671cc --- /dev/null +++ b/backend/btree/src/lib.rs @@ -0,0 +1,78 @@ +//! Persistent B+Tree storage engine. +//! +//! Everything below this module is engine-internal: page and cell layout, the +//! write-ahead log, sidecar locking, the sorted-index sidecar and the v2 +//! compatibility reader. The list of `pub` items in this file is the engine's +//! entire contract with the rest of the backend, and is the surface that becomes +//! the `tuliprox-btree` crate's API. +//! +//! Nothing in here may name application types, `AppState`, `api`, backend +//! `model`, backend `utils` or a Tuliprox-specific error. + +pub(crate) mod codec; +mod common; +pub(crate) mod sorted_index; +#[cfg(test)] +mod stress; +// The v2 line is read-only in production: only the migration reader uses it. +// Its writer is a fixture builder, published under `test-support`. +#[cfg(any(test, feature = "test-support"))] +pub mod v2; +#[cfg(not(any(test, feature = "test-support")))] +pub(crate) mod v2; +pub(crate) mod v3; + +// --- Trees, queries and iterators ----------------------------------------- +// --- Errors ---------------------------------------------------------------- +pub use common::BPlusTreeError; +// --- On-disk layout -------------------------------------------------------- +// A published database has two companion files whose names the engine derives: +// the sorted-index sidecar and the sidecar lock. Callers that stage, publish or +// clean up databases need both, and they must agree with the engine or a +// staging database can end up sharing a lock domain with a published one. +pub use common::{ensure_distinct_sidecar_lock_domains, get_file_path_for_db_index, sidecar_lock_path}; +// --- Sorted-index sidecar -------------------------------------------------- +/// Iterates a database in sorted-index order, falling back to the caller's own +/// handling when the sidecar is unusable. +pub use sorted_index::v4::OwnedIterator as SortedIndexIterator; +// --- Generic typed migration ---------------------------------------------- +// Only the mechanics live here. Which databases exist, which record schema each +// one holds, and what to do when a migration fails are application decisions and +// live in the application that links this crate. +pub use v3::migration as typed_migration; +// --- Staged publication ---------------------------------------------------- +pub use v3::{publish_staged_database, BPlusTreeStagingArtifacts}; +pub use v3::{ + BPlusTree, BPlusTreeDiskIterator, BPlusTreeDiskIteratorOwned, BPlusTreeMetadata, BPlusTreeQuery, + BPlusTreeRangeIterator, BPlusTreeSerialWriter, BPlusTreeUpdate, FlushPolicy, MAGIC, STORAGE_VERSION, +}; + +// --- Test support ---------------------------------------------------------- +/// Fixture builders for the tests of crates that depend on this one. +/// +/// Nothing here is part of the production API. A dependent's tests have to be +/// able to construct the inputs this engine reads - a legacy v2 database, a +/// sorted-index sidecar with a deliberately corrupt entry - and those builders +/// necessarily reach below the public surface. Publishing them behind a feature +/// keeps them out of the production API instead of widening it to accommodate +/// tests: enable `test-support` from `[dev-dependencies]` and it stays off for +/// a normal build. +#[cfg(any(test, feature = "test-support"))] +pub mod test_support { + pub use crate::{sorted_index::v4, v3::Locator}; + use std::io; + + /// Every entry of a database together with its physical locator, for + /// building a sorted-index sidecar by hand. + pub fn collect_with_locators(query: &mut crate::BPlusTreeQuery) -> io::Result> + where + K: Ord + serde::Serialize + for<'de> serde::Deserialize<'de> + Clone, + V: serde::Serialize + for<'de> serde::Deserialize<'de> + Clone, + { + query.collect_with_locators() + } + + /// The identity a sorted-index sidecar has to carry to be accepted for a + /// given database snapshot. + pub fn snapshot_identity(query: &crate::BPlusTreeQuery) -> ([u8; 16], u64) { query.snapshot_identity() } +} diff --git a/backend/src/repository/bplustree/sorted_index.rs b/backend/btree/src/sorted_index.rs similarity index 92% rename from backend/src/repository/bplustree/sorted_index.rs rename to backend/btree/src/sorted_index.rs index e8d1f03dc..4c5dc280e 100644 --- a/backend/src/repository/bplustree/sorted_index.rs +++ b/backend/btree/src/sorted_index.rs @@ -1,7 +1,7 @@ -pub(crate) mod v4 { +pub mod v4 { use crate::{ - repository::bplustree::v3::{BPlusTreeQuery, Locator}, - utils::{binary_deserialize, binary_serialize}, + codec::{binary_deserialize, binary_serialize}, + v3::{BPlusTreeQuery, Locator}, }; use serde::{Deserialize, Serialize}; use std::{ @@ -84,7 +84,7 @@ pub(crate) mod v4 { read_u64(&bytes, 40) } - pub(crate) struct Writer { + pub struct Writer { output: BufWriter, database_id: [u8; 16], generation: u64, @@ -98,14 +98,14 @@ pub(crate) mod v4 { SortKey: Serialize, K: Serialize, { - pub(crate) fn new(path: &Path, database_id: [u8; 16], generation: u64) -> io::Result { + pub fn new(path: &Path, database_id: [u8; 16], generation: u64) -> io::Result { let file = OpenOptions::new().write(true).create_new(true).open(path)?; let mut output = BufWriter::new(file); output.write_all(&header(database_id, generation, 0))?; Ok(Self { output, database_id, generation, count: 0, body: Vec::new(), _marker: PhantomData }) } - pub(crate) fn push(&mut self, sort_key: &SortKey, primary_key: &K, locator: Locator) -> io::Result<()> { + pub fn push(&mut self, sort_key: &SortKey, primary_key: &K, locator: Locator) -> io::Result<()> { let sort_key = binary_serialize(sort_key)?; let primary_key = binary_serialize(primary_key)?; let sort_key_len = u32::try_from(sort_key.len()).map_err(invalid_integer)?; @@ -124,7 +124,7 @@ pub(crate) mod v4 { Ok(()) } - pub(crate) fn finish(mut self) -> io::Result { + pub fn finish(mut self) -> io::Result { self.output.flush()?; let mut file = self.output.into_inner().map_err(std::io::IntoInnerError::into_error)?; file.seek(SeekFrom::Start(0))?; @@ -162,11 +162,7 @@ pub(crate) mod v4 { SortKey: for<'de> Deserialize<'de>, K: for<'de> Deserialize<'de>, { - pub(crate) fn open( - path: &Path, - expected_database_id: [u8; 16], - expected_generation: u64, - ) -> io::Result { + pub(crate) fn open(path: &Path, expected_database_id: [u8; 16], expected_generation: u64) -> io::Result { let file = File::open(path)?; let length = file.metadata()?.len(); if length < HEADER_LEN as u64 { @@ -271,16 +267,14 @@ pub(crate) mod v4 { let sort_key = binary_deserialize( self.body.get(24..sort_key_end).ok_or_else(|| invalid_data("sort key is truncated"))?, )?; - let serialized_primary_key = self - .body - .get(sort_key_end..expected_len) - .ok_or_else(|| invalid_data("primary key is truncated"))?; + let serialized_primary_key = + self.body.get(sort_key_end..expected_len).ok_or_else(|| invalid_data("primary key is truncated"))?; let primary_key = binary_deserialize(serialized_primary_key)?; Ok(Some(BorrowedEntry { sort_key, primary_key, serialized_primary_key, locator })) } } - pub(crate) struct OwnedIterator { + pub struct OwnedIterator { reader: Reader, query: BPlusTreeQuery, previous_sort_key: Option, @@ -293,14 +287,14 @@ pub(crate) mod v4 { V: for<'de> Deserialize<'de>, SortKey: Ord + for<'de> Deserialize<'de>, { - pub(crate) fn open(query: BPlusTreeQuery, index_path: &Path) -> io::Result { + pub fn open(query: BPlusTreeQuery, index_path: &Path) -> io::Result { let (database_id, generation) = query.snapshot_identity(); let reader = Reader::open(index_path, database_id, generation)?; Ok(Self { reader, query, previous_sort_key: None, finished: false }) } #[cfg(test)] - pub(crate) fn remaining(&self) -> u64 { self.reader.remaining() } + pub fn remaining(&self) -> u64 { self.reader.remaining() } } impl Iterator for OwnedIterator @@ -322,9 +316,7 @@ pub(crate) mod v4 { } else { let BorrowedEntry { sort_key, primary_key, serialized_primary_key, locator } = entry; self.previous_sort_key = Some(sort_key); - self.query - .read_locator_value(locator, serialized_primary_key) - .map(|value| (primary_key, value)) + self.query.read_locator_value(locator, serialized_primary_key).map(|value| (primary_key, value)) } } Ok(None) => { @@ -344,8 +336,10 @@ pub(crate) mod v4 { #[cfg(test)] mod tests { use super::*; - use crate::repository::bplustree::v3::{BPlusTree as V3Tree, BPlusTreeQuery as V3Query, Locator}; - use crate::utils::binary_serialize; + use crate::{ + codec::binary_serialize, + v3::{BPlusTree as V3Tree, BPlusTreeQuery as V3Query, Locator}, + }; use std::{fs, io, io::Write}; use tempfile::tempdir; @@ -433,11 +427,7 @@ mod tests { let mut writer = v4::Writer::::new(&index, database_id, generation)?; writer.push(&1, &entries[0].0, entries[0].2)?; - writer.push( - &2, - &entries[1].0, - Locator { slot_index: u16::MAX, ..entries[1].2 }, - )?; + writer.push(&2, &entries[1].0, Locator { slot_index: u16::MAX, ..entries[1].2 })?; writer.finish()?; let query = V3Query::::try_new(&database)?; diff --git a/backend/src/repository/bplustree/stress.rs b/backend/btree/src/stress.rs similarity index 97% rename from backend/src/repository/bplustree/stress.rs rename to backend/btree/src/stress.rs index 5567486f6..4493c6207 100644 --- a/backend/src/repository/bplustree/stress.rs +++ b/backend/btree/src/stress.rs @@ -554,10 +554,7 @@ fn store_comparison_v2(path: &std::path::Path, entries: &[(String, ComparisonVal tree.store(path).map(|_| ()) } -fn store_comparison_v3( - path: &std::path::Path, - entries: &[(String, ComparisonValue)], -) -> std::io::Result<()> { +fn store_comparison_v3(path: &std::path::Path, entries: &[(String, ComparisonValue)]) -> std::io::Result<()> { let mut tree = super::v3::BPlusTree::new(); for (key, value) in entries { tree.insert(key.clone(), value.clone()); @@ -565,10 +562,7 @@ fn store_comparison_v3( tree.store_with_index(path, |value| value.sort_key).map(|_| ()) } -fn comparison_runs( - name: &str, - mut operation: impl FnMut() -> std::io::Result<()>, -) -> std::io::Result> { +fn comparison_runs(name: &str, mut operation: impl FnMut() -> std::io::Result<()>) -> std::io::Result> { let mut runs = Vec::with_capacity(5); for _ in 0..5 { let started = Instant::now(); @@ -667,10 +661,11 @@ fn bplustree_v2_v3_comparison() -> std::io::Result<()> { black_box(values.len()); Ok(()) })?; - let index_path = crate::repository::storage::get_file_path_for_db_index(&v3_path); + let index_path = crate::common::get_file_path_for_db_index(&v3_path); comparison_runs("v3 locator sorted full scan", || { let query = super::v3::BPlusTreeQuery::::try_new(&v3_path)?; - let iterator = super::sorted_index::v4::OwnedIterator::::open(query, &index_path)?; + let iterator = + super::sorted_index::v4::OwnedIterator::::open(query, &index_path)?; black_box(iterator.collect::>>()?.len()); Ok(()) })?; @@ -722,9 +717,13 @@ fn bplustree_v2_v3_comparison() -> std::io::Result<()> { let grown_size = std::fs::metadata(&compact_path)?.len(); let compact_runs = comparison_runs("v3 compaction", || updater.compact())?; let compacted_size = std::fs::metadata(&compact_path)?.len(); - println!("v3 growth/compaction: grown={grown_size}, compacted={compacted_size}, median={:?}", comparison_median(&compact_runs)); + println!( + "v3 growth/compaction: grown={grown_size}, compacted={compacted_size}, median={:?}", + comparison_median(&compact_runs) + ); - let point_regression = comparison_median(&v3_point).as_secs_f64() / comparison_median(&v2_point).as_secs_f64() - 1.0; + let point_regression = + comparison_median(&v3_point).as_secs_f64() / comparison_median(&v2_point).as_secs_f64() - 1.0; let scan_regression = comparison_median(&v3_scan).as_secs_f64() / comparison_median(&v2_scan).as_secs_f64() - 1.0; println!("gate point-latency regression: {:+.2}%", point_regression * 100.0); println!("gate full-scan duration regression: {:+.2}%", scan_regression * 100.0); diff --git a/backend/src/repository/bplustree/v2.rs b/backend/btree/src/v2.rs similarity index 94% rename from backend/src/repository/bplustree/v2.rs rename to backend/btree/src/v2.rs index bcd0ef30f..e2a154989 100644 --- a/backend/src/repository/bplustree/v2.rs +++ b/backend/btree/src/v2.rs @@ -5,23 +5,24 @@ //! rewrite of existing repositories. The future v3 storage line is expected to //! live behind an explicit version boundary and a typed migration path. +#[cfg(any(test, feature = "test-support"))] +use crate::codec::binary_serialize_into; +pub(crate) use crate::common::BPlusTreeError; +// The v2 write path exists only to build fixtures for the migration tests. +#[cfg(any(test, feature = "test-support"))] +use crate::common::{file_writer, rename_or_copy}; use crate::{ - repository::bplustree::common::{mmap_with_advice, read_exact_at_offset, Advice}, - utils, - utils::binary_deserialize, + codec::binary_deserialize, + common::{file_reader, mmap_with_advice, read_exact_at_offset, Advice}, }; -pub(crate) use crate::repository::bplustree::common::BPlusTreeError; -#[cfg(test)] -use crate::utils::binary_serialize_into; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] use log::error; use memmap2::Mmap; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; -use shared::error::to_io_error; -#[cfg(test)] -use shared::error::string_to_io_error; use smallvec::{smallvec, SmallVec}; +#[cfg(any(test, feature = "test-support"))] +use std::{borrow::Cow, io::Write}; use std::{ collections::HashSet, fs::File, @@ -30,12 +31,7 @@ use std::{ ops::Bound, path::Path, }; -#[cfg(test)] -use std::{ - borrow::Cow, - io::Write, -}; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] use tempfile::NamedTempFile; const PAGE_SIZE: u16 = 4096; pub const PAGE_SIZE_USIZE: usize = PAGE_SIZE as usize; @@ -44,7 +40,7 @@ const FLAG_SIZE: usize = 1; pub(crate) const MAGIC: &[u8; 4] = b"BTRE"; pub(crate) const STORAGE_VERSION: u32 = 2; const HEADER_SIZE: u64 = PAGE_SIZE as u64; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const ROOT_OFFSET_POS: u64 = 8; const METADATA_DATA_START_POS: usize = 20; // Reserve space for metadata (e.g. 4096 - 16 = 4080 bytes max, but let's be safe) @@ -54,7 +50,7 @@ const HEADER_FLAG_HAS_TOMBSTONES: u32 = 1 << 30; const HEADER_METADATA_LEN_MASK: u32 = !(HEADER_FLAG_HAS_METADATA_FLAGS | HEADER_FLAG_HAS_TOMBSTONES); #[inline] -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const fn encode_metadata_len_with_flags(metadata_len: u32, has_tombstones: bool) -> u32 { let mut encoded = metadata_len | HEADER_FLAG_HAS_METADATA_FLAGS; if has_tombstones { @@ -79,29 +75,29 @@ const fn decode_metadata_len_and_flags(raw: u32) -> (u32, bool) { // v2 uses conservative runtime fanout instead of pretending that size_of::() // predicts serialized key size. Multi-block nodes keep existing files compatible. -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const DEFAULT_INNER_ORDER: usize = 64; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const DEFAULT_LEAF_ORDER: usize = 64; // Value packing configuration -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const SMALL_VALUE_THRESHOLD: usize = 256; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const PACK_BLOCK_HEADER_SIZE: usize = 4; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const PACK_VALUE_HEADER_SIZE: usize = 4; // LZ4 compression configuration -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const COMPRESSION_MIN_SIZE: usize = 64; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const COMPRESSION_THRESHOLD_PERCENT: usize = 85; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const COMPRESSION_FLAG_NONE: u8 = 0x00; pub const COMPRESSION_FLAG_LZ4: u8 = 0x01; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const MAGIC_METADATA_TARGET_ID_MAPPING: u8 = 0x01; type TraversalStack = SmallVec<[(u64, usize); 8]>; @@ -156,7 +152,9 @@ type TraversalStack = SmallVec<[(u64, usize); 8]>; */ #[inline] -fn u32_from_bytes(bytes: &[u8]) -> io::Result { Ok(u32::from_le_bytes(bytes.try_into().map_err(to_io_error)?)) } +fn u32_from_bytes(bytes: &[u8]) -> io::Result { + Ok(u32::from_le_bytes(bytes.try_into().map_err(io::Error::other)?)) +} #[inline] fn node_flag_to_is_leaf(flag: u8) -> io::Result { @@ -202,7 +200,7 @@ where } #[inline] -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const fn msgpack_array_header_len(count: usize) -> usize { if count <= 0x0f { 1 @@ -214,7 +212,7 @@ const fn msgpack_array_header_len(count: usize) -> usize { } #[inline] -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const fn msgpack_u64_array_upper_bound_len(count: usize) -> usize { // Worst-case per u64: marker + 8 bytes payload. msgpack_array_header_len(count) + count.saturating_mul(9) @@ -223,7 +221,7 @@ const fn msgpack_u64_array_upper_bound_len(count: usize) -> usize { // Adaptively compress value bytes if beneficial. // Returns borrowed raw bytes when compression is not useful to avoid an // allocation on the common uncompressed path. -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] fn compress_if_beneficial(raw_bytes: &[u8]) -> (u8, Cow<'_, [u8]>) { if raw_bytes.len() >= COMPRESSION_MIN_SIZE { let compressed = lz4_flex::compress_prepend_size(raw_bytes); @@ -303,20 +301,20 @@ where V: Serialize + for<'de> Deserialize<'de> + Clone, { #[inline] - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] const fn new(is_leaf: bool) -> Self { Self { is_leaf, keys: vec![], children: vec![], value_info: vec![], values: vec![] } } #[inline] - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn is_overflow(&self, order: usize) -> bool { self.keys.len() > order } #[inline] - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] const fn get_median_index(order: usize) -> usize { order >> 1 } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn find_leaf_entry(node: &Self) -> Option<&K> { if node.is_leaf { node.keys.first() @@ -327,10 +325,10 @@ where } } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn get_entry_index_upper_bound(&self, key: &K) -> usize { get_entry_index_upper_bound::(&self.keys, key) } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn insert(&mut self, key: K, v: V, inner_order: usize, leaf_order: usize) -> Option { if self.is_leaf { // Use single binary search instead of redundant searches @@ -369,7 +367,7 @@ where None } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn split(&mut self, order: usize) -> Self { let median = Self::get_median_index(order); if self.is_leaf { @@ -389,6 +387,7 @@ where } } + #[cfg(any(test, feature = "test-support"))] #[cfg(test)] fn add_historical_fence_key(&mut self) -> bool { if self.is_leaf { @@ -408,7 +407,7 @@ where } /// Write a packed value block to disk - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn write_packed_block( file: &mut W, buffer: &mut [u8], @@ -418,13 +417,13 @@ where file.seek(SeekFrom::Start(offset))?; // Write count - let count = u32::try_from(values.len()).map_err(to_io_error)?; + let count = u32::try_from(values.len()).map_err(io::Error::other)?; buffer[0..4].copy_from_slice(&count.to_le_bytes()); let mut pos = 4; // Write each value: length + data for (_, value_bytes) in values { - let len = u32::try_from(value_bytes.len()).map_err(to_io_error)?; + let len = u32::try_from(value_bytes.len()).map_err(io::Error::other)?; buffer[pos..pos + 4].copy_from_slice(&len.to_le_bytes()); pos += 4; buffer[pos..pos + value_bytes.len()].copy_from_slice(value_bytes); @@ -441,7 +440,7 @@ where } /// Calculate the serialized size of this node in bytes (rounded up to block size) - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn calculate_serialized_size(&self, serial_buf: &mut Vec) -> io::Result { serial_buf.clear(); @@ -471,7 +470,7 @@ where Ok((blocks * PAGE_SIZE_USIZE) as u64) } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_to_block( &self, file: &mut W, @@ -481,13 +480,13 @@ where ) -> io::Result { serial_buf.clear(); binary_serialize_into(&mut *serial_buf, &self.keys)?; - let keys_len = u32::try_from(serial_buf.len()).map_err(to_io_error)?; + let keys_len = u32::try_from(serial_buf.len()).map_err(io::Error::other)?; if self.is_leaf { let keys_end = serial_buf.len(); // Append info_encoded to serial_buf to avoid second allocation binary_serialize_into(&mut *serial_buf, &self.value_info)?; - let info_len = u32::try_from(serial_buf.len() - keys_end).map_err(to_io_error)?; + let info_len = u32::try_from(serial_buf.len() - keys_end).map_err(io::Error::other)?; let info_slice = &serial_buf[keys_end..]; let content_size = FLAG_SIZE + LEN_SIZE + keys_len as usize + LEN_SIZE + info_len as usize; @@ -545,7 +544,7 @@ where // Append pointers to serial_buf let keys_end = serial_buf.len(); binary_serialize_into(&mut *serial_buf, &pointers)?; - let pointers_len = u32::try_from(serial_buf.len() - keys_end).map_err(to_io_error)?; + let pointers_len = u32::try_from(serial_buf.len() - keys_end).map_err(io::Error::other)?; let pointers_slice = &serial_buf[keys_end..]; file.seek(SeekFrom::Start(parent_start))?; @@ -583,7 +582,7 @@ where /// Serialize the tree in breadth-first order for better disk locality /// This improves query performance by keeping nodes at the same level contiguous - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_breadth_first( &mut self, file: &mut W, @@ -602,7 +601,7 @@ where Ok(start_offset) } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_bfs_pass1_populate_value_info(&mut self, serial_buf: &mut Vec) -> io::Result<()> { let mut current_level_mut = vec![self]; while !current_level_mut.is_empty() { @@ -630,7 +629,7 @@ where if current_pack_size + entry_size <= PAGE_SIZE_USIZE { node.value_info.push(ValueInfo { mode: ValueStorageMode::Packed(u64::from(pack_count), current_pack_index), - length: u32::try_from(size).map_err(to_io_error)?, + length: u32::try_from(size).map_err(io::Error::other)?, cache: Mutex::new(None), }); current_pack_index += 1; @@ -642,7 +641,7 @@ where node.value_info.push(ValueInfo { mode: ValueStorageMode::Packed(u64::from(pack_count), 0), - length: u32::try_from(size).map_err(to_io_error)?, + length: u32::try_from(size).map_err(io::Error::other)?, cache: Mutex::new(None), }); } @@ -658,7 +657,7 @@ where node.value_info.push(ValueInfo { mode: ValueStorageMode::Single(u64::MAX), - length: u32::try_from(stored_size).map_err(to_io_error)?, + length: u32::try_from(stored_size).map_err(io::Error::other)?, cache: Mutex::new(cache), }); } @@ -674,7 +673,7 @@ where Ok(()) } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_bfs_pass2_calculate_offsets( &self, serial_buf: &mut Vec, @@ -708,7 +707,7 @@ where Ok((node_offsets, child_ids_by_node, current_offset)) } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_bfs_pass3_assign_value_offsets(&mut self, mut current_offset: u64) { use std::collections::HashMap; let mut current_level_mut = vec![self]; @@ -750,7 +749,7 @@ where } } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_bfs_pass4_write_nodes( &self, file: &mut W, @@ -783,10 +782,7 @@ where node.serialize_to_block(file, buffer, serial_buf, node_offset)?; } else { let node_child_ids = child_ids_by_node.get(node_id).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "B+Tree serialization missing child id table entry", - ) + io::Error::new(io::ErrorKind::InvalidData, "B+Tree serialization missing child id table entry") })?; let mut child_offsets = Vec::with_capacity(node_child_ids.len()); for child_id in node_child_ids { @@ -799,19 +795,13 @@ where child_offsets.push(*child_offset); } - node.serialize_internal_with_offsets( - file, - buffer, - serial_buf, - node_offset, - &child_offsets, - )?; + node.serialize_internal_with_offsets(file, buffer, serial_buf, node_offset, &child_offsets)?; } } Ok(()) } - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_bfs_pass5_write_values( &self, file: &mut W, @@ -871,7 +861,7 @@ where /// Serialize an internal node with pre-calculated child offsets /// Supports multi-block internal nodes when keys + pointers exceed a single page - #[cfg(test)] + #[cfg(any(test, feature = "test-support"))] fn serialize_internal_with_offsets( &self, file: &mut W, @@ -907,14 +897,14 @@ where // Write keys length and data buffer[write_pos..write_pos + LEN_SIZE] - .copy_from_slice(&u32::try_from(keys_len).map_err(to_io_error)?.to_le_bytes()); + .copy_from_slice(&u32::try_from(keys_len).map_err(io::Error::other)?.to_le_bytes()); write_pos += LEN_SIZE; buffer[write_pos..write_pos + keys_len].copy_from_slice(&serial_buf[0..keys_end]); write_pos += keys_len; // Write pointers length and data buffer[write_pos..write_pos + LEN_SIZE] - .copy_from_slice(&u32::try_from(pointer_len).map_err(to_io_error)?.to_le_bytes()); + .copy_from_slice(&u32::try_from(pointer_len).map_err(io::Error::other)?.to_le_bytes()); write_pos += LEN_SIZE; buffer[write_pos..write_pos + pointer_len].copy_from_slice(&serial_buf[keys_end..]); @@ -1026,7 +1016,7 @@ where offset: u64, nested: bool, ) -> io::Result<(Self, Option>)> { - let start = usize::try_from(offset).map_err(to_io_error)?; + let start = usize::try_from(offset).map_err(io::Error::other)?; let header_end = start .checked_add(FLAG_SIZE + LEN_SIZE) .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "Mmap offset overflow"))?; @@ -1269,7 +1259,7 @@ where )); } - let len = u32::from_le_bytes(block_buffer[pos..pos + 4].try_into().map_err(to_io_error)?) as usize; + let len = u32::from_le_bytes(block_buffer[pos..pos + 4].try_into().map_err(io::Error::other)?) as usize; pos += 4; if i == value_index { @@ -1295,14 +1285,14 @@ where // ----------------------------------------------------------------------------- // Metadata Enum // ----------------------------------------------------------------------------- -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] #[derive(Clone, Debug, PartialEq)] pub enum BPlusTreeMetadata { Empty, TargetIdMapping(u32), } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] impl BPlusTreeMetadata { pub fn to_bytes(&self) -> Vec { match self { @@ -1314,10 +1304,9 @@ impl BPlusTreeMetadata { } } } - } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] #[derive(Debug, Clone)] pub struct BPlusTree { root: BPlusTreeNode, @@ -1327,7 +1316,7 @@ pub struct BPlusTree { dirty: bool, } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const fn sanitize_order(order: usize) -> usize { if order < 2 { 2 @@ -1336,10 +1325,10 @@ const fn sanitize_order(order: usize) -> usize { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] const fn default_orders() -> (usize, usize) { (DEFAULT_INNER_ORDER, DEFAULT_LEAF_ORDER) } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] impl Default for BPlusTree where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, @@ -1348,7 +1337,7 @@ where fn default() -> Self { Self::new() } } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] impl BPlusTree where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, @@ -1404,11 +1393,12 @@ where } } + // Only this crate's own migration tests build a tree with a historical fence. + #[cfg(test)] pub(crate) fn add_historical_fence_key(&mut self) -> bool { self.root.add_historical_fence_key() } - pub(crate) fn remove_last_root_child(&mut self) -> bool { - !self.root.is_leaf && self.root.children.pop().is_some() - } + #[cfg(test)] + pub(crate) fn remove_last_root_child(&mut self) -> bool { !self.root.is_leaf && self.root.children.pop().is_some() } pub fn store(&mut self, filepath: &Path) -> io::Result { if self.dirty { @@ -1431,7 +1421,7 @@ where let temp_dir = tempfile::env::temp_dir(); NamedTempFile::new_in(&temp_dir)? }; - let mut file = utils::file_writer(&tempfile); + let mut file = file_writer(&tempfile); let mut buffer = vec![0u8; PAGE_SIZE_USIZE]; // Write header block 0 @@ -1467,8 +1457,8 @@ where file.flush()?; drop(file); - if let Err(err) = utils::rename_or_copy(tempfile.path(), filepath, false) { - return Err(string_to_io_error(format!( + if let Err(err) = rename_or_copy(tempfile.path(), filepath) { + return Err(io::Error::other(format!( "Temp file rename/copy did not work {} {err}", tempfile.path().to_string_lossy() ))); @@ -1479,7 +1469,6 @@ where Err(err) => Err(err), } } - } fn validate_legacy_tree( @@ -1513,7 +1502,10 @@ where BPlusTreeNode::::deserialize_from_block(file, &mut buffer, offset, false)? }; if node.keys.windows(2).any(|keys| keys[0] >= keys[1]) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "legacy B+Tree node keys are not strictly increasing")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "legacy B+Tree node keys are not strictly increasing", + )); } if node.is_leaf { @@ -1531,7 +1523,8 @@ where continue; } - let pointers = pointers.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "internal node has no pointers"))?; + let pointers = + pointers.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "internal node has no pointers"))?; let has_fence_key = pointers.len() == node.keys.len(); if has_fence_key && (is_root || node.keys.last() != upper.as_ref()) { return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid historical internal-node fence key")); @@ -1546,7 +1539,6 @@ where Ok(()) } - /// `BPlusTreeQuery` performs on-disk queries without loading the entire tree into memory. /// For frequent queries, consider using `BPlusTree::load()` instead, which loads the full tree into memory /// at the cost of higher memory usage. @@ -1603,11 +1595,11 @@ where return Err(io::Error::new(io::ErrorKind::InvalidData, format!("Metadata too large: {metadata_len}"))); } - let mut validation_file = utils::file_reader(file.try_clone()?); + let mut validation_file = file_reader(file.try_clone()?); validate_legacy_tree::(mmap.as_ref(), &mut validation_file, root_offset, file_len)?; Ok(Self { - file: if mmap.is_some() { None } else { Some(utils::file_reader(file)) }, + file: if mmap.is_some() { None } else { Some(file_reader(file)) }, mmap, has_tombstones, buffer: vec![0u8; PAGE_SIZE_USIZE], @@ -1617,13 +1609,9 @@ where }) } - pub fn try_new(filepath: &Path) -> io::Result { - Self::try_from_file(File::open(filepath)?) - } - + pub fn try_new(filepath: &Path) -> io::Result { Self::try_from_file(File::open(filepath)?) } } - impl BPlusTreeQuery where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, @@ -1652,7 +1640,6 @@ where }; RangeLeafIterator::new(self, start_cloned, end_cloned) } - } /// Range scan iterator that seeks into the tree and then walks in-order @@ -1732,11 +1719,7 @@ where return Ok(()); } - let child_idx = if let Some(key) = start_key { - get_entry_index_upper_bound(&node.keys, key) - } else { - 0 - }; + let child_idx = if let Some(key) = start_key { get_entry_index_upper_bound(&node.keys, key) } else { 0 }; let Some(ptrs) = pointers else { self.exhausted = true; diff --git a/backend/src/repository/bplustree/v3/format.rs b/backend/btree/src/v3/format.rs similarity index 94% rename from backend/src/repository/bplustree/v3/format.rs rename to backend/btree/src/v3/format.rs index 2586eac3e..4eabb2328 100644 --- a/backend/src/repository/bplustree/v3/format.rs +++ b/backend/btree/src/v3/format.rs @@ -7,12 +7,12 @@ const PAGE_SIZE_U32: u32 = 4096; pub(crate) const STORAGE_VERSION_V1: u32 = 1; #[cfg(test)] pub(crate) const STORAGE_VERSION_V2: u32 = 2; -pub(crate) const STORAGE_VERSION_V3: u32 = 3; +pub const STORAGE_VERSION_V3: u32 = 3; pub(crate) const MAX_ENCODED_KEY_LEN: usize = 2004; pub(crate) const MAX_CELL_FOOTPRINT: usize = 2032; pub(crate) const MAX_INLINE_STORED_VALUE: usize = 512; pub(crate) const OVERFLOW_PAYLOAD_LEN: usize = 4056; -pub(crate) const MAGIC: &[u8; 4] = b"BTRE"; +pub const MAGIC: &[u8; 4] = b"BTRE"; const DATABASE_CHECKSUM_OFFSET: usize = 72; const PAGE_CHECKSUM_OFFSET: usize = 24; @@ -46,10 +46,7 @@ fn bytes_at(bytes: &[u8], offset: usize) -> io::Result<[u8; N]> fn write_at(bytes: &mut [u8], offset: usize, value: &[u8]) -> io::Result<()> { let end = checked_end(offset, value.len())?; - bytes - .get_mut(offset..end) - .ok_or_else(|| invalid_data("truncated format destination"))? - .copy_from_slice(value); + bytes.get_mut(offset..end).ok_or_else(|| invalid_data("truncated format destination"))?.copy_from_slice(value); Ok(()) } @@ -292,7 +289,8 @@ impl PageHeader { } PageType::Internal => { if self.cell_count == 0 - || self.free_start != expected_slot_end(PAGE_HEADER_LEN + INTERNAL_PREAMBLE_LEN, self.cell_count, kind)? + || self.free_start + != expected_slot_end(PAGE_HEADER_LEN + INTERNAL_PREAMBLE_LEN, self.cell_count, kind)? || self.free_end < self.free_start || usize::from(self.free_end) >= PAGE_SIZE || self.left != 0 @@ -426,9 +424,8 @@ impl InternalCellPrefix { } pub(crate) fn decode(bytes: &[u8], page_id: u64, next_page_id: u64) -> io::Result { - let prefix = bytes - .get(..INTERNAL_CELL_PREFIX_LEN) - .ok_or_else(|| invalid_data("truncated internal cell prefix"))?; + let prefix = + bytes.get(..INTERNAL_CELL_PREFIX_LEN).ok_or_else(|| invalid_data("truncated internal cell prefix"))?; require_zero( prefix.get(2..4).ok_or_else(|| invalid_data("missing internal cell reserved bytes"))?, "internal cell reserved bytes must be zero", @@ -562,9 +559,7 @@ impl LeafCellPrefix { } pub(crate) fn decode(bytes: &[u8], page_id: u64, next_page_id: u64) -> io::Result { - let prefix = bytes - .get(..LEAF_CELL_PREFIX_LEN) - .ok_or_else(|| invalid_data("truncated leaf cell prefix"))?; + let prefix = bytes.get(..LEAF_CELL_PREFIX_LEN).ok_or_else(|| invalid_data("truncated leaf cell prefix"))?; let decoded = Self { key_length: read_u16(prefix, 0)?, value_kind: ValueKind::try_from(read_u8(prefix, 2)?)?, @@ -594,9 +589,7 @@ impl OverflowHeader { write_at(page, PAGE_HEADER_LEN, &self.payload_length.to_le_bytes())?; write_at(page, PAGE_HEADER_LEN + 2, &[0; 6])?; let tail_start = checked_end(40, payload_length)?; - page.get_mut(tail_start..) - .ok_or_else(|| invalid_data("overflow payload exceeds page"))? - .fill(0); + page.get_mut(tail_start..).ok_or_else(|| invalid_data("overflow payload exceeds page"))?.fill(0); Ok(()) } @@ -621,9 +614,7 @@ impl OverflowHeader { pub(crate) fn encode_free_body(page: &mut [u8]) -> io::Result<()> { exact_page(page)?; - page.get_mut(PAGE_HEADER_LEN..) - .ok_or_else(|| invalid_data("missing free page body"))? - .fill(0); + page.get_mut(PAGE_HEADER_LEN..).ok_or_else(|| invalid_data("missing free page body"))?.fill(0); Ok(()) } @@ -659,10 +650,9 @@ impl StoredValue<'_> { } fn compression_is_beneficial(raw_length: usize, stored_length: usize) -> io::Result { - let threshold = raw_length - .checked_mul(COMPRESSION_PERCENT) - .ok_or_else(|| invalid_input("compression threshold overflow"))? - / 100; + let threshold = + raw_length.checked_mul(COMPRESSION_PERCENT).ok_or_else(|| invalid_input("compression threshold overflow"))? + / 100; Ok(stored_length < threshold) } @@ -675,9 +665,7 @@ pub(crate) fn encode_value<'a>(raw: &'a [u8], scratch: &'a mut Vec) -> io::R let maximum = lz4_flex::block::get_maximum_output_size(raw.len()) .checked_add(4) .ok_or_else(|| invalid_input("compressed value size overflow"))?; - scratch - .try_reserve(maximum) - .map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + scratch.try_reserve(maximum).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; scratch.resize(maximum, 0); write_at(scratch, 0, &raw_length.to_le_bytes())?; let output = scratch.get_mut(4..).ok_or_else(|| invalid_data("missing compression output"))?; @@ -708,9 +696,7 @@ pub(crate) fn decompress_value_into<'a>( } let payload = stored.get(4..).ok_or_else(|| invalid_data("missing LZ4 payload"))?; scratch.clear(); - scratch - .try_reserve(logical_length) - .map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + scratch.try_reserve(logical_length).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; scratch.resize(logical_length, 0); let decoded_length = lz4_flex::block::decompress_into(payload, scratch.as_mut_slice()) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("LZ4 decompression failed: {err}")))?; @@ -728,19 +714,8 @@ pub(crate) struct LeafCellRef<'a> { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum LeafValueRef<'a> { - Inline { - compression: Compression, - logical_len: u32, - stored: &'a [u8], - crc32: u32, - }, - Overflow { - compression: Compression, - logical_len: u32, - stored_len: u32, - head: u64, - crc32: u32, - }, + Inline { compression: Compression, logical_len: u32, stored: &'a [u8], crc32: u32 }, + Overflow { compression: Compression, logical_len: u32, stored_len: u32, head: u64, crc32: u32 }, Tombstone, } @@ -750,9 +725,7 @@ impl<'a> LeafCellRef<'a> { let key_end = LEAF_CELL_PREFIX_LEN .checked_add(usize::from(prefix.key_length)) .ok_or_else(|| invalid_data("leaf key range overflow"))?; - let key_bytes = cell - .get(LEAF_CELL_PREFIX_LEN..key_end) - .ok_or_else(|| invalid_data("truncated leaf key"))?; + let key_bytes = cell.get(LEAF_CELL_PREFIX_LEN..key_end).ok_or_else(|| invalid_data("truncated leaf key"))?; let value = match prefix.value_kind { ValueKind::Inline => { let cell_end = key_end @@ -813,9 +786,8 @@ impl<'a> InternalCellRef<'a> { if cell_end != cell.len() { return Err(invalid_data("invalid internal cell length")); } - let key_bytes = cell - .get(INTERNAL_CELL_PREFIX_LEN..cell_end) - .ok_or_else(|| invalid_data("truncated internal key"))?; + let key_bytes = + cell.get(INTERNAL_CELL_PREFIX_LEN..cell_end).ok_or_else(|| invalid_data("truncated internal key"))?; Ok(Self { key_bytes, right_child: prefix.right_child }) } } @@ -834,9 +806,7 @@ fn write_cell(output: &mut Vec, prefix: &[u8], key: &[u8], stored: &[u8]) -> .and_then(|value| value.checked_add(stored.len())) .ok_or_else(|| invalid_input("encoded cell length overflow"))?; output.clear(); - output - .try_reserve(length) - .map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + output.try_reserve(length).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; output.extend_from_slice(prefix); output.extend_from_slice(key); output.extend_from_slice(stored); @@ -909,16 +879,16 @@ pub(crate) fn encode_internal_cell( next_page_id: u64, output: &mut Vec, ) -> io::Result<()> { - let prefix = InternalCellPrefix { key_length: encoded_key_length(key)?, right_child } - .encode(page_id, next_page_id)?; + let prefix = + InternalCellPrefix { key_length: encoded_key_length(key)?, right_child }.encode(page_id, next_page_id)?; write_cell(output, &prefix, key, &[]) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct Locator { - pub(crate) leaf_page_id: u64, - pub(crate) slot_index: u16, - pub(crate) serialized_key_crc32: u32, +pub struct Locator { + pub leaf_page_id: u64, + pub slot_index: u16, + pub serialized_key_crc32: u32, } impl Locator { @@ -980,9 +950,7 @@ pub(crate) fn decompress_value_in_place( scratch.resize(total_length, 0); scratch.copy_within(0..stored_length, logical_length); let (output, encoded) = scratch.split_at_mut(logical_length); - let payload = encoded - .get(4..stored_length) - .ok_or_else(|| invalid_data("missing LZ4 payload"))?; + let payload = encoded.get(4..stored_length).ok_or_else(|| invalid_data("missing LZ4 payload"))?; let decoded_length = lz4_flex::block::decompress_into(payload, output) .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("LZ4 decompression failed: {err}")))?; if decoded_length != logical_length { @@ -995,7 +963,7 @@ pub(crate) fn decompress_value_in_place( #[cfg(test)] mod tests { use super::*; - use crate::repository::bplustree::v3::BPlusTreeMetadata; + use crate::v3::BPlusTreeMetadata; use std::io; const PAGE_ID: u64 = 7; @@ -1226,14 +1194,8 @@ mod tests { #[test] fn leaf_page_header_golden_bytes_round_trip() -> io::Result<()> { let expected = page_fixture(1, 2, 40, 4000, 6, 8, 0x88c9_dd55); - let header = PageHeader { - page_type: PageType::Leaf, - cell_count: 2, - free_start: 40, - free_end: 4000, - left: 6, - right: 8, - }; + let header = + PageHeader { page_type: PageType::Leaf, cell_count: 2, free_start: 40, free_end: 4000, left: 6, right: 8 }; let mut encoded = [0u8; PAGE_SIZE]; header.encode_into(&mut encoded, PAGE_ID, NEXT_PAGE_ID)?; assert_eq!(encoded, expected); @@ -1269,14 +1231,8 @@ mod tests { let mut expected = page_fixture(3, 0, 0, 0, 0, 8, 0x8731_d58d); expected[32..34].copy_from_slice(&3u16.to_le_bytes()); expected[40..43].copy_from_slice(b"abc"); - let header = PageHeader { - page_type: PageType::Overflow, - cell_count: 0, - free_start: 0, - free_end: 0, - left: 0, - right: 8, - }; + let header = + PageHeader { page_type: PageType::Overflow, cell_count: 0, free_start: 0, free_end: 0, left: 0, right: 8 }; let overflow = OverflowHeader { payload_length: 3 }; let mut encoded = [0u8; PAGE_SIZE]; encoded[40..43].copy_from_slice(b"abc"); @@ -1291,14 +1247,8 @@ mod tests { #[test] fn free_page_header_golden_bytes_round_trip() -> io::Result<()> { let expected = page_fixture(4, 0, 0, 0, 0, 8, 0x2864_f2d3); - let header = PageHeader { - page_type: PageType::Free, - cell_count: 0, - free_start: 0, - free_end: 0, - left: 0, - right: 8, - }; + let header = + PageHeader { page_type: PageType::Free, cell_count: 0, free_start: 0, free_end: 0, left: 0, right: 8 }; let mut encoded = [0u8; PAGE_SIZE]; encode_free_body(&mut encoded)?; header.encode_into(&mut encoded, PAGE_ID, NEXT_PAGE_ID)?; @@ -1383,9 +1333,7 @@ mod tests { overflow_head: 0, stored_crc32: 0x1122_3344, }; - let leaf_bytes = [ - 3, 0, 0, 1, 128, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x44, 0x33, 0x22, 0x11, - ]; + let leaf_bytes = [3, 0, 0, 1, 128, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x44, 0x33, 0x22, 0x11]; assert_eq!(leaf.encode(PAGE_ID, NEXT_PAGE_ID)?, leaf_bytes); assert_eq!(LeafCellPrefix::decode(&leaf_bytes, PAGE_ID, NEXT_PAGE_ID)?, leaf); assert_eq!(stored_value_checksum(b"abc"), 0x3524_41c2); @@ -1398,9 +1346,7 @@ mod tests { internal[2] = 1; invalid_data(InternalCellPrefix::decode(&internal, PAGE_ID, NEXT_PAGE_ID))?; - let mut leaf = [ - 3, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - ]; + let mut leaf = [3, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0]; leaf[2] = 9; invalid_data(LeafCellPrefix::decode(&leaf, PAGE_ID, NEXT_PAGE_ID))?; leaf[2] = 0; @@ -1494,10 +1440,7 @@ mod tests { assert!(stored.as_slice().len() < compressible.len() * 85 / 100); assert_eq!(stored.as_slice(), expected); let mut decompression_scratch = Vec::new(); - assert_eq!( - decompress_value_into(stored.as_slice(), 128, 128, &mut decompression_scratch)?, - compressible - ); + assert_eq!(decompress_value_into(stored.as_slice(), 128, 128, &mut decompression_scratch)?, compressible); Ok(()) } diff --git a/backend/src/repository/bplustree/v3/migration.rs b/backend/btree/src/v3/migration.rs similarity index 86% rename from backend/src/repository/bplustree/v3/migration.rs rename to backend/btree/src/v3/migration.rs index 65075966c..9c023681b 100644 --- a/backend/src/repository/bplustree/v3/migration.rs +++ b/backend/btree/src/v3/migration.rs @@ -1,5 +1,5 @@ use super::{BPlusTree, BPlusTreeMetadata}; -use crate::repository::bplustree::v2; +use crate::v2; use serde::{Deserialize, Serialize}; use std::{ fs::{File, OpenOptions}, @@ -17,13 +17,13 @@ const HEADER_FLAG_HAS_TOMBSTONES: u32 = 1 << 30; const HEADER_METADATA_LEN_MASK: u32 = !(HEADER_FLAG_HAS_METADATA_FLAGS | HEADER_FLAG_HAS_TOMBSTONES); #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct MigrationValidation { - pub(crate) entries: usize, - pub(crate) database_id: [u8; 16], - pub(crate) generation: u64, +pub struct MigrationValidation { + pub entries: usize, + pub database_id: [u8; 16], + pub generation: u64, } -pub(crate) fn storage_version(path: &Path) -> io::Result> { +pub fn storage_version(path: &Path) -> io::Result> { let mut file = File::open(path)?; if file.metadata()?.len() < 8 { return Ok(None); @@ -83,19 +83,20 @@ fn legacy_metadata(path: &Path) -> io::Result { if metadata_len > METADATA_MAX_SIZE { return Err(io::Error::new(io::ErrorKind::InvalidData, "legacy metadata exceeds header capacity")); } - let metadata_len = usize::try_from(metadata_len).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let metadata_len = + usize::try_from(metadata_len).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; let mut encoded = vec![0; metadata_len]; file.read_exact(&mut encoded)?; match encoded.as_slice() { [] => Ok(BPlusTreeMetadata::Empty), - [1, value0, value1, value2, value3] => Ok(BPlusTreeMetadata::TargetIdMapping(u32::from_le_bytes([ - *value0, *value1, *value2, *value3, - ]))), + [1, value0, value1, value2, value3] => { + Ok(BPlusTreeMetadata::TargetIdMapping(u32::from_le_bytes([*value0, *value1, *value2, *value3]))) + } _ => Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported legacy B+Tree metadata")), } } -pub(crate) fn migrate_v2_typed(source: &Path) -> io::Result +pub fn migrate_v2_typed(source: &Path) -> io::Result where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, @@ -109,10 +110,7 @@ where ) } -pub(crate) fn migrate_v2_typed_with_index( - source: &Path, - sort_key: F, -) -> io::Result +pub fn migrate_v2_typed_with_index(source: &Path, sort_key: F) -> io::Result where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, V: Serialize + for<'de> Deserialize<'de> + Clone, @@ -125,8 +123,8 @@ where |tree, destination| tree.store_with_index_verified(destination, sort_key), |destination, entries| { let query = super::BPlusTreeQuery::::try_new(destination)?; - let index = crate::repository::storage::get_file_path_for_db_index(destination); - let mut iterator = crate::repository::bplustree::sorted_index::v4::OwnedIterator::::open(query, &index)?; + let index = crate::common::get_file_path_for_db_index(destination); + let mut iterator = crate::sorted_index::v4::OwnedIterator::::open(query, &index)?; let indexed_entries = iterator.try_fold(0usize, |count, entry| { let _ = entry?; count @@ -142,10 +140,7 @@ where ) } -pub(crate) fn migrate_v2_typed_map( - source: &Path, - map: Map, -) -> io::Result +pub fn migrate_v2_typed_map(source: &Path, map: Map) -> io::Result where K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, SourceV: Serialize + for<'de> Deserialize<'de> + Clone, @@ -180,7 +175,7 @@ where let normalized_source = migration_source(source, version)?; let read_path = normalized_source.as_deref().unwrap_or(source); let destination = migration_destination(source)?; - let destination_index = crate::repository::storage::get_file_path_for_db_index(&destination); + let destination_index = crate::common::get_file_path_for_db_index(&destination); let converted = (|| { let mut legacy = v2::BPlusTreeQuery::::try_new(read_path)?; let metadata = legacy_metadata(read_path)?; @@ -223,7 +218,7 @@ where drop(query); validate(&destination, entries)?; if indexed { - publish(&destination_index, &crate::repository::storage::get_file_path_for_db_index(source))?; + publish(&destination_index, &crate::common::get_file_path_for_db_index(source))?; } publish(&destination, source)?; Ok(MigrationValidation { entries, database_id, generation }) @@ -248,14 +243,16 @@ fn publish(temporary: &Path, destination: &Path) -> io::Result<()> { fn cleanup_destination(destination: &Path) { let _ = std::fs::remove_file(destination); - let _ = std::fs::remove_file(crate::repository::storage::get_file_path_for_db_index(destination)); - let _ = std::fs::remove_file(crate::repository::bplustree::common::sidecar_lock_path(destination)); + let _ = std::fs::remove_file(crate::common::get_file_path_for_db_index(destination)); + let _ = std::fs::remove_file(crate::common::sidecar_lock_path(destination)); } #[cfg(test)] mod tests { - use super::*; - use super::super::page::{page_open_count, reset_page_open_count}; + use super::{ + super::page::{page_open_count, reset_page_open_count}, + *, + }; #[test] fn typed_migration_validates_the_v3_destination_once() -> io::Result<()> { @@ -300,7 +297,10 @@ mod tests { assert_eq!(validation.generation, 1); assert_eq!(storage_version(&path)?, Some(3)); let mut query = super::super::BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.iter().collect::>>()?, vec![(1, String::from("one")), (2, String::from("two"))]); + assert_eq!( + query.iter().collect::>>()?, + vec![(1, String::from("one")), (2, String::from("two"))] + ); } Ok(()) } @@ -337,7 +337,8 @@ mod tests { let root_offset = u64::from_le_bytes( bytes[8..16].try_into().map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, ); - let root_start = usize::try_from(root_offset).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let root_start = + usize::try_from(root_offset).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; let root_end = root_start .checked_add(v2::PAGE_SIZE_USIZE) .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "fixture root end overflow"))?; @@ -346,7 +347,8 @@ mod tests { .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "fixture root is truncated"))? .to_vec(); bytes.push(0); - let relocated_root = u64::try_from(bytes.len()).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let relocated_root = + u64::try_from(bytes.len()).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; bytes.extend_from_slice(&root); bytes[8..16].copy_from_slice(&relocated_root.to_le_bytes()); std::fs::write(&path, bytes)?; @@ -400,9 +402,9 @@ mod tests { let validation = migrate_v2_typed_with_index::(&path, String::len)?; assert_eq!(validation.entries, 2); let query = super::super::BPlusTreeQuery::::try_new(&path)?; - let index = crate::repository::storage::get_file_path_for_db_index(&path); - let values = crate::repository::bplustree::sorted_index::v4::OwnedIterator::::open(query, &index) - ?.collect::>>()?; + let index = crate::common::get_file_path_for_db_index(&path); + let values = crate::sorted_index::v4::OwnedIterator::::open(query, &index)? + .collect::>>()?; assert_eq!(values, vec![(2, String::from("a")), (1, String::from("bbb"))]); Ok(()) } diff --git a/backend/src/repository/bplustree/v3/mod.rs b/backend/btree/src/v3/mod.rs similarity index 92% rename from backend/src/repository/bplustree/v3/mod.rs rename to backend/btree/src/v3/mod.rs index c80cf7801..f1ddde757 100644 --- a/backend/src/repository/bplustree/v3/mod.rs +++ b/backend/btree/src/v3/mod.rs @@ -1,18 +1,17 @@ mod format; -pub(crate) mod migration; +pub mod migration; mod page; mod publish; mod tree; mod wal; -pub(crate) use publish::{publish_staged_database, BPlusTreeStagingArtifacts}; - +pub use format::{Locator, MAGIC, STORAGE_VERSION_V3 as STORAGE_VERSION}; +pub use publish::{publish_staged_database, BPlusTreeStagingArtifacts}; #[allow(unused_imports)] pub use tree::{ BPlusTree, BPlusTreeDiskIterator, BPlusTreeDiskIteratorOwned, BPlusTreeQuery, BPlusTreeRangeIterator, BPlusTreeSerialWriter, BPlusTreeUpdate, FlushPolicy, }; -pub(crate) use format::{Locator, MAGIC, STORAGE_VERSION_V3 as STORAGE_VERSION}; #[derive(Clone, Debug, Eq, PartialEq)] pub enum BPlusTreeMetadata { @@ -23,7 +22,7 @@ pub enum BPlusTreeMetadata { #[cfg(test)] mod contract_tests { use super::{tree::verify_full, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery}; - use crate::repository::bplustree::v2; + use crate::v2; use std::{io, ops::Bound}; fn incompressible_value() -> Vec { @@ -75,9 +74,7 @@ mod contract_tests { let mut query = Query::try_new(&path)?; assert_eq!( - query - .range_iter(Bound::Included(&20), Bound::Included(&30)) - .collect::>>()?, + query.range_iter(Bound::Included(&20), Bound::Included(&30)).collect::>>()?, expected[1..].to_vec() ); Ok(()) diff --git a/backend/src/repository/bplustree/v3/page.rs b/backend/btree/src/v3/page.rs similarity index 87% rename from backend/src/repository/bplustree/v3/page.rs rename to backend/btree/src/v3/page.rs index 9e9957fce..204513a8c 100644 --- a/backend/src/repository/bplustree/v3/page.rs +++ b/backend/btree/src/v3/page.rs @@ -28,9 +28,9 @@ const _: () = assert!(PAGE_HEADER_LEN + INTERNAL_PREAMBLE_LEN <= PAGE_SIZE); fn slot_base(page_type: PageType) -> io::Result { match page_type { PageType::Leaf => Ok(PAGE_HEADER_LEN), - PageType::Internal => PAGE_HEADER_LEN - .checked_add(INTERNAL_PREAMBLE_LEN) - .ok_or_else(|| invalid_data("slot base overflow")), + PageType::Internal => { + PAGE_HEADER_LEN.checked_add(INTERNAL_PREAMBLE_LEN).ok_or_else(|| invalid_data("slot base overflow")) + } PageType::Overflow | PageType::Free => Err(invalid_input("chain pages do not contain slots")), } } @@ -117,20 +117,12 @@ impl> SlottedPage { .and_then(|size| base.checked_add(size)) .ok_or_else(|| invalid_data("slot offset overflow"))?; let end = checked_end(offset, SLOT_LEN, io::ErrorKind::InvalidData)?; - Slot::decode( - self.bytes - .as_ref() - .get(offset..end) - .ok_or_else(|| invalid_data("slot is outside page"))?, - ) + Slot::decode(self.bytes.as_ref().get(offset..end).ok_or_else(|| invalid_data("slot is outside page"))?) } pub(crate) fn cell(&self, index: usize) -> io::Result<&[u8]> { let range = self.cell_range(index)?; - self.bytes - .as_ref() - .get(range) - .ok_or_else(|| invalid_data("cell is outside page")) + self.bytes.as_ref().get(range).ok_or_else(|| invalid_data("cell is outside page")) } pub(crate) fn cell_range(&self, index: usize) -> io::Result> { @@ -192,30 +184,16 @@ pub(crate) fn encode_overflow_page( payload_length: u16::try_from(payload.len()).map_err(|_| invalid_input("overflow payload exceeds u16"))?, } .encode_into(&mut page)?; - PageHeader { - page_type: PageType::Overflow, - cell_count: 0, - free_start: 0, - free_end: 0, - left: 0, - right: next, - } - .encode_into(&mut page, page_id, next_page_id)?; + PageHeader { page_type: PageType::Overflow, cell_count: 0, free_start: 0, free_end: 0, left: 0, right: next } + .encode_into(&mut page, page_id, next_page_id)?; Ok(page) } pub(crate) fn encode_free_page(page_id: u64, next_page_id: u64, next: u64) -> io::Result<[u8; PAGE_SIZE]> { let mut page = [0; PAGE_SIZE]; encode_free_body(&mut page)?; - PageHeader { - page_type: PageType::Free, - cell_count: 0, - free_start: 0, - free_end: 0, - left: 0, - right: next, - } - .encode_into(&mut page, page_id, next_page_id)?; + PageHeader { page_type: PageType::Free, cell_count: 0, free_start: 0, free_end: 0, left: 0, right: next } + .encode_into(&mut page, page_id, next_page_id)?; Ok(page) } @@ -267,8 +245,7 @@ impl + AsMut<[u8]>> SlottedPage { if self.header.page_type == PageType::Internal { // INTERNAL_PREAMBLE_LEN is a compile-time constant; the range is always in bounds. let preamble_end = PAGE_HEADER_LEN + INTERNAL_PREAMBLE_LEN; // 32 + 8 = 40 - rebuilt[PAGE_HEADER_LEN..preamble_end] - .copy_from_slice(&self.bytes.as_ref()[PAGE_HEADER_LEN..preamble_end]); + rebuilt[PAGE_HEADER_LEN..preamble_end].copy_from_slice(&self.bytes.as_ref()[PAGE_HEADER_LEN..preamble_end]); } let mut count = 0u16; @@ -282,9 +259,8 @@ impl + AsMut<[u8]>> SlottedPage { // SLOT_LEN=4, count<=u16::MAX, base<=40 — overflow is impossible within PAGE_SIZE. let slot_offset = base + usize::from(count) * SLOT_LEN; let slot_end = slot_offset + SLOT_LEN; - let next_cell_start = cell_start - .checked_sub(cell.len()) - .ok_or_else(|| invalid_input("cells exceed page capacity"))?; + let next_cell_start = + cell_start.checked_sub(cell.len()).ok_or_else(|| invalid_input("cells exceed page capacity"))?; if next_cell_start < slot_end { return Err(invalid_input("cells exceed page capacity")); } @@ -302,9 +278,8 @@ impl + AsMut<[u8]>> SlottedPage { if self.header.page_type == PageType::Internal && count == 0 { return Err(invalid_input("internal pages must not be empty")); } - let slot_bytes = usize::from(count) - .checked_mul(SLOT_LEN) - .ok_or_else(|| invalid_input("slot directory size overflow"))?; + let slot_bytes = + usize::from(count).checked_mul(SLOT_LEN).ok_or_else(|| invalid_input("slot directory size overflow"))?; let free_start = base.checked_add(slot_bytes).ok_or_else(|| invalid_input("slot directory overflow"))?; let header = PageHeader { page_type: self.header.page_type, @@ -335,8 +310,7 @@ impl + AsMut<[u8]>> SlottedPage { if page.len() != PAGE_SIZE { return Err(invalid_data("page must be exactly 4096 bytes")); } - page - .get_mut(offset..end) + page.get_mut(offset..end) .ok_or_else(|| invalid_data("cell destination is outside page"))? .copy_from_slice(replacement); write_page_checksum(page) @@ -346,7 +320,7 @@ impl + AsMut<[u8]>> SlottedPage { #[cfg(test)] mod tests { use super::*; - use crate::repository::bplustree::v3::format::{ + use crate::v3::format::{ encode_free_body, page_checksum, write_page_checksum, InternalPreamble, OverflowHeader, PageHeader, PageType, Slot, PAGE_HEADER_LEN, PAGE_SIZE, SLOT_LEN, }; @@ -367,7 +341,11 @@ mod tests { impl AsMut<[u8]> for ChangingMutView { fn as_mut(&mut self) -> &mut [u8] { self.calls += 1; - if self.calls == 1 { &mut self.bytes[..200] } else { &mut self.bytes[..32] } + if self.calls == 1 { + &mut self.bytes[..200] + } else { + &mut self.bytes[..32] + } } } @@ -417,15 +395,11 @@ mod tests { .checked_add(slots.len().checked_mul(SLOT_LEN).ok_or_else(|| io::Error::other("test slot overflow"))?) .and_then(|value| u16::try_from(value).ok()) .ok_or_else(|| io::Error::other("test slot overflow"))?; - PageHeader { - page_type: PageType::Leaf, - cell_count, - free_start, - free_end, - left: 0, - right: 0, - } - .encode_into(&mut page, PAGE_ID, NEXT_PAGE_ID)?; + PageHeader { page_type: PageType::Leaf, cell_count, free_start, free_end, left: 0, right: 0 }.encode_into( + &mut page, + PAGE_ID, + NEXT_PAGE_ID, + )?; Ok(page) } @@ -436,15 +410,11 @@ mod tests { PageType::Free => encode_free_body(&mut page)?, PageType::Leaf | PageType::Internal => return Err(io::Error::other("test requires a chain page")), } - PageHeader { - page_type, - cell_count: 0, - free_start: 0, - free_end: 0, - left: 0, - right: 0, - } - .encode_into(&mut page, PAGE_ID, NEXT_PAGE_ID)?; + PageHeader { page_type, cell_count: 0, free_start: 0, free_end: 0, left: 0, right: 0 }.encode_into( + &mut page, + PAGE_ID, + NEXT_PAGE_ID, + )?; Ok(page) } @@ -565,15 +535,8 @@ mod tests { InternalPreamble { leftmost_child: 3 }.encode_into(&mut page, 2, NEXT_PAGE_ID)?; page[40..44].copy_from_slice(&Slot { offset: 4095, length: 1 }.encode()); page[4095] = b'x'; - PageHeader { - page_type: PageType::Internal, - cell_count: 1, - free_start: 44, - free_end: 4095, - left: 0, - right: 0, - } - .encode_into(&mut page, 2, NEXT_PAGE_ID)?; + PageHeader { page_type: PageType::Internal, cell_count: 1, free_start: 44, free_end: 4095, left: 0, right: 0 } + .encode_into(&mut page, 2, NEXT_PAGE_ID)?; let mut changing = ChangingMutView { bytes: page, calls: 0 }; let mut slotted = SlottedPage::open(&mut changing, 2, NEXT_PAGE_ID)?; @@ -616,30 +579,14 @@ mod tests { (vec![Slot { offset: 4090, length: 0 }], 4090), (vec![Slot { offset: 35, length: 1 }], 36), (vec![Slot { offset: 4090, length: 10 }], 4090), - ( - vec![Slot { offset: 4080, length: 8 }, Slot { offset: 4080, length: 8 }], - 4080, - ), - ( - vec![Slot { offset: 4080, length: 8 }, Slot { offset: 4090, length: 6 }], - 4090, - ), - ( - vec![Slot { offset: 4080, length: 16 }, Slot { offset: 4070, length: 16 }], - 4070, - ), - ( - vec![Slot { offset: 4080, length: 16 }, Slot { offset: 4064, length: 16 }], - 4060, - ), + (vec![Slot { offset: 4080, length: 8 }, Slot { offset: 4080, length: 8 }], 4080), + (vec![Slot { offset: 4080, length: 8 }, Slot { offset: 4090, length: 6 }], 4090), + (vec![Slot { offset: 4080, length: 16 }, Slot { offset: 4070, length: 16 }], 4070), + (vec![Slot { offset: 4080, length: 16 }, Slot { offset: 4064, length: 16 }], 4060), ]; for (slots, free_end) in cases { - invalid_data(SlottedPage::open( - leaf_with_slots(&slots, free_end)?.as_slice(), - PAGE_ID, - NEXT_PAGE_ID, - ))?; + invalid_data(SlottedPage::open(leaf_with_slots(&slots, free_end)?.as_slice(), PAGE_ID, NEXT_PAGE_ID))?; } Ok(()) } @@ -674,15 +621,8 @@ mod tests { fn open_validates_internal_preamble() -> io::Result<()> { let mut page = [0; PAGE_SIZE]; InternalPreamble { leftmost_child: PAGE_ID }.encode_into(&mut page, 2, NEXT_PAGE_ID)?; - PageHeader { - page_type: PageType::Internal, - cell_count: 1, - free_start: 44, - free_end: 4095, - left: 0, - right: 0, - } - .encode_into(&mut page, 2, NEXT_PAGE_ID)?; + PageHeader { page_type: PageType::Internal, cell_count: 1, free_start: 44, free_end: 4095, left: 0, right: 0 } + .encode_into(&mut page, 2, NEXT_PAGE_ID)?; page[40..44].copy_from_slice(&Slot { offset: 4095, length: 1 }.encode()); write_page_checksum(&mut page)?; SlottedPage::open(page.as_slice(), 2, NEXT_PAGE_ID)?; @@ -698,15 +638,8 @@ mod tests { InternalPreamble { leftmost_child: 3 }.encode_into(&mut page, 2, NEXT_PAGE_ID)?; page[40..44].copy_from_slice(&Slot { offset: 4095, length: 1 }.encode()); page[4095] = b'x'; - PageHeader { - page_type: PageType::Internal, - cell_count: 1, - free_start: 44, - free_end: 4095, - left: 0, - right: 0, - } - .encode_into(&mut page, 2, NEXT_PAGE_ID)?; + PageHeader { page_type: PageType::Internal, cell_count: 1, free_start: 44, free_end: 4095, left: 0, right: 0 } + .encode_into(&mut page, 2, NEXT_PAGE_ID)?; { let mut slotted = SlottedPage::open(page.as_mut_slice(), 2, NEXT_PAGE_ID)?; diff --git a/backend/src/repository/bplustree/v3/publish.rs b/backend/btree/src/v3/publish.rs similarity index 88% rename from backend/src/repository/bplustree/v3/publish.rs rename to backend/btree/src/v3/publish.rs index bca6448cb..7c35eeccc 100644 --- a/backend/src/repository/bplustree/v3/publish.rs +++ b/backend/btree/src/v3/publish.rs @@ -1,16 +1,12 @@ use super::{ tree::{publish_database_and_invalidate_sorted_index, BPlusTreeUpdate}, wal::{ - recover_pending_under_existing_lock, sync_parent_directory, wal_path, wal_temporary_path, - ExclusiveSidecarGuard, + recover_pending_under_existing_lock, sync_parent_directory, wal_path, wal_temporary_path, ExclusiveSidecarGuard, }, }; -use crate::repository::{ - bplustree::common::{ - ensure_distinct_sidecar_lock_domains, remove_file_if_exists, require_same_parent_directory, - resolved_path_identity, sidecar_lock_path, - }, - storage::get_file_path_for_db_index, +use crate::common::{ + ensure_distinct_sidecar_lock_domains, get_file_path_for_db_index, remove_file_if_exists, + require_same_parent_directory, resolved_path_identity, sidecar_lock_path, }; use serde::{de::DeserializeOwned, Serialize}; use std::{ @@ -48,9 +44,7 @@ impl BPlusTreeArtifactPaths { first_error.map_or(Ok(()), Err) } - fn paths(&self) -> [&Path; 5] { - [&self.database, &self.index, &self.wal, &self.wal_temporary, &self.sidecar_lock] - } + fn paths(&self) -> [&Path; 5] { [&self.database, &self.index, &self.wal, &self.wal_temporary, &self.sidecar_lock] } fn ensure_disjoint_from(&self, published: &Self) -> io::Result<()> { let mut published_identities = Vec::with_capacity(5); @@ -79,10 +73,10 @@ impl BPlusTreeArtifactPaths { } #[derive(Debug, Eq, PartialEq)] -pub(crate) struct BPlusTreeStagingArtifacts(BPlusTreeArtifactPaths); +pub struct BPlusTreeStagingArtifacts(BPlusTreeArtifactPaths); impl BPlusTreeStagingArtifacts { - pub(crate) fn new(published: &Path, staging: &Path) -> io::Result { + pub fn new(published: &Path, staging: &Path) -> io::Result { ensure_distinct_sidecar_lock_domains(published, staging)?; let published_artifacts = BPlusTreeArtifactPaths::for_database(published); let staging_artifacts = BPlusTreeArtifactPaths::for_database(staging); @@ -90,17 +84,12 @@ impl BPlusTreeStagingArtifacts { Ok(Self(staging_artifacts)) } - pub(crate) fn remove_owned_staging_artifacts(&self) -> io::Result<()> { self.0.remove_all() } + pub fn remove_owned_staging_artifacts(&self) -> io::Result<()> { self.0.remove_all() } - #[cfg(test)] - pub(crate) fn owned_paths(&self) -> [&Path; 5] { - [ - &self.0.database, - &self.0.index, - &self.0.wal, - &self.0.wal_temporary, - &self.0.sidecar_lock, - ] + /// Fixture/assertion helper: the five files this staging set owns. + #[cfg(any(test, feature = "test-support"))] + pub fn owned_paths(&self) -> [&Path; 5] { + [&self.0.database, &self.0.index, &self.0.wal, &self.0.wal_temporary, &self.0.sidecar_lock] } } @@ -137,11 +126,7 @@ where publish_database_and_invalidate_sorted_index(staging, published, sync_published_directory).map_err(|error| { io::Error::new( error.kind(), - format!( - "failed to publish staging B+Tree {} as {}: {error}", - staging.display(), - published.display() - ), + format!("failed to publish staging B+Tree {} as {}: {error}", staging.display(), published.display()), ) })?; drop(final_guard); @@ -160,21 +145,14 @@ where { let staging_artifacts = BPlusTreeStagingArtifacts::new(published, staging)?; require_same_parent_directory(staging, published)?; - let publish_result = publish_staged_database_inner::( - staging, - published, - acquire_final_lock, - sync_published_directory, - ); + let publish_result = + publish_staged_database_inner::(staging, published, acquire_final_lock, sync_published_directory); let cleanup_result = staging_artifacts.remove_owned_staging_artifacts(); match (publish_result, cleanup_result) { (Ok(()), Ok(())) => Ok(()), (Ok(()), Err(cleanup_error)) => Err(io::Error::new( cleanup_error.kind(), - format!( - "B+Tree {} was published, but staging cleanup failed: {cleanup_error}", - published.display() - ), + format!("B+Tree {} was published, but staging cleanup failed: {cleanup_error}", published.display()), )), (Err(publish_error), Ok(())) => Err(publish_error), (Err(publish_error), Err(cleanup_error)) => Err(io::Error::new( @@ -184,7 +162,7 @@ where } } -pub(crate) fn publish_staged_database(staging: &Path, published: &Path) -> io::Result<()> +pub fn publish_staged_database(staging: &Path, published: &Path) -> io::Result<()> where K: Ord + Serialize + DeserializeOwned + Clone, V: Serialize + DeserializeOwned, @@ -236,11 +214,11 @@ where #[cfg(test)] mod tests { use super::{ + super::wal::{leave_uncommitted_test_wal_after_database_write, wal_path}, publish_staged_database, publish_staged_database_after_observed_final_lock_contention, publish_staged_database_with_directory_sync, BPlusTreeArtifactPaths, BPlusTreeStagingArtifacts, }; - use super::super::wal::{leave_uncommitted_test_wal_after_database_write, wal_path}; - use crate::repository::{get_file_path_for_db_index, BPlusTree, BPlusTreeError, BPlusTreeQuery}; + use crate::{common::get_file_path_for_db_index, BPlusTree, BPlusTreeError, BPlusTreeQuery}; use std::{ env, fs, io, process::{Child, Command, ExitStatus, Stdio}, @@ -253,9 +231,7 @@ mod tests { impl KillChildOnDrop { fn spawn(command: &mut Command) -> io::Result { command.spawn().map(|child| Self(Some(child))) } - fn try_wait(&mut self) -> io::Result> { - self.0.as_mut().map_or(Ok(None), Child::try_wait) - } + fn try_wait(&mut self) -> io::Result> { self.0.as_mut().map_or(Ok(None), Child::try_wait) } fn wait_until(&mut self, deadline: Instant) -> io::Result { loop { @@ -333,7 +309,7 @@ mod tests { let mut child = KillChildOnDrop::spawn( Command::new(env::current_exe()?) .arg("--exact") - .arg("repository::bplustree::v3::publish::tests::publish_staged_database_child") + .arg("v3::publish::tests::publish_staged_database_child") .arg("--nocapture") .env("TULIPROX_BPLUS_PUBLISH_STAGING", &staging) .env("TULIPROX_BPLUS_PUBLISH_FINAL", &published) @@ -342,10 +318,7 @@ mod tests { .stderr(Stdio::null()), )?; wait_for_path(&marker, Instant::now() + Duration::from_secs(5))?; - assert_eq!( - final_query.query(&1).map_err(BPlusTreeError::to_io)?, - Some(String::from("published")) - ); + assert_eq!(final_query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("published"))); assert!(staging.exists(), "staging database must exist while the final lock is contended"); assert!(child.try_wait()?.is_none(), "publish must wait for the final shared sidecar lock"); @@ -411,10 +384,7 @@ mod tests { publish_staged_database::(&first_staging, &published)?; let mut first_query = BPlusTreeQuery::::try_new(&published)?; - assert_eq!( - first_query.query(&2).map_err(BPlusTreeError::to_io)?, - Some(String::from("first")) - ); + assert_eq!(first_query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("first"))); drop(first_query); let mut second_tree = BPlusTree::new(); @@ -424,10 +394,7 @@ mod tests { let mut second_query = BPlusTreeQuery::::try_new(&published)?; assert_eq!(second_query.query(&2).map_err(BPlusTreeError::to_io)?, None); - assert_eq!( - second_query.query(&3).map_err(BPlusTreeError::to_io)?, - Some(String::from("second")) - ); + assert_eq!(second_query.query(&3).map_err(BPlusTreeError::to_io)?, Some(String::from("second"))); assert!(BPlusTreeArtifactPaths::for_database(&published).sidecar_lock.exists()); assert!(!BPlusTreeArtifactPaths::for_database(&first_staging).sidecar_lock.exists()); assert!(!BPlusTreeArtifactPaths::for_database(&second_staging).sidecar_lock.exists()); @@ -537,11 +504,9 @@ mod tests { staging_tree.insert(2u32, String::from("staged")); staging_tree.store(&staging)?; - let error = publish_staged_database_with_directory_sync::( - &staging, - &published, - |_| Err(io::Error::other("injected post-rename directory sync failure")), - ) + let error = publish_staged_database_with_directory_sync::(&staging, &published, |_| { + Err(io::Error::other("injected post-rename directory sync failure")) + }) .expect_err("post-rename sync failure must remain visible"); assert!(error.to_string().contains("publication durability remains unknown")); diff --git a/backend/src/repository/bplustree/v3/tree.rs b/backend/btree/src/v3/tree.rs similarity index 92% rename from backend/src/repository/bplustree/v3/tree.rs rename to backend/btree/src/v3/tree.rs index acc3050de..dd5fdbea5 100644 --- a/backend/src/repository/bplustree/v3/tree.rs +++ b/backend/btree/src/v3/tree.rs @@ -2,8 +2,8 @@ use super::{ format::{ decompress_value_in_place, decompress_value_into, encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, encode_tombstone_leaf_cell, encode_value, stored_value_checksum, Compression, - DatabaseHeader, InternalCellRef, InternalPreamble, LeafCellRef, LeafValueRef, Locator, PageHeader, PageType, Slot, - MAX_CELL_FOOTPRINT, MAX_INLINE_STORED_VALUE, OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, SLOT_LEN, + DatabaseHeader, InternalCellRef, InternalPreamble, LeafCellRef, LeafValueRef, Locator, PageHeader, PageType, + Slot, MAX_CELL_FOOTPRINT, MAX_INLINE_STORED_VALUE, OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, SLOT_LEN, }, page::{encode_free_page, encode_overflow_page, overflow_payload, PageValidation, SlottedPage}, wal::{ @@ -14,8 +14,8 @@ use super::{ BPlusTreeMetadata, }; use crate::{ - repository::bplustree::common::{mmap_with_advice, read_exact_at_offset, write_all_at_offset, Advice, BPlusTreeError}, - utils::{binary_deserialize, binary_serialize, binary_serialize_into}, + codec::{binary_deserialize, binary_serialize, binary_serialize_into}, + common::{mmap_with_advice, read_exact_at_offset, write_all_at_offset, Advice, BPlusTreeError}, }; use memmap2::Mmap; use parking_lot::Mutex; @@ -44,9 +44,7 @@ where I: IntoIterator, { cells.into_iter().try_fold(base, |used, cell| { - let footprint = SLOT_LEN - .checked_add(cell.len()) - .ok_or_else(|| invalid_input("cell footprint overflow"))?; + let footprint = SLOT_LEN.checked_add(cell.len()).ok_or_else(|| invalid_input("cell footprint overflow"))?; if footprint > maximum_cell_footprint { return Err(invalid_input("cell footprint exceeds format limit")); } @@ -64,9 +62,8 @@ pub(crate) fn choose_leaf_split>(cells: &[T]) -> io::Result MAX_CELL_FOOTPRINT { return Err(invalid_input("leaf cell footprint exceeds format limit")); } @@ -76,24 +73,16 @@ pub(crate) fn choose_leaf_split>(cells: &[T]) -> io::Result>(cells: &[T]) -> io::Result(cells: I) -> io::Result @@ -145,14 +133,10 @@ pub(crate) fn choose_internal_split>( } } } - let promoted_index = best - .map(|(index, _)| index) - .ok_or_else(|| invalid_input("internal split has no valid boundary"))?; + let promoted_index = + best.map(|(index, _)| index).ok_or_else(|| invalid_input("internal split has no valid boundary"))?; let promoted = InternalCellRef::decode( - cells - .get(promoted_index) - .ok_or_else(|| invalid_input("promoted separator is outside cells"))? - .as_ref(), + cells.get(promoted_index).ok_or_else(|| invalid_input("promoted separator is outside cells"))?.as_ref(), page_id, next_page_id, )?; @@ -161,9 +145,7 @@ pub(crate) fn choose_internal_split>( promoted_index, right_leftmost_child: promoted.right_child, promoted, - left_cells: cells - .get(..promoted_index) - .ok_or_else(|| invalid_input("left split range is outside cells"))?, + left_cells: cells.get(..promoted_index).ok_or_else(|| invalid_input("left split range is outside cells"))?, right_cells: cells .get(promoted_index + 1..) .ok_or_else(|| invalid_input("right split range is outside cells"))?, @@ -229,11 +211,7 @@ pub(crate) fn validate_locator>( if page.header().page_type != PageType::Leaf || page.page_id() != locator.leaf_page_id { return Err(invalid_data("locator does not reference this leaf page")); } - let cell = LeafCellRef::decode( - page.cell(usize::from(locator.slot_index))?, - page.page_id(), - page.next_page_id(), - )?; + let cell = LeafCellRef::decode(page.cell(usize::from(locator.slot_index))?, page.page_id(), page.next_page_id())?; let cell_crc = crc32fast::hash(cell.key_bytes); if cell_crc != locator.serialized_key_crc32 || crc32fast::hash(serialized_primary_key) != locator.serialized_key_crc32 @@ -250,12 +228,8 @@ fn database_page(database: &[u8], page_id: u64, next_page_id: u64) -> io::Result return Err(invalid_data("overflow page id is outside database")); } let page_id = usize::try_from(page_id).map_err(|_| invalid_data("overflow page id exceeds usize"))?; - let offset = page_id - .checked_mul(PAGE_SIZE) - .ok_or_else(|| invalid_data("overflow page offset overflow"))?; - let end = offset - .checked_add(PAGE_SIZE) - .ok_or_else(|| invalid_data("overflow page end overflow"))?; + let offset = page_id.checked_mul(PAGE_SIZE).ok_or_else(|| invalid_data("overflow page offset overflow"))?; + let end = offset.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("overflow page end overflow"))?; database.get(offset..end).ok_or_else(|| invalid_data("truncated overflow page")) } @@ -286,15 +260,14 @@ pub(crate) fn read_leaf_value<'a>( } } LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - let logical_length = usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; + let logical_length = + usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; let stored_length = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; if logical_length > maximum_length || stored_length > maximum_length { return Err(invalid_data("overflow value exceeds allocation limit")); } scratch.clear(); - scratch - .try_reserve(stored_length) - .map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + scratch.try_reserve(stored_length).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; let mut page_id = head; let mut visited = HashSet::new(); while page_id != 0 { @@ -311,9 +284,7 @@ pub(crate) fn read_leaf_value<'a>( if payload.is_empty() { return Err(invalid_data("overflow chain contains an empty payload")); } - visited - .try_reserve(1) - .map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + visited.try_reserve(1).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; if !visited.insert(page_id) { return Err(invalid_data("overflow chain contains a cycle")); } @@ -355,9 +326,7 @@ impl Default for BPlusTree { } impl BPlusTree { - pub const fn new() -> Self { - Self { entries: BTreeMap::new(), metadata: BPlusTreeMetadata::Empty, dirty: true } - } + pub const fn new() -> Self { Self { entries: BTreeMap::new(), metadata: BPlusTreeMetadata::Empty, dirty: true } } pub fn get_metadata(&self) -> &BPlusTreeMetadata { &self.metadata } @@ -443,39 +412,23 @@ fn leaf_page>(page_id: u64, left: u64, right: u64, cells: &[T]) - right, } .encode_into(&mut bytes, page_id, u64::MAX)?; - SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)? - .rebuild_ordered(cells.iter().map(AsRef::as_ref))?; + SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)?.rebuild_ordered(cells.iter().map(AsRef::as_ref))?; Ok(bytes) } -fn internal_page>( - page_id: u64, - leftmost_child: u64, - cells: &[T], -) -> io::Result<[u8; PAGE_SIZE]> { - let first = cells - .first() - .ok_or_else(|| invalid_input("internal page must contain a separator"))? - .as_ref(); - let first_offset = PAGE_SIZE - .checked_sub(first.len()) - .ok_or_else(|| invalid_input("internal cell exceeds page"))?; +fn internal_page>(page_id: u64, leftmost_child: u64, cells: &[T]) -> io::Result<[u8; PAGE_SIZE]> { + let first = cells.first().ok_or_else(|| invalid_input("internal page must contain a separator"))?.as_ref(); + let first_offset = PAGE_SIZE.checked_sub(first.len()).ok_or_else(|| invalid_input("internal cell exceeds page"))?; let mut bytes = [0; PAGE_SIZE]; InternalPreamble { leftmost_child }.encode_into(&mut bytes, page_id, u64::MAX)?; - bytes - .get_mut(40..44) - .ok_or_else(|| invalid_input("internal slot is outside page"))? - .copy_from_slice( - &Slot { - offset: u16::try_from(first_offset).map_err(|_| invalid_input("internal cell offset exceeds u16"))?, - length: u16::try_from(first.len()).map_err(|_| invalid_input("internal cell length exceeds u16"))?, - } - .encode(), - ); - bytes - .get_mut(first_offset..) - .ok_or_else(|| invalid_input("internal cell is outside page"))? - .copy_from_slice(first); + bytes.get_mut(40..44).ok_or_else(|| invalid_input("internal slot is outside page"))?.copy_from_slice( + &Slot { + offset: u16::try_from(first_offset).map_err(|_| invalid_input("internal cell offset exceeds u16"))?, + length: u16::try_from(first.len()).map_err(|_| invalid_input("internal cell length exceeds u16"))?, + } + .encode(), + ); + bytes.get_mut(first_offset..).ok_or_else(|| invalid_input("internal cell is outside page"))?.copy_from_slice(first); PageHeader { page_type: PageType::Internal, cell_count: 1, @@ -485,8 +438,7 @@ fn internal_page>( right: 0, } .encode_into(&mut bytes, page_id, u64::MAX)?; - SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)? - .rebuild_ordered(cells.iter().map(AsRef::as_ref))?; + SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)?.rebuild_ordered(cells.iter().map(AsRef::as_ref))?; Ok(bytes) } @@ -619,14 +571,9 @@ fn group_internal_children(level: &[NodeInfo]) -> io::Result if moved <= previous.0 { return Err(invalid_input("internal page cannot retain two children")); } - groups - .get_mut(last - 1) - .ok_or_else(|| invalid_input("internal grouping is missing its previous group"))? - .1 = moved; - groups - .get_mut(last) - .ok_or_else(|| invalid_input("internal grouping is missing its last group"))? - .0 = moved; + groups.get_mut(last - 1).ok_or_else(|| invalid_input("internal grouping is missing its previous group"))?.1 = + moved; + groups.get_mut(last).ok_or_else(|| invalid_input("internal grouping is missing its last group"))?.0 = moved; } Ok(groups) } @@ -677,10 +624,7 @@ where } fn temporary_path(filepath: &Path) -> io::Result { - let name = filepath - .file_name() - .ok_or_else(|| invalid_input("database path has no file name"))? - .to_string_lossy(); + let name = filepath.file_name().ok_or_else(|| invalid_input("database path has no file name"))?.to_string_lossy(); Ok(filepath.with_file_name(format!("{name}.{}.v3.tmp", uuid::Uuid::new_v4()))) } @@ -691,9 +635,7 @@ pub(super) enum PublishDatabaseError { } impl PublishDatabaseError { - pub(super) const fn database_was_published(&self) -> bool { - matches!(self, Self::PublishedDurabilityUnknown(_)) - } + pub(super) const fn database_was_published(&self) -> bool { matches!(self, Self::PublishedDurabilityUnknown(_)) } fn into_io(self) -> io::Error { match self { @@ -718,10 +660,7 @@ pub(super) fn publish_database( return Err(PublishDatabaseError::NotPublished(error)); } }; - temporary - .persist(destination) - .map_err(io::Error::from) - .map_err(PublishDatabaseError::NotPublished)?; + temporary.persist(destination).map_err(io::Error::from).map_err(PublishDatabaseError::NotPublished)?; sync_directory(destination).map_err(|error| { PublishDatabaseError::PublishedDurabilityUnknown(io::Error::new( error.kind(), @@ -905,10 +844,10 @@ where entries.sort_unstable_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); drop(query); - let index_path = crate::repository::storage::get_file_path_for_db_index(filepath); + let index_path = crate::common::get_file_path_for_db_index(filepath); let temporary = temporary_path(&index_path)?; let prepared = (|| { - let mut writer = crate::repository::bplustree::sorted_index::v4::Writer::new(&temporary, database_id, generation)?; + let mut writer = crate::sorted_index::v4::Writer::new(&temporary, database_id, generation)?; for (sort_key, primary_key, locator) in &entries { writer.push(sort_key, primary_key, *locator)?; } @@ -921,7 +860,6 @@ where } publish_database(&temporary, &index_path, sync_parent_directory).map_err(io::Error::from) } - } struct StoredDatabase { @@ -960,14 +898,12 @@ struct DatabaseImage { impl DatabaseImage { fn open(path: &Path) -> io::Result<(Self, DatabaseHeader)> { let mut file = File::open(path)?; - let file_len = usize::try_from(file.metadata()?.len()) - .map_err(|_| invalid_data("database length exceeds usize"))?; + let file_len = + usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; let mmap = mmap_with_advice(&file, Advice::Normal, "v3 B+Tree update"); let mut fallback = Vec::new(); if mmap.is_none() { - fallback - .try_reserve_exact(file_len) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + fallback.try_reserve_exact(file_len).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; file.read_to_end(&mut fallback)?; } let image = Self { mmap, fallback }; @@ -1079,9 +1015,7 @@ impl WriteTransaction { return Ok(page_id); } let page_id = self.next_header.next_page_id; - self.next_header.next_page_id = page_id - .checked_add(1) - .ok_or_else(|| invalid_input("next page id overflow"))?; + self.next_header.next_page_id = page_id.checked_add(1).ok_or_else(|| invalid_input("next page id overflow"))?; if !self.allocated_pages.insert(page_id) { return Err(invalid_data("appended page was allocated twice")); } @@ -1233,9 +1167,7 @@ where pub fn prepare_upsert_batch(items: &[(&K, &V)]) -> io::Result)>> { let mut prepared = Vec::new(); - prepared - .try_reserve_exact(items.len()) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + prepared.try_reserve_exact(items.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for (key, value) in items { prepared.push(((*key).clone(), binary_serialize(value)?)); } @@ -1424,9 +1356,7 @@ where self.flush_policy = FlushPolicy::Batch; let mut deleted = 0usize; let mut ordered = Vec::new(); - ordered - .try_reserve_exact(keys.len()) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + ordered.try_reserve_exact(keys.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; ordered.extend_from_slice(keys); ordered.sort(); for key in ordered { @@ -1481,9 +1411,7 @@ where ) .map_err(Into::into); } - BPlusTreeQuery::::try_new(&self.filepath) - .and_then(|mut query| query.query_io(key)) - .map_err(Into::into) + BPlusTreeQuery::::try_new(&self.filepath).and_then(|mut query| query.query_io(key)).map_err(Into::into) } pub fn commit(&mut self) -> io::Result<()> { @@ -1682,11 +1610,7 @@ impl Drop for BPlusTreeSerialWriter { } } -fn minimum_tree_key( - transaction: &WriteTransaction, - base: &[u8], - mut page_id: u64, -) -> io::Result>> { +fn minimum_tree_key(transaction: &WriteTransaction, base: &[u8], mut page_id: u64) -> io::Result>> { let mut visited = HashSet::new(); loop { if !visited.insert(page_id) { @@ -1718,11 +1642,8 @@ fn validate_leaf_backlink( if sibling_id == 0 { return Ok(()); } - let sibling = SlottedPage::open( - transaction.page(base, sibling_id)?, - sibling_id, - transaction.next_header.next_page_id, - )?; + let sibling = + SlottedPage::open(transaction.page(base, sibling_id)?, sibling_id, transaction.next_header.next_page_id)?; let backlink = if sibling_points_left { sibling.header().left } else { sibling.header().right }; if sibling.header().page_type != PageType::Leaf || backlink != page_id { return Err(invalid_data("asymmetric leaf sibling link")); @@ -1751,7 +1672,8 @@ fn validate_transaction_free_list(transaction: &WriteTransaction, base: &[u8]) - } for page_id in &transaction.allocated_pages { let page = transaction.page(base, *page_id)?; - if SlottedPage::open(page, *page_id, transaction.next_header.next_page_id)?.header().page_type == PageType::Free { + if SlottedPage::open(page, *page_id, transaction.next_header.next_page_id)?.header().page_type == PageType::Free + { return Err(invalid_data("allocated page still has free-page type")); } } @@ -1790,8 +1712,7 @@ where } previous_key = Some(key); if let LeafValueRef::Overflow { stored_len, head, crc32, .. } = cell.value { - for overflow_page in - validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)? + for overflow_page in validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)? { if !owned_overflow_pages.insert(overflow_page) { return Err(invalid_data("overflow page is owned by multiple values")); @@ -1801,7 +1722,8 @@ where } } PageType::Internal => { - let preamble = InternalPreamble::decode(bytes.as_slice(), *page_id, transaction.next_header.next_page_id)?; + let preamble = + InternalPreamble::decode(bytes.as_slice(), *page_id, transaction.next_header.next_page_id)?; let child = SlottedPage::open( transaction.page(base, preamble.leftmost_child)?, preamble.leftmost_child, @@ -1890,8 +1812,7 @@ fn locate_transaction_leaf Deserialize<'de>>( PageType::Leaf => return Ok((page_id, path)), PageType::Internal => { let (position, child) = internal_child_position(&page, key)?; - path.try_reserve(1) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + path.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; path.push((page_id, position)); page_id = child; } @@ -1900,11 +1821,7 @@ fn locate_transaction_leaf Deserialize<'de>>( } } -fn overflow_chain_pages( - transaction: &WriteTransaction, - base: &[u8], - mut page_id: u64, -) -> io::Result> { +fn overflow_chain_pages(transaction: &WriteTransaction, base: &[u8], mut page_id: u64) -> io::Result> { let mut pages = Vec::new(); let mut visited = HashSet::new(); while page_id != 0 { @@ -1915,9 +1832,7 @@ fn overflow_chain_pages( if page.header().page_type != PageType::Overflow || overflow_payload(&page)?.is_empty() { return Err(invalid_data("invalid overflow chain page")); } - pages - .try_reserve(1) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + pages.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; pages.push(page_id); page_id = page.header().right; } @@ -1939,15 +1854,10 @@ fn validated_overflow_chain_pages( let mut actual = 0usize; let mut hasher = crc32fast::Hasher::new(); for page_id in &pages { - let page = SlottedPage::open( - transaction.page(base, *page_id)?, - *page_id, - transaction.next_header.next_page_id, - )?; + let page = + SlottedPage::open(transaction.page(base, *page_id)?, *page_id, transaction.next_header.next_page_id)?; let payload = overflow_payload(&page)?; - actual = actual - .checked_add(payload.len()) - .ok_or_else(|| invalid_data("overflow chain length overflow"))?; + actual = actual.checked_add(payload.len()).ok_or_else(|| invalid_data("overflow chain length overflow"))?; if actual > expected { return Err(invalid_data("overflow chain exceeds declared length")); } @@ -1977,20 +1887,14 @@ fn write_overflow_chain( return Err(invalid_input("overflow value must not be empty")); } let mut pages = Vec::new(); - pages - .try_reserve_exact(required) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + pages.try_reserve_exact(required).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; if old_pages.len() >= required { pages.extend_from_slice( - old_pages - .get(..required) - .ok_or_else(|| invalid_data("overflow reuse range is invalid"))?, + old_pages.get(..required).ok_or_else(|| invalid_data("overflow reuse range is invalid"))?, ); free_pages( transaction, - old_pages - .get(required..) - .ok_or_else(|| invalid_data("overflow tail range is invalid"))?, + old_pages.get(required..).ok_or_else(|| invalid_data("overflow tail range is invalid"))?, )?; } else { for _ in 0..required { @@ -2001,10 +1905,8 @@ fn write_overflow_chain( for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { let page_id = *pages.get(index).ok_or_else(|| invalid_data("overflow page allocation is missing"))?; let next = pages.get(index + 1).copied().unwrap_or(0); - transaction.write_page( - page_id, - encode_overflow_page(page_id, transaction.next_header.next_page_id, next, payload)?, - )?; + transaction + .write_page(page_id, encode_overflow_page(page_id, transaction.next_header.next_page_id, next, payload)?)?; } pages.first().copied().ok_or_else(|| invalid_data("overflow head is missing")) } @@ -2071,8 +1973,7 @@ fn mutate_leaf( if used_leaf_bytes(&cells)? <= PAGE_SIZE { let next_page_id = transaction.next_header.next_page_id; let dirty = transaction.page_mut(base, leaf_id)?; - SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)? - .rebuild_ordered(cells.iter().copied())?; + SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)?.rebuild_ordered(cells.iter().copied())?; return Ok(None); } @@ -2120,9 +2021,7 @@ fn insert_internal_promotion( cell_scratch, )?; let mut cells = Vec::<&[u8]>::new(); - cells - .try_reserve_exact(count + 1) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + cells.try_reserve_exact(count + 1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for current in 0..count { if current == position { cells.push(cell_scratch); @@ -2135,15 +2034,14 @@ fn insert_internal_promotion( if used_internal_bytes(cells.iter().copied())? <= PAGE_SIZE { let next_page_id = transaction.next_header.next_page_id; let dirty = transaction.page_mut(base, page_id)?; - SlottedPage::open(dirty.as_mut_slice(), page_id, next_page_id)? - .rebuild_ordered(cells.iter().copied())?; + SlottedPage::open(dirty.as_mut_slice(), page_id, next_page_id)?.rebuild_ordered(cells.iter().copied())?; return Ok(None); } let right_page_id = transaction.allocate_page(base)?; let split = choose_internal_split(&cells, page_id, transaction.next_header.next_page_id)?; - let leftmost = InternalPreamble::decode(snapshot.as_slice(), page_id, transaction.next_header.next_page_id)? - .leftmost_child; + let leftmost = + InternalPreamble::decode(snapshot.as_slice(), page_id, transaction.next_header.next_page_id)?.leftmost_child; let promoted = Promotion { key: split.promoted.key_bytes.to_vec(), right_child: right_page_id }; let left_page = internal_page(page_id, leftmost, split.left_cells)?; let right_page = internal_page(right_page_id, split.right_leftmost_child, split.right_cells)?; @@ -2160,7 +2058,8 @@ fn propagate_promotion( cell_scratch: &mut Vec, ) -> io::Result<()> { while let Some((parent, position)) = path.pop() { - let Some(next) = insert_internal_promotion(transaction, base, parent, position, &promotion, cell_scratch)? else { + let Some(next) = insert_internal_promotion(transaction, base, parent, position, &promotion, cell_scratch)? + else { return Ok(()); }; promotion = next; @@ -2279,16 +2178,7 @@ where decompress_value_into(stored, logical_len, transaction_value_limit(transaction)?, scratch)? } LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - read_transaction_overflow( - transaction, - base, - compression, - logical_len, - stored_len, - head, - crc32, - scratch, - )? + read_transaction_overflow(transaction, base, compression, logical_len, stored_len, head, crc32, scratch)? } }; binary_deserialize(bytes).map(Some) @@ -2319,9 +2209,7 @@ fn read_transaction_overflow<'a>( return Err(invalid_data("overflow value exceeds allocation limit")); } scratch.clear(); - scratch - .try_reserve(stored_len) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + scratch.try_reserve(stored_len).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for page_id in overflow_chain_pages(transaction, base, head)? { let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; let payload = overflow_payload(&page)?; @@ -2373,9 +2261,7 @@ where let preamble = InternalPreamble::decode(page.as_bytes(), page.page_id(), page.next_page_id())?; let count = usize::from(page.header().cell_count); let mut separators = Vec::new(); - separators - .try_reserve_exact(count) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + separators.try_reserve_exact(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for index in 0..count { let cell = InternalCellRef::decode(page.cell(index)?, page.page_id(), page.next_page_id())?; separators.push((decode_internal_key(cell.key_bytes)?, cell.right_child)); @@ -2415,7 +2301,8 @@ pub struct BPlusTreeQuery { impl BPlusTreeQuery { fn from_file_unlocked(file: File) -> io::Result { - let file_len = usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; + let file_len = + usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; if file_len < PAGE_SIZE { return Err(invalid_data("database is shorter than its header page")); } @@ -2430,7 +2317,8 @@ impl BPlusTreeQuery { return Err(invalid_data("database file length does not match header")); } let mmap = mmap_with_advice(&file, Advice::Normal, "v3 B+Tree query"); - let page_count = usize::try_from(header.next_page_id).map_err(|_| invalid_data("next page id exceeds usize"))?; + let page_count = + usize::try_from(header.next_page_id).map_err(|_| invalid_data("next page id exceeds usize"))?; let mut page_validations = Vec::new(); page_validations .try_reserve_exact(page_count) @@ -2486,10 +2374,7 @@ impl BPlusTreeQuery { match recover_pending(filepath) { Ok(()) => {} Err(error) - if matches!( - error.kind(), - io::ErrorKind::PermissionDenied | io::ErrorKind::ReadOnlyFilesystem - ) => + if matches!(error.kind(), io::ErrorKind::PermissionDenied | io::ErrorKind::ReadOnlyFilesystem) => { return Err(recovery_required(filepath, error)); } @@ -2521,8 +2406,10 @@ impl BPlusTreeQuery { }) } - #[cfg(test)] - pub(crate) fn clone_error_fixture() -> Self { + /// A query handle over an empty in-memory snapshot, for tests that need a + /// `BPlusTreeQuery` value without a database behind it. + #[cfg(any(test, feature = "test-support"))] + pub fn clone_error_fixture() -> Self { Self { snapshot: Arc::new(QuerySnapshot { file: None, @@ -2551,16 +2438,12 @@ impl BPlusTreeQuery { pub fn filepath(&self) -> &Path { &self.snapshot.filepath } - pub(crate) fn snapshot_identity(&self) -> ([u8; 16], u64) { - (self.header.database_id, self.header.generation) - } + pub(crate) fn snapshot_identity(&self) -> ([u8; 16], u64) { (self.header.database_id, self.header.generation) } pub(crate) fn snapshot_metadata(&self) -> &BPlusTreeMetadata { &self.header.metadata } fn value_allocation_limit(&self) -> usize { - self.file_len - .saturating_mul(256) - .min(usize::try_from(u32::MAX).unwrap_or(usize::MAX)) + self.file_len.saturating_mul(256).min(usize::try_from(u32::MAX).unwrap_or(usize::MAX)) } fn assemble_overflow_chain( @@ -2667,11 +2550,8 @@ impl BPlusTreeQuery { read(&page, scratch).map(|result| (result, validation)) })?; if let Some(validation) = validation { - let slot = self - .snapshot - .page_validations - .get(index) - .ok_or_else(|| invalid_data("page id is outside cache"))?; + let slot = + self.snapshot.page_validations.get(index).ok_or_else(|| invalid_data("page id is outside cache"))?; let _ = slot.set(validation); } Ok(result) @@ -2693,11 +2573,8 @@ impl BPlusTreeQuery { fn cache_internal_route(&self, page_id: u64, route: InternalRoute) -> io::Result<()> { let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; - let slot = self - .snapshot - .internal_routes - .get(index) - .ok_or_else(|| invalid_data("page id is outside route cache"))?; + let slot = + self.snapshot.internal_routes.get(index).ok_or_else(|| invalid_data("page id is outside route cache"))?; let _ = slot.set(route); Ok(()) } @@ -2717,12 +2594,10 @@ impl BPlusTreeQuery { depth += 1; continue; } - let step = self.with_slotted_page(page_id, |page, _| { - match page.header().page_type { - PageType::Leaf => Ok(LocateLeaf::Leaf), - PageType::Internal => decode_internal_route(page).map(LocateLeaf::Internal), - PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), - } + let step = self.with_slotted_page(page_id, |page, _| match page.header().page_type { + PageType::Leaf => Ok(LocateLeaf::Leaf), + PageType::Internal => decode_internal_route(page).map(LocateLeaf::Internal), + PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), })?; match step { LocateLeaf::Internal(route) => { @@ -2751,16 +2626,12 @@ impl BPlusTreeQuery { depth += 1; continue; } - let step = self.with_slotted_page(page_id, |page, _| { - match page.header().page_type { - PageType::Internal => decode_internal_route(page).map(LocateCell::Internal), - PageType::Leaf => search_leaf(page, key)? - .ok() - .map(|index| page.cell_range(index)) - .transpose() - .map(LocateCell::Leaf), - PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), + let step = self.with_slotted_page(page_id, |page, _| match page.header().page_type { + PageType::Internal => decode_internal_route(page).map(LocateCell::Internal), + PageType::Leaf => { + search_leaf(page, key)?.ok().map(|index| page.cell_range(index)).transpose().map(LocateCell::Leaf) } + PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), })?; match step { LocateCell::Internal(route) => { @@ -2784,13 +2655,11 @@ impl BPlusTreeQuery { if depth >= self.header.next_page_id { return Err(invalid_data("tree descent contains a cycle")); } - let result = self.with_slotted_page(page_id, |page, _| { - match page.header().page_type { - PageType::Leaf => Ok(None), - PageType::Internal => InternalPreamble::decode(page.as_bytes(), page_id, page.next_page_id()) - .map(|preamble| Some(preamble.leftmost_child)), - PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), - } + let result = self.with_slotted_page(page_id, |page, _| match page.header().page_type { + PageType::Leaf => Ok(None), + PageType::Internal => InternalPreamble::decode(page.as_bytes(), page_id, page.next_page_id()) + .map(|preamble| Some(preamble.leftmost_child)), + PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), })?; let Some(child) = result else { return Ok(page_id) }; page_id = child; @@ -2873,9 +2742,7 @@ where } let count = usize::from(page.header().cell_count); ranges.clear(); - ranges - .try_reserve(count) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + ranges.try_reserve(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for index in 0..count { ranges.push(page.cell_range(index)?); } @@ -2974,9 +2841,7 @@ where } descending = false; record_page_visit(&mut visited, page_id, "right sibling chain contains a cycle")?; - result - .try_reserve(locators.len()) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + result.try_reserve(locators.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for (locator, range) in locators { let entry = self .decode_entry_range(page_id, range)? @@ -2994,9 +2859,7 @@ where let result: io::Result = (|| { let leaf_id = self.locate_leaf(key)?; let next_page_id = self.header.next_page_id; - let index = self.with_slotted_page(leaf_id, |page, _| { - search_leaf(page, key) - })?; + let index = self.with_slotted_page(leaf_id, |page, _| search_leaf(page, key))?; let Ok(index) = index else { return Ok(false) }; self.with_slotted_page(leaf_id, |page, _| { let cell = LeafCellRef::decode(page.cell(index)?, leaf_id, next_page_id)?; @@ -3108,11 +2971,7 @@ where pub fn disk_iter(self) -> BPlusTreeDiskIteratorOwned { BPlusTreeDiskIteratorOwned::new(self) } - pub fn range_iter( - &mut self, - start: Bound<&K>, - end: Bound<&K>, - ) -> BPlusTreeRangeIterator<'_, K, V> + pub fn range_iter(&mut self, start: Bound<&K>, end: Bound<&K>) -> BPlusTreeRangeIterator<'_, K, V> where K: Clone, { @@ -3143,9 +3002,9 @@ where while result.len() < limit { match iterator.next() { Some(Ok(entry)) => { - result.try_reserve(1).map_err(|error| { - BPlusTreeError::Io(io::Error::new(io::ErrorKind::OutOfMemory, error)) - })?; + result + .try_reserve(1) + .map_err(|error| BPlusTreeError::Io(io::Error::new(io::ErrorKind::OutOfMemory, error)))?; result.push(entry); } Some(Err(err)) => return Err(BPlusTreeError::Io(err)), @@ -3208,9 +3067,7 @@ fn load_cursor_page(query: &mut BPlusTreeQuery, state: &mut CursorSt } let count = usize::from(page.header().cell_count); cell_ranges.clear(); - cell_ranges - .try_reserve(count) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + cell_ranges.try_reserve(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; for index in 0..count { cell_ranges.push(page.cell_range(index)?); } @@ -3248,11 +3105,7 @@ where Bound::Included(key) | Bound::Excluded(key) => query.locate_leaf(key)?, Bound::Unbounded => query.leftmost_leaf()?, }; - record_page_visit( - &mut state.visited_leaves, - state.leaf_page_id, - "right sibling chain contains a cycle", - )?; + record_page_visit(&mut state.visited_leaves, state.leaf_page_id, "right sibling chain contains a cycle")?; if let Bound::Included(key) | Bound::Excluded(key) = &state.start { let page_id = state.leaf_page_id; let next_page_id = query.header.next_page_id; @@ -3315,9 +3168,7 @@ pub struct BPlusTreeDiskIterator<'a, K, V> { } impl<'a, K, V> BPlusTreeDiskIterator<'a, K, V> { - fn new(query: &'a mut BPlusTreeQuery) -> Self { - Self { query, state: CursorState::new(Bound::Unbounded) } - } + fn new(query: &'a mut BPlusTreeQuery) -> Self { Self { query, state: CursorState::new(Bound::Unbounded) } } fn from_bound(query: &'a mut BPlusTreeQuery, start: Bound) -> Self { Self { query, state: CursorState::new(start) } @@ -3339,7 +3190,6 @@ where Some(Err(err)) => Err(err), } } - } impl Iterator for BPlusTreeDiskIterator<'_, K, V> @@ -3376,7 +3226,6 @@ where Some(Err(err)) => Err(err), } } - } impl Iterator for BPlusTreeDiskIteratorOwned @@ -3531,19 +3380,16 @@ where match value { VerifyValue::Tombstone => continue, VerifyValue::Inline => {} - VerifyValue::Overflow(compression, logical_len, stored_len, head, crc32) => query - .assemble_overflow_chain( - compression, - logical_len, - stored_len, - head, - crc32, - Some(overflow_pages), - )?, + VerifyValue::Overflow(compression, logical_len, stored_len, head, crc32) => query.assemble_overflow_chain( + compression, + logical_len, + stored_len, + head, + crc32, + Some(overflow_pages), + )?, } - live_entries = live_entries - .checked_add(1) - .ok_or_else(|| invalid_data("live entry count overflow"))?; + live_entries = live_entries.checked_add(1).ok_or_else(|| invalid_data("live entry count overflow"))?; } Ok((VerifyLeaf { page_id, left, right, minimum, maximum }, live_entries)) } @@ -3578,7 +3424,8 @@ fn push_child_visits( upper: Option<&K>, ) -> io::Result<()> { for index in (0..children.len()).rev() { - let child_lower = if index == 0 { lower.cloned() } else { children.get(index).and_then(|(_, key)| key.clone()) }; + let child_lower = + if index == 0 { lower.cloned() } else { children.get(index).and_then(|(_, key)| key.clone()) }; let child_upper = children.get(index + 1).and_then(|(_, key)| key.clone()).or_else(|| upper.cloned()); let child = children .get(index) @@ -3708,7 +3555,8 @@ where Ok(VerificationReport { live_entries, tree_pages: u64::try_from(tree_pages.len()).map_err(|_| invalid_data("tree page count exceeds u64"))?, - overflow_pages: u64::try_from(overflow_pages.len()).map_err(|_| invalid_data("overflow page count exceeds u64"))?, + overflow_pages: u64::try_from(overflow_pages.len()) + .map_err(|_| invalid_data("overflow page count exceeds u64"))?, free_pages: u64::try_from(free_pages.len()).map_err(|_| invalid_data("free page count exceeds u64"))?, }) } @@ -3717,18 +3565,16 @@ where mod tests { use super::*; use crate::{ - repository::bplustree::v3::{ + codec::binary_serialize, + v3::{ format::{ - encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, - encode_tombstone_leaf_cell, encode_value, Compression, PageHeader, PageType, - OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, - }, - page::{ - encode_free_page, encode_overflow_page, page_open_count, reset_page_open_count, SlottedPage, + encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, encode_tombstone_leaf_cell, + encode_value, Compression, PageHeader, PageType, OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, }, + page::{encode_free_page, encode_overflow_page, page_open_count, reset_page_open_count, SlottedPage}, }, - utils::binary_serialize, }; + use fs2::FileExt as _; use std::{ fs, io, path::{Path, PathBuf}, @@ -3737,7 +3583,6 @@ mod tests { thread::{self, JoinHandle}, time::Duration, }; - use fs2::FileExt as _; const PAGE_ID: u64 = 7; const NEXT_PAGE_ID: u64 = 20; @@ -3978,10 +3823,8 @@ mod tests { fn medium_stored_value_remains_inline() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("medium-inline.db"); - let value = random_value() - .get(..300) - .ok_or_else(|| io::Error::other("random test value is too short"))? - .to_vec(); + let value = + random_value().get(..300).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); let mut tree = BPlusTree::new(); tree.insert(7u32, value.clone()); @@ -4014,10 +3857,7 @@ mod tests { let dir = tempfile::tempdir()?; let path = dir.path().join("overflow-reuse.db"); let large = random_value(); - let smaller = large - .get(..5_000) - .ok_or_else(|| io::Error::other("random test value is too short"))? - .to_vec(); + let smaller = large.get(..5_000).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); let mut tree = BPlusTree::new(); tree.insert(7u32, large); tree.store(&path)?; @@ -4082,7 +3922,10 @@ mod tests { assert_eq!(after.root_page_id, before.root_page_id); assert_eq!(fs::metadata(&path)?.len(), before_len); let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.iter().collect::>>()?, vec![(1, "one".into()), (2, "two".into()), (3, "three".into())]); + assert_eq!( + query.iter().collect::>>()?, + vec![(1, "one".into()), (2, "two".into()), (3, "three".into())] + ); let _ = verify_full(&mut query)?; Ok(()) } @@ -4123,10 +3966,8 @@ mod tests { fn splitting_a_non_rightmost_leaf_repairs_the_former_neighbor() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("middle-leaf-split.db"); - let value = random_value() - .get(..200) - .ok_or_else(|| io::Error::other("random test value is too short"))? - .to_vec(); + let value = + random_value().get(..200).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); let mut tree = BPlusTree::new(); for key in (0..800u32).step_by(2) { tree.insert(key, value.clone()); @@ -4169,9 +4010,7 @@ mod tests { Ok(()) } - fn long_split_key(index: u32, marker: char) -> String { - format!("{index:04}{marker}{}", "k".repeat(1_880)) - } + fn long_split_key(index: u32, marker: char) -> String { format!("{index:04}{marker}{}", "k".repeat(1_880)) } #[test] fn leaf_promotion_recursively_splits_a_full_internal_root() -> io::Result<()> { @@ -4314,7 +4153,6 @@ mod tests { Ok(()) } - /// Throughput benchmark for `BPlusTree::store` across three workload sizes. #[test] #[ignore = "benchmark; run explicitly with --ignored --nocapture"] @@ -4328,7 +4166,8 @@ mod tests { }) .collect() }; - for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] { + for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] + { let dir = tempfile::tempdir()?; let path = dir.path().join("bench.db"); let mut tree = BPlusTree::>::new(); @@ -4347,6 +4186,70 @@ mod tests { Ok(()) } + /// Point-query benchmark for the persisted read path. + /// + /// `bench_store_throughput` only covers writing. The request hot path is + /// `BPlusTreeQuery::query_zero_copy` against an mmapped database, so the + /// extraction gate needs a read-side number too. Everything here is derived + /// from fixed seeds so two runs on the same host compare like for like: the + /// values come from the same LCG the store benchmark uses, and the lookup + /// order is a fixed-stride walk that defeats sequential prefetching without + /// needing a random number generator. + #[test] + #[ignore = "benchmark; run explicitly with --ignored --nocapture"] + fn bench_point_query_throughput() -> io::Result<()> { + // Coprime with every entry count below, so the walk visits each key exactly + // once and never degenerates into a scan. + const STRIDE: u32 = 7_919; + const LOOKUPS: u32 = 200_000; + + let noise = |seed: u32, len: usize| -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + u8::try_from(state >> 24).unwrap_or(0) + }) + .collect() + }; + + for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] + { + let dir = tempfile::tempdir()?; + let path = dir.path().join("bench.db"); + let mut tree = BPlusTree::>::new(); + for key in 0..count { + tree.insert(key, noise(key, size)); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::>::try_new(&path)?; + // Fault the mapping in first; this benchmark measures lookup, not page-in. + let mut checksum = 0u64; + for key in 0..count { + checksum += + query.query_zero_copy(&key).map_err(BPlusTreeError::to_io)?.map_or(0, |value| u64::from(value[0])); + } + + let start = std::time::Instant::now(); + let mut key = 0u32; + for _ in 0..LOOKUPS { + key = (key + STRIDE) % count; + checksum += + query.query_zero_copy(&key).map_err(BPlusTreeError::to_io)?.map_or(0, |value| u64::from(value[0])); + } + let elapsed = start.elapsed(); + + assert!(checksum > 0, "lookups must not be optimized away"); + println!( + "BENCH query {label:6} entries={count:6} bytes={size:5} lookups={LOOKUPS} -> {:>8.1} ms {:>9.0} lookups/s", + elapsed.as_secs_f64() * 1000.0, + f64::from(LOOKUPS) / elapsed.as_secs_f64() + ); + } + Ok(()) + } + #[test] fn streamed_store_writes_every_page_despite_out_of_order_overflow_chains() -> io::Result<()> { // The streaming builder hands out a leaf's page id *before* the overflow chain it @@ -4480,7 +4383,7 @@ mod tests { fn immediate_commit_clears_wal_then_invalidates_sorted_index() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("immediate.db"); - let index_path = crate::repository::storage::get_file_path_for_db_index(&path); + let index_path = crate::common::get_file_path_for_db_index(&path); let mut tree = BPlusTree::new(); tree.insert(1u32, String::from("old")); tree.store(&path)?; @@ -4520,7 +4423,7 @@ mod tests { fn missing_delete_and_empty_commit_are_true_noops() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("no-op.db"); - let index_path = crate::repository::storage::get_file_path_for_db_index(&path); + let index_path = crate::common::get_file_path_for_db_index(&path); let mut tree = BPlusTree::new(); tree.insert(1u32, String::from("one")); tree.store(&path)?; @@ -4558,7 +4461,7 @@ mod tests { fn store_with_index_publishes_an_identity_bound_sorted_snapshot() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("indexed.db"); - let index_path = crate::repository::storage::get_file_path_for_db_index(&path); + let index_path = crate::common::get_file_path_for_db_index(&path); let mut tree = BPlusTree::new(); tree.insert(1u32, String::from("ccc")); tree.insert(2u32, String::from("a")); @@ -4567,10 +4470,7 @@ mod tests { assert_ne!(tree.store_with_index(&path, String::len)?, 0); let query = BPlusTreeQuery::::try_new(&path)?; - let mut sorted = crate::repository::bplustree::sorted_index::v4::OwnedIterator::::open( - query, - &index_path, - )?; + let mut sorted = crate::sorted_index::v4::OwnedIterator::::open(query, &index_path)?; assert_eq!( sorted.by_ref().collect::>>()?, vec![(2, String::from("a")), (3, String::from("bb")), (1, String::from("ccc"))] @@ -4585,7 +4485,7 @@ mod tests { fn compact_rebuilds_only_live_data_with_a_fresh_identity() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("compact.db"); - let index_path = crate::repository::storage::get_file_path_for_db_index(&path); + let index_path = crate::common::get_file_path_for_db_index(&path); let mut tree = BPlusTree::new(); for key in 0..100u32 { tree.insert( @@ -4626,7 +4526,7 @@ mod tests { fn compact_read_failure_preserves_database_and_index() -> io::Result<()> { let dir = tempfile::tempdir()?; let path = dir.path().join("compact-corrupt.db"); - let index_path = crate::repository::storage::get_file_path_for_db_index(&path); + let index_path = crate::common::get_file_path_for_db_index(&path); let mut tree = BPlusTree::new(); tree.insert(1u32, String::from("one")); tree.store(&path)?; @@ -4745,16 +4645,9 @@ mod tests { let active = updater.active.as_mut().ok_or_else(|| io::Error::other("test transaction is missing"))?; let leaf_id = active.transaction.next_header.root_page_id; let snapshot = active.transaction.page_copy(active.base.as_slice(), leaf_id)?; - let page = SlottedPage::open( - snapshot.as_slice(), - leaf_id, - active.transaction.next_header.next_page_id, - )?; + let page = SlottedPage::open(snapshot.as_slice(), leaf_id, active.transaction.next_header.next_page_id)?; let mut cells = page.cells().map(|cell| cell.map(<[u8]>::to_vec)).collect::>>()?; - let duplicate = cells - .first() - .cloned() - .ok_or_else(|| io::Error::other("test leaf has no cells"))?; + let duplicate = cells.first().cloned().ok_or_else(|| io::Error::other("test leaf has no cells"))?; *cells.get_mut(1).ok_or_else(|| io::Error::other("test leaf lacks a second cell"))? = duplicate; let next_page_id = active.transaction.next_header.next_page_id; let dirty = active.transaction.page_mut(active.base.as_slice(), leaf_id)?; @@ -4973,13 +4866,35 @@ mod tests { handle.join().map_err(|_| io::Error::other("replacement writer panicked")) } + /// Waits for the sidecar lock to become free. + /// + /// A writer reporting its result does not mean the lock is already released: + /// publication can finish on the engine's background thread, so the release + /// happens on a different thread from the one the test observes. Asserting + /// `try_exclusive_sidecar` immediately races with that hand-off and fails + /// intermittently. Waiting for the lock to become available tests what the + /// engine actually promises - that it is released - without asserting an + /// instantaneous guarantee it never made. + fn wait_for_exclusive_sidecar(database: &Path) -> io::Result { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if try_exclusive_sidecar(database)? { + return Ok(true); + } + if std::time::Instant::now() >= deadline { + return Ok(false); + } + thread::sleep(Duration::from_millis(10)); + } + } + fn try_exclusive_sidecar(database: &Path) -> io::Result { let file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(false) - .open(crate::repository::bplustree::common::sidecar_lock_path(database))?; + .open(crate::common::sidecar_lock_path(database))?; match file.try_lock_exclusive() { Ok(()) => { fs2::FileExt::unlock(&file)?; @@ -4993,7 +4908,7 @@ mod tests { fn run_exclusive_probe_child(database: &Path, expected: &str) -> io::Result<()> { let status = Command::new(std::env::current_exe()?) .arg("--exact") - .arg("repository::bplustree::v3::tree::tests::exclusive_sidecar_probe_child") + .arg("v3::tree::tests::exclusive_sidecar_probe_child") .arg("--nocapture") .env("TULIPROX_V3_LOCK_PROBE_PATH", database) .env("TULIPROX_V3_LOCK_PROBE_EXPECTED", expected) @@ -5038,7 +4953,7 @@ mod tests { assert_writer_is_blocked(&receiver)?; drop(second); finish_writer(&receiver, handle)?; - assert!(try_exclusive_sidecar(&path)?); + assert!(wait_for_exclusive_sidecar(&path)?); Ok(()) } @@ -5057,7 +4972,7 @@ mod tests { assert_writer_is_blocked(&receiver)?; drop(iterator); finish_writer(&receiver, handle)?; - assert!(try_exclusive_sidecar(&path)?); + assert!(wait_for_exclusive_sidecar(&path)?); Ok(()) } @@ -5172,9 +5087,7 @@ mod tests { } fn leaf_cell_of_footprint(footprint: usize) -> io::Result> { - let key_length = footprint - .checked_sub(4 + 24) - .ok_or_else(|| io::Error::other("footprint is too small"))?; + let key_length = footprint.checked_sub(4 + 24).ok_or_else(|| io::Error::other("footprint is too small"))?; if footprint <= 2032 { let key = vec![b'k'; key_length]; let mut cell = Vec::new(); @@ -5228,8 +5141,21 @@ mod tests { encode_tombstone_leaf_cell(b"deleted", &mut cell)?; assert!(matches!(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, LeafValueRef::Tombstone)); - encode_overflow_leaf_cell(b"large", 5000, Compression::None, 5000, 2, 0x1234_5678, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; - assert!(matches!(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, LeafValueRef::Overflow { head: 2, .. })); + encode_overflow_leaf_cell( + b"large", + 5000, + Compression::None, + 5000, + 2, + 0x1234_5678, + PAGE_ID, + NEXT_PAGE_ID, + &mut cell, + )?; + assert!(matches!( + LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, + LeafValueRef::Overflow { head: 2, .. } + )); Ok(()) } @@ -5279,25 +5205,15 @@ mod tests { #[test] fn adversarial_leaf_splits_reject_old_limit_and_accept_capped_cells() -> io::Result<()> { - let first_witness = [ - leaf_cell_of_footprint(2026)?, - leaf_cell_of_footprint(2040)?, - leaf_cell_of_footprint(2038)?, - ]; + let first_witness = + [leaf_cell_of_footprint(2026)?, leaf_cell_of_footprint(2040)?, leaf_cell_of_footprint(2038)?]; invalid_input(choose_leaf_split(&first_witness))?; - let second_witness = [ - leaf_cell_of_footprint(1984)?, - leaf_cell_of_footprint(2296)?, - leaf_cell_of_footprint(1984)?, - ]; + let second_witness = + [leaf_cell_of_footprint(1984)?, leaf_cell_of_footprint(2296)?, leaf_cell_of_footprint(1984)?]; invalid_input(choose_leaf_split(&second_witness))?; - let capped = [ - leaf_cell_of_footprint(2026)?, - leaf_cell_of_footprint(2032)?, - leaf_cell_of_footprint(2032)?, - ]; + let capped = [leaf_cell_of_footprint(2026)?, leaf_cell_of_footprint(2032)?, leaf_cell_of_footprint(2032)?]; let split = choose_leaf_split(&capped)?; assert_eq!(split, 2); assert!(used_leaf_bytes(&capped[..split])? <= PAGE_SIZE); @@ -5441,10 +5357,7 @@ mod tests { crc32: crc32fast::hash(&stored), }; let mut scratch = Vec::new(); - assert_eq!( - read_leaf_value(&database, &value, next_page_id, raw.len(), &mut scratch)?, - Some(raw.as_slice()) - ); + assert_eq!(read_leaf_value(&database, &value, next_page_id, raw.len(), &mut scratch)?, Some(raw.as_slice())); Ok(()) } @@ -5493,13 +5406,7 @@ mod tests { head: 2, crc32: crc32fast::hash(b"x"), }; - invalid_data(read_leaf_value( - &database, - &empty_chain_value, - NEXT_PAGE_ID, - 6000, - &mut scratch, - )) + invalid_data(read_leaf_value(&database, &empty_chain_value, NEXT_PAGE_ID, 6000, &mut scratch)) } fn page_range(page_id: u64) -> io::Result> { @@ -5512,7 +5419,7 @@ mod tests { fn rewrite_page_checksum(database: &mut [u8], page_id: u64) -> io::Result<()> { let range = page_range(page_id)?; - crate::repository::bplustree::v3::format::write_page_checksum( + crate::v3::format::write_page_checksum( database.get_mut(range).ok_or_else(|| io::Error::other("test page missing"))?, ) } @@ -5612,9 +5519,7 @@ mod tests { tree.store(&path)?; let mut query = BPlusTreeQuery::::try_new(&path)?; - let (page_id, range) = query - .locate_cell(&2)? - .ok_or_else(|| io::Error::other("test key missing"))?; + let (page_id, range) = query.locate_cell(&2)?.ok_or_else(|| io::Error::other("test key missing"))?; let next_page_id = query.header.next_page_id; let stored_offset = query.with_page(page_id, |bytes, _| { let cell = LeafCellRef::decode( @@ -5631,9 +5536,7 @@ mod tests { let mut database = fs::read(&path)?; let absolute = page_range(page_id)?.start + stored_offset; - *database - .get_mut(absolute) - .ok_or_else(|| io::Error::other("test value byte missing"))? = 0xc1; + *database.get_mut(absolute).ok_or_else(|| io::Error::other("test value byte missing"))? = 0xc1; rewrite_page_checksum(&mut database, page_id)?; fs::write(&path, database)?; @@ -5683,9 +5586,8 @@ mod tests { tree.store(&path)?; let mut query = BPlusTreeQuery::::try_new(&path)?; - let (entries, has_more) = query - .range_page(Bound::Unbounded, Bound::Unbounded, 0, usize::MAX) - .map_err(BPlusTreeError::to_io)?; + let (entries, has_more) = + query.range_page(Bound::Unbounded, Bound::Unbounded, 0, usize::MAX).map_err(BPlusTreeError::to_io)?; assert_eq!(entries, vec![(1, String::from("one")), (2, String::from("two"))]); assert!(!has_more); Ok(()) @@ -5726,16 +5628,10 @@ mod tests { let mut duplicate_child = database.clone(); let root_start = page_range(root)?.start; - let leftmost = u64::from_le_bytes( - duplicate_child[root_start + 32..root_start + 40] - .try_into() - .map_err(io::Error::other)?, - ); - let first_cell = u16::from_le_bytes( - duplicate_child[root_start + 40..root_start + 42] - .try_into() - .map_err(io::Error::other)?, - ); + let leftmost = + u64::from_le_bytes(duplicate_child[root_start + 32..root_start + 40].try_into().map_err(io::Error::other)?); + let first_cell = + u16::from_le_bytes(duplicate_child[root_start + 40..root_start + 42].try_into().map_err(io::Error::other)?); let child_offset = root_start + usize::from(first_cell) + 4; duplicate_child[child_offset..child_offset + 8].copy_from_slice(&leftmost.to_le_bytes()); rewrite_page_checksum(&mut duplicate_child, root)?; @@ -5760,15 +5656,11 @@ mod tests { let mut inverted = database.clone(); let first_start = page_range(first)?.start; let second_start = page_range(second)?.start; - let first_count = u16::from_le_bytes( - inverted[first_start + 2..first_start + 4] - .try_into() - .map_err(io::Error::other)?, - ); + let first_count = + u16::from_le_bytes(inverted[first_start + 2..first_start + 4].try_into().map_err(io::Error::other)?); let last_slot = first_start + PAGE_HEADER_LEN + (usize::from(first_count) - 1) * SLOT_LEN; - let left_cell = usize::from(u16::from_le_bytes( - inverted[last_slot..last_slot + 2].try_into().map_err(io::Error::other)?, - )); + let left_cell = + usize::from(u16::from_le_bytes(inverted[last_slot..last_slot + 2].try_into().map_err(io::Error::other)?)); let right_cell = usize::from(u16::from_le_bytes( inverted[second_start + PAGE_HEADER_LEN..second_start + PAGE_HEADER_LEN + 2] .try_into() @@ -5871,9 +5763,8 @@ mod tests { verify_rejects::>(&cycle)?; let mut shared = database; - let start = page_range(second_leaf)?.start - + usize::try_from(second_cell_offset).map_err(io::Error::other)? - + 12; + let start = + page_range(second_leaf)?.start + usize::try_from(second_cell_offset).map_err(io::Error::other)? + 12; shared[start..start + 8].copy_from_slice(&first_head.to_le_bytes()); rewrite_page_checksum(&mut shared, second_leaf)?; let _ = next_page_id; @@ -5894,8 +5785,8 @@ mod tests { drop(query); let mut database = fs::read(&path)?; - let descriptor = page_range(second_leaf)?.start - + usize::try_from(second_cell_offset).map_err(io::Error::other)?; + let descriptor = + page_range(second_leaf)?.start + usize::try_from(second_cell_offset).map_err(io::Error::other)?; database[descriptor + 4..descriptor + 8].copy_from_slice(&u32::MAX.to_le_bytes()); database[descriptor + 8..descriptor + 12].copy_from_slice(&u32::MAX.to_le_bytes()); rewrite_page_checksum(&mut database, second_leaf)?; @@ -5924,9 +5815,7 @@ mod tests { let slot = PAGE_HEADER_LEN .checked_add(index.checked_mul(SLOT_LEN).ok_or_else(|| io::Error::other("test slot overflow"))?) .ok_or_else(|| io::Error::other("test slot overflow"))?; - Ok(u16::from_le_bytes( - bytes[slot..slot + 2].try_into().map_err(io::Error::other)?, - )) + Ok(u16::from_le_bytes(bytes[slot..slot + 2].try_into().map_err(io::Error::other)?)) })?; drop(query); @@ -5951,11 +5840,10 @@ mod tests { new.insert(2u32, vec![2]); new.store(&temporary)?; - let error = publish_database(&temporary, &destination, |_| { - Err(io::Error::other("injected directory sync failure")) - }) - .err() - .ok_or_else(|| io::Error::other("post-commit sync failure was hidden"))?; + let error = + publish_database(&temporary, &destination, |_| Err(io::Error::other("injected directory sync failure"))) + .err() + .ok_or_else(|| io::Error::other("post-commit sync failure was hidden"))?; assert!(error.database_was_published()); let error = io::Error::from(error); assert!(error.to_string().contains("database published but directory sync failed; durability unknown")); @@ -5970,7 +5858,7 @@ mod tests { fn store_sync_failure_after_rename_invalidates_previous_sorted_index() -> io::Result<()> { let dir = tempfile::tempdir()?; let destination = dir.path().join("database.db"); - let index = crate::repository::storage::get_file_path_for_db_index(&destination); + let index = crate::common::get_file_path_for_db_index(&destination); let mut old = BPlusTree::new(); old.insert(1u32, String::from("old")); let _ = old.store_with_index(&destination, String::clone)?; @@ -5998,7 +5886,7 @@ mod tests { fn store_reports_sync_and_index_invalidation_failures_together() -> io::Result<()> { let dir = tempfile::tempdir()?; let destination = dir.path().join("database.db"); - let index = crate::repository::storage::get_file_path_for_db_index(&destination); + let index = crate::common::get_file_path_for_db_index(&destination); let mut old = BPlusTree::new(); old.insert(1u32, String::from("old")); let _ = old.store(&destination)?; diff --git a/backend/src/repository/bplustree/v3/wal.rs b/backend/btree/src/v3/wal.rs similarity index 93% rename from backend/src/repository/bplustree/v3/wal.rs rename to backend/btree/src/v3/wal.rs index d51ea9a27..81a2079e9 100644 --- a/backend/src/repository/bplustree/v3/wal.rs +++ b/backend/btree/src/v3/wal.rs @@ -2,7 +2,7 @@ use super::{ format::{DatabaseHeader, PAGE_SIZE}, page::SlottedPage, }; -use crate::repository::bplustree::common::sidecar_lock_path; +use crate::common::sidecar_lock_path; use fs2::FileExt as _; use log::info; use std::{ @@ -27,9 +27,7 @@ const PAGE_SIZE_U64: u64 = 4096; const WAL_MAGIC: &[u8; 4] = b"BTW3"; const WAL_VERSION: u32 = 1; -fn invalid_data(message: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, message.into()) -} +fn invalid_data(message: impl Into) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, message.into()) } fn invalid_input(message: impl Into) -> io::Error { io::Error::new(io::ErrorKind::InvalidInput, message.into()) @@ -57,7 +55,11 @@ fn read_u32(bytes: &[u8], offset: usize) -> io::Result { Ok(u32::from_le_by fn read_u64(bytes: &[u8], offset: usize) -> io::Result { Ok(u64::from_le_bytes(bytes_at(bytes, offset)?)) } fn require_zero(bytes: &[u8], message: &'static str) -> io::Result<()> { - if bytes.iter().all(|byte| *byte == 0) { Ok(()) } else { Err(invalid_data(message)) } + if bytes.iter().all(|byte| *byte == 0) { + Ok(()) + } else { + Err(invalid_data(message)) + } } fn crc_with_zeroed_field(bytes: &[u8], offset: usize) -> io::Result { @@ -131,9 +133,7 @@ impl WalHeader { if self.transaction_id == 0 { return Err(invalid_data("WAL transaction id must be nonzero")); } - if self.original_database_len < PAGE_SIZE_U64 - || !self.original_database_len.is_multiple_of(PAGE_SIZE_U64) - { + if self.original_database_len < PAGE_SIZE_U64 || !self.original_database_len.is_multiple_of(PAGE_SIZE_U64) { return Err(invalid_data("invalid original database length")); } if self.original_generation == 0 { @@ -170,9 +170,8 @@ impl BeforeImage { return Err(invalid_data("invalid before-image payload length")); } require_zero(&payload[12..16], "before-image reserved bytes must be zero")?; - let page: [u8; PAGE_SIZE] = payload[16..] - .try_into() - .map_err(|_| invalid_data("truncated before-image page"))?; + let page: [u8; PAGE_SIZE] = + payload[16..].try_into().map_err(|_| invalid_data("truncated before-image page"))?; if read_u32(payload, 8)? != crc32fast::hash(&page) { return Err(invalid_data("before-image page checksum mismatch")); } @@ -228,9 +227,8 @@ impl CommitRecord { fn prepare_record(encoded: &mut Vec, kind: u8, payload_length: usize) -> io::Result<()> { let payload_length_u32 = u32::try_from(payload_length).map_err(|_| invalid_input("WAL payload exceeds u32"))?; - let record_length = RECORD_HEADER_LEN - .checked_add(payload_length) - .ok_or_else(|| invalid_input("WAL record length overflow"))?; + let record_length = + RECORD_HEADER_LEN.checked_add(payload_length).ok_or_else(|| invalid_input("WAL record length overflow"))?; encoded.clear(); encoded.try_reserve_exact(record_length).map_err(out_of_memory("WAL record allocation failed"))?; encoded.resize(record_length, 0); @@ -296,9 +294,8 @@ fn read_wal(path: &Path) -> io::Result { if payload_length != expected_length { return Err(invalid_data("WAL record payload length mismatch")); } - let total_length = RECORD_HEADER_LEN - .checked_add(payload_length) - .ok_or_else(|| invalid_data("WAL record length overflow"))?; + let total_length = + RECORD_HEADER_LEN.checked_add(payload_length).ok_or_else(|| invalid_data("WAL record length overflow"))?; let total_length_u64 = u64::try_from(total_length).map_err(|_| invalid_data("WAL record exceeds u64"))?; if remaining < total_length_u64 { torn_tail = true; @@ -340,9 +337,7 @@ fn read_wal(path: &Path) -> io::Result { } _ => return Err(invalid_data("unknown WAL record kind")), } - offset = offset - .checked_add(total_length_u64) - .ok_or_else(|| invalid_data("WAL record offset overflow"))?; + offset = offset.checked_add(total_length_u64).ok_or_else(|| invalid_data("WAL record offset overflow"))?; } Ok(ParsedWal { header, before_images, commit, torn_tail }) } @@ -453,13 +448,7 @@ struct WalErrorContext<'a> { impl WalErrorContext<'_> { fn wrap(&self, phase: &'static str, result: io::Result) -> io::Result { - result.map_err(wal_operation_error( - self.outcome, - self.database, - self.wal, - self.transaction_id, - phase, - )) + result.map_err(wal_operation_error(self.outcome, self.database, self.wal, self.transaction_id, phase)) } } @@ -488,12 +477,7 @@ impl WalReadError { impl fmt::Display for WalReadError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "failed to parse WAL {} for database {}", - self.wal.display(), - self.database.display() - )?; + write!(formatter, "failed to parse WAL {} for database {}", self.wal.display(), self.database.display())?; if let Some(identity) = self.wal_database_id { write!(formatter, ", WAL database identity {identity:02x?}")?; } else { @@ -551,18 +535,14 @@ fn write_page(file: &mut File, page_id: u64, page: &[u8; PAGE_SIZE]) -> io::Resu } fn database_length(next_page_id: u64) -> io::Result { - next_page_id - .checked_mul(PAGE_SIZE_U64) - .ok_or_else(|| invalid_input("database length overflow")) + next_page_id.checked_mul(PAGE_SIZE_U64).ok_or_else(|| invalid_input("database length overflow")) } fn database_header_crc32(page: &[u8; PAGE_SIZE]) -> io::Result { read_u32(page, 72) } fn new_transaction_id() -> io::Result { let bytes = uuid::Uuid::new_v4().as_u128().to_le_bytes(); - let low: [u8; 8] = bytes[..8] - .try_into() - .map_err(|_| io::Error::other("UUID transaction id conversion failed"))?; + let low: [u8; 8] = bytes[..8].try_into().map_err(|_| io::Error::other("UUID transaction id conversion failed"))?; let transaction_id = u64::from_le_bytes(low); Ok(if transaction_id == 0 { 1 } else { transaction_id }) } @@ -575,12 +555,10 @@ pub(super) fn sync_parent_directory(path: &Path) -> io::Result<()> { #[cfg(not(unix))] #[allow(clippy::unnecessary_wraps)] -pub(super) fn sync_parent_directory(_path: &Path) -> io::Result<()> { - Ok(()) -} +pub(super) fn sync_parent_directory(_path: &Path) -> io::Result<()> { Ok(()) } pub(crate) fn invalidate_sorted_index(database: &Path) -> io::Result<()> { - let index = crate::repository::storage::get_file_path_for_db_index(database); + let index = crate::common::get_file_path_for_db_index(database); if index == database { return Ok(()); } @@ -595,12 +573,7 @@ fn clear_wal(database: &Path, transaction_id: u64, outcome: WalOutcome) -> io::R clear_wal_with_hook(database, transaction_id, outcome, &mut |_| Ok(())) } -fn clear_wal_with_hook( - database: &Path, - transaction_id: u64, - outcome: WalOutcome, - hook: &mut H, -) -> io::Result<()> +fn clear_wal_with_hook(database: &Path, transaction_id: u64, outcome: WalOutcome, hook: &mut H) -> io::Result<()> where H: FnMut(RecoveryBoundary) -> io::Result<()>, { @@ -660,9 +633,7 @@ where } #[cfg(test)] -fn ordered_prepared_pages( - prepared: &[(u64, [u8; PAGE_SIZE])], -) -> io::Result> { +fn ordered_prepared_pages(prepared: &[(u64, [u8; PAGE_SIZE])]) -> io::Result> { let mut ordered = Vec::new(); ordered.try_reserve_exact(prepared.len()).map_err(out_of_memory("prepared-page ordering allocation failed"))?; ordered.extend(prepared.iter().map(|(page_id, page)| (*page_id, page))); @@ -721,10 +692,7 @@ fn validate_prepared_pages( } #[cfg(test)] -pub(crate) fn commit_prepared_pages( - database: &Path, - prepared: &[(u64, [u8; PAGE_SIZE])], -) -> io::Result<()> { +pub(crate) fn commit_prepared_pages(database: &Path, prepared: &[(u64, [u8; PAGE_SIZE])]) -> io::Result<()> { with_exclusive_sidecar(database, || { recover_pending_under_existing_lock(database)?; commit_prepared_pages_under_existing_lock(database, prepared) @@ -792,10 +760,7 @@ pub(super) fn leave_uncommitted_test_wal_after_database_write(database: &Path) - ), )), Err(_) => Ok(()), - Ok(()) => Err(io::Error::other(format!( - "test commit for {} unexpectedly completed", - database.display() - ))), + Ok(()) => Err(io::Error::other(format!("test commit for {} unexpectedly completed", database.display()))), } } @@ -883,19 +848,15 @@ fn commit_ordered_page_refs_with_hook_under_existing_lock( context.wrap("sync database", database.sync_all())?; context.wrap("DatabaseSynced hook", hook(CommitBoundary::DatabaseSynced))?; - let final_header_page = ordered - .first() - .map(|(_, page)| page) - .ok_or_else(|| invalid_input("prepared header page is missing")); + let final_header_page = + ordered.first().map(|(_, page)| page).ok_or_else(|| invalid_input("prepared header page is missing")); let final_header_page = context.wrap("locate prepared database header", final_header_page)?; let commit = CommitRecord { transaction_id, new_generation: final_header.generation, database_length: final_length, - database_header_crc32: context.wrap( - "read prepared database header checksum", - database_header_crc32(final_header_page), - )?, + database_header_crc32: context + .wrap("read prepared database header checksum", database_header_crc32(final_header_page))?, }; context.wrap("encode WAL commit record", commit.encode_record_into(&mut record))?; context.wrap("append WAL commit record", wal.write_all(&record))?; @@ -934,7 +895,9 @@ impl fmt::Display for RecoveryRequired { } impl Error for RecoveryRequired { - fn source(&self) -> Option<&(dyn Error + 'static)> { self.cause.as_ref().map(|cause| cause as &(dyn Error + 'static)) } + fn source(&self) -> Option<&(dyn Error + 'static)> { + self.cause.as_ref().map(|cause| cause as &(dyn Error + 'static)) + } } fn append_suffix(path: &Path, suffix: &str) -> PathBuf { @@ -954,12 +917,7 @@ pub(crate) fn recovery_required(database: &Path, cause: io::Error) -> io::Error } fn open_sidecar(database: &Path) -> io::Result { - OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(sidecar_lock_path(database)) + OpenOptions::new().read(true).write(true).create(true).truncate(false).open(sidecar_lock_path(database)) } pub(crate) struct SharedSidecarGuard { @@ -1176,10 +1134,7 @@ pub(crate) fn recover_pending_under_existing_lock(database_path: &Path) -> io::R } #[cfg(test)] -fn recover_pending_with_hook( - database: &Path, - hook: impl FnMut(RecoveryBoundary) -> io::Result<()>, -) -> io::Result<()> { +fn recover_pending_with_hook(database: &Path, hook: impl FnMut(RecoveryBoundary) -> io::Result<()>) -> io::Result<()> { with_exclusive_sidecar(database, || recover_pending_with_hook_under_existing_lock(database, hook)) } @@ -1242,7 +1197,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::repository::bplustree::v3::{ + use crate::v3::{ format::{write_page_checksum, DatabaseHeader, PAGE_SIZE}, page::encode_free_page, tree::{BPlusTree, BPlusTreeQuery}, @@ -1252,9 +1207,8 @@ mod tests { io::{Read, Seek, SeekFrom, Write}, }; - const DATABASE_ID: [u8; 16] = [ - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - ]; + const DATABASE_ID: [u8; 16] = + [0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f]; const TRANSACTION_ID: u64 = 0x0102_0304_0506_0708; fn checksum_with_zeroed_field(bytes: &[u8], offset: usize) -> io::Result { @@ -1318,10 +1272,8 @@ mod tests { fn prepared_header_page(original: &DatabaseHeader) -> io::Result<[u8; PAGE_SIZE]> { let mut updated = original.clone(); - updated.generation = updated - .generation - .checked_add(1) - .ok_or_else(|| io::Error::other("test generation overflow"))?; + updated.generation = + updated.generation.checked_add(1).ok_or_else(|| io::Error::other("test generation overflow"))?; updated.encode() } @@ -1351,9 +1303,7 @@ mod tests { } } - fn create_multi_page_database( - name: &str, - ) -> io::Result<(tempfile::TempDir, PathBuf, DatabaseHeader, Vec)> { + fn create_multi_page_database(name: &str) -> io::Result<(tempfile::TempDir, PathBuf, DatabaseHeader, Vec)> { let directory = tempfile::tempdir()?; let path = directory.path().join(name); let mut tree = BPlusTree::::new(); @@ -1394,20 +1344,14 @@ mod tests { Ok(page) } - fn create_uncommitted_multi_page_wal( - name: &str, - ) -> io::Result<(tempfile::TempDir, PathBuf, Vec, Vec)> { + fn create_uncommitted_multi_page_wal(name: &str) -> io::Result<(tempfile::TempDir, PathBuf, Vec, Vec)> { let (directory, path, original, original_bytes) = create_multi_page_database(name)?; let appended_page_id = original.next_page_id; let mut updated = original.clone(); - updated.generation = updated - .generation - .checked_add(1) - .ok_or_else(|| io::Error::other("test generation overflow"))?; - updated.next_page_id = updated - .next_page_id - .checked_add(1) - .ok_or_else(|| io::Error::other("test page-id overflow"))?; + updated.generation = + updated.generation.checked_add(1).ok_or_else(|| io::Error::other("test generation overflow"))?; + updated.next_page_id = + updated.next_page_id.checked_add(1).ok_or_else(|| io::Error::other("test page-id overflow"))?; updated.free_page_head = appended_page_id; let prepared_len = usize::try_from(updated.next_page_id).map_err(io::Error::other)?; @@ -1418,9 +1362,7 @@ mod tests { .map_err(io::Error::other)? .checked_mul(PAGE_SIZE) .ok_or_else(|| io::Error::other("test page offset overflow"))?; - let end = start - .checked_add(PAGE_SIZE) - .ok_or_else(|| io::Error::other("test page end overflow"))?; + let end = start.checked_add(PAGE_SIZE).ok_or_else(|| io::Error::other("test page end overflow"))?; let page: [u8; PAGE_SIZE] = original_bytes .get(start..end) .ok_or_else(|| io::Error::other("multi-page fixture is truncated"))? @@ -1433,10 +1375,7 @@ mod tests { }; prepared.push((page_id, page)); } - prepared.push(( - appended_page_id, - encode_free_page(appended_page_id, updated.next_page_id, 0)?, - )); + prepared.push((appended_page_id, encode_free_page(appended_page_id, updated.next_page_id, 0)?)); let _ = commit_prepared_pages_with_hook(&path, &prepared, fail_at(CommitBoundary::DatabaseWritten)); let active = wal_path(&path); if !active.try_exists()? { @@ -1450,9 +1389,7 @@ mod tests { .map_err(io::Error::other)? .checked_mul(PAGE_SIZE) .ok_or_else(|| io::Error::other("fixture page offset overflow"))?; - let end = start - .checked_add(PAGE_SIZE) - .ok_or_else(|| io::Error::other("fixture page end overflow"))?; + let end = start.checked_add(PAGE_SIZE).ok_or_else(|| io::Error::other("fixture page end overflow"))?; bytes.get(start..end).ok_or_else(|| io::Error::other("fixture page is truncated")) } @@ -1481,12 +1418,9 @@ mod tests { fn rebuild_record_crc(record: &mut [u8]) -> io::Result<()> { let payload_length = usize::try_from(read_u32(record, 4)?).map_err(io::Error::other)?; - let record_length = 16usize - .checked_add(payload_length) - .ok_or_else(|| io::Error::other("record length overflow"))?; - let bytes = record - .get(..record_length) - .ok_or_else(|| io::Error::other("truncated record fixture"))?; + let record_length = + 16usize.checked_add(payload_length).ok_or_else(|| io::Error::other("record length overflow"))?; + let bytes = record.get(..record_length).ok_or_else(|| io::Error::other("truncated record fixture"))?; let checksum = checksum_with_zeroed_field(bytes, 8)?; record .get_mut(8..12) @@ -1674,7 +1608,7 @@ mod tests { #[test] fn committed_recovery_invalidates_sorted_index_before_clearing_wal() -> io::Result<()> { let (_directory, path, original, _) = create_database("committed-index-recovery.db")?; - let index_path = crate::repository::storage::get_file_path_for_db_index(&path); + let index_path = crate::common::get_file_path_for_db_index(&path); fs::write(&index_path, b"stale sorted index")?; let prepared = [(0, prepared_header_page(&original)?)]; @@ -1836,9 +1770,7 @@ mod tests { let database_before = fs::read(&path)?; let wal_before = fs::read(&active)?; - let error = recover_pending(&path) - .err() - .ok_or_else(|| io::Error::other("foreign WAL was accepted"))?; + let error = recover_pending(&path).err().ok_or_else(|| io::Error::other("foreign WAL was accepted"))?; let message = error.to_string(); assert!(message.contains(&path.display().to_string())); assert!(message.contains(&format!("{:02x?}", [0x88; 16]))); @@ -1855,17 +1787,14 @@ mod tests { let _ = commit_prepared_pages_with_hook(&path, &prepared, fail_at(CommitBoundary::BeforeImagesSynced)); let active = wal_path(&path); let mut bytes = fs::read(&active)?; - let checksum_byte = bytes - .get_mut(WAL_HEADER_LEN + 8) - .ok_or_else(|| io::Error::other("WAL record fixture is truncated"))?; + let checksum_byte = + bytes.get_mut(WAL_HEADER_LEN + 8).ok_or_else(|| io::Error::other("WAL record fixture is truncated"))?; *checksum_byte ^= 1; fs::write(&active, bytes)?; let database_before = fs::read(&path)?; let wal_before = fs::read(&active)?; - let error = recover_pending(&path) - .err() - .ok_or_else(|| io::Error::other("corrupt WAL record was accepted"))?; + let error = recover_pending(&path).err().ok_or_else(|| io::Error::other("corrupt WAL record was accepted"))?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); let context = error .get_ref() @@ -1978,10 +1907,8 @@ mod tests { updated.generation += 1; updated.next_page_id += 1; updated.free_page_head = appended_page_id; - let prepared = [ - (0, updated.encode()?), - (appended_page_id, encode_free_page(appended_page_id, updated.next_page_id, 0)?), - ]; + let prepared = + [(0, updated.encode()?), (appended_page_id, encode_free_page(appended_page_id, updated.next_page_id, 0)?)]; let _ = commit_prepared_pages_with_hook(&path, &prepared, fail_at(CommitBoundary::DatabaseWritten)); assert_eq!(fs::metadata(&path)?.len(), original_length + PAGE_SIZE as u64); @@ -2001,7 +1928,7 @@ mod tests { assert_eq!(database_header(&path)?.generation, original.generation + 1); let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(crate::repository::bplustree::common::BPlusTreeError::to_io)?, Some(String::from("original"))); + assert_eq!(query.query(&1).map_err(crate::common::BPlusTreeError::to_io)?, Some(String::from("original"))); assert_eq!(database_header(&path)?.generation, original.generation); assert!(!wal_path(&path).try_exists()?); Ok(()) diff --git a/backend/config-loader/Cargo.toml b/backend/config-loader/Cargo.toml new file mode 100644 index 000000000..582d068b6 --- /dev/null +++ b/backend/config-loader/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "tuliprox-config-loader" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Application configuration bootstrap for tuliprox" + +[lints] +workspace = true + +[dependencies] +shared = { workspace = true } +tuliprox-core = { workspace = true } +tuliprox-repository = { workspace = true } +arc-swap.workspace = true +chrono.workspace = true +log.workspace = true +serde.workspace = true +serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } +serde-saphyr.workspace = true +tempfile = "3.27.0" +tokio = { workspace = true, features = ["rt-multi-thread", "fs", "macros"] } +url.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +windows-sys = { version = "0.61.2", features = ["Win32_Storage_FileSystem"] } diff --git a/backend/src/utils/file/config_reader.rs b/backend/config-loader/src/lib.rs similarity index 72% rename from backend/src/utils/file/config_reader.rs rename to backend/config-loader/src/lib.rs index feaa3bd53..2fce9607b 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/config-loader/src/lib.rs @@ -1,39 +1,40 @@ -use crate::api::model::AppState; -use crate::model::Config; -use crate::model::{ApiProxyConfig, AppConfig, MediaToolCapabilities, SourcesConfig, UserPlan}; -use crate::repository::{ - csv_read_inputs, csv_write_inputs, get_api_user_db_path, is_csv_file, load_api_user, -}; -use crate::utils; -use crate::utils::{file_exists_async, file_reader}; -use crate::utils::sys_utils::exit; -use crate::utils::{ - open_file, read_mappings_file_unprepared, read_templates_file, EnvResolvingReader, - FileLockManager, -}; +pub mod runtime_config_report; + use arc_swap::{ArcSwap, ArcSwapAny}; use chrono::Local; use log::{error, info, warn}; use serde::Serialize; use shared::{ + concat_string, + defaults::{generate_default_access_secret, generate_default_encrypt_secret, TEMPLATE_FILE}, error::TuliproxError, foundation::prepare_templates, model::{ - ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, - MsgKind, PatternTemplate, PlansConfigDto, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, UserPlanDto, + ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, + InputType, MsgKind, PatternTemplate, PlansConfigDto, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, + UserPlanDto, }, - utils::{CONSTANTS, PROVIDER_SCHEME_PREFIX}, - defaults::{generate_default_access_secret, generate_default_encrypt_secret, TEMPLATE_FILE}, + utils::PROVIDER_SCHEME_PREFIX, +}; +use std::{ + collections::{HashMap, HashSet}, + io::ErrorKind, + path::{Path, PathBuf}, + sync::Arc, }; -use std::collections::{HashMap, HashSet}; -use std::env; -use std::fs::File; -use std::io::{self, Read}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; use tokio::fs; -use shared::concat_string; -use crate::utils::request::{is_uri}; +use tuliprox_core::{ + model::{ApiProxyConfig, AppConfig, Config, MediaToolCapabilities, SourcesConfig, UserPlan}, + utils, + utils::{ + config_file_reader, exit, file_exists_async, open_file, read_mappings_file_unprepared, read_templates_file, + request::is_uri, resolve_env_var, FileLockManager, + }, +}; +use tuliprox_repository::{ + backup_api_user_db_file, csv_read_inputs, csv_write_inputs, get_api_user_db_path, is_csv_file, load_api_user, + merge_api_user, +}; use url::Url; /// Options controlling how configuration files are parsed. @@ -63,46 +64,21 @@ impl ReadConfigOptions { // Expand environment variables and include all computed fields — the typical // production default. #[must_use] - pub const fn resolve_and_compute() -> Self { - Self { resolve_env: true, include_computed: true } - } + pub const fn resolve_and_compute() -> Self { Self { resolve_env: true, include_computed: true } } // Parse the raw YAML without any variable expansion or computed fields — // useful when you need the literal, unexpanded content (e.g. before saving). #[must_use] - pub const fn raw() -> Self { - Self { resolve_env: false, include_computed: false } - } + pub const fn raw() -> Self { Self { resolve_env: false, include_computed: false } } } -pub(crate) struct PreparedTemplateBundle { - pub(crate) definition: Option, - pub(crate) prepared: Option>, - pub(crate) files_used: Option>, -} - -enum EitherReader { - Left(L), - Right(R), +pub struct PreparedTemplateBundle { + pub definition: Option, + pub prepared: Option>, + pub files_used: Option>, } // `Read`-Trait for Either -impl Read for EitherReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - match self { - EitherReader::Left(reader) => reader.read(buf), - EitherReader::Right(reader) => reader.read(buf), - } - } -} - -pub fn config_file_reader(file: File, resolve_env: bool) -> impl Read { - if resolve_env { - EitherReader::Left(EnvResolvingReader::new(file_reader(file))) - } else { - EitherReader::Right(file_reader(file)) - } -} pub async fn read_api_proxy_config( config: &AppConfig, @@ -114,7 +90,7 @@ pub async fn read_api_proxy_config( let mut errors = vec![]; let mut api_proxy: ApiProxyConfig = ApiProxyConfig::from(&api_proxy_dto); apply_authoritative_plans(config, &mut api_proxy, resolve_env).await; - api_proxy.migrate_api_user(config, &mut errors).await; + migrate_api_user(&mut api_proxy, config, &mut errors).await; if !errors.is_empty() { for error in errors { error!("{error}"); @@ -194,7 +170,14 @@ pub async fn read_sources_file_from_path_with_options( hdhr_config: Option<&HdHomeRunDeviceOverview>, prepared_templates: Option<&[shared::model::PatternTemplate]>, ) -> Result { - read_sources_file_from_path_with_templates(sources_file, opts.resolve_env, opts.include_computed, hdhr_config, prepared_templates).await + read_sources_file_from_path_with_templates( + sources_file, + opts.resolve_env, + opts.include_computed, + hdhr_config, + prepared_templates, + ) + .await } pub async fn read_sources_file_from_path( @@ -207,8 +190,9 @@ pub async fn read_sources_file_from_path( sources_file, ReadConfigOptions { resolve_env, include_computed }, hdhr_config, - None - ).await + None, + ) + .await } pub async fn read_sources_file_with_options( @@ -227,13 +211,16 @@ pub async fn read_sources_file( hdhr_config: Option<&HdHomeRunDeviceOverview>, prepared_templates: Option<&[shared::model::PatternTemplate]>, ) -> Result { - read_sources_file_with_options(sources_file, ReadConfigOptions { resolve_env, include_computed }, hdhr_config, prepared_templates).await + read_sources_file_with_options( + sources_file, + ReadConfigOptions { resolve_env, include_computed }, + hdhr_config, + prepared_templates, + ) + .await } -pub fn read_config_file_with_options( - config_file: &str, - opts: ReadConfigOptions, -) -> Result { +pub fn read_config_file_with_options(config_file: &str, opts: ReadConfigOptions) -> Result { match open_file(&std::path::PathBuf::from(config_file)) { Ok(file) => { let maybe_config: Result = @@ -261,7 +248,7 @@ pub fn read_config_file( } #[allow(clippy::too_many_lines)] -pub(crate) fn read_templates( +pub fn read_templates( template_file_path: Option<&str>, resolve_env: bool, sources_inline_templates: Option<&[PatternTemplate]>, @@ -272,17 +259,10 @@ pub(crate) fn read_templates( sources: HashSet, } - fn track_template_name( - usage: &mut HashMap, - template_name: &str, - source: &str, - ) { + fn track_template_name(usage: &mut HashMap, template_name: &str, source: &str) { let entry = usage .entry(template_name.to_string()) - .or_insert_with(|| TemplateNameUsage { - count: 0, - sources: HashSet::new(), - }); + .or_insert_with(|| TemplateNameUsage { count: 0, sources: HashSet::new() }); entry.count = entry.count.saturating_add(1); entry.sources.insert(source.to_string()); } @@ -308,11 +288,7 @@ pub(crate) fn read_templates( ) }; for template in &definition.templates { - track_template_name( - &mut template_name_usage, - template.name.as_str(), - file_source_label.as_str(), - ); + track_template_name(&mut template_name_usage, template.name.as_str(), file_source_label.as_str()); } merged_templates.extend(definition.templates); loaded_template_files.extend(template_paths.iter().map(|p| p.display().to_string())); @@ -321,21 +297,13 @@ pub(crate) fn read_templates( if let Some(templates) = sources_inline_templates { for template in templates { - track_template_name( - &mut template_name_usage, - template.name.as_str(), - "source.yml inline templates", - ); + track_template_name(&mut template_name_usage, template.name.as_str(), "source.yml inline templates"); } merged_templates.extend(templates.iter().cloned()); } if let Some(templates) = mappings_inline_templates { for template in templates { - track_template_name( - &mut template_name_usage, - template.name.as_str(), - "mapping.yml inline templates", - ); + track_template_name(&mut template_name_usage, template.name.as_str(), "mapping.yml inline templates"); } merged_templates.extend(templates.iter().cloned()); } @@ -360,27 +328,17 @@ pub(crate) fn read_templates( ))); } - let files_used = if loaded_template_files.is_empty() { - None - } else { - Some(loaded_template_files) - }; + let files_used = if loaded_template_files.is_empty() { None } else { Some(loaded_template_files) }; if merged_templates.is_empty() { - return Ok(PreparedTemplateBundle { - definition: None, - prepared: None, - files_used, - }); + return Ok(PreparedTemplateBundle { definition: None, prepared: None, files_used }); } let mut prepared_templates = merged_templates.clone(); prepare_templates(&mut prepared_templates)?; Ok(PreparedTemplateBundle { - definition: Some(TemplateDefinitionDto { - templates: merged_templates, - }), + definition: Some(TemplateDefinitionDto { templates: merged_templates }), prepared: Some(prepared_templates), files_used, }) @@ -402,30 +360,21 @@ pub async fn read_app_config_dto( resolve_template_and_mapping_paths(paths, config.template_path.as_deref(), config.mapping_path.as_deref()); let mut sources = parse_sources_file_from_path(&PathBuf::from(sources_file), resolve_env).await?; - let mut mappings = read_mappings_file_unprepared(&effective_mapping_path, resolve_env)? - .map(|(_, mapping)| mapping); + let mut mappings = read_mappings_file_unprepared(&effective_mapping_path, resolve_env)?.map(|(_, mapping)| mapping); let template_bundle = read_templates( Some(&effective_template_path), resolve_env, sources.templates.as_deref(), - mappings - .as_ref() - .and_then(|mapping| mapping.mappings.templates.as_deref()), + mappings.as_ref().and_then(|mapping| mapping.mappings.templates.as_deref()), )?; let templates = template_bundle.definition; let prepared_templates = template_bundle.prepared; if resolve_env { - sources.prepare( - include_computed, - config.get_hdhr_device_overview().as_ref(), - prepared_templates.as_deref(), - )?; + sources.prepare(include_computed, config.get_hdhr_device_overview().as_ref(), prepared_templates.as_deref())?; if let Some(mapping) = mappings.as_mut() { - mapping - .mappings - .prepare(prepared_templates.as_deref())?; + mapping.mappings.prepare(prepared_templates.as_deref())?; } } @@ -444,13 +393,7 @@ pub async fn read_app_config_dto( } }; - Ok(AppConfigDto { - config, - sources, - mappings, - templates, - api_proxy, - }) + Ok(AppConfigDto { config, sources, mappings, templates, api_proxy }) } fn apply_prepared_mappings( @@ -463,19 +406,13 @@ fn apply_prepared_mappings( ) { match mapping.mappings.prepare(prepared_templates) { Ok(()) => { - let mappings: crate::model::Mappings = crate::model::Mappings::from(&*mapping); + let mappings: tuliprox_core::model::Mappings = tuliprox_core::model::Mappings::from(&*mapping); app_config.set_mappings(mappings_file, &mappings); - paths.mapping_files_used = mapping_paths.map(|items| { - items - .iter() - .map(|p| p.display().to_string()) - .collect::>() - }); + paths.mapping_files_used = + mapping_paths.map(|items| items.iter().map(|p| p.display().to_string()).collect::>()); } Err(err) => { - error!( - "Failed to prepare mapping '{mappings_file}': {err}. Skipping mapping registration." - ); + error!("Failed to prepare mapping '{mappings_file}': {err}. Skipping mapping registration."); paths.mapping_files_used = None; } } @@ -490,13 +427,7 @@ pub async fn prepare_sources_batch( .inputs .iter() .flat_map(|item| { - std::iter::once(item.id).chain( - item.aliases - .as_ref() - .into_iter() - .flatten() - .map(|alias| alias.id), - ) + std::iter::once(item.id).chain(item.aliases.as_ref().into_iter().flatten().map(|alias| alias.id)) }) .max() .unwrap_or(0); @@ -538,22 +469,14 @@ Use a batch:// URL or a local CSV path (absolute/relative)." return match csv_read_inputs(input_type, url).await { Ok((file_path, batch_aliases)) => Ok(Some((file_path, batch_aliases))), - Err(err) => { - Err(TuliproxError::Config(format!("{err}"))) - } + Err(err) => Err(TuliproxError::Config(format!("{err}"))), }; } Ok(None) } -pub async fn prepare_users( - app_config_dto: &mut AppConfigDto, - app_config: &AppConfig, -) -> Result<(), TuliproxError> { - let use_user_db = app_config_dto - .api_proxy - .as_ref() - .is_some_and(|p| p.use_user_db); +pub async fn prepare_users(app_config_dto: &mut AppConfigDto, app_config: &AppConfig) -> Result<(), TuliproxError> { + let use_user_db = app_config_dto.api_proxy.as_ref().is_some_and(|p| p.use_user_db); if use_user_db { let user_db_path = get_api_user_db_path(app_config); @@ -561,16 +484,11 @@ pub async fn prepare_users( match load_api_user(app_config).await { Ok(stored_users) => { if let Some(api_proxy) = app_config_dto.api_proxy.as_mut() { - api_proxy - .user - .extend(stored_users.iter().map(TargetUserDto::from)); + api_proxy.user.extend(stored_users.iter().map(TargetUserDto::from)); } } Err(err) => { - warn!( - "Failed to load users from DB at {}: {err}", - user_db_path.display() - ); + warn!("Failed to load users from DB at {}: {err}", user_db_path.display()); } } } @@ -601,14 +519,9 @@ pub async fn read_initial_app_config( .or(config_dto.template_path.as_deref()) .map(|path| if resolve_env { resolve_env_var(path) } else { path.to_string() }); - paths.mapping_file_path = Some(utils::resolve_mapping_file_path( - config_path, - configured_mapping_path.as_deref(), - )); - paths.template_file_path = Some(utils::resolve_template_file_path( - config_path, - configured_template_path.as_deref(), - )); + paths.mapping_file_path = Some(utils::resolve_mapping_file_path(config_path, configured_mapping_path.as_deref())); + paths.template_file_path = + Some(utils::resolve_template_file_path(config_path, configured_template_path.as_deref())); let mut sources_dto = parse_sources_file_from_path(&PathBuf::from(sources_file), resolve_env).await?; @@ -626,9 +539,7 @@ pub async fn read_initial_app_config( paths.template_file_path.as_deref(), resolve_env, sources_dto.templates.as_deref(), - mappings_dto - .as_ref() - .and_then(|mapping| mapping.mappings.templates.as_deref()), + mappings_dto.as_ref().and_then(|mapping| mapping.mappings.templates.as_deref()), )?; let prepared_templates = template_bundle.prepared; let template_files_used = template_bundle.files_used; @@ -714,9 +625,7 @@ pub fn read_api_proxy_file( } Ok(Some(api_proxy_dto)) } - Err(err) => { - Err(TuliproxError::Config(format!("can't read api-proxy-config file: {err}"))) - } + Err(err) => Err(TuliproxError::Config(format!("can't read api-proxy-config file: {err}"))), } }) } @@ -728,7 +637,7 @@ pub async fn read_api_proxy(config: &AppConfig, resolve_env: bool) -> Option Ok(()), @@ -820,9 +735,9 @@ where } #[cfg(windows)] -fn replace_file_windows(source: &Path, target: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use winapi::um::winbase::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; +fn replace_file_windows(source: &Path, target: &Path) -> std::io::Result<()> { + use std::{io, os::windows::ffi::OsStrExt}; + use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; let mut source_wide: Vec = source.as_os_str().encode_wide().collect(); source_wide.push(0); @@ -832,11 +747,7 @@ fn replace_file_windows(source: &Path, target: &Path) -> io::Result<()> { // SAFETY: both paths are NUL-terminated UTF-16 buffers that remain alive for // the call. `MOVEFILE_REPLACE_EXISTING` leaves the target in place on error. let ok = unsafe { - MoveFileExW( - source_wide.as_ptr(), - target_wide.as_ptr(), - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, - ) + MoveFileExW(source_wide.as_ptr(), target_wide.as_ptr(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) }; if ok == 0 { return Err(io::Error::last_os_error()); @@ -918,10 +829,9 @@ async fn apply_authoritative_plans(config: &AppConfig, api_proxy: &mut ApiProxyC cfg.get_backup_dir().to_string() }; match save_plans(&plans_path_str, &backup_dir, &migrated).await { - Ok(()) => info!( - "Migrated {} user plan(s) from api-proxy.yml to {plans_path_str}", - migrated.plans.len() - ), + Ok(()) => { + info!("Migrated {} user plan(s) from api-proxy.yml to {plans_path_str}", migrated.plans.len()); + } Err(err) => error!("Failed to migrate plans to {plans_path_str}: {err}"), } let plans = legacy_plans.iter().map(|plan| Arc::new(UserPlan::from(plan))).collect(); @@ -951,19 +861,11 @@ fn read_legacy_api_proxy_plans(config: &AppConfig, resolve_env: bool) -> Option< } } -pub async fn save_main_config( - file_path: &str, - backup_dir: &str, - config: &ConfigDto, -) -> Result<(), TuliproxError> { +pub async fn save_main_config(file_path: &str, backup_dir: &str, config: &ConfigDto) -> Result<(), TuliproxError> { write_config_file(file_path, backup_dir, config, "config.yml").await } -pub async fn save_sources_config( - file_path: &str, - backup_dir: &str, - config: &T, -) -> Result<(), TuliproxError> +pub async fn save_sources_config(file_path: &str, backup_dir: &str, config: &T) -> Result<(), TuliproxError> where T: ?Sized + Serialize, { @@ -979,28 +881,21 @@ pub async fn save_templates_config( } async fn build_templates_to_persist( - app_state: &Arc, + app_config: &Arc, dto: &SourcesConfigDto, ) -> Result, TuliproxError> { let mut new_dto = dto.clone(); - let config = app_state.app_config.config.load(); - let paths = app_state.app_config.paths.load(); + let config = app_config.config.load(); + let paths = app_config.paths.load(); - let existing_source_inline_templates = match parse_sources_file_from_path( - Path::new(paths.sources_file_path.as_str()), - true, - ) - .await - { - Ok(existing_sources) => existing_sources.templates, - Err(err) => { - warn!( - "Failed to read existing source.yml for template migration '{}': {err}", - paths.sources_file_path - ); - None - } - }; + let existing_source_inline_templates = + match parse_sources_file_from_path(Path::new(paths.sources_file_path.as_str()), true).await { + Ok(existing_sources) => existing_sources.templates, + Err(err) => { + warn!("Failed to read existing source.yml for template migration '{}': {err}", paths.sources_file_path); + None + } + }; let mapping_inline_templates = if let Some(mapping_file_path) = paths.mapping_file_path.as_ref() { read_mappings_file_unprepared(mapping_file_path, true)? @@ -1017,19 +912,11 @@ async fn build_templates_to_persist( }; let template_file_path = paths.template_file_path.as_deref().or(config.template_path.as_deref()); - let template_bundle = read_templates( - template_file_path, - true, - source_inline_templates, - mapping_inline_templates.as_deref(), - )?; + let template_bundle = + read_templates(template_file_path, true, source_inline_templates, mapping_inline_templates.as_deref())?; let prepared_templates = template_bundle.prepared; - new_dto.prepare( - true, - config.get_hdhr_device_overview().as_ref(), - prepared_templates.as_deref(), - )?; + new_dto.prepare(true, config.get_hdhr_device_overview().as_ref(), prepared_templates.as_deref())?; if let Some(templates) = new_dto.templates.clone() { if templates.is_empty() { @@ -1045,18 +932,18 @@ async fn build_templates_to_persist( } pub async fn validate_source_config_for_persist( - app_state: &Arc, + app_config: &Arc, dto: &SourcesConfigDto, ) -> Result, TuliproxError> { - build_templates_to_persist(app_state, dto).await + build_templates_to_persist(app_config, dto).await } pub async fn persist_templates_config( - app_state: &Arc, + app_config: &Arc, template_definition: &TemplateDefinitionDto, ) -> Result<(), TuliproxError> { - let config = app_state.app_config.config.load(); - let paths = app_state.app_config.paths.load(); + let config = app_config.config.load(); + let paths = app_config.paths.load(); let template_file = utils::resolve_template_persist_file_path( paths.template_file_path.as_deref().or(config.template_path.as_deref()), &paths.config_path, @@ -1065,21 +952,21 @@ pub async fn persist_templates_config( } pub async fn persist_source_config( - app_state: &Arc, + app_config: &Arc, source_file_path: Option<&Path>, doc: SourcesConfigDto, ) -> Result { let source_file = { source_file_path.and_then(|p| p.to_str()).map_or_else( || { - let paths = app_state.app_config.paths.load(); + let paths = app_config.paths.load(); paths.sources_file_path.clone() }, ToString::to_string, ) }; let backup_dir = { - let config = app_state.app_config.config.load(); + let config = app_config.config.load(); config.get_backup_dir().to_string() }; @@ -1092,11 +979,7 @@ pub async fn persist_source_config( pub async fn sanitize_sources_for_persist(mut source_config: SourcesConfigDto) -> SourcesConfigDto { source_config.templates = None; for input in &mut source_config.inputs { - if input - .panel_api - .as_ref() - .is_some_and(|panel| panel.alias_pool.is_some()) - { + if input.panel_api.as_ref().is_some_and(|panel| panel.alias_pool.is_some()) { if let Some(aliases) = input.aliases.as_mut() { aliases.sort_by(|a, b| { let a_ts = a.exp_date.unwrap_or(i64::MIN); @@ -1105,10 +988,8 @@ pub async fn sanitize_sources_for_persist(mut source_config: SourcesConfigDto) - }); } } - if matches!( - input.input_type, - InputType::XtreamBatch | InputType::M3uBatch | InputType::StalkerBatch - ) && is_csv_file(input.url.as_str()) + if matches!(input.input_type, InputType::XtreamBatch | InputType::M3uBatch | InputType::StalkerBatch) + && is_csv_file(input.url.as_str()) { if let Some(aliases) = &input.aliases { if let Err(err) = csv_write_inputs(input.url.as_str(), aliases).await { @@ -1133,37 +1014,24 @@ pub async fn sanitize_sources_for_persist(mut source_config: SourcesConfigDto) - } pub async fn validate_and_persist_source_config( - app_state: &Arc, + app_config: &Arc, dto: SourcesConfigDto, ) -> Result { - let templates_to_persist = validate_source_config_for_persist(app_state, &dto).await?; + let templates_to_persist = validate_source_config_for_persist(app_config, &dto).await?; if let Some(template_definition) = templates_to_persist.as_ref() { - persist_templates_config(app_state, template_definition).await?; + persist_templates_config(app_config, template_definition).await?; } - persist_source_config(app_state, None, dto).await + persist_source_config(app_config, None, dto).await } -pub fn resolve_env_var(value: &str) -> String { - if value.is_empty() { - return String::new(); - } - CONSTANTS - .re_env_var - .replace_all(value, |caps: ®ex::Captures| { - let var_name = &caps["var"]; - env::var(var_name).unwrap_or_else(|e| { - error!("Could not resolve env var '{var_name}': {e}"); - format!("${{env:{var_name}}}") - }) - }) - .to_string() -} - -pub async fn persist_messaging_templates(app_state: &Arc, cfg: &mut ConfigDto) -> Result<(), TuliproxError> { +pub async fn persist_messaging_templates( + app_config: &Arc, + cfg: &mut ConfigDto, +) -> Result<(), TuliproxError> { let templates_dir = { - let paths = app_state.app_config.paths.load(); + let paths = app_config.paths.load(); PathBuf::from(&paths.config_path).join("messaging_templates") }; @@ -1190,8 +1058,12 @@ pub async fn persist_messaging_templates(app_state: &Arc, cfg: &mut Co Ok(()) } - -async fn persist_single_template(prefix: &str, kind: Option<&MsgKind>, template: &str, templates_dir: &Path) -> Result { +async fn persist_single_template( + prefix: &str, + kind: Option<&MsgKind>, + template: &str, + templates_dir: &Path, +) -> Result { if template.is_empty() || is_uri(template) { return Ok(template.to_string()); } @@ -1208,35 +1080,158 @@ async fn persist_single_template(prefix: &str, kind: Option<&MsgKind>, template: // It's a raw string, persist it if !file_exists_async(templates_dir).await { - tokio::fs::create_dir_all(templates_dir) - .await - .map_err(|e| TuliproxError::Config(format!("Messaging templates dir: failed to create dir: {} {e}", templates_dir.display())))?; + tokio::fs::create_dir_all(templates_dir).await.map_err(|e| { + TuliproxError::Config(format!( + "Messaging templates dir: failed to create dir: {} {e}", + templates_dir.display() + )) + })?; } - let filename = if let Some(k) = kind { - k.template_filename(prefix) - } else { - concat_string!(prefix, "_default.templ") - }; + let filename = + if let Some(k) = kind { k.template_filename(prefix) } else { concat_string!(prefix, "_default.templ") }; let file_path = templates_dir.join(filename); - fs::write(&file_path, template).await.map_err(|e| TuliproxError::Config(format!("Failed to write template file: {e}")))?; + fs::write(&file_path, template) + .await + .map_err(|e| TuliproxError::Config(format!("Failed to write template file: {e}")))?; - Url::from_file_path(&file_path) - .map(|u| u.to_string()) - .map_err(|()| TuliproxError::Config(format!("Failed to convert persisted path to file URL: {}", file_path.display()))) + Url::from_file_path(&file_path).map(|u| u.to_string()).map_err(|()| { + TuliproxError::Config(format!("Failed to convert persisted path to file URL: {}", file_path.display())) + }) } +// ---- API user migration (config file <-> user database) ---- +// +// Was a method on `ApiProxyConfig`, then briefly a repository module. It writes +// the api-proxy config file, so it belongs with the rest of the configuration +// bootstrap; living in `repository` made that layer call back into this one. +fn serialize_api_proxy_config(config: &ApiProxyConfigDto) -> Result { + let mut serialized = String::new(); + let options = serde_saphyr::ser_options! {prefer_block_scalars: false}; + serde_saphyr::to_fmt_writer_with_options(&mut serialized, config, options) + .map_err(|err| format!("Could not serialize api proxy config: {err}"))?; + Ok(serialized) +} + +async fn api_proxy_file_would_change(api_proxy_file: &str, config: &ApiProxyConfigDto) -> Result { + let serialized = serialize_api_proxy_config(config)?; + match tokio::fs::read_to_string(api_proxy_file).await { + Ok(existing) => Ok(existing != serialized), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(true), + Err(err) => Err(format!("Could not read api proxy file {api_proxy_file}: {err}")), + } +} + +async fn backfill_output_clusters_to_file(api_proxy: &ApiProxyConfig, cfg: &AppConfig, errors: &mut Vec) { + if api_proxy.user.is_empty() { + return; + } + let paths = ArcSwap::load(&cfg.paths); + let api_proxy_file = paths.api_proxy_file_path.as_str(); + let dto = ApiProxyConfigDto::from(api_proxy); + match api_proxy_file_would_change(api_proxy_file, &dto).await { + Ok(true) => {} + Ok(false) => return, + Err(err) => { + errors.push(err); + return; + } + } + let config = ArcSwap::load(&cfg.config); + let backup_dir = config.get_backup_dir(); + if let Err(err) = save_api_proxy(api_proxy_file, backup_dir.as_ref(), &dto).await { + errors.push(format!("Error saving api proxy file: {err}")); + } +} + +// we have the option to store user in the config file or in the user_db +// When we switch from one to other we need to migrate the existing data. +/// # Panics +pub async fn migrate_api_user(api_proxy: &mut ApiProxyConfig, cfg: &AppConfig, errors: &mut Vec) { + let paths = ArcSwap::load(&cfg.paths); + let api_proxy_file = paths.api_proxy_file_path.as_str(); + if api_proxy.use_user_db { + // we have user defined in config file. + // we migrate them to the db and delete them from the config file + if !&api_proxy.user.is_empty() { + if let Err(err) = merge_api_user(cfg, &api_proxy.user).await { + errors.push(err.to_string()); + } else { + let config = ArcSwap::load(&cfg.config); + let backup_dir = config.get_backup_dir(); + api_proxy.user = vec![]; + if let Err(err) = + save_api_proxy(api_proxy_file, backup_dir.as_ref(), &ApiProxyConfigDto::from(&*api_proxy)).await + { + errors.push(format!("Error saving api proxy file: {err}")); + } + } + } + match load_api_user(cfg).await { + Ok(users) => { + let mut users = users; + api_proxy.resolve_target_users(&mut users); + api_proxy.user = users; + } + Err(err) => { + println!("{err}"); + errors.push(err.to_string()); + } + } + } else { + backfill_output_clusters_to_file(api_proxy, cfg, errors).await; + let user_db_path = get_api_user_db_path(cfg); + if file_exists_async(&user_db_path).await { + // we can't have user defined in db file. + // we need to load them and save them into the config file + if let Ok(stored_users) = load_api_user(cfg).await { + let mut stored_users = stored_users; + api_proxy.resolve_target_users(&mut stored_users); + for stored_user in stored_users { + if let Some(target_user) = api_proxy.user.iter_mut().find(|t| t.target == stored_user.target) { + for stored_credential in &stored_user.credentials { + if !target_user.credentials.iter().any(|c| c.username == stored_credential.username) { + target_user.credentials.push(stored_credential.clone()); + } + } + } else { + api_proxy.user.push(stored_user); + } + } + } + + let config = ArcSwap::load(&cfg.config); + let backup_dir = config.get_backup_dir(); + let dto = ApiProxyConfigDto::from(&*api_proxy); + match api_proxy_file_would_change(api_proxy_file, &dto).await { + Ok(true) => { + if let Err(err) = save_api_proxy(api_proxy_file, backup_dir.as_ref(), &dto).await { + errors.push(format!("Error saving api proxy file: {err}")); + } else { + backup_api_user_db_file(cfg, &user_db_path).await; + let _ = tokio::fs::remove_file(&user_db_path).await; + } + } + Ok(false) => { + backup_api_user_db_file(cfg, &user_db_path).await; + let _ = tokio::fs::remove_file(&user_db_path).await; + } + Err(err) => errors.push(err), + } + } + } +} #[cfg(test)] mod tests { - use super::{prepare_sources_batch, sanitize_sources_for_persist, write_config_file}; - use crate::utils::{file::config_reader::get_batch_aliases, resolve_env_var}; + use super::{get_batch_aliases, prepare_sources_batch, sanitize_sources_for_persist, write_config_file}; use shared::{ model::{ConfigInputAliasDto, ConfigInputDto, InputType, SourcesConfigDto}, utils::Internable, }; use tempfile::tempdir; + use tuliprox_core::utils::resolve_env_var; #[test] #[allow(clippy::manual_unwrap_or_default)] @@ -1254,9 +1249,7 @@ mod tests { async fn batch_provider_scheme_returns_clear_error() { let result = get_batch_aliases(InputType::XtreamBatch, "provider://my_provider").await; let err = result.expect_err("provider:// must not be accepted for batch inputs"); - assert!(err - .to_string() - .contains("does not support provider:// URLs")); + assert!(err.to_string().contains("does not support provider:// URLs")); } #[tokio::test] @@ -1284,11 +1277,8 @@ mod tests { async fn stalker_batch_loads_and_persists_csv_aliases() { let dir = tempdir().expect("temp dir"); let path = dir.path().join("stalker.csv"); - std::fs::write( - &path, - "#name;username;password;url;max_connections\nalias;user;pass;http://portal.example;1\n", - ) - .expect("write csv"); + std::fs::write(&path, "#name;username;password;url;max_connections\nalias;user;pass;http://portal.example;1\n") + .expect("write csv"); let (_, aliases) = get_batch_aliases(InputType::StalkerBatch, path.to_string_lossy().as_ref()) .await @@ -1330,5 +1320,4 @@ mod tests { assert_eq!(tokio::fs::read_to_string(config_path).await?, "old"); Ok(()) } - } diff --git a/backend/src/runtime_config_report.rs b/backend/config-loader/src/runtime_config_report.rs similarity index 85% rename from backend/src/runtime_config_report.rs rename to backend/config-loader/src/runtime_config_report.rs index 225af0e7e..fcf985eaa 100644 --- a/backend/src/runtime_config_report.rs +++ b/backend/config-loader/src/runtime_config_report.rs @@ -1,15 +1,22 @@ -use crate::model::{AppConfig, Config, HdHomeRunConfig, HdHomeRunFlags, LibraryConfig}; -use crate::utils::read_app_config_dto; +//! Human-readable summary of the resolved runtime configuration. +//! +//! Logged once at startup. It reads the same config this crate loads, so it +//! belongs beside the loader rather than in the binary. + +use crate::read_app_config_dto; use arc_swap::{access::Access, ArcSwap}; use log::{error, info}; use serde::Serialize; use serde_json::Value; -use shared::error::TuliproxError; -use shared::model::{ - ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigPaths, HdHomeRunConfigDto, HdHomeRunDeviceConfigDto, - LibraryConfigDto, LibraryMetadataConfigDto, LibraryMetadataReadConfigDto, LibraryPlaylistConfigDto, - LogConfigDto, RuntimeConfigReportFormat, ThumbnailConfigDto, +use shared::{ + error::TuliproxError, + model::{ + ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigPaths, HdHomeRunConfigDto, HdHomeRunDeviceConfigDto, + LibraryConfigDto, LibraryMetadataConfigDto, LibraryMetadataReadConfigDto, LibraryPlaylistConfigDto, + LogConfigDto, RuntimeConfigReportFormat, ThumbnailConfigDto, + }, }; +use tuliprox_core::model::{AppConfig, Config, HdHomeRunConfig, HdHomeRunFlags, LibraryConfig}; #[derive(Debug, Serialize)] struct RuntimeConfigReport { @@ -21,7 +28,7 @@ struct RuntimeConfigReport { paths: ConfigPaths, } -pub(crate) async fn log_runtime_config_report(app_config: &AppConfig) { +pub async fn log_runtime_config_report(app_config: &AppConfig) { let config = > as Access>::load(&app_config.config); let Some(log_config) = config.log.clone() else { return; @@ -49,32 +56,13 @@ async fn render_runtime_config_report( async fn build_runtime_config_report(app_config: &AppConfig) -> Result { let paths = > as Access>::load(&app_config.paths).clone(); - let AppConfigDto { - sources, - mappings, - templates, - api_proxy, - .. - } = read_app_config_dto(&paths, true, true).await?; + let AppConfigDto { sources, mappings, templates, api_proxy, .. } = read_app_config_dto(&paths, true, true).await?; - let config = runtime_config_to_dto( - &> as Access>::load(&app_config.config), - ); - let api_proxy = app_config - .api_proxy - .load() - .as_ref() - .map(|runtime| ApiProxyConfigDto::from(runtime.as_ref())) - .or(api_proxy); + let config = runtime_config_to_dto(&> as Access>::load(&app_config.config)); + let api_proxy = + app_config.api_proxy.load().as_ref().map(|runtime| ApiProxyConfigDto::from(runtime.as_ref())).or(api_proxy); - Ok(RuntimeConfigReport { - config, - sources, - mappings, - templates, - api_proxy, - paths, - }) + Ok(RuntimeConfigReport { config, sources, mappings, templates, api_proxy, paths }) } fn runtime_config_to_dto(config: &Config) -> ConfigDto { @@ -92,10 +80,7 @@ fn runtime_config_to_dto(config: &Config) -> ConfigDto { custom_stream_response_enabled: config.custom_stream_response_enabled, custom_stream_response_error_status: config.custom_stream_response_error_status, video: config.video.as_ref().map(shared::model::VideoConfigDto::from), - metadata_update: config - .metadata_update - .as_ref() - .map(shared::model::MetadataUpdateConfigDto::from), + metadata_update: config.metadata_update.as_ref().map(shared::model::MetadataUpdateConfigDto::from), schedules: config .schedules .as_ref() @@ -112,10 +97,7 @@ fn runtime_config_to_dto(config: &Config) -> ConfigDto { accept_insecure_ssl_certificates: config.accept_insecure_ssl_certificates, web_ui: config.web_ui.as_ref().map(shared::model::WebUiConfigDto::from), messaging: config.messaging.as_ref().map(shared::model::MessagingConfigDto::from), - reverse_proxy: config - .reverse_proxy - .as_ref() - .map(shared::model::ReverseProxyConfigDto::from), + reverse_proxy: config.reverse_proxy.as_ref().map(shared::model::ReverseProxyConfigDto::from), hdhomerun: config.hdhomerun.as_ref().map(hdhomerun_config_to_dto), proxy: config.proxy.as_ref().map(shared::model::ProxyConfigDto::from), ipcheck: config.ipcheck.as_ref().map(shared::model::IpCheckConfigDto::from), @@ -192,8 +174,9 @@ fn serialize_report_value(value: &Value, format: RuntimeConfigReportFormat) -> R RuntimeConfigReportFormat::Yaml => { let mut serialized = String::new(); let options = serde_saphyr::ser_options! {prefer_block_scalars: false}; - serde_saphyr::to_fmt_writer_with_options(&mut serialized, value, options) - .map_err(|err| TuliproxError::Config(format!("Failed to serialize runtime config report as YAML: {err}")))?; + serde_saphyr::to_fmt_writer_with_options(&mut serialized, value, options).map_err(|err| { + TuliproxError::Config(format!("Failed to serialize runtime config report as YAML: {err}")) + })?; Ok(serialized) } } @@ -251,10 +234,7 @@ fn should_redact_field(current_key: Option<&str>, parent_key: Option<&str>) -> b ); } - key.ends_with("_password") - || key.ends_with("_secret") - || key.ends_with("_token") - || key.ends_with("_api_key") + key.ends_with("_password") || key.ends_with("_secret") || key.ends_with("_token") || key.ends_with("_api_key") } fn should_redact_as_url(current_key: Option<&str>) -> bool { @@ -269,11 +249,11 @@ fn redact_url_like_value(value: &str) -> String { shared::utils::sanitize_sensit #[cfg(test)] mod tests { use super::{build_runtime_config_report, redact_value, render_runtime_config_report}; - use crate::model::AppConfig; - use crate::utils::read_initial_app_config; + use crate::read_initial_app_config; use shared::model::{ConfigPaths, RuntimeConfigReportFormat}; use tempfile::tempdir; use tokio::fs; + use tuliprox_core::model::AppConfig; fn test_paths(config_dir: &std::path::Path) -> ConfigPaths { ConfigPaths { @@ -329,9 +309,7 @@ sources: fs::write(config_dir.join("source.yml"), source_yml).await.expect("source file"); let mut paths = test_paths(&config_dir); - let app_config = read_initial_app_config(&mut paths, true, true, false) - .await - .expect("app config"); + let app_config = read_initial_app_config(&mut paths, true, true, false).await.expect("app config"); std::mem::forget(temp_dir); app_config } @@ -342,13 +320,7 @@ sources: let report = build_runtime_config_report(&app_config).await.expect("report"); - assert!( - report - .config - .storage_dir - .as_deref() - .is_some_and(|value| value.contains("/config/home/data")) - ); + assert!(report.config.storage_dir.as_deref().is_some_and(|value| value.contains("/config/home/data"))); assert_eq!(report.config.api.web_root, report.paths.config_path.clone() + "/home/web"); } @@ -375,9 +347,7 @@ sources: async fn runtime_config_report_formats_yaml() { let app_config = create_test_app_config().await; - let rendered = render_runtime_config_report(&app_config, RuntimeConfigReportFormat::Yaml) - .await - .expect("yaml"); + let rendered = render_runtime_config_report(&app_config, RuntimeConfigReportFormat::Yaml).await.expect("yaml"); assert!(rendered.contains("config:")); assert!(rendered.contains("sources:")); @@ -388,9 +358,7 @@ sources: async fn runtime_config_report_masks_runtime_secrets() { let app_config = create_test_app_config().await; - let rendered = render_runtime_config_report(&app_config, RuntimeConfigReportFormat::Json) - .await - .expect("json"); + let rendered = render_runtime_config_report(&app_config, RuntimeConfigReportFormat::Json).await.expect("json"); assert!(rendered.contains("\"password\": \"***\"")); assert!(!rendered.contains("proxy-pass")); diff --git a/backend/config-loader/tests/temp_dir_override.rs b/backend/config-loader/tests/temp_dir_override.rs new file mode 100644 index 000000000..196b99307 --- /dev/null +++ b/backend/config-loader/tests/temp_dir_override.rs @@ -0,0 +1,13 @@ +use tuliprox_core::model::Config; + +#[test] +fn explicit_test_setting_disables_temp_dir_override() { + let storage_dir = tempfile::tempdir().expect("storage temp dir"); + std::env::set_var("TULIPROX_DISABLE_TEMP_DIR_OVERRIDE", "1"); + + let config = Config { storage_dir: storage_dir.path().to_string_lossy().into_owned(), ..Config::default() }; + config.update_runtime(); + + let temp_dir = tempfile::tempdir().expect("process temp dir"); + assert!(!temp_dir.path().starts_with(storage_dir.path())); +} diff --git a/backend/core/Cargo.toml b/backend/core/Cargo.toml new file mode 100644 index 000000000..c59af361f --- /dev/null +++ b/backend/core/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "tuliprox-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Resolved configuration model and shared utilities for tuliprox" + +[lints] +workspace = true + +[dependencies] +http-body-util = "0.1.5" +serde_html_form = "0.4.1" +shared = { workspace = true } +tuliprox-mpegts = { workspace = true } +tuliprox-media-server = { workspace = true } +aes = "0.8.4" +arc-swap.workspace = true +async-compression = { version = "0.4.43", features = ["tokio", "brotli", "deflate", "gzip", "zlib", "zstd"] } +axum = { version = "0.8.9", features = ["macros", "default", "ws"] } +base64.workspace = true +blake3.workspace = true +bytes.workspace = true +chrono.workspace = true +chrono-tz = "0.10.4" +ctr = "0.9.2" +env_logger = "0.11.11" +flate2 = "1.1.9" +futures.workspace = true +indexmap.workspace = true +ipnet = "2" +mime = "0.3.17" +jsonwebtoken = { version = "10.4.0", default-features = false, features = ["aws_lc_rs"] } +log.workspace = true +parking_lot = "0.12.5" +paste.workspace = true +path-clean.workspace = true +quick-xml = { version = "0.41", features = ["async-tokio", "serialize"] } +rand = "0.9.5" +rmp-serde = "1.3.1" +regex.workspace = true +reqwest = { version = "0.13.4", features = ["json", "stream", "rustls", "socks", "form", "query"] } +serde.workspace = true +serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } +serde-saphyr.workspace = true +strum.workspace = true +tempfile = "3.27.0" +thiserror.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs", "process", "signal", "time", "macros"] } +tokio-stream = { version = "0.1.19", features = ["sync"] } +tokio-util = { version = "0.7.19", features = ["io", "io-util"] } +url.workspace = true +zeroize.workspace = true + +[target.'cfg(not(target_os = "linux"))'.dependencies] +sysinfo = { version = "0.38.4", default-features = false, features = ["system", "network"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/backend/core/src/lib.rs b/backend/core/src/lib.rs new file mode 100644 index 000000000..0a95de2ad --- /dev/null +++ b/backend/core/src/lib.rs @@ -0,0 +1,14 @@ +//! Resolved configuration model and shared utilities. +//! +//! `model` holds the runtime configuration and domain types; `utils` holds the +//! helpers that read, write and transform them. The two reference each other - +//! configured helpers need the configuration, and configuration resolution uses +//! the helpers - so they are one package rather than two joined by an interface +//! invented to keep a layer diagram. +//! +//! Nothing here names the API layer, a repository, or the playlist pipeline. + +pub mod response_macros; + +pub mod model; +pub mod utils; diff --git a/backend/src/api/model/batch_result_collector.rs b/backend/core/src/model/batch_result_collector.rs similarity index 87% rename from backend/src/api/model/batch_result_collector.rs rename to backend/core/src/model/batch_result_collector.rs index 3b034d53c..f99f8bedb 100644 --- a/backend/src/api/model/batch_result_collector.rs +++ b/backend/core/src/model/batch_result_collector.rs @@ -1,4 +1,4 @@ -use crate::api::model::ProviderIdType; +use crate::model::ProviderIdType; use shared::model::{LiveStreamProperties, SeriesStreamProperties, VideoStreamProperties}; use std::mem; @@ -6,9 +6,9 @@ const BATCH_THRESHOLD: usize = 200; #[derive(Debug, Default)] pub struct BatchResultCollector { - pub(crate) vod: Vec<(ProviderIdType, VideoStreamProperties)>, - pub(crate) series: Vec<(ProviderIdType, SeriesStreamProperties)>, - pub(crate) live: Vec<(ProviderIdType, LiveStreamProperties)>, + pub vod: Vec<(ProviderIdType, VideoStreamProperties)>, + pub series: Vec<(ProviderIdType, SeriesStreamProperties)>, + pub live: Vec<(ProviderIdType, LiveStreamProperties)>, } impl BatchResultCollector { diff --git a/backend/src/model/config/api.rs b/backend/core/src/model/config/api.rs similarity index 58% rename from backend/src/model/config/api.rs rename to backend/core/src/model/config/api.rs index c837417bc..c6dca99c0 100644 --- a/backend/src/model/config/api.rs +++ b/backend/core/src/model/config/api.rs @@ -1,5 +1,5 @@ -use shared::model::ConfigApiDto; use crate::model::macros; +use shared::model::ConfigApiDto; #[derive(Debug, Clone, Default)] pub struct ConfigApi { @@ -11,20 +11,12 @@ pub struct ConfigApi { macros::from_impl!(ConfigApi); impl From<&ConfigApiDto> for ConfigApi { fn from(dto: &ConfigApiDto) -> Self { - Self { - host:dto.host.clone(), - port: dto.port, - web_root: dto.web_root.clone(), - } + Self { host: dto.host.clone(), port: dto.port, web_root: dto.web_root.clone() } } } impl From<&ConfigApi> for ConfigApiDto { fn from(instance: &ConfigApi) -> Self { - Self { - host: instance.host.clone(), - port: instance.port, - web_root: instance.web_root.clone(), - } + Self { host: instance.host.clone(), port: instance.port, web_root: instance.web_root.clone() } } -} \ No newline at end of file +} diff --git a/backend/src/model/config/api_proxy.rs b/backend/core/src/model/config/api_proxy.rs similarity index 55% rename from backend/src/model/config/api_proxy.rs rename to backend/core/src/model/config/api_proxy.rs index 32aef55ca..b7451f00d 100644 --- a/backend/src/model/config/api_proxy.rs +++ b/backend/core/src/model/config/api_proxy.rs @@ -1,19 +1,12 @@ -use crate::model::{macros, AppConfig, Config, ProxyUserCredentials, TargetUser}; -use crate::repository::{backup_api_user_db_file, get_api_user_db_path, load_api_user, merge_api_user}; -use crate::utils; -use crate::utils::file_exists_async; -use arc_swap::access::Access; -use arc_swap::ArcSwap; +use crate::model::{macros, ProxyUserCredentials, TargetUser}; use log::{debug, error}; -use shared::foundation::{get_filter, Filter}; -use shared::model::{ - ApiProxyConfigDto, ApiProxyServerInfoDto, ClusterFlags, ConfigPaths, ProxyType, TargetUserDto, UserPlanDto, - UserPlanTrialDto, +use shared::{ + foundation::{get_filter, Filter}, + model::{ + ApiProxyConfigDto, ApiProxyServerInfoDto, ClusterFlags, ProxyType, TargetUserDto, UserPlanDto, UserPlanTrialDto, + }, }; -use std::cmp::PartialEq; -use std::collections::HashMap; -use std::io::ErrorKind; -use std::sync::Arc; +use std::{cmp::PartialEq, collections::HashMap, sync::Arc}; const API_USER: &str = "api"; const TEST_USER: &str = "test"; @@ -181,23 +174,6 @@ impl From<&ApiProxyConfig> for ApiProxyConfigDto { } } -fn serialize_api_proxy_config(config: &ApiProxyConfigDto) -> Result { - let mut serialized = String::new(); - let options = serde_saphyr::ser_options! {prefer_block_scalars: false}; - serde_saphyr::to_fmt_writer_with_options(&mut serialized, config, options) - .map_err(|err| format!("Could not serialize api proxy config: {err}"))?; - Ok(serialized) -} - -async fn api_proxy_file_would_change(api_proxy_file: &str, config: &ApiProxyConfigDto) -> Result { - let serialized = serialize_api_proxy_config(config)?; - match tokio::fs::read_to_string(api_proxy_file).await { - Ok(existing) => Ok(existing != serialized), - Err(err) if err.kind() == ErrorKind::NotFound => Ok(true), - Err(err) => Err(format!("Could not read api proxy file {api_proxy_file}: {err}")), - } -} - impl ApiProxyConfig { pub fn plan_map(&self) -> HashMap> { self.plans.iter().map(|plan| (plan.name.clone(), Arc::clone(plan))).collect() @@ -225,107 +201,6 @@ impl ApiProxyConfig { } } - async fn backfill_output_clusters_to_file(&self, cfg: &AppConfig, errors: &mut Vec) { - if self.user.is_empty() { - return; - } - let paths = > as Access>::load(&cfg.paths); - let api_proxy_file = paths.api_proxy_file_path.as_str(); - let dto = ApiProxyConfigDto::from(self); - match api_proxy_file_would_change(api_proxy_file, &dto).await { - Ok(true) => {} - Ok(false) => return, - Err(err) => { - errors.push(err); - return; - } - } - let config = > as Access>::load(&cfg.config); - let backup_dir = config.get_backup_dir(); - if let Err(err) = utils::save_api_proxy(api_proxy_file, backup_dir.as_ref(), &dto).await { - errors.push(format!("Error saving api proxy file: {err}")); - } - } - - // we have the option to store user in the config file or in the user_db - // When we switch from one to other we need to migrate the existing data. - /// # Panics - pub async fn migrate_api_user(&mut self, cfg: &AppConfig, errors: &mut Vec) { - let paths = > as Access>::load(&cfg.paths); - let api_proxy_file = paths.api_proxy_file_path.as_str(); - if self.use_user_db { - // we have user defined in config file. - // we migrate them to the db and delete them from the config file - if !&self.user.is_empty() { - if let Err(err) = merge_api_user(cfg, &self.user).await { - errors.push(err.to_string()); - } else { - let config = > as Access>::load(&cfg.config); - let backup_dir = config.get_backup_dir(); - self.user = vec![]; - if let Err(err) = - utils::save_api_proxy(api_proxy_file, backup_dir.as_ref(), &ApiProxyConfigDto::from(&*self)) - .await - { - errors.push(format!("Error saving api proxy file: {err}")); - } - } - } - match load_api_user(cfg).await { - Ok(users) => { - let mut users = users; - self.resolve_target_users(&mut users); - self.user = users; - } - Err(err) => { - println!("{err}"); - errors.push(err.to_string()); - } - } - } else { - self.backfill_output_clusters_to_file(cfg, errors).await; - let user_db_path = get_api_user_db_path(cfg); - if file_exists_async(&user_db_path).await { - // we can't have user defined in db file. - // we need to load them and save them into the config file - if let Ok(stored_users) = load_api_user(cfg).await { - let mut stored_users = stored_users; - self.resolve_target_users(&mut stored_users); - for stored_user in stored_users { - if let Some(target_user) = self.user.iter_mut().find(|t| t.target == stored_user.target) { - for stored_credential in &stored_user.credentials { - if !target_user.credentials.iter().any(|c| c.username == stored_credential.username) { - target_user.credentials.push(stored_credential.clone()); - } - } - } else { - self.user.push(stored_user); - } - } - } - - let config = > as Access>::load(&cfg.config); - let backup_dir = config.get_backup_dir(); - let dto = ApiProxyConfigDto::from(&*self); - match api_proxy_file_would_change(api_proxy_file, &dto).await { - Ok(true) => { - if let Err(err) = utils::save_api_proxy(api_proxy_file, backup_dir.as_ref(), &dto).await { - errors.push(format!("Error saving api proxy file: {err}")); - } else { - backup_api_user_db_file(cfg, &user_db_path).await; - let _ = tokio::fs::remove_file(&user_db_path).await; - } - } - Ok(false) => { - backup_api_user_db_file(cfg, &user_db_path).await; - let _ = tokio::fs::remove_file(&user_db_path).await; - } - Err(err) => errors.push(err), - } - } - } - } - pub fn get_target_name(&self, username: &str, password: &str) -> Option<(Arc, String)> { for target_user in &self.user { if let Some((credentials, target_name)) = target_user.get_target_name(username, password) { diff --git a/backend/src/model/config/api_user.rs b/backend/core/src/model/config/api_user.rs similarity index 86% rename from backend/src/model/config/api_user.rs rename to backend/core/src/model/config/api_user.rs index 11efe6bd5..60f59fcf6 100644 --- a/backend/src/model/config/api_user.rs +++ b/backend/core/src/model/config/api_user.rs @@ -1,16 +1,15 @@ -use crate::api::model::AppState; -use crate::model::{macros, Config, UserPlan}; -use arc_swap::access::Access; -use arc_swap::ArcSwap; +use crate::model::{macros, AppConfig, Config, UserPlan}; +use arc_swap::{access::Access, ArcSwap}; use chrono::Local; use log::{debug, error, warn}; -use shared::foundation::{get_filter, BinaryOperator, Filter}; -use shared::model::{ - ClusterFlags, NetworkAccessDto, ProxyType, ProxyUserCredentialsDto, ProxyUserStatus, TargetUserDto, - UserConnectionPermission, XtreamCluster, +use shared::{ + foundation::{get_filter, BinaryOperator, Filter}, + model::{ + ClusterFlags, NetworkAccessDto, ProxyType, ProxyUserCredentialsDto, ProxyUserStatus, TargetUserDto, + XtreamCluster, + }, }; -use std::collections::HashMap; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use zeroize::Zeroize; #[derive(Debug, Clone, PartialEq, Eq)] @@ -31,9 +30,7 @@ pub struct NetworkAccess { } impl NetworkAccess { - pub fn is_empty(&self) -> bool { - self.allowed_countries.is_empty() && self.allowed_networks.is_empty() - } + pub fn is_empty(&self) -> bool { self.allowed_countries.is_empty() && self.allowed_networks.is_empty() } } impl From<&NetworkAccessDto> for NetworkAccess { @@ -82,11 +79,8 @@ impl From<&NetworkAccessDto> for NetworkAccess { impl From<&NetworkAccess> for NetworkAccessDto { fn from(instance: &NetworkAccess) -> Self { - let allowed_countries = if instance.allowed_countries.is_empty() { - None - } else { - Some(instance.allowed_countries.clone()) - }; + let allowed_countries = + if instance.allowed_countries.is_empty() { None } else { Some(instance.allowed_countries.clone()) }; let allowed_networks = if instance.allowed_networks.is_empty() { None } else { @@ -218,18 +212,13 @@ impl ProxyUserCredentials { self.plan, self.username ); } - self.output_clusters = self - .raw_output_clusters - .or_else(|| plan.and_then(|p| p.output_clusters)) - .unwrap_or_else(ClusterFlags::all); + self.output_clusters = + self.raw_output_clusters.or_else(|| plan.and_then(|p| p.output_clusters)).unwrap_or_else(ClusterFlags::all); if let Some(plan_proxy) = plan.and_then(|p| p.proxy) { self.proxy = self.raw_proxy.unwrap_or(plan_proxy); } - self.max_connections = if self.raw_max_connections > 0 { - self.raw_max_connections - } else { - plan.map_or(0, |p| p.max_connections) - }; + self.max_connections = + if self.raw_max_connections > 0 { self.raw_max_connections } else { plan.map_or(0, |p| p.max_connections) }; self.soft_connections = if self.raw_soft_connections > 0 { self.raw_soft_connections } else { @@ -272,26 +261,24 @@ impl ProxyUserCredentials { pub fn matches_token(&self, token: &str) -> bool { if let Some(tkn) = &self.token { - return crate::auth::constant_time_eq(tkn.as_bytes(), token.as_bytes()); + return crate::utils::constant_time_eq(tkn.as_bytes(), token.as_bytes()); } false } pub fn matches(&self, username: &str, password: &str) -> bool { - self.username.eq(username) && crate::auth::constant_time_eq(self.password.as_bytes(), password.as_bytes()) + self.username.eq(username) && crate::utils::constant_time_eq(self.password.as_bytes(), password.as_bytes()) } #[inline] - pub fn has_permissions(&self, app_state: &AppState) -> bool { - self.permission_denied_reason(app_state).is_none() + pub fn has_permissions(&self, app_config: &AppConfig) -> bool { + self.permission_denied_reason(app_config).is_none() } #[inline] - pub fn permission_denied(&self, app_state: &AppState) -> bool { - !self.has_permissions(app_state) - } + pub fn permission_denied(&self, app_config: &AppConfig) -> bool { !self.has_permissions(app_config) } - pub fn permission_denied_reason(&self, app_state: &AppState) -> Option { + pub fn permission_denied_reason(&self, app_config: &AppConfig) -> Option { // A plan reference that cannot be resolved must never fall back to // default clusters, unlimited connections and no filter. if self.t_has_unresolved_plan { @@ -303,7 +290,7 @@ impl ProxyUserCredentials { debug!("User access denied, invalid filter: {}", self.username); return Some(ProxyUserPermissionDenyReason::InvalidFilter); } - let config = > as Access>::load(&app_state.app_config.config); + let config = > as Access>::load(&app_config.config); if config.user_access_control { if let Some(exp_date) = self.exp_date.as_ref() { let now = Local::now(); @@ -338,23 +325,11 @@ impl ProxyUserCredentials { None } - pub fn allows_cluster(&self, cluster: XtreamCluster) -> bool { - self.output_clusters.has_cluster(cluster.into()) - } + pub fn allows_cluster(&self, cluster: XtreamCluster) -> bool { self.output_clusters.has_cluster(cluster.into()) } pub fn allows_item_type(&self, item_type: shared::model::PlaylistItemType) -> bool { self.output_clusters.has_cluster(item_type) } - - pub async fn connection_permission(&self, app_state: &AppState) -> UserConnectionPermission { - let config = > as Access>::load(&app_state.app_config.config); - if (self.max_connections > 0 || self.soft_connections > 0) && config.user_access_control { - return app_state - .get_connection_permission(&self.username, self.max_connections, self.soft_connections) - .await; - } - UserConnectionPermission::Allowed - } } impl Drop for ProxyUserCredentials { @@ -375,10 +350,7 @@ pub struct TargetUser { macros::from_impl!(TargetUser); impl From<&TargetUserDto> for TargetUser { fn from(dto: &TargetUserDto) -> Self { - Self { - target: dto.target.clone(), - credentials: dto.credentials.iter().map(|c| Arc::new(c.into())).collect(), - } + Self { target: dto.target.clone(), credentials: dto.credentials.iter().map(|c| Arc::new(c.into())).collect() } } } diff --git a/backend/src/model/config/app.rs b/backend/core/src/model/config/app.rs similarity index 84% rename from backend/src/model/config/app.rs rename to backend/core/src/model/config/app.rs index 029e59b80..e0bf3394f 100644 --- a/backend/src/model/config/app.rs +++ b/backend/core/src/model/config/app.rs @@ -1,26 +1,32 @@ -use crate::api::model::TransportStreamBuffer; -use crate::model::{ - ApiProxyConfig, ApiProxyServerInfo, Config, ConfigInput, ConfigInputOptions, ConfigTarget, CustomStreamResponse, - GracePeriodOptions, HdHomeRunConfig, HdHomeRunFlags, Mappings, MediaToolCapabilities, - ProxyUserCredentials, ReverseProxyDisabledHeaderConfig, SourcesConfig, TargetOutput, +use crate::{ + model::{ + ApiProxyConfig, ApiProxyServerInfo, Config, ConfigInput, ConfigInputOptions, ConfigTarget, + CustomStreamResponse, GracePeriodOptions, HdHomeRunConfig, HdHomeRunFlags, Mappings, MediaToolCapabilities, + ProxyUserCredentials, ReverseProxyDisabledHeaderConfig, SourcesConfig, TargetOutput, + }, + utils, }; -use crate::utils; use arc_swap::{ArcSwap, ArcSwapOption}; use log::{error, warn}; use rand::Rng; -use shared::error::TuliproxError; -use shared::model::{ConfigPaths, GeoIpUnavailablePolicy}; -use shared::defaults::{ - CHANNEL_UNAVAILABLE, HLS_SESSION_OR_LEASE_EXPIRED, LOW_PRIORITY_PREEMPTED, PANEL_API_PROVISIONING, - PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT, PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX, - PROVIDER_CONNECTIONS_EXHAUSTED, USER_ACCOUNT_EXPIRED, USER_CONNECTIONS_EXHAUSTED, +use shared::{ + defaults::{ + CHANNEL_UNAVAILABLE, HLS_SESSION_OR_LEASE_EXPIRED, LOW_PRIORITY_PREEMPTED, PANEL_API_PROVISIONING, + PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT, PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX, + PROVIDER_CONNECTIONS_EXHAUSTED, USER_ACCOUNT_EXPIRED, USER_CONNECTIONS_EXHAUSTED, + }, + error::TuliproxError, + model::{ConfigPaths, GeoIpUnavailablePolicy}, }; -use std::borrow::Cow; -use std::collections::HashSet; -use std::fs::File; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::{ + borrow::Cow, + collections::HashSet, + fs::File, + io::Read, + path::{Path, PathBuf}, + sync::Arc, +}; +use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; fn generate_secret() -> [u8; 32] { let mut rng = rand::rng(); @@ -40,7 +46,7 @@ pub struct AppConfig { pub custom_stream_response: Arc>, pub access_token_secret: [u8; 32], pub encrypt_secret: [u8; 16], - pub(crate) media_tools: Arc, + pub media_tools: Arc, } impl AppConfig { @@ -109,7 +115,11 @@ impl AppConfig { target_mappings.push(mappings); } } - target.mapping.store(if target_mappings.is_empty() { None } else { Some(Arc::new(target_mappings)) }); + target.mapping.store(if target_mappings.is_empty() { + None + } else { + Some(Arc::new(target_mappings)) + }); } } } @@ -119,7 +129,9 @@ impl AppConfig { if let Some(username) = output_username { if let Some((_, config_target)) = self.get_target_for_username(username) { if config_target.name != target_name { - return Err(TuliproxError::Config(format!("User:{username} does not belong to target: {target_name}"))); + return Err(TuliproxError::Config(format!( + "User:{username} does not belong to target: {target_name}" + ))); } } else { return Err(TuliproxError::Config(format!("User: {username} does not exist"))); @@ -133,10 +145,7 @@ impl AppConfig { let check_homerun = { let config = self.config.load(); self.hdhomerun.store(config.hdhomerun.as_ref().map(|h| Arc::new(h.clone()))); - config - .hdhomerun - .as_ref() - .is_some_and(|h| h.flags.contains(HdHomeRunFlags::Enabled)) + config.hdhomerun.as_ref().is_some_and(|h| h.flags.contains(HdHomeRunFlags::Enabled)) }; let sources = self.sources.load(); for source in &sources.sources { @@ -184,12 +193,22 @@ impl AppConfig { config.reverse_proxy.as_ref().is_none_or(|r| !r.resource_rewrite_disabled) } + /// The secret used to rewrite resource URLs, falling back to the configured + /// encrypt secret. Derived entirely from this configuration, so callers do + /// not need the server state to obtain it. + pub fn get_encrypt_secret(&self) -> [u8; 16] { + self.get_reverse_proxy_rewrite_secret().unwrap_or(self.encrypt_secret) + } + pub fn get_reverse_proxy_rewrite_secret(&self) -> Option<[u8; 16]> { let config = self.config.load(); config.reverse_proxy.as_ref().map(|r| r.rewrite_secret) } - fn intern_get_target_for_user(&self, user_target: Option<(Arc, String)>) -> Option<(Arc, Arc)> { + fn intern_get_target_for_user( + &self, + user_target: Option<(Arc, String)>, + ) -> Option<(Arc, Arc)> { match user_target { Some((user, target_name)) => { let sources = self.sources.load(); @@ -202,19 +221,15 @@ impl AppConfig { } None } - None => None + None => None, } } pub fn get_inputs_for_target(&self, target_name: &str) -> Option>> { let sources = self.sources.load(); if let Some(inputs) = sources.get_source_inputs_by_target_by_name(target_name) { - let result: Vec> = sources - .inputs - .iter() - .filter(|s| inputs.contains(&s.name)) - .map(Arc::clone) - .collect(); + let result: Vec> = + sources.inputs.iter().filter(|s| inputs.contains(&s.name)).map(Arc::clone).collect(); if !result.is_empty() { return Some(result); } @@ -224,18 +239,29 @@ impl AppConfig { pub fn get_target_for_username(&self, username: &str) -> Option<(Arc, Arc)> { if let Some(credentials) = self.get_user_credentials(username) { - return self.api_proxy.load().as_ref() - .and_then(|api_proxy| self.intern_get_target_for_user(api_proxy.get_target_name(&credentials.username, &credentials.password))); + return self.api_proxy.load().as_ref().and_then(|api_proxy| { + self.intern_get_target_for_user(api_proxy.get_target_name(&credentials.username, &credentials.password)) + }); } None } - pub fn get_target_for_user(&self, username: &str, password: &str) -> Option<(Arc, Arc)> { - self.api_proxy.load().as_ref().and_then(|api_proxy| self.intern_get_target_for_user(api_proxy.get_target_name(username, password))) + pub fn get_target_for_user( + &self, + username: &str, + password: &str, + ) -> Option<(Arc, Arc)> { + self.api_proxy + .load() + .as_ref() + .and_then(|api_proxy| self.intern_get_target_for_user(api_proxy.get_target_name(username, password))) } pub fn get_target_for_user_by_token(&self, token: &str) -> Option<(Arc, Arc)> { - self.api_proxy.load().as_ref().and_then(|api_proxy| self.intern_get_target_for_user(api_proxy.get_target_name_by_token(token))) + self.api_proxy + .load() + .as_ref() + .and_then(|api_proxy| self.intern_get_target_for_user(api_proxy.get_target_name_by_token(token))) } pub fn get_user_credentials(&self, username: &str) -> Option> { @@ -312,7 +338,9 @@ impl AppConfig { return Err(TuliproxError::Config("input name required".to_string())); } if seen_names.contains(&input_name) { - return Err(TuliproxError::Config(format!("input and alias names should be unique: {input_name}"))); + return Err(TuliproxError::Config(format!( + "input and alias names should be unique: {input_name}" + ))); } seen_names.insert(input_name.clone()); } @@ -322,7 +350,6 @@ impl AppConfig { Ok(()) } - fn check_scheduled_targets(&self, target_names: &HashSet>) -> Result<(), TuliproxError> { let config = self.config.load(); if let Some(schedules) = &config.schedules { @@ -330,7 +357,9 @@ impl AppConfig { if let Some(targets) = &schedule.targets { for target_name in targets { if !target_names.contains(target_name.as_str()) { - return Err(TuliproxError::Config(format!("Unknown target name in scheduler: {target_name}"))); + return Err(TuliproxError::Config(format!( + "Unknown target name in scheduler: {target_name}" + ))); } } } @@ -340,12 +369,13 @@ impl AppConfig { } /** - * if `include_computed` set to true for `app_state` - */ + * if `include_computed` set to true for `app_state` + */ pub fn prepare(&mut self, include_computed: bool) -> Result<(), TuliproxError> { if include_computed { self.access_token_secret = generate_secret(); - self.encrypt_secret = <&[u8] as TryInto<[u8; 16]>>::try_into(&generate_secret()[0..16]).map_err(|err| TuliproxError::Crypto(err.to_string()))?; + self.encrypt_secret = <&[u8] as TryInto<[u8; 16]>>::try_into(&generate_secret()[0..16]) + .map_err(|err| TuliproxError::Crypto(err.to_string()))?; self.prepare_paths(); } else { self.prepare_mapping_path(); @@ -388,13 +418,8 @@ impl AppConfig { /// Resolves a candidate file path, compares it with the current value /// (read via `getter`), and only stores a new `ConfigPaths` when the /// value actually changed. - fn set_optional_path( - &self, - requested: Option<&str>, - resolver: R, - getter: G, - setter: S, - ) where + fn set_optional_path(&self, requested: Option<&str>, resolver: R, getter: G, setter: S) + where R: Fn(&str, Option<&str>) -> String, G: Fn(&ConfigPaths) -> &Option, S: Fn(&mut ConfigPaths, Option), @@ -433,8 +458,11 @@ impl AppConfig { if let Ok(meta) = std::fs::metadata(file_path) { const MAX_RESPONSE_SIZE: u64 = 10 * 1024 * 1024; if meta.len() > MAX_RESPONSE_SIZE { - error!("Custom stream response file too large ({} bytes): {}", - meta.len(), file_path.display()); + error!( + "Custom stream response file too large ({} bytes): {}", + meta.len(), + file_path.display() + ); return None; } } @@ -495,15 +523,11 @@ impl AppConfig { } } - pub fn get_server_info(&self, server_info_name: &str) -> Option { let guard = self.api_proxy.load(); if let Some(api_proxy) = guard.as_ref() { let server_info_list = &api_proxy.server; - server_info_list - .iter() - .find(|c| c.name.eq(server_info_name)) - .cloned() + server_info_list.iter().find(|c| c.name.eq(server_info_name)).cloned() } else { None } @@ -519,9 +543,7 @@ impl AppConfig { config.get_disabled_headers() } - pub fn get_grace_options(&self) -> GracePeriodOptions { - self.config.load().get_grace_options() - } + pub fn get_grace_options(&self) -> GracePeriodOptions { self.config.load().get_grace_options() } pub fn get_geoip_unavailable_policy(&self) -> GeoIpUnavailablePolicy { self.config.load().get_geoip_unavailable_policy() @@ -539,18 +561,20 @@ impl AppConfig { self.media_tools.is_ffprobe_available().await } - pub async fn is_ffmpeg_available(&self) -> bool { - self.media_tools.is_ffmpeg_available().await - } + pub async fn is_ffmpeg_available(&self) -> bool { self.media_tools.is_ffmpeg_available().await } } #[cfg(test)] mod tests { use super::*; - use crate::model::{ConfigSource, Mapping, MappingDefinition, Mapper}; - use crate::utils::FileLockManager; - use shared::foundation::Filter; - use shared::model::{ConfigPaths, ProcessingOrder}; + use crate::{ + model::{ConfigSource, Mapper, Mapping, MappingDefinition}, + utils::FileLockManager, + }; + use shared::{ + foundation::Filter, + model::{ConfigPaths, ProcessingOrder}, + }; fn empty_paths() -> ConfigPaths { ConfigPaths { @@ -619,9 +643,7 @@ mod tests { } fn empty_mappings() -> Mappings { - Mappings { - mappings: MappingDefinition { templates: None, mapping: Vec::new() }, - } + Mappings { mappings: MappingDefinition { templates: None, mapping: Vec::new() } } } #[test] diff --git a/backend/src/model/config/base.rs b/backend/core/src/model/config/base.rs similarity index 78% rename from backend/src/model/config/base.rs rename to backend/core/src/model/config/base.rs index 179dabf5f..c4d4ea25c 100644 --- a/backend/src/model/config/base.rs +++ b/backend/core/src/model/config/base.rs @@ -1,18 +1,27 @@ -use crate::model::{ - macros, ConfigApi, HdHomeRunConfig, HdHomeRunFlags, IpCheckConfig, LibraryConfig, - LogConfig, MetadataUpdateConfig, MessagingConfig, ProxyConfig, ReverseProxyConfig, - ReverseProxyDisabledHeaderConfig, ScheduleConfig, VideoConfig, WebUiConfig, +use crate::{ + model::{ + macros, ConfigApi, HdHomeRunConfig, HdHomeRunFlags, IpCheckConfig, LibraryConfig, LogConfig, MessagingConfig, + MetadataUpdateConfig, ProxyConfig, ReverseProxyConfig, ReverseProxyDisabledHeaderConfig, ScheduleConfig, + VideoConfig, WebUiConfig, + }, + utils, + utils::get_default_path_for_home, }; -use crate::utils; use log::{error, info}; use path_clean::PathClean; -use shared::error::TuliproxError; -use shared::model::{ConfigDto, GeoIpUnavailablePolicy, HdHomeRunDeviceOverview}; -use shared::utils::{ set_sanitize_sensitive_info}; -use shared::defaults::{default_grace_period_millis, default_grace_period_timeout_secs, DEFAULT_BACKUP_DIR, DEFAULT_CACHE_DIR, DEFAULT_DOWNLOAD_DIR, DEFAULT_STORAGE_DIR, DEFAULT_STORAGE_TEMP_DIR, DEFAULT_USER_CONFIG_DIR}; -use std::borrow::Cow; -use std::path::{Path, PathBuf}; -use crate::utils::get_default_path_for_home; +use shared::{ + defaults::{ + default_grace_period_millis, default_grace_period_timeout_secs, DEFAULT_BACKUP_DIR, DEFAULT_CACHE_DIR, + DEFAULT_DOWNLOAD_DIR, DEFAULT_STORAGE_DIR, DEFAULT_STORAGE_TEMP_DIR, DEFAULT_USER_CONFIG_DIR, + }, + error::TuliproxError, + model::{ConfigDto, GeoIpUnavailablePolicy, HdHomeRunDeviceOverview}, + utils::set_sanitize_sensitive_info, +}; +use std::{ + borrow::Cow, + path::{Path, PathBuf}, +}; fn create_directories(cfg: &Config, temp_path: &Path) { // Collect the paths into a vector. @@ -21,13 +30,17 @@ fn create_directories(cfg: &Config, temp_path: &Path) { cfg.backup_dir.clone(), cfg.user_config_dir.clone(), cfg.video.as_ref().and_then(|v| v.download.as_ref()).map(|d| d.directory.clone()), - cfg.reverse_proxy.as_ref().and_then(|r| r.cache.as_ref().and_then(|c| if c.enabled { Some(c.directory.clone()) } else { None })), + cfg.reverse_proxy + .as_ref() + .and_then(|r| r.cache.as_ref().and_then(|c| if c.enabled { Some(c.directory.clone()) } else { None })), cfg.metadata_update.as_ref().map(|m| m.cache_path.clone()), ]; - let mut paths: Vec = paths_strings.iter() + let mut paths: Vec = paths_strings + .iter() .filter_map(|opt| opt.as_ref()) // Get rid of the `Option` - .map(PathBuf::from).collect(); + .map(PathBuf::from) + .collect(); paths.push(temp_path.to_path_buf()); // Iterate over the paths, filter out `None` values, and process the `Some(path)` values. @@ -99,11 +112,14 @@ pub struct Config { } impl Config { - fn lexical_normalize(path: &Path) -> PathBuf { - path.components().collect::() - } + fn lexical_normalize(path: &Path) -> PathBuf { path.components().collect::() } - fn normalize_storage_path(raw_storage_dir: &str, storage_dir: &str, input: &str, default_dir: &str) -> Option { + fn normalize_storage_path( + raw_storage_dir: &str, + storage_dir: &str, + input: &str, + default_dir: &str, + ) -> Option { let trimmed_input = input.trim(); if trimmed_input.is_empty() { return Some(PathBuf::from(storage_dir).join(default_dir)); @@ -137,7 +153,6 @@ impl Config { } pub fn prepare(&mut self, config_path: &str, home_path: &str) -> Result<(), TuliproxError> { - self.prepare_directories(home_path); if let Some(ref mut webui) = &mut self.web_ui { webui.prepare(config_path)?; @@ -149,11 +164,8 @@ impl Config { if let Some(metadata_update) = self.metadata_update.as_mut() { let meta_path = PathBuf::from(&metadata_update.cache_path); - let meta_path = if meta_path.is_relative() { - PathBuf::from(&self.storage_dir).join(meta_path) - } else { - meta_path - }; + let meta_path = + if meta_path.is_relative() { PathBuf::from(&self.storage_dir).join(meta_path) } else { meta_path }; metadata_update.cache_path = meta_path.to_string_lossy().to_string(); } @@ -171,11 +183,8 @@ impl Config { .to_string_lossy() .to_string(); } else if download_path.is_relative() { - download.directory = PathBuf::from(home_path) - .join(download_path) - .clean() - .to_string_lossy() - .to_string(); + download.directory = + PathBuf::from(home_path).join(download_path).clean().to_string_lossy().to_string(); } } } @@ -194,8 +203,9 @@ impl Config { return; } - let normalized = Self::normalize_storage_path(raw_storage_dir, &self.storage_dir, &cache.directory, DEFAULT_CACHE_DIR) - .unwrap_or_else(|| PathBuf::from(cache.directory.trim())); + let normalized = + Self::normalize_storage_path(raw_storage_dir, &self.storage_dir, &cache.directory, DEFAULT_CACHE_DIR) + .unwrap_or_else(|| PathBuf::from(cache.directory.trim())); cache.directory = normalized.clean().to_string_lossy().to_string(); } @@ -234,9 +244,7 @@ impl Config { fn prepare_api_web_root(&mut self, home_path: &str) { let web_root = self.api.web_root.trim(); if web_root.is_empty() { - self.api.web_root = utils::get_default_web_root_path_for_home(Path::new(home_path)) - .display() - .to_string(); + self.api.web_root = utils::get_default_web_root_path_for_home(Path::new(home_path)).display().to_string(); } else { let web_root_path = PathBuf::from(web_root); let resolved = if web_root_path.is_absolute() { @@ -252,23 +260,18 @@ impl Config { set_sanitize_sensitive_info(self.log.as_ref().is_none_or(|l| l.sanitize_sensitive_info)); let temp_path = PathBuf::from(&self.storage_dir).join(DEFAULT_STORAGE_TEMP_DIR); create_directories(self, &temp_path); - // `tempfile::env::override_temp_dir` mutates a process-global - // default. In production that is the point — every tempfile - // helper routes through the configured storage temp dir. In - // tests it poisons every parallel test in the same process: - // the first test to call `update_runtime` wins, and every - // later `tempdir()` from a sibling test lands in that test's - // `storage_dir/tmp/`. Gate the override to non-test builds. - #[cfg(not(test))] - let _ = tempfile::env::override_temp_dir(&temp_path); + // The override is process-global. Test runners disable it explicitly because + // this crate is compiled without `cfg(test)` when used as a dependency. + if std::env::var_os("TULIPROX_DISABLE_TEMP_DIR_OVERRIDE").is_none() { + let _ = tempfile::env::override_temp_dir(&temp_path); + } } pub fn get_hdhr_device_overview(&self) -> Option { - self.hdhomerun.as_ref().map(|hdhr| - HdHomeRunDeviceOverview { - enabled: hdhr.flags.contains(HdHomeRunFlags::Enabled), - devices: hdhr.devices.iter().map(|d| d.name.clone()).collect::>(), - }) + self.hdhomerun.as_ref().map(|hdhr| HdHomeRunDeviceOverview { + enabled: hdhr.flags.contains(HdHomeRunFlags::Enabled), + devices: hdhr.devices.iter().map(|d| d.name.clone()).collect::>(), + }) } pub fn is_geoip_enabled(&self) -> bool { @@ -283,21 +286,17 @@ impl Config { } pub fn get_disabled_headers(&self) -> Option { - self.reverse_proxy - .as_ref() - .and_then(|r| r.disabled_header.clone()) + self.reverse_proxy.as_ref().and_then(|r| r.disabled_header.clone()) } pub fn get_grace_options(&self) -> GracePeriodOptions { - self.reverse_proxy - .as_ref() - .and_then(|r| r.stream.as_ref()) - .map_or_else(GracePeriodOptions::default, - |s| GracePeriodOptions { - period_millis: s.grace_period_millis, - timeout_secs: s.grace_period_timeout_secs, - hold_stream: s.grace_period_hold_stream, - }) + self.reverse_proxy.as_ref().and_then(|r| r.stream.as_ref()).map_or_else(GracePeriodOptions::default, |s| { + GracePeriodOptions { + period_millis: s.grace_period_millis, + timeout_secs: s.grace_period_timeout_secs, + hold_stream: s.grace_period_hold_stream, + } + }) } } @@ -354,23 +353,14 @@ mod tests { let home_path = temp_dir.path().join("home"); let config_path = temp_dir.path().join("config"); - let mut config = Config { - storage_dir: "data".to_string(), - ..Default::default() - }; + let mut config = Config { storage_dir: "data".to_string(), ..Default::default() }; config.api.web_root = "./web".to_string(); config - .prepare( - config_path.to_string_lossy().as_ref(), - home_path.to_string_lossy().as_ref(), - ) + .prepare(config_path.to_string_lossy().as_ref(), home_path.to_string_lossy().as_ref()) .expect("prepare should succeed"); - assert_eq!( - config.api.web_root, - home_path.join("web").to_string_lossy() - ); + assert_eq!(config.api.web_root, home_path.join("web").to_string_lossy()); } #[test] @@ -380,23 +370,14 @@ mod tests { let config_path = temp_dir.path().join("config"); let absolute_web_root = temp_dir.path().join("custom_web_root"); - let mut config = Config { - storage_dir: "data".to_string(), - ..Default::default() - }; + let mut config = Config { storage_dir: "data".to_string(), ..Default::default() }; config.api.web_root = absolute_web_root.to_string_lossy().to_string(); config - .prepare( - config_path.to_string_lossy().as_ref(), - home_path.to_string_lossy().as_ref(), - ) + .prepare(config_path.to_string_lossy().as_ref(), home_path.to_string_lossy().as_ref()) .expect("prepare should succeed"); - assert_eq!( - config.api.web_root, - absolute_web_root.to_string_lossy() - ); + assert_eq!(config.api.web_root, absolute_web_root.to_string_lossy()); } #[test] @@ -436,10 +417,8 @@ mod tests { #[test] fn get_geoip_unavailable_policy_no_geoip_returns_deny() { - let dto = ConfigDto { - reverse_proxy: Some(shared::model::ReverseProxyConfigDto::default()), - ..Default::default() - }; + let dto = + ConfigDto { reverse_proxy: Some(shared::model::ReverseProxyConfigDto::default()), ..Default::default() }; let config = Config::from(&dto); assert_eq!(config.get_geoip_unavailable_policy(), GeoIpUnavailablePolicy::Deny); } diff --git a/backend/src/model/config/cache.rs b/backend/core/src/model/config/cache.rs similarity index 97% rename from backend/src/model/config/cache.rs rename to backend/core/src/model/config/cache.rs index b1af3bf4b..4d814be3e 100644 --- a/backend/src/model/config/cache.rs +++ b/backend/core/src/model/config/cache.rs @@ -1,5 +1,5 @@ -use shared::model::{ByteSize, CacheConfigDto}; use crate::model::macros; +use shared::model::{ByteSize, CacheConfigDto}; #[derive(Debug, Clone)] pub struct CacheConfig { @@ -17,7 +17,7 @@ impl From<&CacheConfigDto> for CacheConfig { // Dto prepare should have set the right path directory: dto.directory.as_ref().map_or_else(Default::default, std::string::ToString::to_string), size_str: dto.size.clone(), - size: get_size(dto) + size: get_size(dto), } } } diff --git a/backend/src/model/config/epg.rs b/backend/core/src/model/config/epg.rs similarity index 94% rename from backend/src/model/config/epg.rs rename to backend/core/src/model/config/epg.rs index 66b1dcdb9..b5b657832 100644 --- a/backend/src/model/config/epg.rs +++ b/backend/core/src/model/config/epg.rs @@ -1,9 +1,10 @@ use crate::model::{macros, EpgSmartMatchConfig}; -use shared::model::{ - EpgConfigDto, EpgSourceDto, EpgSourceTypeDto, IcsDummyConfigDto, IcsEpgSourceConfigDto, - IcsEventMappingDto, +use shared::{ + model::{ + EpgConfigDto, EpgSourceDto, EpgSourceTypeDto, IcsDummyConfigDto, IcsEpgSourceConfigDto, IcsEventMappingDto, + }, + utils::Internable, }; -use shared::utils::Internable; use std::sync::Arc; #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -180,10 +181,7 @@ mod tests { assert_eq!(source.source_type, EpgSourceType::Ics); assert_eq!(source.channel_id.as_deref(), Some("f1.calendar")); assert_eq!(source.channel_title.as_deref(), Some("Formula 1")); - assert_eq!( - source.match_names.iter().map(AsRef::as_ref).collect::>(), - vec!["F1", "Formel 1"] - ); + assert_eq!(source.match_names.iter().map(AsRef::as_ref).collect::>(), vec!["F1", "Formel 1"]); assert_eq!(source.ics.as_ref().map(|config| config.max_events), Some(50_000)); } } diff --git a/backend/src/model/config/epg_smart_match.rs b/backend/core/src/model/config/epg_smart_match.rs similarity index 90% rename from backend/src/model/config/epg_smart_match.rs rename to backend/core/src/model/config/epg_smart_match.rs index 0ee789119..cd9fd9d78 100644 --- a/backend/src/model/config/epg_smart_match.rs +++ b/backend/core/src/model/config/epg_smart_match.rs @@ -1,9 +1,11 @@ -use std::sync::Arc; -use regex::Regex; -use shared::model::{EpgNamePrefix, EpgSmartMatchConfigDto}; -use shared::utils::{CONSTANTS}; -use shared::defaults::{default_epg_name_prefix_separator, default_epg_strip}; use crate::model::macros; +use regex::Regex; +use shared::{ + defaults::{default_epg_name_prefix_separator, default_epg_strip}, + model::{EpgNamePrefix, EpgSmartMatchConfigDto}, + utils::CONSTANTS, +}; +use std::sync::Arc; #[derive(Debug, Clone)] pub struct EpgSmartMatchConfig { diff --git a/backend/src/model/config/favourites.rs b/backend/core/src/model/config/favourites.rs similarity index 91% rename from backend/src/model/config/favourites.rs rename to backend/core/src/model/config/favourites.rs index 4616a1b32..30182d6fa 100644 --- a/backend/src/model/config/favourites.rs +++ b/backend/core/src/model/config/favourites.rs @@ -1,7 +1,9 @@ -use std::sync::Arc; use crate::model::macros; -use shared::foundation::{CompiledRegex, Filter}; -use shared::model::{ConfigFavouritesDto, ItemField, XtreamCluster}; +use shared::{ + foundation::{CompiledRegex, Filter}, + model::{ConfigFavouritesDto, ItemField, XtreamCluster}, +}; +use std::sync::Arc; #[derive(Debug, Clone)] pub struct ConfigFavourites { @@ -23,7 +25,6 @@ impl ConfigFavourites { } } - macros::from_impl!(ConfigFavourites); impl From<&ConfigFavouritesDto> for ConfigFavourites { fn from(dto: &ConfigFavouritesDto) -> Self { @@ -46,4 +47,4 @@ impl From<&ConfigFavourites> for ConfigFavouritesDto { t_filter: Some(instance.filter.clone()), } } -} \ No newline at end of file +} diff --git a/backend/src/model/config/geoip.rs b/backend/core/src/model/config/geoip.rs similarity index 70% rename from backend/src/model/config/geoip.rs rename to backend/core/src/model/config/geoip.rs index 23f8ddb92..767331e0a 100644 --- a/backend/src/model/config/geoip.rs +++ b/backend/core/src/model/config/geoip.rs @@ -1,32 +1,24 @@ -use shared::model::{GeoIpConfigDto, GeoIpUnavailablePolicy}; use crate::model::macros; +use shared::model::{GeoIpConfigDto, GeoIpUnavailablePolicy}; #[derive(Debug, Clone)] pub struct GeoIpConfig { - pub(crate) enabled: bool, - pub(crate) url: String, - pub(crate) unavailable_policy: GeoIpUnavailablePolicy, + pub enabled: bool, + pub url: String, + pub unavailable_policy: GeoIpUnavailablePolicy, } macros::from_impl!(GeoIpConfig); impl From<&GeoIpConfigDto> for GeoIpConfig { fn from(dto: &GeoIpConfigDto) -> Self { - Self { - enabled: dto.enabled, - url: dto.url.clone(), - unavailable_policy: dto.unavailable_policy, - } + Self { enabled: dto.enabled, url: dto.url.clone(), unavailable_policy: dto.unavailable_policy } } } impl From<&GeoIpConfig> for GeoIpConfigDto { fn from(instance: &GeoIpConfig) -> Self { - Self { - enabled: instance.enabled, - url: instance.url.clone(), - unavailable_policy: instance.unavailable_policy, - } + Self { enabled: instance.enabled, url: instance.url.clone(), unavailable_policy: instance.unavailable_policy } } } @@ -36,11 +28,8 @@ mod tests { #[test] fn geoip_unavailable_policy_dto_default_converts_to_deny() { - let dto = GeoIpConfigDto { - enabled: false, - url: String::new(), - unavailable_policy: GeoIpUnavailablePolicy::Deny, - }; + let dto = + GeoIpConfigDto { enabled: false, url: String::new(), unavailable_policy: GeoIpUnavailablePolicy::Deny }; let config: GeoIpConfig = (&dto).into(); assert_eq!(config.unavailable_policy, GeoIpUnavailablePolicy::Deny); } diff --git a/backend/src/model/config/hdhomerun.rs b/backend/core/src/model/config/hdhomerun.rs similarity index 85% rename from backend/src/model/config/hdhomerun.rs rename to backend/core/src/model/config/hdhomerun.rs index 379ed78df..fbcf0d587 100644 --- a/backend/src/model/config/hdhomerun.rs +++ b/backend/core/src/model/config/hdhomerun.rs @@ -1,6 +1,8 @@ -use shared::model::{HdHomeRunConfigDto, HdHomeRunDeviceConfigDto}; use crate::model::macros; -use shared::{apply_flags, create_bitset}; +use shared::{ + apply_flags, create_bitset, + model::{HdHomeRunConfigDto, HdHomeRunDeviceConfigDto}, +}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct HdHomeRunDeviceConfig { @@ -42,14 +44,7 @@ impl From<&HdHomeRunDeviceConfigDto> for HdHomeRunDeviceConfig { } } -create_bitset!( - u8, - HdHomeRunFlags, - Enabled, - Auth, - SsdpDiscovery, - ProprietaryDiscovery -); +create_bitset!(u8, HdHomeRunFlags, Enabled, Auth, SsdpDiscovery, ProprietaryDiscovery); #[derive(Debug, Clone, PartialEq, Eq)] pub struct HdHomeRunConfig { @@ -68,9 +63,6 @@ impl From<&HdHomeRunConfigDto> for HdHomeRunConfig { (ssdp_discovery, SsdpDiscovery), (proprietary_discovery, ProprietaryDiscovery) ); - Self { - flags, - devices: dto.devices.iter().map(Into::into).collect(), - } + Self { flags, devices: dto.devices.iter().map(Into::into).collect() } } } diff --git a/backend/src/model/config/healthcheck.rs b/backend/core/src/model/config/healthcheck.rs similarity index 100% rename from backend/src/model/config/healthcheck.rs rename to backend/core/src/model/config/healthcheck.rs diff --git a/backend/src/model/config/input.rs b/backend/core/src/model/config/input.rs similarity index 96% rename from backend/src/model/config/input.rs rename to backend/core/src/model/config/input.rs index d6abfd211..a5d3d98d9 100644 --- a/backend/src/model/config/input.rs +++ b/backend/core/src/model/config/input.rs @@ -1,6 +1,6 @@ use crate::{ model::{macros, ConfigProvider, EpgConfig, PanelApiConfig}, - repository::get_csv_file_path, + utils::get_csv_file_path, }; use chrono::Utc; use log::warn; @@ -28,6 +28,11 @@ use std::{ path::PathBuf, sync::{Arc, LazyLock}, }; +use tuliprox_media_server::{ + client::MediaServerHttpClient, + errors::{MediaServerError, MediaServerErrorKind}, + plex::client::{PlexCatalogClient, PlexClientSettings}, +}; use url::Url; create_bitset!( @@ -124,6 +129,44 @@ pub struct MediaServerInputConfig { pub allow_relay: bool, } +impl MediaServerInputConfig { + /// The subset of this configuration a Plex catalog client actually needs. + /// + /// The adapter lives here rather than in `media_server` so that the + /// media-server module never names the application's configuration model. + pub fn plex_client_settings(&self) -> PlexClientSettings { + PlexClientSettings { + token: self.token.clone(), + account_token: self.account_token.clone(), + server_id: self.server_id.clone(), + server_name: self.server_name.clone(), + prefer_https: self.prefer_https, + allow_relay: self.allow_relay, + libraries: self.libraries.clone(), + } + } +} + +impl ConfigInput { + /// Builds the Plex catalog client this input describes. + /// + /// Previously `PlexCatalogClient::from_input`; it moved out of `media_server` + /// because it is the only thing there that knew what a `ConfigInput` is. + pub fn plex_catalog_client(&self, http: MediaServerHttpClient) -> Result { + if self.input_type != InputType::Plex { + return Err(MediaServerError::new(MediaServerErrorKind::MediaServerDiscoveryFailed) + .provider("plex") + .detail("plex catalog client requires a plex input")); + } + let media_server = self.media_server.as_ref().ok_or_else(|| { + MediaServerError::new(MediaServerErrorKind::MediaServerDiscoveryFailed) + .provider("plex") + .detail("plex input is missing media_server configuration") + })?; + Ok(PlexCatalogClient::new(self.name.clone(), self.url.as_str(), &media_server.plex_client_settings(), http)) + } +} + #[derive(Debug, Clone)] pub struct ConfigInputStaged { pub for_input: Option>, @@ -1048,10 +1091,7 @@ mod tests { #[test] fn input_options_conversion_sets_disable_hls_streaming_flag() { - let dto = ConfigInputOptionsDto { - disable_hls_streaming: true, - ..ConfigInputOptionsDto::default() - }; + let dto = ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }; let options = ConfigInputOptions::from(&dto); diff --git a/backend/src/model/config/ip_check.rs b/backend/core/src/model/config/ip_check.rs similarity index 81% rename from backend/src/model/config/ip_check.rs rename to backend/core/src/model/config/ip_check.rs index 7098285c1..f73f6645b 100644 --- a/backend/src/model/config/ip_check.rs +++ b/backend/core/src/model/config/ip_check.rs @@ -1,7 +1,7 @@ -use std::sync::Arc; -use regex::Regex; -use shared::model::{IpCheckConfigDto}; use crate::model::macros; +use regex::Regex; +use shared::model::IpCheckConfigDto; +use std::sync::Arc; #[derive(Debug, Clone)] pub struct IpCheckConfig { @@ -20,12 +20,14 @@ impl From<&IpCheckConfigDto> for IpCheckConfig { url_ipv4: dto.url_ipv4.clone(), url_ipv6: dto.url_ipv6.clone(), pattern_ipv4: dto.pattern_ipv4.as_ref().and_then(|s| { - shared::model::REGEX_CACHE.get_or_compile(s) + shared::model::REGEX_CACHE + .get_or_compile(s) .map_err(|e| log::warn!("Invalid pattern_ipv4 regex '{s}': {e}")) .ok() }), pattern_ipv6: dto.pattern_ipv6.as_ref().and_then(|s| { - shared::model::REGEX_CACHE.get_or_compile(s) + shared::model::REGEX_CACHE + .get_or_compile(s) .map_err(|e| log::warn!("Invalid pattern_ipv6 regex '{s}': {e}")) .ok() }), @@ -39,7 +41,7 @@ impl From<&IpCheckConfig> for IpCheckConfigDto { url_ipv4: dto.url_ipv4.clone(), url_ipv6: dto.url_ipv6.clone(), pattern_ipv4: dto.pattern_ipv4.as_ref().map(std::string::ToString::to_string), - pattern_ipv6: dto.pattern_ipv6.as_ref().map(std::string::ToString::to_string), + pattern_ipv6: dto.pattern_ipv6.as_ref().map(std::string::ToString::to_string), } } -} \ No newline at end of file +} diff --git a/backend/src/model/config/library.rs b/backend/core/src/model/config/library.rs similarity index 88% rename from backend/src/model/config/library.rs rename to backend/core/src/model/config/library.rs index 0ca27347f..6e53de6d3 100644 --- a/backend/src/model/config/library.rs +++ b/backend/core/src/model/config/library.rs @@ -1,10 +1,11 @@ use crate::model::macros; -use shared::error::TuliproxError; -use shared::model::{ConfigDto, LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; -use shared::defaults::DEFAULT_STORAGE_DIR; -use shared::utils::Internable; -use std::path::PathBuf; -use std::sync::Arc; +use shared::{ + defaults::DEFAULT_STORAGE_DIR, + error::TuliproxError, + model::{ConfigDto, LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}, + utils::Internable, +}; +use std::{path::PathBuf, sync::Arc}; #[derive(Debug, Clone, Default)] pub struct LibraryScanDirectory { @@ -133,12 +134,8 @@ pub fn validate_library_paths_from_dto(cfg: &ConfigDto) -> Result<(), TuliproxEr }; let mut library_cfg = LibraryConfig::from(library_dto); - let storage_dir = cfg - .storage_dir - .as_deref() - .map(str::trim) - .filter(|dir| !dir.is_empty()) - .unwrap_or(DEFAULT_STORAGE_DIR); + let storage_dir = + cfg.storage_dir.as_deref().map(str::trim).filter(|dir| !dir.is_empty()).unwrap_or(DEFAULT_STORAGE_DIR); // Always validate configured scan directories, even when library is disabled. // This prevents persisting invalid paths that would later break startup when enabled. if library_cfg.enabled { @@ -165,10 +162,7 @@ mod tests { fn validate_library_paths_rejects_missing_scan_directory() { let cfg = ConfigDto { storage_dir: Some( - std::env::current_dir() - .expect("current_dir should be available") - .to_string_lossy() - .to_string(), + std::env::current_dir().expect("current_dir should be available").to_string_lossy().to_string(), ), library: Some(LibraryConfigDto { enabled: true, @@ -186,10 +180,7 @@ mod tests { fn validate_library_paths_rejects_missing_scan_directory_even_when_disabled() { let cfg = ConfigDto { storage_dir: Some( - std::env::current_dir() - .expect("current_dir should be available") - .to_string_lossy() - .to_string(), + std::env::current_dir().expect("current_dir should be available").to_string_lossy().to_string(), ), library: Some(LibraryConfigDto { enabled: false, @@ -208,10 +199,7 @@ mod tests { fn validate_library_paths_accepts_existing_scan_directory() { let cfg = ConfigDto { storage_dir: Some( - std::env::current_dir() - .expect("current_dir should be available") - .to_string_lossy() - .to_string(), + std::env::current_dir().expect("current_dir should be available").to_string_lossy().to_string(), ), library: Some(LibraryConfigDto { enabled: true, diff --git a/backend/src/model/config/log.rs b/backend/core/src/model/config/log.rs similarity index 94% rename from backend/src/model/config/log.rs rename to backend/core/src/model/config/log.rs index f3dbbe3eb..0f901d338 100644 --- a/backend/src/model/config/log.rs +++ b/backend/core/src/model/config/log.rs @@ -1,6 +1,8 @@ -use shared::model::{LogConfigDto, RuntimeConfigReportFormat}; -use shared::defaults::default_as_true; use crate::model::macros; +use shared::{ + defaults::default_as_true, + model::{LogConfigDto, RuntimeConfigReportFormat}, +}; // We need serde for these structs to read them during // start from the yaml file without reading the whole config. diff --git a/backend/src/model/config/macros.rs b/backend/core/src/model/config/macros.rs similarity index 95% rename from backend/src/model/config/macros.rs rename to backend/core/src/model/config/macros.rs index 5955d6f56..c383d2bd0 100644 --- a/backend/src/model/config/macros.rs +++ b/backend/core/src/model/config/macros.rs @@ -8,7 +8,7 @@ macro_rules! from_impl { } } } - } + }; } pub use from_impl; @@ -24,7 +24,7 @@ macro_rules! try_from_impl { } } } - } + }; } -pub use try_from_impl; \ No newline at end of file +pub use try_from_impl; diff --git a/backend/src/model/config/media_tools.rs b/backend/core/src/model/config/media_tools.rs similarity index 74% rename from backend/src/model/config/media_tools.rs rename to backend/core/src/model/config/media_tools.rs index 486f579d5..fe0c65fe8 100644 --- a/backend/src/model/config/media_tools.rs +++ b/backend/core/src/model/config/media_tools.rs @@ -11,19 +11,11 @@ pub struct MediaToolCapabilities { impl MediaToolCapabilities { #[must_use] - pub fn new() -> Self { - Self { - available: OnceCell::new(), - } - } + pub fn new() -> Self { Self { available: OnceCell::new() } } - pub async fn is_ffmpeg_available(&self) -> bool { - self.available().await.contains(MediaToolCapability::Ffmpeg) - } + pub async fn is_ffmpeg_available(&self) -> bool { self.available().await.contains(MediaToolCapability::Ffmpeg) } - pub async fn is_ffprobe_available(&self) -> bool { - self.available().await.contains(MediaToolCapability::Ffprobe) - } + pub async fn is_ffprobe_available(&self) -> bool { self.available().await.contains(MediaToolCapability::Ffprobe) } async fn available(&self) -> MediaToolCapabilitySet { *self.available.get_or_init(Self::detect_available_tools).await @@ -45,4 +37,3 @@ impl MediaToolCapabilities { available } } - diff --git a/backend/src/model/config/messaging.rs b/backend/core/src/model/config/messaging.rs similarity index 86% rename from backend/src/model/config/messaging.rs rename to backend/core/src/model/config/messaging.rs index ca3a6df8a..b0c24051c 100644 --- a/backend/src/model/config/messaging.rs +++ b/backend/core/src/model/config/messaging.rs @@ -1,6 +1,9 @@ -use log::warn; use crate::model::macros; -use shared::model::{DiscordMessagingConfigDto, DiskAlertConfigDto, MessagingConfigDto, MsgKind, PushoverMessagingConfigDto, RestMessagingConfigDto, TelegramMessagingConfigDto}; +use log::warn; +use shared::model::{ + DiscordMessagingConfigDto, DiskAlertConfigDto, MessagingConfigDto, MsgKind, PushoverMessagingConfigDto, + RestMessagingConfigDto, TelegramMessagingConfigDto, +}; use std::path::{Path, PathBuf}; /// Runtime configuration for the threshold-driven disk-space alert. @@ -92,9 +95,7 @@ pub struct RestMessagingConfig { } impl RestMessagingConfig { - pub fn prepare(&mut self, templates_dir: &Path) { - discover_templates("rest", &mut self.templates, templates_dir); - } + pub fn prepare(&mut self, templates_dir: &Path) { discover_templates("rest", &mut self.templates, templates_dir); } } macros::from_impl!(RestMessagingConfig); @@ -119,15 +120,8 @@ impl From<&RestMessagingConfigDto> for RestMessagingConfig { impl From<&RestMessagingConfig> for RestMessagingConfigDto { fn from(model: &RestMessagingConfig) -> Self { - let headers = model.headers.iter() - .map(|(k, v)| format!("{k}: {v}")) - .collect(); - Self { - url: model.url.clone(), - method: Some(model.method.clone()), - headers, - templates: model.templates.clone(), - } + let headers = model.headers.iter().map(|(k, v)| format!("{k}: {v}")).collect(); + Self { url: model.url.clone(), method: Some(model.method.clone()), headers, templates: model.templates.clone() } } } @@ -145,20 +139,12 @@ impl DiscordMessagingConfig { macros::from_impl!(DiscordMessagingConfig); impl From<&DiscordMessagingConfigDto> for DiscordMessagingConfig { - fn from(dto: &DiscordMessagingConfigDto) -> Self { - Self { - url: dto.url.clone(), - templates: dto.templates.clone(), - } - } + fn from(dto: &DiscordMessagingConfigDto) -> Self { Self { url: dto.url.clone(), templates: dto.templates.clone() } } } impl From<&DiscordMessagingConfig> for DiscordMessagingConfigDto { fn from(instance: &DiscordMessagingConfig) -> Self { - Self { - url: instance.url.clone(), - templates: instance.templates.clone(), - } + Self { url: instance.url.clone(), templates: instance.templates.clone() } } } @@ -173,7 +159,10 @@ macros::from_impl!(PushoverMessagingConfig); impl From<&PushoverMessagingConfigDto> for PushoverMessagingConfig { fn from(dto: &PushoverMessagingConfigDto) -> Self { Self { - url: dto.url.as_ref().map_or_else(|| String::from("https://api.pushover.net/1/messages.json"), ToString::to_string), + url: dto + .url + .as_ref() + .map_or_else(|| String::from("https://api.pushover.net/1/messages.json"), ToString::to_string), token: dto.token.clone(), user: dto.user.clone(), } @@ -182,11 +171,7 @@ impl From<&PushoverMessagingConfigDto> for PushoverMessagingConfig { impl From<&PushoverMessagingConfig> for PushoverMessagingConfigDto { fn from(instance: &PushoverMessagingConfig) -> Self { - Self { - url: Some(instance.url.clone()), - token: instance.token.clone(), - user: instance.user.clone(), - } + Self { url: Some(instance.url.clone()), token: instance.token.clone(), user: instance.user.clone() } } } diff --git a/backend/src/model/config/metadata_update.rs b/backend/core/src/model/config/metadata_update.rs similarity index 90% rename from backend/src/model/config/metadata_update.rs rename to backend/core/src/model/config/metadata_update.rs index 018f8cc2c..a77270b92 100644 --- a/backend/src/model/config/metadata_update.rs +++ b/backend/core/src/model/config/metadata_update.rs @@ -1,10 +1,23 @@ use crate::model::macros; -use shared::model::{ - ByteSize, FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, - TmdbConfigDto, +use shared::{ + defaults::{ + default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, + default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, + default_metadata_max_resolve_retry_backoff, default_metadata_probe_cooldown, + default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, + default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, + default_metadata_progress_log_interval, default_metadata_queue_log_interval, + default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, + default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, + default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_match_threshold, + default_tmdb_rate_limit_ms, + }, + model::{ + ByteSize, FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, + TmdbConfigDto, + }, + utils::parse_duration_seconds, }; -use shared::utils::{parse_duration_seconds}; -use shared::defaults::{default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_max_resolve_retry_backoff, default_metadata_probe_cooldown, default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_match_threshold, default_tmdb_rate_limit_ms}; #[derive(Debug, Clone)] pub struct MetadataUpdateConfig { @@ -102,9 +115,7 @@ impl Default for TmdbConfig { } impl Default for MetadataUpdateConfig { - fn default() -> Self { - Self::from(&MetadataUpdateConfigDto::default()) - } + fn default() -> Self { Self::from(&MetadataUpdateConfigDto::default()) } } macros::from_impl!(MetadataUpdateConfig); @@ -198,10 +209,7 @@ impl From<&MetadataLogConfigDto> for MetadataLogConfig { impl From<&MetadataLogConfig> for MetadataLogConfigDto { fn from(instance: &MetadataLogConfig) -> Self { - Self { - queue_interval: instance.queue_interval.clone(), - progress_interval: instance.progress_interval.clone(), - } + Self { queue_interval: instance.queue_interval.clone(), progress_interval: instance.progress_interval.clone() } } } diff --git a/backend/src/model/config/mod.rs b/backend/core/src/model/config/mod.rs similarity index 100% rename from backend/src/model/config/mod.rs rename to backend/core/src/model/config/mod.rs index f5562a27d..b1d2faa7e 100644 --- a/backend/src/model/config/mod.rs +++ b/backend/core/src/model/config/mod.rs @@ -1,40 +1,40 @@ -mod trakt; +mod api; +mod api_proxy; +mod api_user; +mod app; mod base; -mod web_ui; -mod web_auth; -mod messaging; -mod metadata_update; -mod media_tools; -mod hdhomerun; -mod ip_check; -mod source; -mod target; -mod sort; -mod video_download; -mod log; -mod input; -mod stream; -mod stream_history; +mod cache; mod epg; mod epg_smart_match; -mod reverse_proxy; -mod cache; -mod rate_limit; -mod proxy; -mod schedule; -mod api_proxy; -mod rename; -mod healthcheck; -mod app; -mod api; -mod api_user; -mod trakt_api; -pub mod macros; mod favourites; mod geoip; +mod hdhomerun; +mod healthcheck; +mod input; +mod ip_check; mod library; +mod log; +pub mod macros; +mod media_tools; +mod messaging; +mod metadata_update; mod panel_api; +mod proxy; mod qos_aggregation; +mod rate_limit; +mod rename; +mod reverse_proxy; +mod schedule; +mod sort; +mod source; +mod stream; +mod stream_history; +mod target; +mod trakt; +mod trakt_api; +mod video_download; +mod web_auth; +mod web_ui; pub use api::*; pub use api_proxy::*; @@ -43,19 +43,23 @@ pub use app::*; pub use base::*; pub use epg::*; pub use epg_smart_match::*; +pub use favourites::*; +pub use geoip::*; pub use hdhomerun::*; pub use healthcheck::*; pub use input::*; pub use ip_check::*; +pub use library::*; pub use log::*; -pub use messaging::*; pub use media_tools::*; +pub use messaging::*; pub use metadata_update::*; +pub use panel_api::*; pub use proxy::*; +pub use qos_aggregation::*; pub use rate_limit::*; pub use rename::*; pub use reverse_proxy::*; -pub use geoip::*; pub use schedule::*; pub use sort::*; pub use source::*; @@ -67,7 +71,3 @@ pub use trakt_api::*; pub use video_download::*; pub use web_auth::*; pub use web_ui::*; -pub use favourites::*; -pub use library::*; -pub use panel_api::*; -pub use qos_aggregation::*; diff --git a/backend/src/model/config/panel_api.rs b/backend/core/src/model/config/panel_api.rs similarity index 83% rename from backend/src/model/config/panel_api.rs rename to backend/core/src/model/config/panel_api.rs index f529bdeba..c37e8b165 100644 --- a/backend/src/model/config/panel_api.rs +++ b/backend/core/src/model/config/panel_api.rs @@ -1,8 +1,13 @@ use crate::model::macros; -use shared::error::TuliproxError; -use shared::model::{PanelApiAliasPoolDto, PanelApiAliasPoolSizeDto, PanelApiAliasPoolSizeValue, PanelApiConfigDto, PanelApiProvisioningDto, PanelApiQueryParamDto, PanelApiQueryParametersDto}; +use shared::{ + error::TuliproxError, + model::{ + PanelApiAliasPoolDto, PanelApiAliasPoolSizeDto, PanelApiAliasPoolSizeValue, PanelApiConfigDto, + PanelApiProvisioningDto, PanelApiQueryParamDto, PanelApiQueryParametersDto, + }, + utils::Internable, +}; use std::sync::Arc; -use shared::utils::Internable; #[derive(Debug, Clone)] pub struct PanelApiQueryParam { @@ -12,21 +17,11 @@ pub struct PanelApiQueryParam { macros::from_impl!(PanelApiQueryParam); impl From<&PanelApiQueryParamDto> for PanelApiQueryParam { - fn from(dto: &PanelApiQueryParamDto) -> Self { - Self { - key: dto.key.clone(), - value: dto.value.clone(), - } - } + fn from(dto: &PanelApiQueryParamDto) -> Self { Self { key: dto.key.clone(), value: dto.value.clone() } } } impl From<&PanelApiQueryParam> for PanelApiQueryParamDto { - fn from(instance: &PanelApiQueryParam) -> Self { - Self { - key: instance.key.clone(), - value: instance.value.clone(), - } - } + fn from(instance: &PanelApiQueryParam) -> Self { Self { key: instance.key.clone(), value: instance.value.clone() } } } #[derive(Debug, Clone)] @@ -63,14 +58,9 @@ impl From<&PanelApiQueryParameters> for PanelApiQueryParametersDto { } } - impl PanelApiQueryParameters { - fn validate_type_is_m3u(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - let typ = params - .iter() - .find(|p| p.key.trim().eq_ignore_ascii_case("type")) - .map(|p| p.value.trim().to_string()); + let typ = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("type")).map(|p| p.value.trim().to_string()); match typ { Some(v) if v.eq_ignore_ascii_case("m3u") => Ok(()), Some(v) => Err(TuliproxError::Config(format!("panel_api: unsupported type={v}, only m3u is supported"))), @@ -81,19 +71,28 @@ impl PanelApiQueryParameters { fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); let Some(api_key) = api_key else { - return Err(TuliproxError::Config(format!("panel_api: {section} must contain query param 'api_key' (use value 'auto')"))); + return Err(TuliproxError::Config(format!( + "panel_api: {section} must contain query param 'api_key' (use value 'auto')" + ))); }; if api_key.value.trim().is_empty() { - return Err(TuliproxError::Config(format!("panel_api: {section} query param 'api_key' must not be empty (use value 'auto')"))); + return Err(TuliproxError::Config(format!( + "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" + ))); } Ok(()) } - fn require_username_password_params_auto(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { + fn require_username_password_params_auto( + params: &[PanelApiQueryParam], + section: &str, + ) -> Result<(), TuliproxError> { let username = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("username")); let password = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("password")); if username.is_none() || password.is_none() { - return Err(TuliproxError::Config(format!("panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')"))); + return Err(TuliproxError::Config(format!( + "panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')" + ))); } if !username.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) || !password.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) @@ -175,12 +174,7 @@ pub struct PanelApiAliasPoolSize { macros::from_impl!(PanelApiAliasPoolSize); impl From<&PanelApiAliasPoolSizeDto> for PanelApiAliasPoolSize { - fn from(dto: &PanelApiAliasPoolSizeDto) -> Self { - Self { - min: dto.min.clone(), - max: dto.max.clone(), - } - } + fn from(dto: &PanelApiAliasPoolSizeDto) -> Self { Self { min: dto.min.clone(), max: dto.max.clone() } } } #[derive(Debug, Clone)] @@ -192,10 +186,7 @@ pub struct PanelApiAliasPool { macros::from_impl!(PanelApiAliasPool); impl From<&PanelApiAliasPoolDto> for PanelApiAliasPool { fn from(dto: &PanelApiAliasPoolDto) -> Self { - Self { - size: dto.size.as_ref().map(PanelApiAliasPoolSize::from), - remove_expired: dto.remove_expired, - } + Self { size: dto.size.as_ref().map(PanelApiAliasPoolSize::from), remove_expired: dto.remove_expired } } } @@ -233,10 +224,7 @@ impl From<&PanelApiConfig> for PanelApiConfigDto { query_parameter: PanelApiQueryParametersDto::from(&instance.query_parameter), credits: None, alias_pool: instance.alias_pool.as_ref().map(|p| PanelApiAliasPoolDto { - size: p.size.as_ref().map(|s| PanelApiAliasPoolSizeDto { - min: s.min.clone(), - max: s.max.clone(), - }), + size: p.size.as_ref().map(|s| PanelApiAliasPoolSizeDto { min: s.min.clone(), max: s.max.clone() }), remove_expired: p.remove_expired, }), } @@ -244,7 +232,6 @@ impl From<&PanelApiConfig> for PanelApiConfigDto { } impl PanelApiConfig { - pub fn prepare(&mut self) -> Result<(), TuliproxError> { if !self.enabled { return Ok(()); diff --git a/backend/core/src/model/config/proxy.rs b/backend/core/src/model/config/proxy.rs new file mode 100644 index 000000000..a01183f07 --- /dev/null +++ b/backend/core/src/model/config/proxy.rs @@ -0,0 +1,22 @@ +use crate::model::macros; +use shared::model::ProxyConfigDto; + +#[derive(Debug, Clone)] +pub struct ProxyConfig { + pub url: String, + pub username: Option, + pub password: Option, +} + +macros::from_impl!(ProxyConfig); +impl From<&ProxyConfigDto> for ProxyConfig { + fn from(dto: &ProxyConfigDto) -> Self { + Self { url: dto.url.clone(), username: dto.username.clone(), password: dto.password.clone() } + } +} + +impl From<&ProxyConfig> for ProxyConfigDto { + fn from(dto: &ProxyConfig) -> Self { + Self { url: dto.url.clone(), username: dto.username.clone(), password: dto.password.clone() } + } +} diff --git a/backend/src/model/config/qos_aggregation.rs b/backend/core/src/model/config/qos_aggregation.rs similarity index 100% rename from backend/src/model/config/qos_aggregation.rs rename to backend/core/src/model/config/qos_aggregation.rs diff --git a/backend/src/model/config/rate_limit.rs b/backend/core/src/model/config/rate_limit.rs similarity index 59% rename from backend/src/model/config/rate_limit.rs rename to backend/core/src/model/config/rate_limit.rs index 5021f3ee5..fdd743acc 100644 --- a/backend/src/model/config/rate_limit.rs +++ b/backend/core/src/model/config/rate_limit.rs @@ -1,5 +1,5 @@ -use shared::model::RateLimitConfigDto; use crate::model::macros; +use shared::model::RateLimitConfigDto; #[derive(Debug, Clone)] pub struct RateLimitConfig { @@ -11,20 +11,12 @@ pub struct RateLimitConfig { macros::from_impl!(RateLimitConfig); impl From<&RateLimitConfigDto> for RateLimitConfig { fn from(dto: &RateLimitConfigDto) -> Self { - Self { - enabled: dto.enabled, - period_millis: dto.period_millis, - burst_size: dto.burst_size, - } + Self { enabled: dto.enabled, period_millis: dto.period_millis, burst_size: dto.burst_size } } } impl From<&RateLimitConfig> for RateLimitConfigDto { fn from(instance: &RateLimitConfig) -> Self { - Self { - enabled: instance.enabled, - period_millis: instance.period_millis, - burst_size: instance.burst_size, - } + Self { enabled: instance.enabled, period_millis: instance.period_millis, burst_size: instance.burst_size } } -} \ No newline at end of file +} diff --git a/backend/src/model/config/rename.rs b/backend/core/src/model/config/rename.rs similarity index 92% rename from backend/src/model/config/rename.rs rename to backend/core/src/model/config/rename.rs index 030ae7ced..ce7c7fe6f 100644 --- a/backend/src/model/config/rename.rs +++ b/backend/core/src/model/config/rename.rs @@ -24,9 +24,9 @@ impl From<&ConfigRenameDto> for ConfigRename { }) .unwrap_or_else(|_| { // Final fallback that avoids panicking on malformed user input. - shared::model::REGEX_CACHE.get_or_compile("$^").unwrap_or_else(|err| { - unreachable!("hardcoded fallback regex '$^' must compile: {err}") - }) + shared::model::REGEX_CACHE + .get_or_compile("$^") + .unwrap_or_else(|err| unreachable!("hardcoded fallback regex '$^' must compile: {err}")) }); Self { field: dto.field, new_name: dto.new_name.clone(), raw_pattern: dto.pattern.clone(), pattern } } diff --git a/backend/src/model/config/reverse_proxy.rs b/backend/core/src/model/config/reverse_proxy.rs similarity index 84% rename from backend/src/model/config/reverse_proxy.rs rename to backend/core/src/model/config/reverse_proxy.rs index 89c14370d..00353c4d4 100644 --- a/backend/src/model/config/reverse_proxy.rs +++ b/backend/core/src/model/config/reverse_proxy.rs @@ -1,16 +1,20 @@ -use crate::model::config::cache::CacheConfig; -use crate::model::{macros, GeoIpConfig, QosAggregationConfig, RateLimitConfig, StreamConfig}; -use regex::Regex; -use shared::model::{ - HlsCacheConfigDto, HlsCorruptSegmentWatchdogConfigDto, HlsManifestRecoveryBurstConfigDto, - HlsSegmentRepairConfigDto, HlsSegmentRepairSizeIncreaseConfigDto, ResourceRetryConfigDto, ReverseProxyConfigDto, - ReverseProxyDisabledHeaderConfigDto, HlsStripMode, REGEX_CACHE, +use crate::model::{ + config::cache::CacheConfig, macros, GeoIpConfig, QosAggregationConfig, RateLimitConfig, StreamConfig, }; -use shared::defaults::{default_resource_retry_attempts, default_resource_retry_backoff_ms, default_resource_retry_backoff_multiplier, }; -use shared::utils::{hex_to_u8_16, u8_16_to_hex}; -use std::cmp::max; -use std::sync::Arc; -use shared::defaults::HLS_CACHE_DIR_SUFFIX; +use regex::Regex; +use shared::{ + defaults::{ + default_resource_retry_attempts, default_resource_retry_backoff_ms, default_resource_retry_backoff_multiplier, + HLS_CACHE_DIR_SUFFIX, + }, + model::{ + HlsCacheConfigDto, HlsCorruptSegmentWatchdogConfigDto, HlsManifestRecoveryBurstConfigDto, + HlsSegmentRepairConfigDto, HlsSegmentRepairSizeIncreaseConfigDto, HlsStripMode, ResourceRetryConfigDto, + ReverseProxyConfigDto, ReverseProxyDisabledHeaderConfigDto, REGEX_CACHE, + }, + utils::{hex_to_u8_16, u8_16_to_hex}, +}; +use std::{cmp::max, sync::Arc}; #[derive(Debug, Clone)] pub struct ReverseProxyDisabledHeaderConfig { @@ -32,9 +36,7 @@ impl ReverseProxyDisabledHeaderConfig { if self.cloudflare_header && header_lc.starts_with("cf-") { return true; } - self.custom_header - .iter() - .any(|h| h.trim().eq_ignore_ascii_case(&header_lc)) + self.custom_header.iter().any(|h| h.trim().eq_ignore_ascii_case(&header_lc)) } } @@ -69,11 +71,7 @@ impl ResourceRetryConfig { ( max(1, self.max_attempts), self.backoff_millis.max(1), - if self.backoff_multiplier.is_finite() { - self.backoff_multiplier.max(1.0) - } else { - 1.0 - }, + if self.backoff_multiplier.is_finite() { self.backoff_multiplier.max(1.0) } else { 1.0 }, ) } @@ -90,25 +88,27 @@ macros::from_impl!(ResourceRetryConfig); impl From<&ResourceRetryConfigDto> for ResourceRetryConfig { fn from(dto: &ResourceRetryConfigDto) -> Self { - let multiplier = if dto.backoff_multiplier.is_finite() { - dto.backoff_multiplier.max(1.0) - } else { - 1.0 - }; - + let multiplier = if dto.backoff_multiplier.is_finite() { dto.backoff_multiplier.max(1.0) } else { 1.0 }; + // Compile patterns, default to service-abuse if none or empty - let patterns = dto.failover_redirect_patterns - .as_ref() - .filter(|v| !v.is_empty()) - .map_or_else(default_failover_redirect_patterns, |patterns| { - patterns.iter() - .filter_map(|p| REGEX_CACHE.get_or_compile(p).map_err(|e| { - log::warn!("Failed to compile failover redirect pattern '{p}': {e}"); - e - }).ok()) + let patterns = dto.failover_redirect_patterns.as_ref().filter(|v| !v.is_empty()).map_or_else( + default_failover_redirect_patterns, + |patterns| { + patterns + .iter() + .filter_map(|p| { + REGEX_CACHE + .get_or_compile(p) + .map_err(|e| { + log::warn!("Failed to compile failover redirect pattern '{p}': {e}"); + e + }) + .ok() + }) .collect() - }); - + }, + ); + Self { max_attempts: dto.max_attempts, backoff_millis: dto.backoff_millis, @@ -120,10 +120,7 @@ impl From<&ResourceRetryConfigDto> for ResourceRetryConfig { impl From<&ResourceRetryConfig> for ResourceRetryConfigDto { fn from(cfg: &ResourceRetryConfig) -> Self { - let patterns: Vec = cfg.failover_redirect_patterns - .iter() - .map(|re| re.as_str().to_string()) - .collect(); + let patterns: Vec = cfg.failover_redirect_patterns.iter().map(|re| re.as_str().to_string()).collect(); Self { max_attempts: cfg.max_attempts, backoff_millis: cfg.backoff_millis, @@ -146,15 +143,11 @@ pub struct StripConfig { } impl From<&shared::model::HlsStripConfigDto> for StripConfig { - fn from(dto: &shared::model::HlsStripConfigDto) -> Self { - Self { mode: dto.mode, value: dto.value } - } + fn from(dto: &shared::model::HlsStripConfigDto) -> Self { Self { mode: dto.mode, value: dto.value } } } impl From<&StripConfig> for shared::model::HlsStripConfigDto { - fn from(config: &StripConfig) -> Self { - Self { mode: config.mode, value: config.value } - } + fn from(config: &StripConfig) -> Self { Self { mode: config.mode, value: config.value } } } #[derive(Debug, Clone, Eq, PartialEq)] @@ -167,15 +160,11 @@ impl Default for HlsManifestRecoveryBurstConfig { } impl From<&HlsManifestRecoveryBurstConfigDto> for HlsManifestRecoveryBurstConfig { - fn from(dto: &HlsManifestRecoveryBurstConfigDto) -> Self { - Self { level: dto.level } - } + fn from(dto: &HlsManifestRecoveryBurstConfigDto) -> Self { Self { level: dto.level } } } impl From<&HlsManifestRecoveryBurstConfig> for HlsManifestRecoveryBurstConfigDto { - fn from(config: &HlsManifestRecoveryBurstConfig) -> Self { - Self { level: config.level } - } + fn from(config: &HlsManifestRecoveryBurstConfig) -> Self { Self { level: config.level } } } #[derive(Debug, Clone, Eq, PartialEq)] @@ -191,11 +180,7 @@ impl Default for HlsSegmentRepairSizeIncreaseConfig { impl From<&HlsSegmentRepairSizeIncreaseConfigDto> for HlsSegmentRepairSizeIncreaseConfig { fn from(dto: &HlsSegmentRepairSizeIncreaseConfigDto) -> Self { - Self { - low_percent: dto.low_percent, - medium_percent: dto.medium_percent, - high_percent: dto.high_percent, - } + Self { low_percent: dto.low_percent, medium_percent: dto.medium_percent, high_percent: dto.high_percent } } } @@ -261,19 +246,13 @@ impl Default for HlsCorruptSegmentWatchdogConfig { impl From<&HlsCorruptSegmentWatchdogConfigDto> for HlsCorruptSegmentWatchdogConfig { fn from(dto: &HlsCorruptSegmentWatchdogConfigDto) -> Self { - Self { - mode: dto.mode, - max_parallel_jobs: dto.max_parallel_jobs, - } + Self { mode: dto.mode, max_parallel_jobs: dto.max_parallel_jobs } } } impl From<&HlsCorruptSegmentWatchdogConfig> for HlsCorruptSegmentWatchdogConfigDto { fn from(config: &HlsCorruptSegmentWatchdogConfig) -> Self { - Self { - mode: config.mode, - max_parallel_jobs: config.max_parallel_jobs, - } + Self { mode: config.mode, max_parallel_jobs: config.max_parallel_jobs } } } @@ -298,11 +277,9 @@ pub struct HlsCacheConfig { } fn parse_hls_byte_size_or_default(value: &shared::model::ByteSize, default_value: &str) -> u64 { - value.parse_bytes().unwrap_or_else(|_| { - shared::model::ByteSize::new(default_value) - .parse_bytes() - .unwrap_or_default() - }) + value + .parse_bytes() + .unwrap_or_else(|_| shared::model::ByteSize::new(default_value).parse_bytes().unwrap_or_default()) } pub fn default_hls_cache_path() -> String { @@ -375,10 +352,7 @@ impl From<&ReverseProxyConfigDto> for ReverseProxyConfig { Self { resource_rewrite_disabled: dto.resource_rewrite_disabled, rewrite_secret: hex_to_u8_16(&dto.rewrite_secret).unwrap_or_default(), - resource_retry: dto - .resource_retry - .as_ref() - .map_or_else(ResourceRetryConfig::default, Into::into), + resource_retry: dto.resource_retry.as_ref().map_or_else(ResourceRetryConfig::default, Into::into), disabled_header: dto.disabled_header.as_ref().map(|d| ReverseProxyDisabledHeaderConfig { referer_header: d.referer_header, x_header: d.x_header, @@ -425,7 +399,10 @@ mod tests { default_hls_cache_path, HlsCacheConfig, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, ReverseProxyConfig, }; - use shared::model::{ByteSize, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, QosAggregationConfigDto, ReverseProxyConfigDto, StreamHistoryConfigDto}; + use shared::model::{ + ByteSize, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, + QosAggregationConfigDto, ReverseProxyConfigDto, StreamHistoryConfigDto, + }; #[test] fn reverse_proxy_config_preserves_nested_stream_history() { @@ -453,15 +430,8 @@ mod tests { fn reverse_proxy_config_preserves_nested_qos_aggregation() { let dto = ReverseProxyConfigDto { rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), - stream_history: Some(StreamHistoryConfigDto { - stream_history_enabled: true, - ..Default::default() - }), - qos_aggregation: Some(QosAggregationConfigDto { - enabled: true, - interval_secs: 300, - ..Default::default() - }), + stream_history: Some(StreamHistoryConfigDto { stream_history_enabled: true, ..Default::default() }), + qos_aggregation: Some(QosAggregationConfigDto { enabled: true, interval_secs: 300, ..Default::default() }), ..Default::default() }; @@ -499,10 +469,7 @@ mod tests { hls, HlsCacheConfig { cache_path: default_hls_cache_path(), - strip: super::StripConfig { - mode: HlsStripMode::Segments, - value: 0, - }, + strip: super::StripConfig { mode: HlsStripMode::Segments, value: 0 }, cache_duration: 300, cache_bytes: 10_737_418_240, // 10 * 1024^3 (was 10 * 1e9 under SI decimal before consolidation) cache_bytes_str: "10GB".to_string(), diff --git a/backend/src/model/config/schedule.rs b/backend/core/src/model/config/schedule.rs similarity index 61% rename from backend/src/model/config/schedule.rs rename to backend/core/src/model/config/schedule.rs index c309f1604..d69d2f9e7 100644 --- a/backend/src/model/config/schedule.rs +++ b/backend/core/src/model/config/schedule.rs @@ -1,5 +1,5 @@ -use shared::model::{ScheduleConfigDto, ScheduleTaskType}; use crate::model::macros; +use shared::model::{ScheduleConfigDto, ScheduleTaskType}; #[derive(Debug, Clone)] pub struct ScheduleConfig { @@ -11,19 +11,11 @@ pub struct ScheduleConfig { macros::from_impl!(ScheduleConfig); impl From<&ScheduleConfigDto> for ScheduleConfig { fn from(dto: &ScheduleConfigDto) -> Self { - Self { - schedule: dto.schedule.clone(), - task_type: dto.task_type, - targets: dto.targets.clone(), - } + Self { schedule: dto.schedule.clone(), task_type: dto.task_type, targets: dto.targets.clone() } } } impl From<&ScheduleConfig> for ScheduleConfigDto { fn from(dto: &ScheduleConfig) -> Self { - Self { - schedule: dto.schedule.clone(), - task_type: dto.task_type, - targets: dto.targets.clone(), - } + Self { schedule: dto.schedule.clone(), task_type: dto.task_type, targets: dto.targets.clone() } } } diff --git a/backend/src/model/config/sort.rs b/backend/core/src/model/config/sort.rs similarity index 79% rename from backend/src/model/config/sort.rs rename to backend/core/src/model/config/sort.rs index 13a2d2fb1..e26a26a99 100644 --- a/backend/src/model/config/sort.rs +++ b/backend/core/src/model/config/sort.rs @@ -1,10 +1,10 @@ -use std::sync::Arc; -use regex::Regex; -use shared::model::{ConfigSortRuleDto, ConfigSortDto, ItemField, SortOrder, SortTarget}; -use shared::foundation::Filter; use crate::model::macros; - - +use regex::Regex; +use shared::{ + foundation::Filter, + model::{ConfigSortDto, ConfigSortRuleDto, ItemField, SortOrder, SortTarget}, +}; +use std::sync::Arc; #[derive(Debug, Clone)] pub struct ConfigSortRule { @@ -54,18 +54,12 @@ pub struct ConfigSort { macros::from_impl!(ConfigSort); impl From<&ConfigSortDto> for ConfigSort { fn from(dto: &ConfigSortDto) -> Self { - Self { - match_as_ascii: dto.match_as_ascii, - rules: dto.rules.iter().map(Into::into).collect(), - } + Self { match_as_ascii: dto.match_as_ascii, rules: dto.rules.iter().map(Into::into).collect() } } } impl From<&ConfigSort> for ConfigSortDto { fn from(instance: &ConfigSort) -> Self { - Self { - match_as_ascii: instance.match_as_ascii, - rules: instance.rules.iter().map(Into::into).collect(), - } + Self { match_as_ascii: instance.match_as_ascii, rules: instance.rules.iter().map(Into::into).collect() } } } diff --git a/backend/src/model/config/source.rs b/backend/core/src/model/config/source.rs similarity index 88% rename from backend/src/model/config/source.rs rename to backend/core/src/model/config/source.rs index 44f7c9f40..6228c45f0 100644 --- a/backend/src/model/config/source.rs +++ b/backend/core/src/model/config/source.rs @@ -1,19 +1,24 @@ use crate::model::{macros, ConfigInput, ConfigTarget, ProcessTargets}; use indexmap::IndexMap; use parking_lot::RwLock; -use shared::error::TuliproxError; -use shared::model::provider_saturation::build_group_lookup; -use shared::model::{ - ConfigProviderDto, ConfigSourceDto, DnsPrefer, DnsScheme, OnConnectErrorPolicy, OnResolveErrorPolicy, PatternTemplate, - ProviderUrlSelectionPolicy, SourcesConfigDto, +use shared::{ + error::TuliproxError, + model::{ + provider_saturation::build_group_lookup, ConfigProviderDto, ConfigSourceDto, DnsPrefer, DnsScheme, + OnConnectErrorPolicy, OnResolveErrorPolicy, PatternTemplate, ProviderUrlSelectionPolicy, SourcesConfigDto, + }, +}; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + net::IpAddr, + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::SystemTime, }; -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; -use std::path::PathBuf; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::SystemTime; use url::Url; #[derive(Debug, Clone, PartialEq, Eq)] @@ -94,9 +99,7 @@ impl ProviderDnsCache { } let len = ips.len(); let guard = self.by_host.read(); - let idx = guard - .get(&host.to_ascii_lowercase()) - .map_or(0, |entry| entry.rr_index.load(Ordering::Relaxed) % len); + let idx = guard.get(&host.to_ascii_lowercase()).map_or(0, |entry| entry.rr_index.load(Ordering::Relaxed) % len); Some(ips[idx]) } @@ -108,7 +111,11 @@ impl ProviderDnsCache { let mut guard = self.by_host.write(); let entry = guard.entry(host.to_ascii_lowercase()).or_default(); // `% len` guards against a stale index when the override list changed length. - let idx = entry.rr_index.try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)).unwrap_or_else(|i| i) % len; + let idx = entry + .rr_index + .try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)) + .unwrap_or_else(|i| i) + % len; Some(ips[idx]) } @@ -119,7 +126,11 @@ impl ProviderDnsCache { return None; } let len = entry.ips.len(); - let idx = entry.rr_index.try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)).unwrap_or_else(|i| i) % len; + let idx = entry + .rr_index + .try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)) + .unwrap_or_else(|i| i) + % len; Some(entry.ips[idx]) } @@ -173,9 +184,7 @@ impl ProviderDnsCache { pub fn ip_count(&self, host: &str) -> usize { let guard = self.by_host.read(); - guard - .get(&host.to_ascii_lowercase()) - .map_or(0, |entry| entry.ips.len()) + guard.get(&host.to_ascii_lowercase()).map_or(0, |entry| entry.ips.len()) } } @@ -202,7 +211,6 @@ impl Clone for ConfigProvider { } } - macros::from_impl!(ConfigProvider); impl From<&ConfigProviderDto> for ConfigProvider { fn from(dto: &ConfigProviderDto) -> Self { @@ -231,9 +239,7 @@ impl ConfigProvider { } /// Resets the current URL index to 0 - pub fn reset_index(&self) { - self.current_url_index.store(0, Ordering::Relaxed); - } + pub fn reset_index(&self) { self.current_url_index.store(0, Ordering::Relaxed); } /// Gets the current URL index #[inline] @@ -296,11 +302,7 @@ impl ConfigProvider { let mut seen = HashSet::new(); for raw in &self.urls { let raw = raw.as_ref(); - let candidate = if raw.contains("://") { - raw.to_string() - } else { - format!("http://{raw}") - }; + let candidate = if raw.contains("://") { raw.to_string() } else { format!("http://{raw}") }; let Ok(parsed) = Url::parse(&candidate) else { continue; }; @@ -322,7 +324,9 @@ impl ConfigProvider { pub fn clear_resolved(&self, host: &str) { self.dns_cache.clear_resolved(host); } - pub fn mark_resolve_error(&self, host: &str, err: impl Into) { self.dns_cache.mark_resolve_error(host, err); } + pub fn mark_resolve_error(&self, host: &str, err: impl Into) { + self.dns_cache.mark_resolve_error(host, err); + } pub fn snapshot_resolved(&self) -> HashMap> { self.dns_cache.snapshot_resolved() } @@ -358,7 +362,8 @@ impl ConfigProvider { return None; } - let previous = self.current_url_index + let previous = self + .current_url_index .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { let next = (current + 1) % len; // If we've cycled back to start, we've tried all URLs. @@ -421,7 +426,9 @@ impl TryFrom<&SourcesConfigDto> for SourcesConfig { let mut inputs = Vec::::new(); let mut batch_files = Vec::::new(); let mut input_names = HashSet::new(); - let provider: Vec<_> = dto.provider.as_ref() + let provider: Vec<_> = dto + .provider + .as_ref() .map(|list| list.iter().map(ConfigProvider::from).map(Arc::new).collect()) .unwrap_or_default(); @@ -460,29 +467,16 @@ impl TryFrom<&SourcesConfigDto> for SourcesConfig { sources.push(ConfigSource::from(source_dto)); } - Ok(Self { - batch_files, - templates: dto.templates.clone(), - provider, - inputs, - sources, - group_lookup, - }) + Ok(Self { batch_files, templates: dto.templates.clone(), provider, inputs, sources, group_lookup }) } } impl SourcesConfig { - pub(crate) fn get_source_at(&self, idx: usize) -> Option<&ConfigSource> { - self.sources.get(idx) - } + pub fn get_source_at(&self, idx: usize) -> Option<&ConfigSource> { self.sources.get(idx) } - pub(crate) fn get_staged_input_for_provider(&self, provider_name: &Arc) -> Option<&Arc> { + pub fn get_staged_input_for_provider(&self, provider_name: &Arc) -> Option<&Arc> { self.inputs.iter().find(|input| { - input - .staged - .as_ref() - .and_then(|staged| staged.for_input.as_ref()) - .is_some_and(|name| name == provider_name) + input.staged.as_ref().and_then(|staged| staged.for_input.as_ref()).is_some_and(|name| name == provider_name) }) } @@ -537,7 +531,8 @@ impl SourcesConfig { } } - let missing_targets: Vec = check_targets.iter().filter(|&(_, v)| *v == 0).map(|(k, _)| k.clone()).collect(); + let missing_targets: Vec = + check_targets.iter().filter(|&(_, v)| *v == 0).map(|(k, _)| k.clone()).collect(); if !missing_targets.is_empty() { return Err(TuliproxError::ConfigSource(format!("No target found for {}", missing_targets.join(", ")))); } @@ -547,12 +542,7 @@ impl SourcesConfig { enabled = false; } - Ok(ProcessTargets { - enabled, - inputs, - targets, - target_names, - }) + Ok(ProcessTargets { enabled, inputs, targets, target_names }) } pub fn get_unique_target_names(&self) -> HashSet> { @@ -623,14 +613,10 @@ mod tests { dns: None, }); - provider.store_resolved( - "cdn-c.example.net", - vec!["203.0.113.30".parse::().expect("ip parse should work")], - ); - provider.store_resolved( - "cdn-a.example.net", - vec!["203.0.113.10".parse::().expect("ip parse should work")], - ); + provider + .store_resolved("cdn-c.example.net", vec!["203.0.113.30".parse::().expect("ip parse should work")]); + provider + .store_resolved("cdn-a.example.net", vec!["203.0.113.10".parse::().expect("ip parse should work")]); provider.store_resolved( "extra-b.example.net", vec!["203.0.113.50".parse::().expect("ip parse should work")], @@ -662,9 +648,6 @@ mod tests { dns: None, }); - assert_eq!( - provider.provider_url_selection_policy(), - ProviderUrlSelectionPolicy::RestartFromFirst - ); + assert_eq!(provider.provider_url_selection_policy(), ProviderUrlSelectionPolicy::RestartFromFirst); } } diff --git a/backend/src/model/config/stream.rs b/backend/core/src/model/config/stream.rs similarity index 88% rename from backend/src/model/config/stream.rs rename to backend/core/src/model/config/stream.rs index ba20d5b4a..2b93baf92 100644 --- a/backend/src/model/config/stream.rs +++ b/backend/core/src/model/config/stream.rs @@ -1,8 +1,9 @@ -use shared::model::{AdmissionStrategy, StreamBufferConfigDto, StreamConfigDto}; -use shared::utils::parse_to_kbps; -use crate::api::model::TransportStreamBuffer; use crate::model::macros; - +use shared::{ + model::{AdmissionStrategy, StreamBufferConfigDto, StreamConfigDto}, + utils::parse_to_kbps, +}; +use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; #[derive(Debug, Clone)] pub struct StreamBufferConfig { @@ -14,21 +15,13 @@ pub struct StreamBufferConfig { macros::from_impl!(StreamBufferConfig); impl From<&StreamBufferConfigDto> for StreamBufferConfig { fn from(dto: &StreamBufferConfigDto) -> Self { - Self { - enabled: dto.enabled, - size: dto.size, - max_bytes_mb: dto.max_bytes_mb, - } + Self { enabled: dto.enabled, size: dto.size, max_bytes_mb: dto.max_bytes_mb } } } impl From<&StreamBufferConfig> for StreamBufferConfigDto { fn from(dto: &StreamBufferConfig) -> Self { - Self { - enabled: dto.enabled, - size: dto.size, - max_bytes_mb: dto.max_bytes_mb, - } + Self { enabled: dto.enabled, size: dto.size, max_bytes_mb: dto.max_bytes_mb } } } @@ -112,7 +105,7 @@ impl From<&StreamConfig> for StreamConfigDto { #[cfg(test)] mod tests { - use super::{StreamConfig}; + use super::StreamConfig; use shared::model::{AdmissionStrategy, StreamConfigDto}; #[test] @@ -123,10 +116,8 @@ mod tests { #[test] fn stream_config_preserves_explicit_empty_admission_strategies() { - let domain = StreamConfig::from(&StreamConfigDto { - admission_strategies: Some(vec![]), - ..StreamConfigDto::default() - }); + let domain = + StreamConfig::from(&StreamConfigDto { admission_strategies: Some(vec![]), ..StreamConfigDto::default() }); assert_eq!(domain.admission_strategies, Some(vec![])); } @@ -169,7 +160,7 @@ pub struct CustomStreamResponse { pub channel_unavailable: Option, pub user_connections_exhausted: Option, // user has no more connections pub provider_connections_exhausted: Option, // provider limit reached, has no more connections - pub low_priority_preempted: Option, // stream was preempted by a higher-priority user + pub low_priority_preempted: Option, // stream was preempted by a higher-priority user pub user_account_expired: Option, pub panel_api_provisioning: Option, pub hls_session_or_lease_expired: Option, diff --git a/backend/src/model/config/stream_history.rs b/backend/core/src/model/config/stream_history.rs similarity index 91% rename from backend/src/model/config/stream_history.rs rename to backend/core/src/model/config/stream_history.rs index 858cd7b8f..4f412de87 100644 --- a/backend/src/model/config/stream_history.rs +++ b/backend/core/src/model/config/stream_history.rs @@ -1,8 +1,8 @@ use crate::model::macros; -use shared::model::{ - StreamHistoryConfigDto, +use shared::{ + defaults::{default_stream_history_batch_size, default_stream_history_retention_days}, + model::StreamHistoryConfigDto, }; -use shared::defaults::{default_stream_history_batch_size, default_stream_history_retention_days}; #[derive(Debug, Clone)] pub struct StreamHistoryConfig { diff --git a/backend/src/model/config/target.rs b/backend/core/src/model/config/target.rs similarity index 90% rename from backend/src/model/config/target.rs rename to backend/core/src/model/config/target.rs index cb7c24902..ddefcb8aa 100644 --- a/backend/src/model/config/target.rs +++ b/backend/core/src/model/config/target.rs @@ -1,16 +1,19 @@ -use crate::model::config::favourites::ConfigFavourites; -use crate::model::config::trakt::TraktConfig; -use crate::model::mapping::Mapping; -use crate::model::{macros, ConfigRename, ConfigSort}; -use arc_swap::ArcSwapOption; -use shared::foundation::Filter; -use shared::foundation::ValueProvider; -use shared::model::PlaylistItemType; -use shared::model::{ - ConfigTargetDto, ConfigTargetOptions, HdHomeRunTargetOutputDto, M3uTargetOutputDto, ProcessingOrder, - StrmExportStyle, StrmTargetOutputDto, TargetOutputDto, TargetType, TraktConfigDto, XtreamTargetOutputDto, +use crate::model::{ + config::{favourites::ConfigFavourites, trakt::TraktConfig}, + macros, + mapping::Mapping, + ConfigRename, ConfigSort, +}; +use arc_swap::ArcSwapOption; +use shared::{ + apply_flags, create_bitset, + foundation::{Filter, ValueProvider}, + model::{ + ConfigTargetDto, ConfigTargetOptions, HdHomeRunTargetOutputDto, M3uTargetOutputDto, PlaylistItemType, + ProcessingOrder, StrmExportStyle, StrmTargetOutputDto, TargetOutputDto, TargetType, TraktConfigDto, + XtreamTargetOutputDto, + }, }; -use shared::{apply_flags, create_bitset}; use std::sync::Arc; create_bitset!(u8, XtreamTargetFlags, SkipLiveDirectSource, SkipVideoDirectSource, SkipSeriesDirectSource); @@ -29,9 +32,7 @@ impl ProcessTargets { !self.enabled || self.targets.is_empty() || self.targets.contains(&tid) } - pub fn has_input(&self, tid: u16) -> bool { - !self.enabled || self.inputs.is_empty() || self.inputs.contains(&tid) - } + pub fn has_input(&self, tid: u16) -> bool { !self.enabled || self.inputs.is_empty() || self.inputs.contains(&tid) } } #[derive(Debug, Clone)] @@ -255,34 +256,30 @@ pub struct ConfigTarget { } impl ConfigTarget { - pub fn filter(&self, provider: &ValueProvider) -> bool { - self.filter.filter(provider) - } + pub fn filter(&self, provider: &ValueProvider) -> bool { self.filter.filter(provider) } - pub(crate) fn get_xtream_output(&self) -> Option<&XtreamTargetOutput> { + pub fn get_xtream_output(&self) -> Option<&XtreamTargetOutput> { self.output.iter().find_map(|o| match o { TargetOutput::Xtream(output) => Some(output), _ => None, }) } - pub(crate) fn get_m3u_output(&self) -> Option<&M3uTargetOutput> { + pub fn get_m3u_output(&self) -> Option<&M3uTargetOutput> { self.output.iter().find_map(|o| match o { TargetOutput::M3u(output) => Some(output), _ => None, }) } - pub(crate) fn get_hdhomerun_output(&self) -> Option<&HdHomeRunTargetOutput> { + pub fn get_hdhomerun_output(&self) -> Option<&HdHomeRunTargetOutput> { self.output.iter().find_map(|o| match o { TargetOutput::HdHomeRun(output) => Some(output), _ => None, }) } - pub fn has_output(&self, tt: TargetType) -> bool { - self.output.iter().any(|o| TargetType::from(o) == tt) - } + pub fn has_output(&self, tt: TargetType) -> bool { self.output.iter().any(|o| TargetType::from(o) == tt) } pub fn is_force_redirect(&self, item_type: PlaylistItemType) -> bool { if item_type.is_local() { diff --git a/backend/src/model/config/trakt.rs b/backend/core/src/model/config/trakt.rs similarity index 96% rename from backend/src/model/config/trakt.rs rename to backend/core/src/model/config/trakt.rs index 5a455255b..7d24bdc68 100644 --- a/backend/src/model/config/trakt.rs +++ b/backend/core/src/model/config/trakt.rs @@ -1,9 +1,8 @@ +use crate::model::{config::trakt_api::TraktMatchItem, macros}; use shared::model::{ PlaylistItem, TraktApiConfigDto, TraktChartConfigDto, TraktChartKind, TraktChartType, TraktConfigDto, TraktContentType, TraktListConfigDto, }; -use crate::model::config::trakt_api::TraktMatchItem; -use crate::model::macros; #[derive(Debug, Clone)] pub struct TraktApiConfig { @@ -146,7 +145,7 @@ pub struct TraktConfig { } macros::from_impl!(TraktConfig); -impl From<&TraktConfigDto> for TraktConfig { +impl From<&TraktConfigDto> for TraktConfig { fn from(dto: &TraktConfigDto) -> Self { Self { enabled: dto.enabled, @@ -156,7 +155,7 @@ impl From<&TraktConfigDto> for TraktConfig { } } } -impl From<&TraktConfig> for TraktConfigDto { +impl From<&TraktConfig> for TraktConfigDto { fn from(dto: &TraktConfig) -> Self { Self { enabled: dto.enabled, diff --git a/backend/src/model/config/trakt_api.rs b/backend/core/src/model/config/trakt_api.rs similarity index 68% rename from backend/src/model/config/trakt_api.rs rename to backend/core/src/model/config/trakt_api.rs index f048825df..6d30fabf8 100644 --- a/backend/src/model/config/trakt_api.rs +++ b/backend/core/src/model/config/trakt_api.rs @@ -1,7 +1,6 @@ -use shared::utils::is_blank_optional_string; -use serde::{Deserialize, Serialize}; -use shared::model::TraktContentType; use crate::utils::normalize_title_for_matching; +use serde::{Deserialize, Serialize}; +use shared::{model::TraktContentType, utils::is_blank_optional_string}; // Trakt API Response structures #[derive(Debug, Clone, Serialize, Deserialize)] @@ -51,8 +50,20 @@ impl TraktListItem { pub fn prepare(&mut self) { self.content_type = match self.item_type.as_str() { - "movie" => if self.movie.is_some() { TraktContentType::Vod } else { TraktContentType::Both }, - "show" => if self.show.is_some() { TraktContentType::Series } else { TraktContentType::Both }, + "movie" => { + if self.movie.is_some() { + TraktContentType::Vod + } else { + TraktContentType::Both + } + } + "show" => { + if self.show.is_some() { + TraktContentType::Series + } else { + TraktContentType::Both + } + } _ => TraktContentType::Both, } } @@ -97,29 +108,25 @@ pub struct TraktMatchItem<'a> { impl<'a> TraktMatchItem<'a> { pub fn from_trakt_list_item(item: &'a TraktListItem) -> Option { match item.item_type.as_str() { - "movie" => { - item.movie.as_ref().map(|movie| TraktMatchItem { - title: movie.title.as_str(), - normalized_title: normalize_title_for_matching(movie.title.as_str()), - year: movie.year, - tmdb_id: movie.ids.tmdb, - trakt_id: movie.ids.trakt, - content_type: TraktContentType::Vod, - rank: item.rank, - }) - } - "show" => { - item.show.as_ref().map(|show| TraktMatchItem { - title: show.title.as_str(), - normalized_title: normalize_title_for_matching(show.title.as_str()), - year: show.year, - tmdb_id: show.ids.tmdb, - trakt_id: show.ids.trakt, - content_type: TraktContentType::Series, - rank: item.rank, - }) - } + "movie" => item.movie.as_ref().map(|movie| TraktMatchItem { + title: movie.title.as_str(), + normalized_title: normalize_title_for_matching(movie.title.as_str()), + year: movie.year, + tmdb_id: movie.ids.tmdb, + trakt_id: movie.ids.trakt, + content_type: TraktContentType::Vod, + rank: item.rank, + }), + "show" => item.show.as_ref().map(|show| TraktMatchItem { + title: show.title.as_str(), + normalized_title: normalize_title_for_matching(show.title.as_str()), + year: show.year, + tmdb_id: show.ids.tmdb, + trakt_id: show.ids.trakt, + content_type: TraktContentType::Series, + rank: item.rank, + }), _ => None, } } -} \ No newline at end of file +} diff --git a/backend/src/model/config/video_download.rs b/backend/core/src/model/config/video_download.rs similarity index 89% rename from backend/src/model/config/video_download.rs rename to backend/core/src/model/config/video_download.rs index 7c9b55418..0b5db29ea 100644 --- a/backend/src/model/config/video_download.rs +++ b/backend/core/src/model/config/video_download.rs @@ -1,17 +1,16 @@ use crate::model::macros; use chrono_tz::Tz; use regex::Regex; -use shared::defaults::DEFAULT_DOWNLOAD_DIR; -use shared::model::{ - default_recording_notification_backoff_initial_secs, - default_recording_notification_backoff_max_secs, - default_recording_notification_max_attempts, default_recording_notification_outbox_buffer, - RecordingConfigDto, RecordingContainerFormat, RecordingDiskConfigDto, - RecordingNotificationConfigDto, RecordingQuotaConfigDto, RecordingRetentionConfigDto, - VideoConfigDto, VideoDownloadConfigDto, +use shared::{ + defaults::DEFAULT_DOWNLOAD_DIR, + model::{ + default_recording_notification_backoff_initial_secs, default_recording_notification_backoff_max_secs, + default_recording_notification_max_attempts, default_recording_notification_outbox_buffer, RecordingConfigDto, + RecordingContainerFormat, RecordingDiskConfigDto, RecordingNotificationConfigDto, RecordingQuotaConfigDto, + RecordingRetentionConfigDto, VideoConfigDto, VideoDownloadConfigDto, + }, }; -use std::collections::HashMap; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; #[derive(Debug, Clone)] pub struct VideoDownloadConfig { @@ -38,9 +37,12 @@ impl From<&VideoDownloadConfigDto> for VideoDownloadConfig { headers: dto.headers.clone(), directory: dto.directory.as_ref().map_or_else(|| DEFAULT_DOWNLOAD_DIR.to_string(), ToString::to_string), organize_into_directories: dto.organize_into_directories, - episode_pattern: dto.episode_pattern.as_ref().and_then(|s| shared::model::REGEX_CACHE.get_or_compile(s) - .map_err(|e| log::warn!("Invalid episode_pattern regex '{s}': {e}")) - .ok()), + episode_pattern: dto.episode_pattern.as_ref().and_then(|s| { + shared::model::REGEX_CACHE + .get_or_compile(s) + .map_err(|e| log::warn!("Invalid episode_pattern regex '{s}': {e}")) + .ok() + }), download_priority: dto.download_priority, recording_priority: dto.recording_priority, reserve_slots_for_users: dto.reserve_slots_for_users, @@ -103,9 +105,7 @@ pub struct RecordingRetentionConfig { } impl Default for RecordingRetentionConfig { - fn default() -> Self { - Self::from(&RecordingRetentionConfigDto::default()) - } + fn default() -> Self { Self::from(&RecordingRetentionConfigDto::default()) } } /// Runtime notification-delivery knobs. Always present: an absent @@ -120,9 +120,7 @@ pub struct RecordingNotificationConfig { } impl Default for RecordingNotificationConfig { - fn default() -> Self { - Self::from(&RecordingNotificationConfigDto::default()) - } + fn default() -> Self { Self::from(&RecordingNotificationConfigDto::default()) } } impl RecordingNotificationConfig { @@ -198,11 +196,7 @@ impl From<&RecordingConfigDto> for RecordingConfig { retention: dto.retention.as_ref().map(Into::into), disk: dto.disk.as_ref().map(Into::into), quota: dto.quota.as_ref().map(Into::into), - notifications: dto - .notifications - .as_ref() - .map(Into::into) - .unwrap_or_default(), + notifications: dto.notifications.as_ref().map(Into::into).unwrap_or_default(), fallback_bytes_per_minute: dto.fallback_bytes_per_minute, } } @@ -216,14 +210,26 @@ impl From<&RecordingConfig> for RecordingConfigDto { directory: Some(instance.directory.clone()), timezone: Some(instance.timezone.name().to_string()), filename_template: Some(instance.filename_template.clone()), - default_pre_roll_secs: if instance.default_pre_roll_secs == 0 { None } else { Some(instance.default_pre_roll_secs) }, + default_pre_roll_secs: if instance.default_pre_roll_secs == 0 { + None + } else { + Some(instance.default_pre_roll_secs) + }, max_pre_roll_secs: instance.max_pre_roll_secs, - default_post_roll_secs: if instance.default_post_roll_secs == 0 { None } else { Some(instance.default_post_roll_secs) }, + default_post_roll_secs: if instance.default_post_roll_secs == 0 { + None + } else { + Some(instance.default_post_roll_secs) + }, max_post_roll_secs: instance.max_post_roll_secs, retention: instance.retention.as_ref().map(Into::into), disk: instance.disk.as_ref().map(Into::into), quota: instance.quota.as_ref().map(Into::into), - notifications: if instance.notifications.is_empty() { None } else { Some((&instance.notifications).into()) }, + notifications: if instance.notifications.is_empty() { + None + } else { + Some((&instance.notifications).into()) + }, fallback_bytes_per_minute: instance.fallback_bytes_per_minute, } } diff --git a/backend/src/model/config/web_auth.rs b/backend/core/src/model/config/web_auth.rs similarity index 91% rename from backend/src/model/config/web_auth.rs rename to backend/core/src/model/config/web_auth.rs index 552ea755b..c547bba32 100644 --- a/backend/src/model/config/web_auth.rs +++ b/backend/core/src/model/config/web_auth.rs @@ -1,16 +1,19 @@ -use std::fmt; -use std::fs::File; -use std::io::{BufRead, ErrorKind}; -use std::path::{Path, PathBuf}; - +use crate::{model::macros, utils}; use log::{debug, error, trace, warn}; -use shared::defaults::{is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path}; -use shared::error::TuliproxError; -use shared::model::permission::{permission_from_name, PermissionSet, PERM_ALL}; -use shared::model::WebAuthConfigDto; - -use crate::model::macros; -use crate::utils; +use shared::{ + defaults::{is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path}, + error::TuliproxError, + model::{ + permission::{permission_from_name, PermissionSet, PERM_ALL}, + WebAuthConfigDto, + }, +}; +use std::{ + fmt, + fs::File, + io::{BufRead, ErrorKind}, + path::{Path, PathBuf}, +}; #[derive(Clone)] pub struct WebUiUser { @@ -143,7 +146,10 @@ impl WebAuthConfig { pub fn get_user_password(&self, username: &str) -> Option<&str> { self.t_users.as_ref().and_then(|users| { - users.iter().find(|credential| credential.username.eq_ignore_ascii_case(username)).map(|credential| credential.password_hash.as_str()) + users + .iter() + .find(|credential| credential.username.eq_ignore_ascii_case(username)) + .map(|credential| credential.password_hash.as_str()) }) } @@ -257,7 +263,10 @@ impl WebAuthConfig { let name = name.trim(); if name.eq_ignore_ascii_case("admin") { - warn!("groups file {}: 'admin' is reserved and cannot be redefined, skipping", groupfile_path.display()); + warn!( + "groups file {}: 'admin' is reserved and cannot be redefined, skipping", + groupfile_path.display() + ); continue; } @@ -265,12 +274,19 @@ impl WebAuthConfig { for perm_name in perms_str.split(',').map(str::trim).filter(|perm| !perm.is_empty()) { match permission_from_name(perm_name) { Some(permission) => permissions.set(permission), - None => warn!("groups file {}: unknown permission '{perm_name}' in group '{name}', ignoring", groupfile_path.display()), + None => warn!( + "groups file {}: unknown permission '{perm_name}' in group '{name}', ignoring", + groupfile_path.display() + ), } } - if let Some(existing_group) = groups.iter_mut().find(|group| group.name.eq_ignore_ascii_case(name)) { - warn!("groups file {}: duplicate group '{name}' redefined, replacing prior definition", groupfile_path.display()); + if let Some(existing_group) = groups.iter_mut().find(|group| group.name.eq_ignore_ascii_case(name)) + { + warn!( + "groups file {}: duplicate group '{name}' redefined, replacing prior definition", + groupfile_path.display() + ); existing_group.permissions = permissions; } else { groups.push(RbacGroup { name: name.to_string(), permissions }); @@ -283,7 +299,10 @@ impl WebAuthConfig { vec![] } Err(err) => { - error!("Could not read groups file {}: {err}, using only built-in admin permissions", groupfile_path.display()); + error!( + "Could not read groups file {}: {err}, using only built-in admin permissions", + groupfile_path.display() + ); vec![] } } @@ -300,9 +319,10 @@ impl WebAuthConfig { continue; } - let group_exists = self.t_groups.as_ref().is_some_and(|groups| { - groups.iter().any(|group| group.name.eq_ignore_ascii_case(group_name)) - }); + let group_exists = self + .t_groups + .as_ref() + .is_some_and(|groups| groups.iter().any(|group| group.name.eq_ignore_ascii_case(group_name))); if !group_exists { warn!("user '{}' references unknown group '{}', it will have no effect", user.username, group_name); @@ -332,9 +352,7 @@ mod tests { } impl Drop for DirGuard { - fn drop(&mut self) { - let _ = std::env::set_current_dir(&self.previous_dir); - } + fn drop(&mut self) { let _ = std::env::set_current_dir(&self.previous_dir); } } // --- parse_user_line tests --- @@ -507,10 +525,7 @@ mod tests { password_hash: "hash".to_string(), groups: vec!["viewer".to_string()], }]), - t_groups: Some(vec![RbacGroup { - name: "viewer".to_string(), - permissions: viewer_perms, - }]), + t_groups: Some(vec![RbacGroup { name: "viewer".to_string(), permissions: viewer_perms }]), }; let perms = config.resolve_permissions("alice"); assert!(perms.contains(Permission::ConfigRead)); @@ -539,14 +554,8 @@ mod tests { groups: vec!["viewer".to_string(), "editor".to_string()], }]), t_groups: Some(vec![ - RbacGroup { - name: "viewer".to_string(), - permissions: viewer_perms, - }, - RbacGroup { - name: "editor".to_string(), - permissions: editor_perms, - }, + RbacGroup { name: "viewer".to_string(), permissions: viewer_perms }, + RbacGroup { name: "editor".to_string(), permissions: editor_perms }, ]), }; let perms = config.resolve_permissions("bob"); @@ -571,10 +580,7 @@ mod tests { password_hash: "hash".to_string(), groups: vec!["admin".to_string(), "viewer".to_string()], }]), - t_groups: Some(vec![RbacGroup { - name: "viewer".to_string(), - permissions: viewer_perms, - }]), + t_groups: Some(vec![RbacGroup { name: "viewer".to_string(), permissions: viewer_perms }]), }; let perms = config.resolve_permissions("superuser"); assert_eq!(perms, PERM_ALL); diff --git a/backend/src/model/config/web_ui.rs b/backend/core/src/model/config/web_ui.rs similarity index 73% rename from backend/src/model/config/web_ui.rs rename to backend/core/src/model/config/web_ui.rs index 06d75cc66..aa1d8464f 100644 --- a/backend/src/model/config/web_ui.rs +++ b/backend/core/src/model/config/web_ui.rs @@ -1,8 +1,12 @@ use crate::model::{macros, WebAuthConfig}; -use shared::error::TuliproxError; -use shared::model::view_type::ViewType; -use shared::model::{ContentSecurityPolicyConfigDto, StreamInfoConfigDto, StreamInfoFields, StreamInfoFieldsSet, WebUiConfigDto}; -use shared::defaults::default_kick_secs; +use shared::{ + defaults::default_kick_secs, + error::TuliproxError, + model::{ + view_type::ViewType, ContentSecurityPolicyConfigDto, StreamInfoConfigDto, StreamInfoFields, + StreamInfoFieldsSet, WebUiConfigDto, + }, +}; #[derive(Debug, Clone)] pub struct StreamInfoConfig { @@ -10,55 +14,37 @@ pub struct StreamInfoConfig { } impl StreamInfoConfig { - pub fn is_none(&self) -> bool { - self.flags.is_empty() - } + pub fn is_none(&self) -> bool { self.flags.is_empty() } #[inline] - pub fn hide_group(&self) -> bool { - self.flags.contains(StreamInfoFields::HideGroup) - } + pub fn hide_group(&self) -> bool { self.flags.contains(StreamInfoFields::HideGroup) } #[inline] - pub fn hide_ip(&self) -> bool { - self.flags.contains(StreamInfoFields::HideIp) - } + pub fn hide_ip(&self) -> bool { self.flags.contains(StreamInfoFields::HideIp) } #[inline] - pub fn hide_country(&self) -> bool { - self.flags.contains(StreamInfoFields::HideCountry) - } + pub fn hide_country(&self) -> bool { self.flags.contains(StreamInfoFields::HideCountry) } #[inline] - pub fn hide_shared(&self) -> bool { - self.flags.contains(StreamInfoFields::HideShared) - } + pub fn hide_shared(&self) -> bool { self.flags.contains(StreamInfoFields::HideShared) } #[inline] - pub fn hide_duration(&self) -> bool { - self.flags.contains(StreamInfoFields::HideDuration) - } + pub fn hide_duration(&self) -> bool { self.flags.contains(StreamInfoFields::HideDuration) } #[inline] - pub fn hide_bandwidth(&self) -> bool { - self.flags.contains(StreamInfoFields::HideBandwidth) - } + pub fn hide_bandwidth(&self) -> bool { self.flags.contains(StreamInfoFields::HideBandwidth) } #[inline] pub fn hide_transferred(&self) -> bool { self.flags.contains(StreamInfoFields::HideTransferred) } #[inline] - pub fn hide_player(&self) -> bool { - self.flags.contains(StreamInfoFields::HidePlayer) - } + pub fn hide_player(&self) -> bool { self.flags.contains(StreamInfoFields::HidePlayer) } #[inline] pub fn hide_user_comment(&self) -> bool { self.flags.contains(StreamInfoFields::HideUserComment) } #[inline] - pub fn hide_epg(&self) -> bool { - self.flags.contains(StreamInfoFields::HideEpg) - } + pub fn hide_epg(&self) -> bool { self.flags.contains(StreamInfoFields::HideEpg) } } #[derive(Debug, Clone)] @@ -101,19 +87,13 @@ macros::from_impl!(ContentSecurityPolicyConfig); impl From<&ContentSecurityPolicyConfigDto> for ContentSecurityPolicyConfig { fn from(dto: &ContentSecurityPolicyConfigDto) -> Self { - Self { - enabled: dto.enabled, - custom_attributes: dto.custom_attributes.clone(), - } + Self { enabled: dto.enabled, custom_attributes: dto.custom_attributes.clone() } } } impl From<&ContentSecurityPolicyConfig> for ContentSecurityPolicyConfigDto { fn from(e: &ContentSecurityPolicyConfig) -> Self { - Self { - enabled: e.enabled, - custom_attributes: e.custom_attributes.clone(), - } + Self { enabled: e.enabled, custom_attributes: e.custom_attributes.clone() } } } @@ -170,13 +150,8 @@ impl From<&WebUiConfigDto> for WebUiConfig { } impl From<&WebUiConfig> for WebUiConfigDto { fn from(instance: &WebUiConfig) -> Self { - let stream_info = instance.stream_info.as_ref().and_then(|cfg| { - if cfg.is_none() { - None - } else { - Some(cfg.into()) - } - }); + let stream_info = + instance.stream_info.as_ref().and_then(|cfg| if cfg.is_none() { None } else { Some(cfg.into()) }); Self { enabled: instance.enabled, user_ui_enabled: instance.user_ui_enabled, diff --git a/backend/core/src/model/custom_stream_flags.rs b/backend/core/src/model/custom_stream_flags.rs new file mode 100644 index 000000000..5886a7732 --- /dev/null +++ b/backend/core/src/model/custom_stream_flags.rs @@ -0,0 +1,9 @@ +//! Config predicates for the custom-video substitution feature. +//! +//! Read by the streaming layer, the HLS proxy and the provisioning path, so +//! they live with the configuration rather than with any one caller. + +use crate::model::AppConfig; + +/// `true` when the server may answer a request with a canned video clip. +pub fn is_custom_video_stream_enabled(cfg: &AppConfig) -> bool { cfg.config.load().custom_stream_response_enabled } diff --git a/backend/src/auth/fingerprint.rs b/backend/core/src/model/fingerprint.rs similarity index 74% rename from backend/src/auth/fingerprint.rs rename to backend/core/src/model/fingerprint.rs index 5eb965b55..2cba8b2cf 100644 --- a/backend/src/auth/fingerprint.rs +++ b/backend/core/src/model/fingerprint.rs @@ -1,20 +1,25 @@ +type Rejection = (StatusCode, &'static str); +use axum::{ + extract::{ConnectInfo, FromRequestParts}, + http::{request::Parts, StatusCode}, +}; use std::net::SocketAddr; -use axum::extract::{ConnectInfo, FromRequestParts}; -use axum::http::request::Parts; -use axum::http::StatusCode; -use crate::auth::Rejection; const MAX_HEADER_LENGTH: usize = 512; fn validate_header(value: &str) -> Option { // TODO i think this is unnecessary because axum validates the headers ? if value.len() <= MAX_HEADER_LENGTH && !value.contains('\0') { - Some(value.to_string()) + Some(value.to_string()) } else { None } } +/// The client identity a request is attributed to. +/// +/// Derived from the request's headers and peer address. Both the auth layer and +/// the connection layer key on it, so it lives here rather than with either. #[derive(Debug, PartialEq, Eq, Clone)] pub struct Fingerprint { pub key: String, @@ -23,13 +28,7 @@ pub struct Fingerprint { } impl Fingerprint { - pub fn new(key: String, client_ip: String, addr: SocketAddr) -> Self { - Self { - key, - client_ip, - addr, - } - } + pub fn new(key: String, client_ip: String, addr: SocketAddr) -> Self { Self { key, client_ip, addr } } } impl FromRequestParts for Fingerprint @@ -44,7 +43,6 @@ where } impl Fingerprint { - async fn decode_request_parts(req: &mut Parts, state: &B) -> Result where B: Send + Sync, @@ -57,15 +55,15 @@ impl Fingerprint { let mut forwarded_for = None; let mut real_ip = None; for header in &req.headers { - if header.0.as_str().eq_ignore_ascii_case(axum::http::header::USER_AGENT.as_str()) { + if header.0.as_str().eq_ignore_ascii_case(axum::http::header::USER_AGENT.as_str()) { if let Ok(val) = header.1.to_str() { user_agent = validate_header(val); } - } else if header.0.as_str().eq_ignore_ascii_case("x-forwarded-for") { + } else if header.0.as_str().eq_ignore_ascii_case("x-forwarded-for") { if let Ok(val) = header.1.to_str() { forwarded_for = validate_header(val); } - } else if header.0.as_str().eq_ignore_ascii_case("x-real-ip") { + } else if header.0.as_str().eq_ignore_ascii_case("x-real-ip") { if let Ok(val) = header.1.to_str() { real_ip = validate_header(val); } @@ -80,8 +78,8 @@ impl Fingerprint { let ua = user_agent.unwrap_or_else(String::new); let key = format!("{client_ip}|{ua}"); - // debug!("{key}, {client_ip}, {addr}"); + // debug!("{key}, {client_ip}, {addr}"); Ok(Fingerprint::new(key, client_ip, addr)) } -} \ No newline at end of file +} diff --git a/backend/src/model/healthcheck.rs b/backend/core/src/model/healthcheck.rs similarity index 100% rename from backend/src/model/healthcheck.rs rename to backend/core/src/model/healthcheck.rs index d8c1e9f56..46b5534c3 100644 --- a/backend/src/model/healthcheck.rs +++ b/backend/core/src/model/healthcheck.rs @@ -1,5 +1,5 @@ -use shared::utils::is_blank_optional_string; use serde::{Deserialize, Serialize}; +use shared::utils::is_blank_optional_string; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Healthcheck { diff --git a/backend/src/model/input_source.rs b/backend/core/src/model/input_source.rs similarity index 94% rename from backend/src/model/input_source.rs rename to backend/core/src/model/input_source.rs index 4fcf30656..1ac66728c 100644 --- a/backend/src/model/input_source.rs +++ b/backend/core/src/model/input_source.rs @@ -1,10 +1,9 @@ -use std::collections::HashMap; -use std::sync::Arc; -use shared::model::InputFetchMethod; use crate::model::{ConfigInput, ConfigProvider}; +use shared::model::InputFetchMethod; +use std::{collections::HashMap, sync::Arc}; /// Represents an input source for fetching content. -/// +/// /// When created from a `ConfigInput` that uses the `provider://` scheme, /// the provider context is preserved to enable URL failover on errors. #[derive(Clone, Debug)] @@ -22,7 +21,7 @@ pub struct InputSource { impl InputSource { /// Creates a new `InputSource` with a different URL while preserving provider context. - /// + /// /// The provider is preserved so that failover can occur even when the URL /// is derived from the original (e.g., adding query parameters, changing paths). pub fn with_url(&self, url: String) -> Self { @@ -38,12 +37,10 @@ impl InputSource { } /// Returns the provider for this input source, if available. - /// + /// /// This can be used to pass to `send_with_retry_and_provider` for failover support. #[inline] - pub fn get_provider(&self) -> Option<&Arc> { - self.provider.as_ref() - } + pub fn get_provider(&self) -> Option<&Arc> { self.provider.as_ref() } } impl From<&ConfigInput> for InputSource { @@ -58,4 +55,4 @@ impl From<&ConfigInput> for InputSource { headers: input.headers.clone(), } } -} \ No newline at end of file +} diff --git a/backend/src/model/mapping.rs b/backend/core/src/model/mapping.rs similarity index 78% rename from backend/src/model/mapping.rs rename to backend/core/src/model/mapping.rs index 40ac1860a..f751e9216 100644 --- a/backend/src/model/mapping.rs +++ b/backend/core/src/model/mapping.rs @@ -1,10 +1,11 @@ -use shared::foundation::Filter; -use shared::foundation::MapperScript; -use shared::model::{ - MapperDto, MappingCounter, MappingCounterDefinition, MappingDefinitionDto, MappingDto, MappingStage, - MappingsDto, PatternTemplate, -}; use crate::model::macros; +use shared::{ + foundation::{Filter, MapperScript}, + model::{ + MapperDto, MappingCounter, MappingCounterDefinition, MappingDefinitionDto, MappingDto, MappingStage, + MappingsDto, PatternTemplate, + }, +}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] pub struct Mapper { @@ -17,7 +18,7 @@ pub struct Mapper { } macros::from_impl!(Mapper); -impl From<&MapperDto> for Mapper { +impl From<&MapperDto> for Mapper { fn from(dto: &MapperDto) -> Self { Self { filter: dto.filter.clone(), @@ -40,10 +41,10 @@ pub struct Mapping { #[serde(skip_serializing, skip_deserializing)] pub t_counter: Option>, #[serde(skip_serializing, skip_deserializing)] - pub(crate) templates: Option> + pub templates: Option>, } -impl From<&MappingDto> for Mapping { +impl From<&MappingDto> for Mapping { fn from(dto: &MappingDto) -> Self { Self { id: dto.id.clone(), @@ -66,10 +67,7 @@ pub struct MappingDefinition { macros::from_impl!(MappingDefinition); impl From<&MappingDefinitionDto> for MappingDefinition { fn from(dto: &MappingDefinitionDto) -> Self { - Self { - templates: dto.templates.clone(), - mapping: dto.mapping.iter().map(Into::into).collect(), - } + Self { templates: dto.templates.clone(), mapping: dto.mapping.iter().map(Into::into).collect() } } } @@ -79,7 +77,6 @@ pub struct Mappings { } impl Mappings { - pub fn get_mapping(&self, mapping_id: &str) -> Option { for mapping in &self.mappings.mapping { if mapping.id.eq(mapping_id) { @@ -92,9 +89,5 @@ impl Mappings { macros::from_impl!(Mappings); impl From<&MappingsDto> for Mappings { - fn from(dto: &MappingsDto) -> Self { - Mappings { - mappings: MappingDefinition::from(&dto.mappings), - } - } + fn from(dto: &MappingsDto) -> Self { Mappings { mappings: MappingDefinition::from(&dto.mappings) } } } diff --git a/backend/src/model/messaging.rs b/backend/core/src/model/messaging.rs similarity index 82% rename from backend/src/model/messaging.rs rename to backend/core/src/model/messaging.rs index 6f6d98ebc..ed38ba002 100644 --- a/backend/src/model/messaging.rs +++ b/backend/core/src/model/messaging.rs @@ -18,13 +18,9 @@ pub struct ProcessingStats { } impl ProcessingStats { - pub fn new_stats(stats: Vec) -> Self { - Self { stats: Some(stats), errors: None } - } + pub fn new_stats(stats: Vec) -> Self { Self { stats: Some(stats), errors: None } } - pub fn new_error(error: String) -> Self { - Self { stats: None, errors: Some(error) } - } + pub fn new_error(error: String) -> Self { Self { stats: None, errors: Some(error) } } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -49,22 +45,18 @@ pub struct RecordingLifecycleMessage { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind", content = "data")] pub enum MessageContent { - Info(String), - Error(String), - Watch(WatchChanges), - ProcessingStats(ProcessingStats), - DiskAlert(DiskAlert), - RecordingLifecycle(RecordingLifecycleMessage), + Info(String), + Error(String), + Watch(WatchChanges), + ProcessingStats(ProcessingStats), + DiskAlert(DiskAlert), + RecordingLifecycle(RecordingLifecycleMessage), } impl MessageContent { - pub fn event_stats(stats: Vec) -> Self { - Self::ProcessingStats(ProcessingStats::new_stats(stats)) - } + pub fn event_stats(stats: Vec) -> Self { Self::ProcessingStats(ProcessingStats::new_stats(stats)) } - pub fn event_error(error: String) -> Self { - Self::ProcessingStats(ProcessingStats::new_error(error)) - } + pub fn event_error(error: String) -> Self { Self::ProcessingStats(ProcessingStats::new_error(error)) } pub fn kind(&self) -> MsgKind { match self { diff --git a/backend/core/src/model/mod.rs b/backend/core/src/model/mod.rs new file mode 100644 index 000000000..dac3c00d0 --- /dev/null +++ b/backend/core/src/model/mod.rs @@ -0,0 +1,34 @@ +mod config; +mod healthcheck; +mod input_source; +mod mapping; +pub mod messaging; +pub mod readiness; +mod stream_history; +mod xmltv; +mod xtream; +// Streaming error type. No dependencies of its own, and named by both the +// streaming layer and the buffer it reports on, so it belongs here rather than +// in `api`. +// Playlist/library update semaphores. No dependencies of their own, and named +// by both `api` and `processing`. +pub mod batch_result_collector; +pub mod custom_stream_flags; +pub mod fingerprint; +pub mod playlist_filter; +pub mod playlist_key; +pub mod provider; +pub mod proxy_redirect; +pub mod stalker_record; +pub mod stream_error; +pub mod update_guard; +pub mod update_task; +pub mod user_api_request; +pub mod xtream_response; + +pub use self::{ + batch_result_collector::*, config::*, custom_stream_flags::*, fingerprint::*, healthcheck::*, input_source::*, + mapping::*, messaging::*, playlist_filter::*, playlist_key::*, provider::*, proxy_redirect::*, stalker_record::*, + stream_error::*, stream_history::*, update_guard::*, update_task::*, xmltv::*, xtream::*, +}; +pub use shared::model::xtream_const::*; diff --git a/backend/core/src/model/playlist_filter.rs b/backend/core/src/model/playlist_filter.rs new file mode 100644 index 000000000..3cbcdcd7c --- /dev/null +++ b/backend/core/src/model/playlist_filter.rs @@ -0,0 +1,41 @@ +//! Filtering playlist groups against a configured filter expression. +//! +//! Pure predicate work over shared playlist types. It lived in +//! `processing::processor::playlist`, which is what made `repository` reach up +//! into `processing` to filter a playlist before persisting it. + +use shared::{ + foundation::{Filter, ValueProvider}, + model::{PlaylistGroup, PlaylistItem}, +}; + +pub fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { + let provider = ValueProvider { pli, match_as_ascii }; + filter.filter(&provider) +} + +pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) -> Option> { + // NOTE: the source `playlist` is intentionally cloned (not drained) here because + // the caller reuses the same slice for every target output and for the no-filter + // fallback path, so the survivors cannot be moved out of it. Cap the initial + // allocation so selective filters do not retain capacity for every source item. + const INITIAL_FILTERED_GROUP_CAPACITY: usize = 256; + let mut new_playlist = Vec::with_capacity(playlist.len()); + for pg in playlist.iter() { + let mut channels = Vec::with_capacity(pg.channels.len().min(INITIAL_FILTERED_GROUP_CAPACITY)); + channels.extend(pg.channels.iter().filter(|&pli| is_valid(pli, filter, false)).cloned()); + if !channels.is_empty() { + new_playlist.push(PlaylistGroup { + id: pg.id, + title: pg.title.clone(), + channels, + xtream_cluster: pg.xtream_cluster, + }); + } + } + if new_playlist.is_empty() { + None + } else { + Some(new_playlist) + } +} diff --git a/backend/core/src/model/playlist_key.rs b/backend/core/src/model/playlist_key.rs new file mode 100644 index 000000000..af140acb4 --- /dev/null +++ b/backend/core/src/model/playlist_key.rs @@ -0,0 +1,19 @@ +//! Small shared keys and URL helpers for playlist handling. +//! +//! Both are named by the parsers and by the repositories, so they sit below +//! both rather than in either. + +use shared::model::XtreamCluster; +use std::sync::Arc; + +/// Identifies a category within one cluster of a target's playlist. +pub type CategoryKey = (XtreamCluster, Arc); + +/// Base URL of an Xtream provider's `player_api.php`, with credentials applied. +pub fn get_xtream_stream_url_base(url: &str, username: &str, password: &str) -> String { + let query = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("username", username) + .append_pair("password", password) + .finish(); + format!("{url}/player_api.php?{query}") +} diff --git a/backend/src/api/model/provider_config.rs b/backend/core/src/model/provider.rs similarity index 77% rename from backend/src/api/model/provider_config.rs rename to backend/core/src/model/provider.rs index 0c3d0b39e..95d2051bd 100644 --- a/backend/src/api/model/provider_config.rs +++ b/backend/core/src/model/provider.rs @@ -1,13 +1,101 @@ +//! Provider configuration, allocation and the handle that represents a claim. +//! +//! These are the value types of provider brokerage: what a provider is, whether +//! one is available, and the handle a caller holds while using it. The manager +//! that allocates them stays in `api`; these do not, because playlist processing +//! passes them around and would otherwise have to name the API layer. + use crate::{ - api::model::ProviderAllocation, model::{is_input_expired, ConfigInput, ConfigInputAlias, InputUserInfo}, utils::debug_if_enabled, }; use jsonwebtoken::get_current_timestamp; use log::debug; use shared::{model::InputType, utils::sanitize_sensitive_info, write_if_some}; -use std::{fmt, ops::Deref, sync::Arc}; +use std::{fmt, net::SocketAddr, ops::Deref, sync::Arc}; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; + +/// Address of the client holding a provider connection. +pub type ClientConnectionId = SocketAddr; +/// Identifier of one allocation within a provider's lineup. +pub type AllocationId = u64; + +#[derive(Debug, Clone)] +pub enum ProviderAllocation { + Exhausted, + Available(Arc), + GracePeriod(Arc), +} + +impl ProviderAllocation { + pub fn short_key(&self) -> &str { + match self { + ProviderAllocation::Exhausted => "exhausted", + ProviderAllocation::Available(_) => "available", + ProviderAllocation::GracePeriod(_) => "grace_period", + } + } + + pub fn new_available(config: Arc) -> Self { ProviderAllocation::Available(config) } + + pub fn new_grace_period(config: Arc) -> Self { ProviderAllocation::GracePeriod(config) } + + pub fn get_provider_name(&self) -> Option> { + match self { + ProviderAllocation::Exhausted => None, + ProviderAllocation::Available(ref cfg) | ProviderAllocation::GracePeriod(ref cfg) => Some(cfg.name.clone()), + } + } + + pub fn get_provider_id(&self) -> Option { + match self { + ProviderAllocation::Exhausted => None, + ProviderAllocation::Available(ref cfg) | ProviderAllocation::GracePeriod(ref cfg) => Some(cfg.id), + } + } + + pub fn get_provider_config(&self) -> Option> { + match self { + ProviderAllocation::Exhausted => None, + ProviderAllocation::Available(ref cfg) | ProviderAllocation::GracePeriod(ref cfg) => Some(Arc::clone(cfg)), + } + } + + pub async fn release(&self) { + match &self { + ProviderAllocation::Exhausted => {} + ProviderAllocation::Available(config) | ProviderAllocation::GracePeriod(config) => { + config.release().await; + } + } + } + + #[inline] + pub fn is_unlimited_provider(&self) -> bool { + matches!(self, Self::Available(c) | Self::GracePeriod(c) if c.is_unlimited()) + } +} + +#[derive(Debug, Clone)] +pub struct ProviderHandle { + pub client_id: ClientConnectionId, + pub allocation_id: AllocationId, + pub allocation: ProviderAllocation, + // Token to cancel the background task (e.g. internal probe) if preempted + pub cancel_token: Option, +} + +impl ProviderHandle { + pub fn new( + client_id: ClientConnectionId, + allocation_id: AllocationId, + allocation: ProviderAllocation, + cancel_token: Option, + ) -> Self { + Self { client_id, allocation_id, allocation, cancel_token } + } +} pub type ProviderConnectionChangeCallback = Arc, usize) + Send + Sync>; @@ -20,9 +108,9 @@ pub enum ProviderConfigAllocation { #[derive(Debug, Default, Copy, Clone)] pub struct ProviderConfigConnection { - pub(crate) current_connections: usize, - pub(crate) granted_grace: bool, - pub(crate) grace_ts: u64, + pub current_connections: usize, + pub granted_grace: bool, + pub grace_ts: u64, } /// This struct represents an individual provider configuration with fields like: @@ -165,7 +253,7 @@ impl ProviderConfig { pub fn is_unlimited(&self) -> bool { self.max_connections == 0 } #[inline] - pub(crate) fn exp_date(&self) -> Option { self.exp_date } + pub fn exp_date(&self) -> Option { self.exp_date } pub fn get_user_info(&self) -> Option { InputUserInfo::new(self.input_type, self.username.as_deref(), self.password.as_deref(), &self.url) @@ -313,14 +401,14 @@ impl ProviderConfig { } #[inline] - pub(crate) async fn get_current_connections(&self) -> usize { self.connection.read().await.current_connections } + pub async fn get_current_connections(&self) -> usize { self.connection.read().await.current_connections } #[inline] - pub(crate) fn get_priority(&self) -> i16 { self.priority } + pub fn get_priority(&self) -> i16 { self.priority } } #[derive(Clone, Debug)] -pub(in crate::api::model) struct ProviderConfigWrapper { +pub struct ProviderConfigWrapper { inner: Arc, } @@ -331,7 +419,7 @@ impl fmt::Display for ProviderConfigWrapper { impl ProviderConfigWrapper { pub fn new(cfg: ProviderConfig) -> Self { Self { inner: Arc::new(cfg) } } - pub(in crate::api::model) fn config(&self) -> Arc { Arc::clone(&self.inner) } + pub fn config(&self) -> Arc { Arc::clone(&self.inner) } pub async fn try_allocate(&self, grace: bool, grace_period_timeout_secs: u64) -> ProviderAllocation { match self.inner.try_allocate(grace, grace_period_timeout_secs).await { @@ -354,12 +442,26 @@ impl Deref for ProviderConfigWrapper { fn deref(&self) -> &Self::Target { &self.inner } } +impl PartialEq for ProviderAllocation { + fn eq(&self, other: &Self) -> bool { + // Note: released flag ignored + match (self, other) { + (ProviderAllocation::Exhausted, ProviderAllocation::Exhausted) => true, + (ProviderAllocation::Available(cfg1), ProviderAllocation::Available(cfg2)) + | (ProviderAllocation::GracePeriod(cfg1), ProviderAllocation::GracePeriod(cfg2)) => cfg1 == cfg2, + _ => false, + } + } +} + #[cfg(test)] mod tests { use super::*; use shared::model::{InputFetchMethod, InputType, StagedInputType}; - use std::collections::HashMap; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::{ + collections::HashMap, + sync::atomic::{AtomicUsize, Ordering}, + }; fn build_test_config(max_connections: u16) -> ProviderConfig { let input = ConfigInput { @@ -408,14 +510,8 @@ mod tests { assert!(provider.is_unlimited()); // Drive connection count up so `release` executes its normal path. - assert!(matches!( - provider.try_allocate(false, 0).await, - ProviderConfigAllocation::Available - )); - assert!(matches!( - provider.try_allocate(false, 0).await, - ProviderConfigAllocation::Available - )); + assert!(matches!(provider.try_allocate(false, 0).await, ProviderConfigAllocation::Available)); + assert!(matches!(provider.try_allocate(false, 0).await, ProviderConfigAllocation::Available)); assert_eq!(provider.get_current_connections().await, 2); { diff --git a/backend/core/src/model/proxy_redirect.rs b/backend/core/src/model/proxy_redirect.rs new file mode 100644 index 000000000..96ce4587e --- /dev/null +++ b/backend/core/src/model/proxy_redirect.rs @@ -0,0 +1,73 @@ +//! Deciding whether outbound requests must follow redirects manually. +//! +//! A proxy that speaks HTTP(S) makes automatic redirect handling unreliable, so +//! the affected call sites drive redirects themselves. The predicate reads the +//! configured proxy plus the standard proxy environment variables and nothing +//! else, which is why it lives here rather than on the server's root state. + +use crate::model::AppConfig; +use std::ffi::OsStr; +use url::Url; + +/// `true` when requests through this configuration must follow redirects +/// manually. +pub fn should_use_manual_redirects(app_config: &AppConfig) -> bool { + let config = app_config.config.load(); + config.proxy.as_ref().is_some_and(|proxy| should_use_manual_redirect_for_proxy(proxy.url.as_str())) + || proxy_env_present() +} + +pub fn proxy_env_present() -> bool { should_use_manual_redirects_for_env_vars(std::env::vars_os()) } + +pub fn parse_proxy_url_with_http_fallback(proxy_url: &str) -> Option { + let trimmed = proxy_url.trim(); + if trimmed.is_empty() { + return None; + } + + if let Ok(url) = Url::parse(trimmed) { + if matches!(url.scheme().to_ascii_lowercase().as_str(), "http" | "https") { + return Some(url); + } + if trimmed.contains("://") { + return None; + } + } + + if trimmed.contains("://") { + return None; + } + if trimmed.starts_with('/') || trimmed.starts_with('\\') { + return None; + } + + Url::parse(format!("http://{trimmed}").as_str()).ok() +} + +pub fn should_use_manual_redirect_for_proxy(proxy_url: &str) -> bool { + parse_proxy_url_with_http_fallback(proxy_url).is_some_and(|url| { + matches!(url.scheme().to_ascii_lowercase().as_str(), "http" | "https") && url.host_str().is_some() + }) +} + +pub fn should_use_manual_redirects_for_env_vars(vars: I) -> bool +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ + const ENV_KEYS: [&str; 3] = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; + + vars.into_iter().any(|(key, value)| { + let Some(key) = key.as_ref().to_str() else { + return false; + }; + let Some(value) = value.as_ref().to_str() else { + return false; + }; + let value = value.trim(); + ENV_KEYS.iter().any(|candidate| candidate.eq_ignore_ascii_case(key)) + && !value.is_empty() + && should_use_manual_redirect_for_proxy(value) + }) +} diff --git a/backend/src/model/readiness.rs b/backend/core/src/model/readiness.rs similarity index 78% rename from backend/src/model/readiness.rs rename to backend/core/src/model/readiness.rs index ab0bce407..13d8a7900 100644 --- a/backend/src/model/readiness.rs +++ b/backend/core/src/model/readiness.rs @@ -1,8 +1,6 @@ -use std::collections::HashMap; -use std::sync::Arc; - use crate::model::config::{ConfigInput, ConfigInputAlias}; use shared::model::provider_saturation::{CapacityGroup, ProviderSlot}; +use std::{collections::HashMap, sync::Arc}; #[allow(clippy::implicit_hasher)] pub fn build_provider_slots( @@ -16,11 +14,7 @@ pub fn build_provider_slots( continue; } let current = connections.get(&input.name).copied().unwrap_or(0); - slots.push(ProviderSlot { - name: input.name.clone(), - max_connections: input.max_connections, - current, - }); + slots.push(ProviderSlot { name: input.name.clone(), max_connections: input.max_connections, current }); push_aliases(&mut slots, input.aliases.as_deref(), connections); } slots @@ -45,11 +39,7 @@ fn push_aliases( continue; } let current = connections.get(&alias.name).copied().unwrap_or(0); - slots.push(ProviderSlot { - name: alias.name.clone(), - max_connections: alias.max_connections, - current, - }); + slots.push(ProviderSlot { name: alias.name.clone(), max_connections: alias.max_connections, current }); } } @@ -59,12 +49,7 @@ mod tests { use shared::model::provider_saturation::{build_group_lookup, is_exhausted}; fn input(name: &str, max: u16) -> Arc { - Arc::new(ConfigInput { - name: Arc::from(name), - max_connections: max, - enabled: true, - ..ConfigInput::default() - }) + Arc::new(ConfigInput { name: Arc::from(name), max_connections: max, enabled: true, ..ConfigInput::default() }) } #[test] @@ -91,12 +76,8 @@ mod tests { #[test] fn disabled_members_offer_no_capacity() { - let mut full_input = ConfigInput { - name: Arc::from("a"), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; + let mut full_input = + ConfigInput { name: Arc::from("a"), max_connections: 1, enabled: true, ..ConfigInput::default() }; full_input.aliases = Some(vec![ConfigInputAlias { id: 2, name: Arc::from("a-off"), @@ -110,11 +91,8 @@ mod tests { stalker: None, }]); // The disabled alias and the disabled input must not add spare capacity. - let disabled_input = Arc::new(ConfigInput { - name: Arc::from("off"), - max_connections: 9, - ..ConfigInput::default() - }); + let disabled_input = + Arc::new(ConfigInput { name: Arc::from("off"), max_connections: 9, ..ConfigInput::default() }); let inputs = vec![Arc::new(full_input), disabled_input]; let mut conns = HashMap::new(); conns.insert(Arc::from("a"), 1usize); diff --git a/backend/core/src/model/stalker_record.rs b/backend/core/src/model/stalker_record.rs new file mode 100644 index 000000000..907ea2692 --- /dev/null +++ b/backend/core/src/model/stalker_record.rs @@ -0,0 +1,16 @@ +//! Persisted shape of one Stalker EPG programme. +//! +//! Written and read by the repository, produced by the Stalker client. Plain +//! data with no behaviour, so it sits below both. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StalkerProgramRecord { + pub channel_id: Option, + pub title: String, + pub start_epoch: Option, + pub stop_epoch: Option, + pub description: Option, + pub category: Option, +} diff --git a/backend/src/api/model/stream_error.rs b/backend/core/src/model/stream_error.rs similarity index 96% rename from backend/src/api/model/stream_error.rs rename to backend/core/src/model/stream_error.rs index 47a06f093..8f5b25599 100644 --- a/backend/src/api/model/stream_error.rs +++ b/backend/core/src/model/stream_error.rs @@ -125,11 +125,7 @@ mod tests { #[test] fn reqwest_stream_error_exposes_provider_failure_metadata() { - let err = StreamError::Reqwest { - message: "upstream failed".to_string(), - class: "http_5xx", - status: Some(503), - }; + let err = StreamError::Reqwest { message: "upstream failed".to_string(), class: "http_5xx", status: Some(503) }; assert_eq!(err.provider_error_class(), "http_5xx"); assert_eq!(err.provider_http_status(), Some(503)); diff --git a/backend/src/model/stream_history.rs b/backend/core/src/model/stream_history.rs similarity index 86% rename from backend/src/model/stream_history.rs rename to backend/core/src/model/stream_history.rs index 9bbf57b44..73bcf0552 100644 --- a/backend/src/model/stream_history.rs +++ b/backend/core/src/model/stream_history.rs @@ -1,8 +1,10 @@ -use crate::utils::arc_str_option_serde; -use std::sync::Arc; +use crate::utils::{arc_str_option_serde, encode_base64_hash, now_utc_secs, utc_day_from_secs}; use serde::{Deserialize, Serialize}; -use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage, PlaylistItemType, StreamHistoryEventType, StreamHistoryRecordDto, StreamInfo}; -use crate::utils::{encode_base64_hash, now_utc_secs, utc_day_from_secs}; +use shared::model::{ + ConnectFailureReason, DisconnectReason, FailureStage, PlaylistItemType, StreamHistoryEventType, + StreamHistoryRecordDto, StreamInfo, +}; +use std::sync::Arc; pub const RECORD_SCHEMA_VERSION: u8 = 1; @@ -115,11 +117,7 @@ impl StreamHistoryRecord { api_username: Some(info.username.clone()), provider_name: Some(info.provider.clone()), provider_username: None, - input_name: if info.channel.input_name.is_empty() { - None - } else { - Some(info.channel.input_name.clone()) - }, + input_name: if info.channel.input_name.is_empty() { None } else { Some(info.channel.input_name.clone()) }, virtual_id: Some(info.channel.virtual_id), item_type: Some(info.channel.item_type), title: Some(info.channel.title.to_string()), @@ -131,18 +129,52 @@ impl StreamHistoryRecord { shared_stream_id: info.channel.shared_stream_id, provider_id: Some(info.channel.provider_id), cluster: Some(info.channel.cluster.to_string()), - container: info.channel.technical.as_ref().and_then(|t| if t.container.is_empty() { None } else { Some(t.container.clone()) }), + container: info.channel.technical.as_ref().and_then(|t| { + if t.container.is_empty() { + None + } else { + Some(t.container.clone()) + } + }), stream_url_hash: if info.channel.url.is_empty() { None } else { Some(encode_base64_hash(info.channel.url.as_ref())) }, stream_identity_key: Self::build_stream_identity_key(info), - video_codec: info.channel.technical.as_ref().and_then(|t| if t.video_codec.is_empty() { None } else { Some(t.video_codec.clone()) }), - audio_codec: info.channel.technical.as_ref().and_then(|t| if t.audio_codec.is_empty() { None } else { Some(t.audio_codec.clone()) }), - audio_channels: info.channel.technical.as_ref().and_then(|t| if t.audio_channels.is_empty() { None } else { Some(t.audio_channels.clone()) }), - resolution: info.channel.technical.as_ref().and_then(|t| if t.resolution.is_empty() { None } else { Some(t.resolution.clone()) }), - fps: info.channel.technical.as_ref().and_then(|t| if t.fps.is_empty() { None } else { Some(t.fps.clone()) }), + video_codec: info.channel.technical.as_ref().and_then(|t| { + if t.video_codec.is_empty() { + None + } else { + Some(t.video_codec.clone()) + } + }), + audio_codec: info.channel.technical.as_ref().and_then(|t| { + if t.audio_codec.is_empty() { + None + } else { + Some(t.audio_codec.clone()) + } + }), + audio_channels: info.channel.technical.as_ref().and_then(|t| { + if t.audio_channels.is_empty() { + None + } else { + Some(t.audio_channels.clone()) + } + }), + resolution: info.channel.technical.as_ref().and_then(|t| { + if t.resolution.is_empty() { + None + } else { + Some(t.resolution.clone()) + } + }), + fps: info + .channel + .technical + .as_ref() + .and_then(|t| if t.fps.is_empty() { None } else { Some(t.fps.clone()) }), connect_ts_utc: None, disconnect_ts_utc: None, session_duration: None, @@ -184,7 +216,11 @@ impl StreamHistoryRecord { record } - pub fn with_provider_failure(mut self, provider_http_status: Option, provider_error_class: Option<&str>) -> Self { + pub fn with_provider_failure( + mut self, + provider_http_status: Option, + provider_error_class: Option<&str>, + ) -> Self { self.provider_http_status = provider_http_status; self.provider_error_class = provider_error_class.map(ToString::to_string); self @@ -247,4 +283,4 @@ impl From<&StreamHistoryRecord> for StreamHistoryRecordDto { target_name: record.target_name.clone(), } } -} \ No newline at end of file +} diff --git a/backend/src/api/model/update_guard.rs b/backend/core/src/model/update_guard.rs similarity index 100% rename from backend/src/api/model/update_guard.rs rename to backend/core/src/model/update_guard.rs diff --git a/backend/core/src/model/update_task.rs b/backend/core/src/model/update_task.rs new file mode 100644 index 000000000..2bf9b3a1e --- /dev/null +++ b/backend/core/src/model/update_task.rs @@ -0,0 +1,99 @@ +//! Value types describing queued metadata-update work. +//! +//! Plain data - an identifier, a reason set, a task description. They are named +//! by the manager that queues them and by the processing code that produces +//! them, so they belong below both rather than inside `api`. + +use shared::{create_bitset, model::PlaylistItemType}; +use std::sync::Arc; + +create_bitset!(u8, ResolveReason, Info, Tmdb, Date, Probe, MissingDetails); + +/// `PlaylistItemIdType` ID can be either a String (M3U) or u32 (Xtream/TargetDB) +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ProviderIdType { + Text(Arc), + Id(u32), +} + +impl std::fmt::Display for ProviderIdType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProviderIdType::Text(s) => write!(f, "{s}"), + ProviderIdType::Id(id) => write!(f, "{id}"), + } + } +} + +impl From for ProviderIdType { + fn from(id: u32) -> Self { ProviderIdType::Id(id) } +} + +impl From<&str> for ProviderIdType { + fn from(s: &str) -> Self { ProviderIdType::Text(Arc::from(s)) } +} + +impl From for ProviderIdType { + fn from(s: String) -> Self { ProviderIdType::Text(Arc::from(s.as_str())) } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum UpdateTask { + ResolveVod { + id: ProviderIdType, + reason: ResolveReasonSet, + delay: u16, + source_last_modified: Option, + }, + ResolveSeries { + id: ProviderIdType, + reason: ResolveReasonSet, + delay: u16, + source_last_modified: Option, + }, + ProbeLive { + id: ProviderIdType, + reason: ResolveReasonSet, + delay: u16, + interval: u64, + }, + // Generic probe for M3U/Library/etc. + ProbeStream { + probe_scope: Arc, + unique_id: String, + url: String, + item_type: PlaylistItemType, + reason: ResolveReasonSet, + delay: u16, + }, +} + +impl UpdateTask { + pub fn delay(&self) -> u16 { + match self { + UpdateTask::ResolveVod { delay, .. } + | UpdateTask::ResolveSeries { delay, .. } + | UpdateTask::ProbeLive { delay, .. } + | UpdateTask::ProbeStream { delay, .. } => *delay, + } + } +} + +impl std::fmt::Display for UpdateTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + UpdateTask::ResolveVod { id, reason, delay, .. } => { + write!(f, "Resolve VOD {id} (Reason: {reason}, Delay: {delay}sec)") + } + UpdateTask::ResolveSeries { id, reason, delay, .. } => { + write!(f, "Resolve Series {id} (Reason: {reason}, Delay: {delay}sec)") + } + UpdateTask::ProbeLive { id, reason, delay, interval } => { + write!(f, "Probe Live {id} (Reason: {reason}, Delay: {delay}sec, Interval: {interval}secs )") + } + UpdateTask::ProbeStream { probe_scope, unique_id, reason, delay, .. } => { + write!(f, "Probe Stream {probe_scope}/{unique_id} (Reason: {reason}, Delay: {delay}sec)") + } + } + } +} diff --git a/backend/src/api/model/request.rs b/backend/core/src/model/user_api_request.rs similarity index 80% rename from backend/src/api/model/request.rs rename to backend/core/src/model/user_api_request.rs index 4721fda1e..6e0e1e535 100644 --- a/backend/src/api/model/request.rs +++ b/backend/core/src/model/user_api_request.rs @@ -1,6 +1,8 @@ -use axum::extract::{FromRequest, Request}; -use axum::http::StatusCode; -use axum::response::{IntoResponse, Response}; +use axum::{ + extract::{FromRequest, Request}, + http::StatusCode, + response::{IntoResponse, Response}, +}; use http_body_util::LengthLimitError; use log::log_enabled; use std::error::Error as StdError; @@ -68,14 +70,9 @@ where async fn from_request(req: Request, _state: &S) -> Result { let (parts, body) = req.into_parts(); - let query_req = parse_query_request(parts.uri.query()) - .map_err(ParseBodyError::into_response)?; + let query_req = parse_query_request(parts.uri.query()).map_err(ParseBodyError::into_response)?; - let content_type = parts - .headers - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); + let content_type = parts.headers.get("content-type").and_then(|v| v.to_str().ok()).unwrap_or(""); let body_req = match parse_body(body, content_type).await { Ok(body_req) => Some(body_req), @@ -87,15 +84,13 @@ where } async fn parse_body(body: axum::body::Body, content_type: &str) -> Result { - let bytes = axum::body::to_bytes(body, MAX_BODY_SIZE_BYTES) - .await - .map_err(|e| { - if is_length_limit_error(&e) { - ParseBodyError::PayloadTooLarge(format!("Request body too large (max {MAX_BODY_SIZE_BYTES} bytes)")) - } else { - ParseBodyError::BadRequest(format!("Failed to read request body: {e}")) - } - })?; + let bytes = axum::body::to_bytes(body, MAX_BODY_SIZE_BYTES).await.map_err(|e| { + if is_length_limit_error(&e) { + ParseBodyError::PayloadTooLarge(format!("Request body too large (max {MAX_BODY_SIZE_BYTES} bytes)")) + } else { + ParseBodyError::BadRequest(format!("Failed to read request body: {e}")) + } + })?; if bytes.is_empty() { return Ok(UserApiRequest::default()); @@ -105,7 +100,8 @@ async fn parse_body(body: axum::body::Body, content_type: &str) -> Result Option<&str> { let (name, value) = param.split_once('=')?; if name.trim().eq_ignore_ascii_case("boundary") { let trimmed = value.trim(); - let unquoted = trimmed - .strip_prefix('"') - .and_then(|inner| inner.strip_suffix('"')) - .unwrap_or(trimmed); + let unquoted = trimmed.strip_prefix('"').and_then(|inner| inner.strip_suffix('"')).unwrap_or(trimmed); if !unquoted.is_empty() { return Some(unquoted); } @@ -262,20 +255,48 @@ impl UserApiRequest { if log_enabled!(log::Level::Debug) { use std::fmt::Write; let mut msg = endpoint.to_string(); - if !self.username.is_empty() { let _ = write!(msg, " username={}", self.username); } - if !self.password.is_empty() { msg.push_str(" password=***"); } - if !self.token.is_empty() { msg.push_str(" token=***"); } - if !self.action.is_empty() { let _ = write!(msg, " action={}", self.action); } - if !self.series_id.is_empty() { let _ = write!(msg, " series_id={}", self.series_id); } - if !self.vod_id.is_empty() { let _ = write!(msg, " vod_id={}", self.vod_id); } - if !self.stream_id.is_empty() { let _ = write!(msg, " stream_id={}", self.stream_id); } - if !self.category_id.is_empty() { let _ = write!(msg, " category_id={}", self.category_id); } - if !self.limit.is_empty() { let _ = write!(msg, " limit={}", self.limit); } - if !self.start.is_empty() { let _ = write!(msg, " start={}", self.start); } - if !self.end.is_empty() { let _ = write!(msg, " end={}", self.end); } - if !self.stream.is_empty() { let _ = write!(msg, " stream={}", self.stream); } - if !self.duration.is_empty() { let _ = write!(msg, " duration={}", self.duration); } - if !self.content_type.is_empty() { let _ = write!(msg, " type={}", self.content_type); } + if !self.username.is_empty() { + let _ = write!(msg, " username={}", self.username); + } + if !self.password.is_empty() { + msg.push_str(" password=***"); + } + if !self.token.is_empty() { + msg.push_str(" token=***"); + } + if !self.action.is_empty() { + let _ = write!(msg, " action={}", self.action); + } + if !self.series_id.is_empty() { + let _ = write!(msg, " series_id={}", self.series_id); + } + if !self.vod_id.is_empty() { + let _ = write!(msg, " vod_id={}", self.vod_id); + } + if !self.stream_id.is_empty() { + let _ = write!(msg, " stream_id={}", self.stream_id); + } + if !self.category_id.is_empty() { + let _ = write!(msg, " category_id={}", self.category_id); + } + if !self.limit.is_empty() { + let _ = write!(msg, " limit={}", self.limit); + } + if !self.start.is_empty() { + let _ = write!(msg, " start={}", self.start); + } + if !self.end.is_empty() { + let _ = write!(msg, " end={}", self.end); + } + if !self.stream.is_empty() { + let _ = write!(msg, " stream={}", self.stream); + } + if !self.duration.is_empty() { + let _ = write!(msg, " duration={}", self.duration); + } + if !self.content_type.is_empty() { + let _ = write!(msg, " type={}", self.content_type); + } log::debug!("{msg}"); } } @@ -292,9 +313,11 @@ impl UserApiRequest { #[cfg(test)] mod tests { use super::{parse_body, UserApiRequest, MAX_BODY_SIZE_BYTES}; - use axum::body::Body; - use axum::extract::FromRequest; - use axum::http::{Request as HttpRequest, StatusCode}; + use axum::{ + body::Body, + extract::FromRequest, + http::{Request as HttpRequest, StatusCode}, + }; #[test] fn merge_prefer_primary_uses_fallback_for_empty_fields() { @@ -403,7 +426,7 @@ mod tests { super::ParseBodyError::PayloadTooLarge(msg) => { assert!(msg.contains("body too large"), "unexpected error: {msg}"); } - other@ super::ParseBodyError::BadRequest(_) => panic!("unexpected error: {other:?}"), + other @ super::ParseBodyError::BadRequest(_) => panic!("unexpected error: {other:?}"), } } @@ -416,7 +439,9 @@ mod tests { .body(Body::from(oversized)) .expect("request should build"); - let Err(response) = super::UserApiRequestQueryOrBody::from_request(request, &()).await else { panic!("oversized body should reject extractor") }; + let Err(response) = super::UserApiRequestQueryOrBody::from_request(request, &()).await else { + panic!("oversized body should reject extractor") + }; assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); } @@ -432,12 +457,9 @@ mod tests { "--abc123--\r\n", ); - let parsed = parse_body( - Body::from(body), - "Multipart/Form-Data; charset=utf-8; boundary=\"abc123\"", - ) - .await - .expect("multipart body should parse"); + let parsed = parse_body(Body::from(body), "Multipart/Form-Data; charset=utf-8; boundary=\"abc123\"") + .await + .expect("multipart body should parse"); assert_eq!(parsed.username, " alice "); assert_eq!(parsed.password, "\t secret \t"); @@ -452,12 +474,9 @@ mod tests { "--abc123--\r\n", ); - let parsed = parse_body( - Body::from(body), - "multipart/form-data; boundary=abc123", - ) - .await - .expect("multipart body should parse"); + let parsed = parse_body(Body::from(body), "multipart/form-data; boundary=abc123") + .await + .expect("multipart body should parse"); assert_eq!(parsed.content_type, "m3u_plus"); } @@ -471,14 +490,10 @@ mod tests { "--abc123--\r\n", ); - let parsed = parse_body( - Body::from(body), - "multipart/form-data; boundary=abc123", - ) - .await - .expect("multipart body should parse"); + let parsed = parse_body(Body::from(body), "multipart/form-data; boundary=abc123") + .await + .expect("multipart body should parse"); assert_eq!(parsed.username, "alice"); } - } diff --git a/backend/src/model/xmltv.rs b/backend/core/src/model/xmltv.rs similarity index 82% rename from backend/src/model/xmltv.rs rename to backend/core/src/model/xmltv.rs index b86a5f5ee..d0d90d4dd 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/core/src/model/xmltv.rs @@ -1,20 +1,22 @@ -use crate::api::model::AppState; -use crate::model::IcsEpgSourceConfig; -use crate::model::xmltv::XmlTagIcon::Undefined; -use crate::model::InputSource; -use crate::utils::request::get_remote_content_as_stream; -use crate::utils::{async_file_reader, parse_xmltv_time, FileLockManager}; +use crate::{ + model::{xmltv::XmlTagIcon::Undefined, AppConfig, IcsEpgSourceConfig, InputSource}, + utils::{async_file_reader, parse_xmltv_time, request::get_remote_content_as_stream}, +}; use chrono::{Datelike, TimeZone, Utc}; use futures::TryFutureExt; -use quick_xml::events::{Event}; -use shared::concat_string; -use shared::error::TuliproxError; -use shared::model::{EpgCategory, EpgChannel, EpgProgramme, InputFetchMethod}; -use shared::utils::{sanitize_sensitive_info, Internable}; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::io::{AsyncRead}; +use quick_xml::events::Event; +use shared::{ + concat_string, + error::TuliproxError, + model::{EpgCategory, EpgChannel, EpgProgramme, InputFetchMethod}, + utils::{sanitize_sensitive_info, Internable}, +}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::io::AsyncRead; use url::Url; pub const EPG_TAG_TV: &str = "tv"; @@ -42,7 +44,6 @@ pub const EPG_ATTRIB_LANG: &str = "lang"; // https://github.com/XMLTV/xmltv/blob/master/xmltv.dtd - #[derive(Debug, Clone, Eq, PartialEq, Default)] pub enum XmlTagIcon { #[default] @@ -62,15 +63,8 @@ pub struct XmlTag { } impl XmlTag { - pub(crate) fn new(name: Arc, attribs: Option, Arc>>) -> Self { - Self { - name, - value: None, - attributes: attribs, - children: None, - icon: Undefined, - normalized_epg_ids: None, - } + pub fn new(name: Arc, attribs: Option, Arc>>) -> Self { + Self { name, value: None, attributes: attribs, children: None, icon: Undefined, normalized_epg_ids: None } } pub fn get_attribute_value(&self, attr_name: &Arc) -> Option<&Arc> { @@ -78,7 +72,6 @@ impl XmlTag { } } - #[derive(Debug, Clone)] pub struct Epg { pub priority: i16, @@ -106,37 +99,7 @@ pub struct PersistedEpgSource { pub kind: PersistedEpgSourceKind, } -#[derive(Debug, Clone)] -pub struct TVGuide { - epg_sources: Vec, - file_locks: Option>, -} - -impl TVGuide { - pub fn new(mut epg_sources: Vec) -> Self { - epg_sources.sort_by_key(|a| a.priority); - Self { epg_sources, file_locks: None } - } - - /// Uses the shared cache lock manager while reading persisted EPG sources. - pub fn with_file_locks(mut self, file_locks: Arc) -> Self { - self.file_locks = Some(file_locks); - self - } - - #[inline] - pub fn get_epg_sources(&self) -> &Vec { - &self.epg_sources - } - - pub(crate) fn get_file_locks(&self) -> Option<&FileLockManager> { self.file_locks.as_deref() } -} - - -fn filter_channels_and_programmes( - channels: &mut Vec, - programmes: &mut Vec, -) { +fn filter_channels_and_programmes(channels: &mut Vec, programmes: &mut Vec) { let mut prog_map: HashMap, Vec> = HashMap::new(); for prog in programmes.drain(..) { prog_map.entry(prog.get_transient_channel_id().clone()).or_default().push(prog); @@ -157,9 +120,13 @@ pub async fn parse_xmltv_for_web_ui_from_file(path: &Path) -> Result, url: &str) -> Result, TuliproxError> { +pub async fn parse_xmltv_for_web_ui_from_url( + app_config: &Arc, + http_client: &reqwest::Client, + url: &str, +) -> Result, TuliproxError> { if let Ok(request_url) = Url::parse(url) { - let client = app_state.http_client.load(); + let client = http_client; let input_source: InputSource = InputSource { name: "xmltv".intern(), url: request_url.to_string(), @@ -170,17 +137,11 @@ pub async fn parse_xmltv_for_web_ui_from_url(app_state: &Arc, url: &st headers: HashMap::default(), }; - match get_remote_content_as_stream( - &app_state.app_config, - &client, - &input_source, - None, - &request_url, - ).await { - Ok((stream, _url)) => { - parse_xmltv_for_web_ui(stream).await + match get_remote_content_as_stream(app_config, client, &input_source, None, &request_url).await { + Ok((stream, _url)) => parse_xmltv_for_web_ui(stream).await, + Err(err) => { + Err(TuliproxError::Download(format!("Failed to download: {} {err}", sanitize_sensitive_info(url)))) } - Err(err) => Err(TuliproxError::Download(format!("Failed to download: {} {err}", sanitize_sensitive_info(url)))) } } else { Err(TuliproxError::UrlParse(format!("Invalid url: {}", sanitize_sensitive_info(url)))) @@ -231,8 +192,10 @@ async fn parse_xmltv_for_web_ui(reader: R) -> Resul // only 1 day old epg let now = Utc::now(); - let yesterday_start = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 0, 0, 0) - .single().expect("Current date at midnight should always be valid") + let yesterday_start = Utc + .with_ymd_and_hms(now.year(), now.month(), now.day(), 0, 0, 0) + .single() + .expect("Current date at midnight should always be valid") - chrono::Duration::days(1); let threshold_ts = yesterday_start.timestamp(); @@ -288,7 +251,9 @@ async fn parse_xmltv_for_web_ui(reader: R) -> Resul } } if let (Some(pstart), Some(pstop), Some(pchannel)) = (start, stop, channel) { - if let (Some(start_time), Some(stop_time)) = (parse_xmltv_time(&pstart), parse_xmltv_time(&pstop)) { + if let (Some(start_time), Some(stop_time)) = + (parse_xmltv_time(&pstart), parse_xmltv_time(&pstop)) + { if stop_time >= threshold_ts { let mut epg_programme = EpgProgramme::new(start_time, stop_time, pchannel); epg_programme.catchup_id = catchup_id; @@ -351,10 +316,9 @@ async fn parse_xmltv_for_web_ui(reader: R) -> Resul } else if current_tag == TextTag::Desc { program.desc = Some(concat_text(program.desc.as_ref(), text)); } else if current_tag == TextTag::Category { - program.categories.push(EpgCategory { - value: text.intern(), - lang: current_category_lang.take(), - }); + program + .categories + .push(EpgCategory { value: text.intern(), lang: current_category_lang.take() }); } } } diff --git a/backend/src/model/xtream.rs b/backend/core/src/model/xtream.rs similarity index 73% rename from backend/src/model/xtream.rs rename to backend/core/src/model/xtream.rs index ab3518160..fa954e366 100644 --- a/backend/src/model/xtream.rs +++ b/backend/core/src/model/xtream.rs @@ -1,20 +1,23 @@ -use std::sync::Arc; -use crate::model::{AppConfig, ProxyUserCredentials}; -use crate::model::{ConfigTarget, XtreamTargetOutput}; -use shared::error::TuliproxError; +use crate::model::{AppConfig, ConfigTarget, ProxyUserCredentials, XtreamTargetFlags, XtreamTargetOutput}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use shared::model::{ - PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, XtreamMappingFlags, XtreamMappingFlagsSet, - XtreamMappingOptions, +use shared::{ + error::TuliproxError, + model::{ + PlaylistItem, PlaylistItemType, PlaylistItemTypeSet, XtreamMappingFlags, XtreamMappingFlagsSet, + XtreamMappingOptions, + }, + utils::{arc_str_serde, concat_path_leading_slash, deserialize_number_from_string_or_zero}, }; -use shared::utils::{arc_str_serde, concat_path_leading_slash, deserialize_number_from_string_or_zero}; -use crate::model::XtreamTargetFlags; +use std::sync::Arc; use strum::IntoEnumIterator; #[derive(Deserialize, Default)] pub struct XtreamCategory { - #[serde(deserialize_with = "deserialize_number_from_string_or_zero", serialize_with = "shared::utils::serialize_number_as_string")] + #[serde( + deserialize_with = "deserialize_number_from_string_or_zero", + serialize_with = "shared::utils::serialize_number_as_string" + )] pub category_id: u32, #[serde(with = "arc_str_serde")] pub category_name: Arc, @@ -24,16 +27,16 @@ pub struct XtreamCategory { } impl XtreamCategory { - pub fn add(&mut self, item: PlaylistItem) { - self.channels.push(item); - } + pub fn add(&mut self, item: PlaylistItem) { self.channels.push(item); } } - -pub fn xtream_mapping_option_from_target_options(target: &ConfigTarget, target_output: &XtreamTargetOutput, - app_config: &AppConfig, user: &ProxyUserCredentials, - encrypt_secret: [u8; 16]) -> Result { - +pub fn xtream_mapping_option_from_target_options( + target: &ConfigTarget, + target_output: &XtreamTargetOutput, + app_config: &AppConfig, + user: &ProxyUserCredentials, + encrypt_secret: [u8; 16], +) -> Result { let force_redirect = target.options.as_ref().and_then(|o| o.force_redirect); let mut reverse_item_types = PlaylistItemTypeSet::empty(); let mut resource_proxy_item_types = PlaylistItemTypeSet::empty(); @@ -51,50 +54,32 @@ pub fn xtream_mapping_option_from_target_options(target: &ConfigTarget, target_o } let mut flags = XtreamMappingFlagsSet::new(); - if target_output - .flags - .contains(XtreamTargetFlags::SkipLiveDirectSource) - { + if target_output.flags.contains(XtreamTargetFlags::SkipLiveDirectSource) { flags.set(XtreamMappingFlags::SkipLiveDirectSource); } - if target_output - .flags - .contains(XtreamTargetFlags::SkipVideoDirectSource) - { + if target_output.flags.contains(XtreamTargetFlags::SkipVideoDirectSource) { flags.set(XtreamMappingFlags::SkipVideoDirectSource); } - if target_output - .flags - .contains(XtreamTargetFlags::SkipSeriesDirectSource) - { + if target_output.flags.contains(XtreamTargetFlags::SkipSeriesDirectSource) { flags.set(XtreamMappingFlags::SkipSeriesDirectSource); } if app_config.is_reverse_proxy_resource_rewrite_enabled() { flags.set(XtreamMappingFlags::RewriteResourceUrl); } - let base_url = if user.t_is_api_user { let config = app_config.config.load(); - let web_ui_path = config - .web_ui - .as_ref() - .and_then(|w| w.path.as_ref()) - .map_or("", String::as_str); + let web_ui_path = config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource") } else { match app_config.get_user_server_info(user) { Some(server_info) => server_info.get_base_url(), None => { - return Err(TuliproxError::ApiXtream(format!( - "No server info configured for user '{}'", - user.username - ))) + return Err(TuliproxError::ApiXtream(format!("No server info configured for user '{}'", user.username))) } } }; - Ok(XtreamMappingOptions { flags, force_redirect, @@ -104,13 +89,14 @@ pub fn xtream_mapping_option_from_target_options(target: &ConfigTarget, target_o password: user.password.clone(), base_url, web_ui_request: user.t_is_api_user, - encrypt_secret + encrypt_secret, }) } pub fn normalize_release_date(document: &mut serde_json::Map) { // Find the first non-empty release date key - let date_value = document.get("release_date") + let date_value = document + .get("release_date") .or_else(|| document.get("releaseDate")) .or_else(|| document.get("releasedate")) .filter(|v| v.as_str().is_some_and(|s| !s.is_empty())) @@ -139,16 +125,21 @@ pub struct PlaylistXtreamCategory { #[cfg(test)] mod tests { use super::xtream_mapping_option_from_target_options; - use crate::model::{AppConfig, Config, ConfigInput, ConfigTarget, MediaToolCapabilities, ProxyUserCredentials, SourcesConfig, TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput}; - use crate::utils::FileLockManager; - use crate::ConfigPaths; + use crate::{ + model::{ + AppConfig, Config, ConfigInput, ConfigTarget, MediaToolCapabilities, ProxyUserCredentials, SourcesConfig, + TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, + }, + utils::FileLockManager, + }; use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::foundation::Filter; - use shared::error::TuliproxError; - use shared::model::{InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder}; - use shared::utils::Internable; - use std::collections::HashMap; - use std::sync::Arc; + use shared::{ + error::TuliproxError, + foundation::Filter, + model::{ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder}, + utils::Internable, + }; + use std::{collections::HashMap, sync::Arc}; fn create_test_app_config() -> AppConfig { let input = Arc::new(ConfigInput { @@ -225,10 +216,7 @@ mod tests { assert!(result.is_err(), "missing server info must not degrade to an empty base_url"); let err = result.err().unwrap_or_else(|| unreachable!()); - assert!( - matches!(err, TuliproxError::ApiXtream(_)), - "missing server info should surface as ApiXtream error" - ); + assert!(matches!(err, TuliproxError::ApiXtream(_)), "missing server info should surface as ApiXtream error"); assert!( err.to_string().contains("No server info configured"), "error should explicitly mention missing server info: {err}" @@ -244,9 +232,8 @@ mod tests { user.password = "secret".to_string(); user.t_is_api_user = true; - let options = - xtream_mapping_option_from_target_options(&target, &xtream_output, &app_config, &user, [0; 16]) - .expect("mapping options"); + let options = xtream_mapping_option_from_target_options(&target, &xtream_output, &app_config, &user, [0; 16]) + .expect("mapping options"); assert!(!options.reverse_item_types.is_set(PlaylistItemType::Live)); assert!(options.resource_proxy_item_types.is_set(PlaylistItemType::Live)); diff --git a/backend/src/api/model/xtream.rs b/backend/core/src/model/xtream_response.rs similarity index 100% rename from backend/src/api/model/xtream.rs rename to backend/core/src/model/xtream_response.rs diff --git a/backend/core/src/response_macros.rs b/backend/core/src/response_macros.rs new file mode 100644 index 000000000..8b904a333 --- /dev/null +++ b/backend/core/src/response_macros.rs @@ -0,0 +1,12 @@ +//! Small response-building macros shared by every crate that builds HTTP +//! responses. + +/// Turn a `Result` into a response, mapping the error case to a +/// 500. The expansion names only `axum`, so it works in any crate. +#[macro_export] +macro_rules! try_unwrap_body { + ($body:expr) => { + $body + .map_or_else(|_| axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), |resp| resp.into_response()) + }; +} diff --git a/backend/src/utils/atomic_json_store.rs b/backend/core/src/utils/atomic_json_store.rs similarity index 89% rename from backend/src/utils/atomic_json_store.rs rename to backend/core/src/utils/atomic_json_store.rs index e6e1b08dc..1654404ab 100644 --- a/backend/src/utils/atomic_json_store.rs +++ b/backend/core/src/utils/atomic_json_store.rs @@ -30,9 +30,7 @@ pub struct AtomicWriteError { } impl AtomicWriteError { - pub fn new(stage: AtomicWriteStage, source: std::io::Error) -> Self { - Self { stage, source } - } + pub fn new(stage: AtomicWriteStage, source: std::io::Error) -> Self { Self { stage, source } } } impl std::fmt::Display for AtomicWriteError { @@ -42,15 +40,11 @@ impl std::fmt::Display for AtomicWriteError { } impl std::error::Error for AtomicWriteError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.source) - } + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { Some(&self.source) } } impl From for std::io::Error { - fn from(err: AtomicWriteError) -> Self { - std::io::Error::new(err.source.kind(), err) - } + fn from(err: AtomicWriteError) -> Self { std::io::Error::new(err.source.kind(), err) } } /// Filesystem operations needed for an atomic write. Default impls of the @@ -62,10 +56,7 @@ pub trait AtomicWriteOps: Send + Sync { tmp: &Path, content: &[u8], ) -> impl std::future::Future> + Send; - fn sync_file( - &self, - _tmp: &Path, - ) -> impl std::future::Future> + Send { + fn sync_file(&self, _tmp: &Path) -> impl std::future::Future> + Send { async { Ok(()) } } fn rename( @@ -73,10 +64,7 @@ pub trait AtomicWriteOps: Send + Sync { tmp: &Path, final_path: &Path, ) -> impl std::future::Future> + Send; - fn sync_parent( - &self, - _parent: &Path, - ) -> impl std::future::Future> + Send { + fn sync_parent(&self, _parent: &Path) -> impl std::future::Future> + Send { async { Ok(()) } } } @@ -86,15 +74,11 @@ pub struct RealAtomicWriteOps; impl AtomicWriteOps for RealAtomicWriteOps { async fn write_temp(&self, tmp: &Path, content: &[u8]) -> Result<(), AtomicWriteError> { - fs::write(tmp, content) - .await - .map_err(|e| AtomicWriteError::new(AtomicWriteStage::WriteTemp, e)) + fs::write(tmp, content).await.map_err(|e| AtomicWriteError::new(AtomicWriteStage::WriteTemp, e)) } async fn rename(&self, tmp: &Path, final_path: &Path) -> Result<(), AtomicWriteError> { - fs::rename(tmp, final_path) - .await - .map_err(|e| AtomicWriteError::new(AtomicWriteStage::Rename, e)) + fs::rename(tmp, final_path).await.map_err(|e| AtomicWriteError::new(AtomicWriteStage::Rename, e)) } } @@ -132,19 +116,13 @@ pub async fn write_json_atomic_with_ops( /// Atomic write using the production filesystem ops. Production callers should /// use this entry point. pub async fn write_json_atomic(final_path: &Path, content: &[u8]) -> std::io::Result<()> { - write_json_atomic_with_ops(final_path, content, &RealAtomicWriteOps) - .await - .map_err(std::io::Error::from) + write_json_atomic_with_ops(final_path, content, &RealAtomicWriteOps).await.map_err(std::io::Error::from) } /// Atomic write with a caller-supplied temp file path. Use this when /// concurrent writers need disjoint temp names — the caller is /// responsible for producing a unique path per call. -pub async fn write_json_atomic_to_tmp( - final_path: &Path, - tmp: &Path, - content: &[u8], -) -> Result<(), AtomicWriteError> { +pub async fn write_json_atomic_to_tmp(final_path: &Path, tmp: &Path, content: &[u8]) -> Result<(), AtomicWriteError> { write_json_atomic_with_ops_to_tmp(final_path, tmp, content, &RealAtomicWriteOps).await } @@ -168,8 +146,10 @@ pub async fn write_json_atomic_with_ops_to_tmp( #[cfg(test)] mod tests { use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; use tempfile::TempDir; #[derive(Default)] @@ -207,15 +187,15 @@ mod tests { async fn rename(&self, tmp: &Path, final_path: &Path) -> Result<(), AtomicWriteError> { self.rename_calls.fetch_add(1, Ordering::SeqCst); if self.fail_at == Some(AtomicWriteStage::Rename) { - return Err(AtomicWriteError::new( - AtomicWriteStage::Rename, - std::io::Error::other("rename injection"), - )); + return Err(AtomicWriteError::new(AtomicWriteStage::Rename, std::io::Error::other("rename injection"))); } tokio::fs::rename(tmp, final_path).await.map_err(|e| AtomicWriteError::new(AtomicWriteStage::Rename, e)) } - fn sync_parent(&self, _parent: &Path) -> impl std::future::Future> + Send { + fn sync_parent( + &self, + _parent: &Path, + ) -> impl std::future::Future> + Send { self.sync_parent_calls.fetch_add(1, Ordering::SeqCst); if self.fail_at == Some(AtomicWriteStage::SyncParent) { return std::future::ready(Err(AtomicWriteError::new( diff --git a/backend/src/utils/binary_utils.rs b/backend/core/src/utils/binary_utils.rs similarity index 100% rename from backend/src/utils/binary_utils.rs rename to backend/core/src/utils/binary_utils.rs index cdd3eb556..932910ee4 100644 --- a/backend/src/utils/binary_utils.rs +++ b/backend/core/src/utils/binary_utils.rs @@ -1,6 +1,6 @@ -use std::io; use log::error; use shared::error::to_io_error; +use std::io; #[inline] pub fn binary_serialize(value: &T) -> io::Result> diff --git a/backend/src/api/model/byte_range.rs b/backend/core/src/utils/byte_range.rs similarity index 90% rename from backend/src/api/model/byte_range.rs rename to backend/core/src/utils/byte_range.rs index 658c0c8ed..a7bded4b3 100644 --- a/backend/src/api/model/byte_range.rs +++ b/backend/core/src/utils/byte_range.rs @@ -1,16 +1,18 @@ +//! Parsing a single HTTP `Range` header. +//! +//! Pure header arithmetic; both the streaming layer and the HLS proxy serve +//! ranged responses from it. + use axum::http::HeaderValue; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::api) enum SingleByteRange { +pub enum SingleByteRange { Full, Partial { start: u64, end: u64, length: u64 }, Unsatisfiable, } -pub(in crate::api) fn resolve_single_byte_range( - range_header: Option<&HeaderValue>, - full_size: u64, -) -> SingleByteRange { +pub fn resolve_single_byte_range(range_header: Option<&HeaderValue>, full_size: u64) -> SingleByteRange { let Some(range_header) = range_header else { return SingleByteRange::Full; }; @@ -113,9 +115,6 @@ mod tests { resolve_single_byte_range(Some(&HeaderValue::from_bytes(&[0xFF]).expect("opaque header")), 10), SingleByteRange::Full ); - assert_eq!( - resolve_single_byte_range(Some(&header("items=0-1")), 10), - SingleByteRange::Full - ); + assert_eq!(resolve_single_byte_range(Some(&header("items=0-1")), 10), SingleByteRange::Full); } } diff --git a/backend/src/utils/compression/compressed_file_reader.rs b/backend/core/src/utils/compression/compressed_file_reader.rs similarity index 100% rename from backend/src/utils/compression/compressed_file_reader.rs rename to backend/core/src/utils/compression/compressed_file_reader.rs diff --git a/backend/src/utils/compression/compressed_file_reader_async.rs b/backend/core/src/utils/compression/compressed_file_reader_async.rs similarity index 100% rename from backend/src/utils/compression/compressed_file_reader_async.rs rename to backend/core/src/utils/compression/compressed_file_reader_async.rs diff --git a/backend/src/utils/compression/compression_utils.rs b/backend/core/src/utils/compression/compression_utils.rs similarity index 100% rename from backend/src/utils/compression/compression_utils.rs rename to backend/core/src/utils/compression/compression_utils.rs diff --git a/backend/src/utils/compression/mod.rs b/backend/core/src/utils/compression/mod.rs similarity index 100% rename from backend/src/utils/compression/mod.rs rename to backend/core/src/utils/compression/mod.rs index 0014d2cf0..df2a49efa 100644 --- a/backend/src/utils/compression/mod.rs +++ b/backend/core/src/utils/compression/mod.rs @@ -1,3 +1,3 @@ pub mod compressed_file_reader; -pub mod compression_utils; pub mod compressed_file_reader_async; +pub mod compression_utils; diff --git a/backend/src/utils/crypto_utils.rs b/backend/core/src/utils/crypto_utils.rs similarity index 95% rename from backend/src/utils/crypto_utils.rs rename to backend/core/src/utils/crypto_utils.rs index 36f24d790..674b05c91 100644 --- a/backend/src/utils/crypto_utils.rs +++ b/backend/core/src/utils/crypto_utils.rs @@ -1,9 +1,16 @@ +/// Length-checked, branch-free byte comparison. +/// +/// Lives here rather than in `auth` because the configuration model compares +/// credentials with it, and `model` must not depend on `auth`. +pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b.iter()).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + use aes::Aes128; use base64::{engine::general_purpose, Engine as _}; use ctr::cipher::{KeyIvInit, StreamCipher}; use rand::{rngs::OsRng, RngCore, TryRngCore}; -use shared::error::TuliproxError; -use shared::utils::encode_base64_string; +use shared::{error::TuliproxError, utils::encode_base64_string}; type Aes128Ctr = ctr::Ctr128BE; const AUTH_TOKEN_VERSION: u8 = 1; @@ -139,7 +146,8 @@ pub fn deobscure_text(secret: &[u8; 16], encoded: &str) -> Result Self { + pub const fn from_lowercase(lowercase: bool) -> Self { if lowercase { Self::LowercaseAscii } else { @@ -22,7 +22,7 @@ impl EpgIdOutputCase { /// Runs `f` with an ASCII-lowercase EPG ID. /// /// Non-ASCII characters remain unchanged, matching `eq_ignore_ascii_case` semantics. -pub(crate) fn with_folded_epg_id(id: &str, f: impl FnOnce(&str) -> R) -> R { +pub fn with_folded_epg_id(id: &str, f: impl FnOnce(&str) -> R) -> R { if !id.bytes().any(|byte| byte.is_ascii_uppercase()) { return f(id); } @@ -42,25 +42,19 @@ pub(crate) fn with_folded_epg_id(id: &str, f: impl FnOnce(&str) -> R) -> R { } /// Folds a trusted, persisted EPG ID and interns only a changed value. -pub(crate) fn fold_epg_id_arc(id: &Arc) -> Arc { - with_folded_epg_id(id, |folded| { - if folded == id.as_ref() { - Arc::clone(id) - } else { - folded.intern() - } - }) +pub fn fold_epg_id_arc(id: &Arc) -> Arc { + with_folded_epg_id(id, |folded| if folded == id.as_ref() { Arc::clone(id) } else { folded.intern() }) } /// Folds an untrusted request ID without adding it to the global interner. -pub(crate) fn fold_untrusted_epg_id(id: &str) -> Arc { +pub fn fold_untrusted_epg_id(id: &str) -> Arc { let mut folded = id.to_owned(); folded.make_ascii_lowercase(); Arc::from(folded) } /// Applies the target output casing to an untrusted request ID without interning it. -pub(crate) fn canonicalize_untrusted_epg_id(id: &str, output_case: EpgIdOutputCase) -> Arc { +pub fn canonicalize_untrusted_epg_id(id: &str, output_case: EpgIdOutputCase) -> Arc { match output_case { EpgIdOutputCase::Preserve => Arc::from(id), EpgIdOutputCase::LowercaseAscii => fold_untrusted_epg_id(id), @@ -68,7 +62,7 @@ pub(crate) fn canonicalize_untrusted_epg_id(id: &str, output_case: EpgIdOutputCa } /// Applies the configured casing to a visible, persisted EPG ID. -pub(crate) fn canonicalize_output_epg_id(id: &Arc, output_case: EpgIdOutputCase) -> Arc { +pub fn canonicalize_output_epg_id(id: &Arc, output_case: EpgIdOutputCase) -> Arc { match output_case { EpgIdOutputCase::Preserve => Arc::clone(id), EpgIdOutputCase::LowercaseAscii => fold_epg_id_arc(id), @@ -76,7 +70,7 @@ pub(crate) fn canonicalize_output_epg_id(id: &Arc, output_case: EpgIdOutput } /// Applies Unicode lowercase only to visible XMLTV text. -pub(crate) fn lowercase_xmltv_text(value: &str, enabled: bool) -> Cow<'_, str> { +pub fn lowercase_xmltv_text(value: &str, enabled: bool) -> Cow<'_, str> { if !enabled { return Cow::Borrowed(value); } @@ -92,8 +86,8 @@ pub(crate) fn lowercase_xmltv_text(value: &str, enabled: bool) -> Cow<'_, str> { #[cfg(test)] mod tests { use super::{ - canonicalize_output_epg_id, canonicalize_untrusted_epg_id, fold_epg_id_arc, - fold_untrusted_epg_id, lowercase_xmltv_text, EpgIdOutputCase, + canonicalize_output_epg_id, canonicalize_untrusted_epg_id, fold_epg_id_arc, fold_untrusted_epg_id, + lowercase_xmltv_text, EpgIdOutputCase, }; use quick_xml::{ events::{BytesText, Event}, diff --git a/backend/src/utils/epg_parser.rs b/backend/core/src/utils/epg_parser.rs similarity index 85% rename from backend/src/utils/epg_parser.rs rename to backend/core/src/utils/epg_parser.rs index e40f3ea69..3bdc499d1 100644 --- a/backend/src/utils/epg_parser.rs +++ b/backend/core/src/utils/epg_parser.rs @@ -1,5 +1,4 @@ -use crate::api::model::AppState; -use crate::model::{ConfigTarget, ProxyUserCredentials}; +use crate::model::{AppConfig, ConfigTarget, ProxyUserCredentials}; use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; use chrono_tz::Tz; use shared::model::PlaylistItemType; @@ -38,7 +37,6 @@ pub fn parse_timeshift(time_shift: Option<&str>) -> EpgTimeShift { } } - #[derive(Debug, Clone)] pub enum EpgTimeShift { None, @@ -53,9 +51,13 @@ pub struct EpgProcessingOptions { pub encrypt_secret: [u8; 16], } -pub fn get_epg_processing_options(app_state: &Arc, user: &ProxyUserCredentials, target: &Arc) -> EpgProcessingOptions { - let rewrite_resources = app_state.app_config.is_reverse_proxy_resource_rewrite_enabled(); - let encrypt_secret = app_state.get_encrypt_secret(); +pub fn get_epg_processing_options( + app_config: &Arc, + user: &ProxyUserCredentials, + target: &Arc, +) -> EpgProcessingOptions { + let rewrite_resources = app_config.is_reverse_proxy_resource_rewrite_enabled(); + let encrypt_secret = app_config.get_encrypt_secret(); // If redirect is true -> rewrite_urls = false -> keep original // If redirect is false and rewrite_resources is true -> rewrite_urls = true -> rewriting allowed @@ -64,16 +66,10 @@ pub fn get_epg_processing_options(app_state: &Arc, user: &ProxyUserCre let rewrite_urls = !redirect && rewrite_resources; let timeshift = parse_timeshift(user.epg_timeshift.as_deref()); - EpgProcessingOptions { - rewrite_urls, - time_shift: timeshift, - encrypt_secret, - } + EpgProcessingOptions { rewrite_urls, time_shift: timeshift, encrypt_secret } } -pub fn apply_offset(ts_utc: i64, offset_minutes: i32) -> i64 { - ts_utc + i64::from(offset_minutes) * 60 -} +pub fn apply_offset(ts_utc: i64, offset_minutes: i32) -> i64 { ts_utc + i64::from(offset_minutes) * 60 } pub fn format_offset(offset_minutes: i32) -> String { let sign = if offset_minutes < 0 { '-' } else { '+' }; @@ -84,9 +80,7 @@ pub fn format_offset(offset_minutes: i32) -> String { } pub fn parse_xmltv_time(t: &str) -> Option { - DateTime::parse_from_str(t, "%Y%m%d%H%M%S %z") - .ok() - .map(|dt| dt.with_timezone(&Utc).timestamp()) + DateTime::parse_from_str(t, "%Y%m%d%H%M%S %z").ok().map(|dt| dt.with_timezone(&Utc).timestamp()) } pub fn format_xmltv_time_utc(ts: i64, time_shift: &EpgTimeShift) -> String { @@ -95,19 +89,11 @@ pub fn format_xmltv_time_utc(ts: i64, time_shift: &EpgTimeShift) -> String { }; match time_shift { EpgTimeShift::None => dt.format("%Y%m%d%H%M%S %z").to_string(), - EpgTimeShift::Fixed(minutes) => { - match chrono::FixedOffset::east_opt(minutes * 60) { - Some(offset) => { - dt.with_timezone(&offset).format("%Y%m%d%H%M%S %z").to_string() - } - None => { - dt.format("%Y%m%d%H%M%S %z").to_string() - } - } - } - EpgTimeShift::TimeZone(tz) => { - dt.with_timezone(tz).format("%Y%m%d%H%M%S %z").to_string() - } + EpgTimeShift::Fixed(minutes) => match chrono::FixedOffset::east_opt(minutes * 60) { + Some(offset) => dt.with_timezone(&offset).format("%Y%m%d%H%M%S %z").to_string(), + None => dt.format("%Y%m%d%H%M%S %z").to_string(), + }, + EpgTimeShift::TimeZone(tz) => dt.with_timezone(tz).format("%Y%m%d%H%M%S %z").to_string(), } } @@ -149,9 +135,7 @@ pub fn apply_timeshift(date_str: &str, shift: &EpgTimeShift) -> String { // 2. Calculate offset let shifted_dt = match shift { - EpgTimeShift::Fixed(minutes) => { - dt + Duration::minutes(i64::from(*minutes)) - } + EpgTimeShift::Fixed(minutes) => dt + Duration::minutes(i64::from(*minutes)), EpgTimeShift::TimeZone(tz) => { // We assume it is UTC and want to know the target TZ let utc_dt = Utc.from_utc_datetime(&dt); @@ -242,7 +226,7 @@ mod tests { let shifted = apply_timeshift(date_str, &shift); assert_eq!(shifted, "2026-02-08 12:30"); - // Format with dashes: 2026-02-08-11-30 + // Format with dashes: 2026-02-08-11-30 let date_str = "2026-02-08-11-30"; let shifted = apply_timeshift(date_str, &shift); assert_eq!(shifted, "2026-02-08-12-30"); diff --git a/backend/src/utils/ffmpeg.rs b/backend/core/src/utils/ffmpeg.rs similarity index 82% rename from backend/src/utils/ffmpeg.rs rename to backend/core/src/utils/ffmpeg.rs index 4d10f96ae..6e3c1f960 100644 --- a/backend/src/utils/ffmpeg.rs +++ b/backend/core/src/utils/ffmpeg.rs @@ -6,14 +6,16 @@ use reqwest::{ }; use serde_json::Value; use shared::{ + defaults::{default_thumbnail_height, default_thumbnail_width}, model::MediaQuality, utils::{is_dash_url, is_hls_url, sanitize_sensitive_info}, - defaults::{default_thumbnail_height, default_thumbnail_width}, }; -use std::io::ErrorKind; -use std::path::{Path, PathBuf}; -use std::process::{Output, Stdio}; -use std::time::Duration; +use std::{ + io::ErrorKind, + path::{Path, PathBuf}, + process::{Output, Stdio}, + time::Duration, +}; use tokio::{ fs::{self, OpenOptions}, io::{AsyncSeekExt, AsyncWriteExt}, @@ -78,24 +80,20 @@ impl FfmpegExecutor { pub const fn new() -> Self { Self } /// Checks if the system `ffmpeg` binary is available. - pub async fn check_ffmpeg_availability(&self) -> bool { - self.check_binary_availability("ffmpeg").await - } + pub async fn check_ffmpeg_availability(&self) -> bool { self.check_binary_availability("ffmpeg").await } // Checks if ffprobe is available in the system path - pub async fn check_ffprobe_availability(&self) -> bool { - self.check_binary_availability("ffprobe").await - } + pub async fn check_ffprobe_availability(&self) -> bool { self.check_binary_availability("ffprobe").await } /// Extracts a JPEG thumbnail from a local file. /// Attempts a frame at 180s first and falls back to 0s for short videos. pub async fn create_thumbnail(&self, input_path: &str, width: u32, height: u32) -> Result, String> { - let temp_dir = tempfile::tempdir() - .map_err(|e| format!("Failed to create temp dir: {e}"))?; + let temp_dir = tempfile::tempdir().map_err(|e| format!("Failed to create temp dir: {e}"))?; let output_path = temp_dir.path().join("thumb.jpg"); let scale_filter = build_thumbnail_scale_filter(width, height); - let output = self.run_ffmpeg_with_timeout(&build_thumbnail_args(input_path, &output_path, &scale_filter, 180)) + let output = self + .run_ffmpeg_with_timeout(&build_thumbnail_args(input_path, &output_path, &scale_filter, 180)) .await .map_err(|e| format!("Failed to run ffmpeg: {e}"))?; @@ -103,7 +101,8 @@ impl FfmpegExecutor { let stderr = String::from_utf8_lossy(&output.stderr); if stderr.contains("Output file is empty") || stderr.contains("nothing was encoded") { debug!("Video shorter than 180s, retrying at position 0: {input_path}"); - let retry = self.run_ffmpeg_with_timeout(&build_thumbnail_args(input_path, &output_path, &scale_filter, 0)) + let retry = self + .run_ffmpeg_with_timeout(&build_thumbnail_args(input_path, &output_path, &scale_filter, 0)) .await .map_err(|e| format!("Failed to run ffmpeg retry: {e}"))?; @@ -116,16 +115,10 @@ impl FfmpegExecutor { } } - tokio::fs::read(&output_path) - .await - .map_err(|e| format!("Failed to read thumbnail: {e}")) + tokio::fs::read(&output_path).await.map_err(|e| format!("Failed to read thumbnail: {e}")) } - pub async fn probe_url( - &self, - params: &ProbeParams<'_>, - proxy_cfg: Option<&ProxyConfig>, - ) -> ProbeUrlOutcome { + pub async fn probe_url(&self, params: &ProbeParams<'_>, proxy_cfg: Option<&ProxyConfig>) -> ProbeUrlOutcome { let ProbeParams { url, user_agent, analyze_duration, probe_size, timeout_secs } = *params; // Determine timeout: Ensure it's at least as long as the analyze duration + buffer, // but respect the user setting if it's longer. @@ -139,12 +132,16 @@ impl FfmpegExecutor { command.kill_on_drop(true); command - .arg("-v").arg("error") + .arg("-v") + .arg("error") .arg("-show_streams") .arg("-show_format") - .arg("-of").arg("json") - .arg("-analyzeduration").arg(analyze_duration.to_string()) - .arg("-probesize").arg(probe_size.to_string()); + .arg("-of") + .arg("json") + .arg("-analyzeduration") + .arg(analyze_duration.to_string()) + .arg("-probesize") + .arg(probe_size.to_string()); apply_proxy_to_ffprobe(&mut command, proxy_cfg); @@ -169,11 +166,7 @@ impl FfmpegExecutor { ProbeUrlOutcome::Failed(ProbeFailureKind::Other) } - pub async fn probe_remote_url( - &self, - client: &Client, - params: &ProbeParams<'_>, - ) -> ProbeUrlOutcome { + pub async fn probe_remote_url(&self, client: &Client, params: &ProbeParams<'_>) -> ProbeUrlOutcome { let ProbeParams { url, user_agent, analyze_duration, probe_size, timeout_secs } = *params; let analyze_overhead = Duration::from_micros(analyze_duration) + Duration::from_secs(5); let config_timeout = Duration::from_secs(timeout_secs); @@ -188,20 +181,12 @@ impl FfmpegExecutor { if let Ok(outcome) = probe_result { outcome } else { - warn!( - "ffprobe remote stdin probe timed out after {:?} for {}", - timeout_val, - sanitize_sensitive_info(url) - ); + warn!("ffprobe remote stdin probe timed out after {:?} for {}", timeout_val, sanitize_sensitive_info(url)); ProbeUrlOutcome::Failed(ProbeFailureKind::Other) } } - pub async fn probe_remote_seekable_url( - &self, - client: &Client, - params: &ProbeParams<'_>, - ) -> ProbeUrlOutcome { + pub async fn probe_remote_seekable_url(&self, client: &Client, params: &ProbeParams<'_>) -> ProbeUrlOutcome { let ProbeParams { url, user_agent, analyze_duration, probe_size, timeout_secs } = *params; let analyze_overhead = Duration::from_micros(analyze_duration) + Duration::from_secs(5); let config_timeout = Duration::from_secs(timeout_secs); @@ -288,9 +273,7 @@ impl FfmpegExecutor { async fn check_binary_availability(&self, binary: &str) -> bool { let mut command = Command::new(binary); - command - .arg("-version") - .kill_on_drop(true); + command.arg("-version").kill_on_drop(true); match tokio::time::timeout(FFMPEG_TIMEOUT, command.output()).await { Ok(Ok(output)) => output.status.success(), @@ -354,7 +337,8 @@ impl FfmpegExecutor { }; let max_bytes = if probe_size == 0 { usize::MAX } else { usize::try_from(probe_size).unwrap_or(usize::MAX) }; - let fetch_result = stream_probe_bytes_to_stdin(&mut stdin, client, url, user_agent, probe_size, max_bytes).await; + let fetch_result = + stream_probe_bytes_to_stdin(&mut stdin, client, url, user_agent, probe_size, max_bytes).await; let stdin_err = match fetch_result { Ok(()) => None, @@ -379,7 +363,10 @@ impl FfmpegExecutor { match child.wait_with_output().await { Ok(output) => { if let Some(kind) = stdin_err { - debug!("ffprobe remote stdin probe had fetch error, checking child output for {}", sanitize_sensitive_info(url)); + debug!( + "ffprobe remote stdin probe had fetch error, checking child output for {}", + sanitize_sensitive_info(url) + ); let outcome = parse_ffprobe_output(url, &output); if matches!(outcome, ProbeUrlOutcome::Success(..)) { return outcome; @@ -416,10 +403,7 @@ impl FfmpegExecutor { }; if !stage.fully_materialized { - debug!( - "Running seekable ffprobe probe with staged head/tail data for {}", - sanitize_sensitive_info(url) - ); + debug!("Running seekable ffprobe probe with staged head/tail data for {}", sanitize_sensitive_info(url)); } probe_local_path_with_ffprobe(&temp_path, analyze_duration, probe_size).await } @@ -436,10 +420,7 @@ pub fn is_supported_probe_url(url: &str) -> bool { fn format_ffmpeg_timeout_error(args: &[String]) -> String { let summary = args.join(" "); - format!( - "Timed out running ffmpeg after {}s: {summary}", - FFMPEG_TIMEOUT.as_secs() - ) + format!("Timed out running ffmpeg after {}s: {summary}", FFMPEG_TIMEOUT.as_secs()) } fn probe_bytes_limit(probe_size: u64) -> usize { @@ -467,21 +448,13 @@ async fn create_seekable_probe_temp_file() -> Result std::io::Result<()> { @@ -525,37 +498,19 @@ async fn ensure_probe_file_length(path: &Path, content_length: Option) -> R let Some(content_length) = content_length else { return Ok(()); }; - let file = OpenOptions::new() - .create(true) - .write(true) - .truncate(false) - .open(path) - .await - .map_err(|err| { - warn!( - "Failed to open seekable ffprobe temp file {}: {}", - path.display(), - err - ); - ProbeFailureKind::Other - })?; + let file = OpenOptions::new().create(true).write(true).truncate(false).open(path).await.map_err(|err| { + warn!("Failed to open seekable ffprobe temp file {}: {}", path.display(), err); + ProbeFailureKind::Other + })?; file.set_len(content_length).await.map_err(|err| { - warn!( - "Failed to resize seekable ffprobe temp file {}: {}", - path.display(), - err - ); + warn!("Failed to resize seekable ffprobe temp file {}: {}", path.display(), err); ProbeFailureKind::Other }) } async fn flush_probe_file(file: &mut fs::File, path: &Path) -> Result<(), ProbeFailureKind> { file.flush().await.map_err(|err| { - warn!( - "Failed to flush seekable ffprobe temp file {}: {}", - path.display(), - err - ); + warn!("Failed to flush seekable ffprobe temp file {}: {}", path.display(), err); ProbeFailureKind::Other }) } @@ -580,11 +535,7 @@ async fn probe_local_path_with_ffprobe(path: &Path, analyze_duration: u64, probe match command.output().await { Ok(output) => parse_ffprobe_output(&display_path, &output), Err(err) => { - warn!( - "ffprobe execution failed for {}: {}", - sanitize_sensitive_info(&display_path), - err - ); + warn!("ffprobe execution failed for {}: {}", sanitize_sensitive_info(&display_path), err); ProbeUrlOutcome::Failed(ProbeFailureKind::Other) } } @@ -619,9 +570,7 @@ async fn stage_seekable_probe_file( tail_start: None, fully_materialized: head_result.status != StatusCode::PARTIAL_CONTENT && head_result.response_exhausted - && head_result - .total_length - .is_none_or(|content_length| head_result.bytes_written >= content_length), + && head_result.total_length.is_none_or(|content_length| head_result.bytes_written >= content_length), range_supported: head_result.status == StatusCode::PARTIAL_CONTENT, }; @@ -695,7 +644,8 @@ struct RemoteSpanRequest<'a> { } async fn fetch_remote_span_to_file(req: &RemoteSpanRequest<'_>) -> Result { - let RemoteSpanRequest { client, url, user_agent, range, path, offset, max_bytes, allow_full_body_write_on_200 } = *req; + let RemoteSpanRequest { client, url, user_agent, range, path, offset, max_bytes, allow_full_body_write_on_200 } = + *req; let mut request = client.get(url); if let Some(ua) = user_agent { request = request.header(USER_AGENT, ua); @@ -717,44 +667,21 @@ async fn fetch_remote_span_to_file(req: &RemoteSpanRequest<'_>) -> Result) -> Result 0 { match stdin.write_all(&chunk[..to_write]).await { Ok(()) => {} Err(err) if err.kind() == ErrorKind::BrokenPipe => { - debug!( - "ffprobe closed stdin early for {}, stopping fetch", - sanitize_sensitive_info(url) - ); + debug!("ffprobe closed stdin early for {}, stopping fetch", sanitize_sensitive_info(url)); return Ok(()); } Err(err) => { @@ -933,11 +845,7 @@ async fn fetch_probe_bytes( return Err(ProbeFailureKind::NotFound); } if !response.status().is_success() && response.status() != StatusCode::PARTIAL_CONTENT { - warn!( - "ffprobe fetch returned {} for {}", - response.status(), - sanitize_sensitive_info(url) - ); + warn!("ffprobe fetch returned {} for {}", response.status(), sanitize_sensitive_info(url)); return Err(ProbeFailureKind::Other); } @@ -972,10 +880,7 @@ where match stdin.write_all(probe_bytes).await { Ok(()) => Ok(()), Err(err) if err.kind() == ErrorKind::BrokenPipe => { - debug!( - "ffprobe closed stdin early for {}, continuing with child output", - sanitize_sensitive_info(url) - ); + debug!("ffprobe closed stdin early for {}, continuing with child output", sanitize_sensitive_info(url)); Ok(()) } Err(err) => { @@ -1033,23 +938,13 @@ fn apply_proxy_to_ffprobe(command: &mut Command, proxy_cfg: Option<&ProxyConfig> }; let Some(proxy_url) = build_ffprobe_proxy_url(proxy_cfg) else { - warn!( - "Ignoring invalid ffprobe proxy URL: {}", - sanitize_sensitive_info(proxy_cfg.url.as_str()) - ); + warn!("Ignoring invalid ffprobe proxy URL: {}", sanitize_sensitive_info(proxy_cfg.url.as_str())); return; }; // ffprobe is an external process and does not consume the app's reqwest proxy config. // Export proxy env vars explicitly so all probe requests honor the configured upstream proxy. - for key in [ - "HTTP_PROXY", - "HTTPS_PROXY", - "ALL_PROXY", - "http_proxy", - "https_proxy", - "all_proxy", - ] { + for key in ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] { command.env(key, proxy_url.as_str()); } } @@ -1111,44 +1006,36 @@ fn parse_ffprobe_output(url: &str, output: &Output) -> ProbeUrlOutcome { /// ffprobe reports these as `codec_type: "video"` but with /// `disposition.attached_pic: 1`. fn is_attached_pic(stream: &Value) -> bool { - stream - .get("disposition") - .and_then(|d| d.get("attached_pic")) - .and_then(Value::as_u64) - == Some(1) + stream.get("disposition").and_then(|d| d.get("attached_pic")).and_then(Value::as_u64) == Some(1) } fn is_not_found_probe_error(stderr: &str) -> bool { let normalized = stderr.to_ascii_lowercase(); - [ - "http error 404", - "404 not found", - "http/1.1 404", - "http/2 404", - "server returned 404", - ] - .iter() - .any(|needle| normalized.contains(needle)) + ["http error 404", "404 not found", "http/1.1 404", "http/2 404", "server returned 404"] + .iter() + .any(|needle| normalized.contains(needle)) } -fn extract_probe_stream_stats(json: &Value, video_stream: Option<&Value>, audio_stream: Option<&Value>) -> ProbeStreamStats { +fn extract_probe_stream_stats( + json: &Value, + video_stream: Option<&Value>, + audio_stream: Option<&Value>, +) -> ProbeStreamStats { let format = json.get("format"); ProbeStreamStats { duration_secs: format .and_then(|value| parse_duration_secs(value.get("duration"))) .or_else(|| parse_duration_secs(video_stream.and_then(|value| value.get("duration")))) .or_else(|| parse_duration_secs(audio_stream.and_then(|value| value.get("duration")))), - bitrate: format - .and_then(|value| parse_u32_field(value.get("bit_rate"))) - .or_else(|| { - let video_bitrate = parse_u32_field(video_stream.and_then(|value| value.get("bit_rate"))); - let audio_bitrate = parse_u32_field(audio_stream.and_then(|value| value.get("bit_rate"))); - match (video_bitrate, audio_bitrate) { - (Some(video), Some(audio)) => Some(video.saturating_add(audio)), - (Some(bitrate), None) | (None, Some(bitrate)) => Some(bitrate), - (None, None) => None, - } - }), + bitrate: format.and_then(|value| parse_u32_field(value.get("bit_rate"))).or_else(|| { + let video_bitrate = parse_u32_field(video_stream.and_then(|value| value.get("bit_rate"))); + let audio_bitrate = parse_u32_field(audio_stream.and_then(|value| value.get("bit_rate"))); + match (video_bitrate, audio_bitrate) { + (Some(video), Some(audio)) => Some(video.saturating_add(audio)), + (Some(bitrate), None) | (None, Some(bitrate)) => Some(bitrate), + (None, None) => None, + } + }), } } @@ -1172,9 +1059,7 @@ fn parse_duration_secs(value: Option<&Value>) -> Option { } #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] -fn rounded_duration_secs_to_u32(rounded_seconds: f64) -> u32 { - rounded_seconds as u32 -} +fn rounded_duration_secs_to_u32(rounded_seconds: f64) -> u32 { rounded_seconds as u32 } fn parse_u32_field(value: Option<&Value>) -> Option { let raw = value.and_then(|value| match value { @@ -1189,9 +1074,7 @@ fn parse_u32_field(value: Option<&Value>) -> Option { fn build_thumbnail_scale_filter(width: u32, height: u32) -> String { let w = if width < 1 { default_thumbnail_width() } else { width }; let h = if height < 1 { default_thumbnail_height() } else { height }; - format!( - "scale={w}:{h}:force_original_aspect_ratio=increase,crop={w}:{h}" - ) + format!("scale={w}:{h}:force_original_aspect_ratio=increase,crop={w}:{h}") } fn build_thumbnail_args(input_path: &str, output_path: &Path, scale_filter: &str, seek_seconds: u32) -> Vec { @@ -1214,17 +1097,24 @@ fn build_thumbnail_args(input_path: &str, output_path: &Path, scale_filter: &str #[cfg(test)] mod tests { use super::{ - build_ffprobe_proxy_url, build_thumbnail_args, build_thumbnail_scale_filter, extract_probe_stream_stats, - ensure_probe_file_length, fetch_probe_bytes, format_ffmpeg_timeout_error, parse_content_range_total_length, + build_ffprobe_proxy_url, build_thumbnail_args, build_thumbnail_scale_filter, ensure_probe_file_length, + extract_probe_stream_stats, fetch_probe_bytes, format_ffmpeg_timeout_error, parse_content_range_total_length, run_ffprobe_with_stdin, seekable_probe_window_bytes, stage_seekable_probe_file, write_probe_bytes_to_stdin, - FFMPEG_TIMEOUT, - ProbeFailureKind, ProbeStreamStats, ProbeUrlOutcome, + ProbeFailureKind, ProbeStreamStats, ProbeUrlOutcome, FFMPEG_TIMEOUT, }; use crate::model::ProxyConfig; use serde_json::json; use shared::defaults::{default_thumbnail_height, default_thumbnail_width}; - use std::{io, path::Path, pin::Pin, task::{Context, Poll}}; - use std::sync::{atomic::{AtomicUsize, Ordering}, Arc, Mutex}; + use std::{ + io, + path::Path, + pin::Pin, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, + task::{Context, Poll}, + }; use tokio::{ io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}, net::TcpListener, @@ -1260,9 +1150,8 @@ mod tests { guard.extend_from_slice(&buf[..read]); } } - let response_head = format!( - "{status_line}\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n" - ); + let response_head = + format!("{status_line}\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); let _ = socket.write_all(response_head.as_bytes()).await; let _ = socket.write_all(body).await; let _ = socket.shutdown().await; @@ -1319,12 +1208,7 @@ mod tests { let bounded_start = start.min(bounded_end); ( "HTTP/1.1 206 Partial Content".to_string(), - Some(format!( - "Content-Range: bytes {}-{}/{}\r\n", - bounded_start, - bounded_end, - body.len() - )), + Some(format!("Content-Range: bytes {}-{}/{}\r\n", bounded_start, bounded_end, body.len())), body[bounded_start..=bounded_end].to_vec(), ) } else { @@ -1365,11 +1249,7 @@ mod tests { struct ErroringWriter; impl AsyncWrite for ErroringWriter { - fn poll_write( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - _buf: &[u8], - ) -> Poll> { + fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll> { Poll::Ready(Err(io::Error::new(io::ErrorKind::ConnectionReset, "synthetic write failure"))) } @@ -1395,11 +1275,8 @@ mod tests { #[test] fn build_ffprobe_proxy_url_keeps_existing_inline_credentials() { - let proxy_cfg = ProxyConfig { - url: "socks5://bob:pass@proxy.local:1080".to_string(), - username: None, - password: None, - }; + let proxy_cfg = + ProxyConfig { url: "socks5://bob:pass@proxy.local:1080".to_string(), username: None, password: None }; let resolved = build_ffprobe_proxy_url(&proxy_cfg).expect("proxy url should parse"); assert!(resolved.contains("bob:pass@proxy.local:1080")); } @@ -1444,7 +1321,12 @@ mod tests { #[test] fn format_ffmpeg_timeout_error_includes_binary_timeout_and_args() { - let msg = format_ffmpeg_timeout_error(&["-ss".to_string(), "180".to_string(), "-i".to_string(), "/tmp/in.mkv".to_string()]); + let msg = format_ffmpeg_timeout_error(&[ + "-ss".to_string(), + "180".to_string(), + "-i".to_string(), + "/tmp/in.mkv".to_string(), + ]); assert!(msg.contains("ffmpeg")); assert!(msg.contains(&FFMPEG_TIMEOUT.as_secs().to_string())); assert!(msg.contains("-ss 180 -i /tmp/in.mkv")); @@ -1481,10 +1363,7 @@ mod tests { assert_eq!( extract_probe_stream_stats(&payload, Some(&video), Some(&audio)), - ProbeStreamStats { - duration_secs: Some(1541), - bitrate: Some(3_100_000), - } + ProbeStreamStats { duration_secs: Some(1541), bitrate: Some(3_100_000) } ); } @@ -1498,10 +1377,7 @@ mod tests { assert_eq!( extract_probe_stream_stats(&payload, Some(&video), None), - ProbeStreamStats { - duration_secs: Some(120), - bitrate: Some(1500), - } + ProbeStreamStats { duration_secs: Some(120), bitrate: Some(1500) } ); } @@ -1511,10 +1387,7 @@ mod tests { let video = json!({ "bit_rate": "2500000" }); let audio = json!({ "bit_rate": 192_000 }); - assert_eq!( - extract_probe_stream_stats(&payload, Some(&video), Some(&audio)).bitrate, - Some(2_692_000) - ); + assert_eq!(extract_probe_stream_stats(&payload, Some(&video), Some(&audio)).bitrate, Some(2_692_000)); } #[test] @@ -1523,10 +1396,7 @@ mod tests { let video = json!({ "bit_rate": u32::MAX }); let audio = json!({ "bit_rate": 1 }); - assert_eq!( - extract_probe_stream_stats(&payload, Some(&video), Some(&audio)).bitrate, - Some(u32::MAX) - ); + assert_eq!(extract_probe_stream_stats(&payload, Some(&video), Some(&audio)).bitrate, Some(u32::MAX)); } #[test] @@ -1542,50 +1412,38 @@ mod tests { let temp_path = tempdir.path().join("probe.bin"); let remote_logical_length = 10 * 1024 * 1024; - ensure_probe_file_length(&temp_path, Some(remote_logical_length)) - .await - .expect("file length should be set"); + ensure_probe_file_length(&temp_path, Some(remote_logical_length)).await.expect("file length should be set"); - let metadata = tokio::fs::metadata(&temp_path) - .await - .expect("probe file metadata should be readable"); + let metadata = tokio::fs::metadata(&temp_path).await.expect("probe file metadata should be readable"); assert_eq!(metadata.len(), remote_logical_length); } #[test] fn parse_content_range_total_length_extracts_total() { - assert_eq!( - parse_content_range_total_length("bytes 10-20/999"), - Some(999) - ); + assert_eq!(parse_content_range_total_length("bytes 10-20/999"), Some(999)); assert_eq!(parse_content_range_total_length("bytes 0-1/*"), None); } #[tokio::test] async fn fetch_probe_bytes_caps_response_and_sets_headers() { - let (url, request_capture, accepted, handle) = match start_test_http_server("HTTP/1.1 200 OK", b"abcdefghij").await - { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping fetch_probe_bytes_caps_response_and_sets_headers: {err}"); - return; - } - Err(err) => panic!("failed to start test http server: {err}"), - }; + let (url, request_capture, accepted, handle) = + match start_test_http_server("HTTP/1.1 200 OK", b"abcdefghij").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping fetch_probe_bytes_caps_response_and_sets_headers: {err}"); + return; + } + Err(err) => panic!("failed to start test http server: {err}"), + }; - let client = reqwest::Client::builder() - .no_proxy() - .build() - .expect("http client should build"); - let bytes = fetch_probe_bytes(&client, &url, Some("test-agent/1.0"), 5) - .await - .expect("fetch should succeed"); + let client = reqwest::Client::builder().no_proxy().build().expect("http client should build"); + let bytes = fetch_probe_bytes(&client, &url, Some("test-agent/1.0"), 5).await.expect("fetch should succeed"); assert_eq!(bytes, b"abcde"); assert_eq!(accepted.load(Ordering::SeqCst), 1); - let request_text = - String::from_utf8_lossy(&request_capture.lock().expect("request capture mutex poisoned")).to_ascii_lowercase(); + let request_text = String::from_utf8_lossy(&request_capture.lock().expect("request capture mutex poisoned")) + .to_ascii_lowercase(); assert!(request_text.contains("range: bytes=0-4")); assert!(request_text.contains("user-agent: test-agent/1.0")); @@ -1604,10 +1462,7 @@ mod tests { Err(err) => panic!("failed to start test http server: {err}"), }; - let client = reqwest::Client::builder() - .no_proxy() - .build() - .expect("http client should build"); + let client = reqwest::Client::builder().no_proxy().build().expect("http client should build"); let result = fetch_probe_bytes(&client, &url, None, 16).await; assert!(matches!(result, Err(ProbeFailureKind::NotFound))); @@ -1618,15 +1473,11 @@ mod tests { #[tokio::test] async fn stage_seekable_probe_file_writes_head_and_tail_segments() { let body = Arc::new(b"abcdefghijklmnopqrstuvwxyz".to_vec()); - let (url, accepted, handle) = start_range_test_http_server(Arc::clone(&body)) - .await - .expect("range server should start"); + let (url, accepted, handle) = + start_range_test_http_server(Arc::clone(&body)).await.expect("range server should start"); let tempdir = tempfile::tempdir().expect("tempdir should succeed"); let temp_path = tempdir.path().join("probe.bin"); - let client = reqwest::Client::builder() - .no_proxy() - .build() - .expect("http client should build"); + let client = reqwest::Client::builder().no_proxy().build().expect("http client should build"); let stage = stage_seekable_probe_file(&client, &url, Some("test-agent/1.0"), 5, &temp_path) .await @@ -1650,15 +1501,11 @@ mod tests { #[tokio::test] async fn stage_seekable_probe_file_with_zero_probe_size_uses_bounded_default_window() { let body = Arc::new(b"abcdefghijklmnopqrstuvwxyz".to_vec()); - let (url, accepted, handle) = start_range_test_http_server(Arc::clone(&body)) - .await - .expect("range server should start"); + let (url, accepted, handle) = + start_range_test_http_server(Arc::clone(&body)).await.expect("range server should start"); let tempdir = tempfile::tempdir().expect("tempdir should succeed"); let temp_path = tempdir.path().join("probe.bin"); - let client = reqwest::Client::builder() - .no_proxy() - .build() - .expect("http client should build"); + let client = reqwest::Client::builder().no_proxy().build().expect("http client should build"); let stage = stage_seekable_probe_file(&client, &url, Some("test-agent/1.0"), 0, &temp_path) .await diff --git a/backend/core/src/utils/file/config_reader.rs b/backend/core/src/utils/file/config_reader.rs new file mode 100644 index 000000000..a7e54fcac --- /dev/null +++ b/backend/core/src/utils/file/config_reader.rs @@ -0,0 +1,54 @@ +//! Reading a configuration file with environment-variable resolution. +//! +//! Only the two primitives the other `utils` file readers share live here. The +//! application configuration bootstrap that used to sit alongside them - reading +//! sources, preparing batches and users, persisting config - moved to +//! `crate::config_loader`, because it reads and writes the repository and +//! `utils` must not depend on that layer. + +use crate::utils::{file_reader, EnvResolvingReader}; +use log::error; +use shared::utils::CONSTANTS; +use std::{ + env, + fs::File, + io::{self, Read}, +}; + +pub fn config_file_reader(file: File, resolve_env: bool) -> impl Read { + if resolve_env { + EitherReader::Left(EnvResolvingReader::new(file_reader(file))) + } else { + EitherReader::Right(file_reader(file)) + } +} + +pub fn resolve_env_var(value: &str) -> String { + if value.is_empty() { + return String::new(); + } + CONSTANTS + .re_env_var + .replace_all(value, |caps: ®ex::Captures| { + let var_name = &caps["var"]; + env::var(var_name).unwrap_or_else(|e| { + error!("Could not resolve env var '{var_name}': {e}"); + format!("${{env:{var_name}}}") + }) + }) + .to_string() +} + +enum EitherReader { + Left(L), + Right(R), +} + +impl Read for EitherReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + EitherReader::Left(reader) => reader.read(buf), + EitherReader::Right(reader) => reader.read(buf), + } + } +} diff --git a/backend/src/utils/file/env_resolving_reader.rs b/backend/core/src/utils/file/env_resolving_reader.rs similarity index 77% rename from backend/src/utils/file/env_resolving_reader.rs rename to backend/core/src/utils/file/env_resolving_reader.rs index daa7b1781..38c947579 100644 --- a/backend/src/utils/file/env_resolving_reader.rs +++ b/backend/core/src/utils/file/env_resolving_reader.rs @@ -1,5 +1,5 @@ -use std::io::{self, BufRead, BufReader, Read, Cursor}; use crate::utils::resolve_env_var; +use std::io::{self, BufRead, BufReader, Cursor, Read}; pub struct EnvResolvingReader { inner: BufReader, @@ -7,12 +7,7 @@ pub struct EnvResolvingReader { } impl EnvResolvingReader { - pub(crate) fn new(reader: BufReader) -> Self { - Self { - inner: reader, - buffer: Cursor::new(Vec::new()), - } - } + pub fn new(reader: BufReader) -> Self { Self { inner: reader, buffer: Cursor::new(Vec::new()) } } fn fill_buffer(&mut self) -> io::Result<()> { let mut line = String::new(); @@ -46,7 +41,5 @@ impl BufRead for EnvResolvingReader { self.buffer.fill_buf() } - fn consume(&mut self, amt: usize) { - self.buffer.consume(amt); - } -} \ No newline at end of file + fn consume(&mut self, amt: usize) { self.buffer.consume(amt); } +} diff --git a/backend/src/utils/file/file_lock_manager.rs b/backend/core/src/utils/file/file_lock_manager.rs similarity index 92% rename from backend/src/utils/file/file_lock_manager.rs rename to backend/core/src/utils/file/file_lock_manager.rs index 39bff30bc..6bf51ab12 100644 --- a/backend/src/utils/file/file_lock_manager.rs +++ b/backend/core/src/utils/file/file_lock_manager.rs @@ -1,14 +1,14 @@ -use std::collections::HashMap; -use std::env; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Weak}; -use std::{fmt, io}; -use std::time::Duration; -use tokio::sync::{Mutex, RwLock}; -use tokio::sync::{OwnedRwLockReadGuard, OwnedRwLockWriteGuard}; -use shared::error::str_to_io_error; +use crate::model::AppConfig; use path_clean::PathClean; -use crate::api::model::AppState; +use shared::error::str_to_io_error; +use std::{ + collections::HashMap, + env, fmt, io, + path::{Path, PathBuf}, + sync::{Arc, Weak}, + time::Duration, +}; +use tokio::sync::{Mutex, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock}; fn revision_is_missing(result: &io::Result) -> bool { matches!(result, Ok(false)) } @@ -115,7 +115,7 @@ impl FileLockManager { let file_lock = self.get_or_create_lock(Self::get_lock_key_for_path(path)).await; match Arc::clone(&file_lock).try_write_owned() { Ok(lock_guard) => Ok(FileWriteGuard::new(lock_guard)), - Err(_) => Err(str_to_io_error("Failed to acquire write lock")) + Err(_) => Err(str_to_io_error("Failed to acquire write lock")), } } @@ -144,11 +144,10 @@ impl FileLockManager { let file_lock = self.get_or_create_lock(lock_key).await; match Arc::clone(&file_lock).try_write_owned() { Ok(lock_guard) => Ok(FileWriteGuard::new(lock_guard)), - Err(_) => Err(str_to_io_error("Failed to acquire write lock")) + Err(_) => Err(str_to_io_error("Failed to acquire write lock")), } } - fn get_lock_key_for_path(path: &Path) -> LockKey { let normalized_path = normalize_path(path); LockKey::Path(normalized_path) @@ -158,7 +157,6 @@ impl FileLockManager { async fn get_or_create_lock(&self, lock_key: LockKey) -> Arc> { let mut locks = self.locks.lock().await; - if let Some(weak_lock) = locks.get(&lock_key) { if let Some(strong_lock) = weak_lock.upgrade() { return strong_lock; @@ -173,9 +171,7 @@ impl FileLockManager { } impl Default for FileLockManager { - fn default() -> Self { - Self::new() - } + fn default() -> Self { Self::new() } } impl fmt::Debug for FileLockManager { @@ -192,9 +188,7 @@ pub struct FileReadGuard { } impl FileReadGuard { - fn new(guard: OwnedRwLockReadGuard<()>) -> Self { - Self { _guard: guard } - } + fn new(guard: OwnedRwLockReadGuard<()>) -> Self { Self { _guard: guard } } } // Define FileWriteGuard to hold both the lock reference and the actual write guard. @@ -203,9 +197,7 @@ pub struct FileWriteGuard { } impl FileWriteGuard { - fn new(guard: OwnedRwLockWriteGuard<()>) -> Self { - Self { _guard: guard } - } + fn new(guard: OwnedRwLockWriteGuard<()>) -> Self { Self { _guard: guard } } } fn normalize_path(path: &Path) -> PathBuf { @@ -222,13 +214,13 @@ fn normalize_path(path: &Path) -> PathBuf { base.clean() } -pub fn exec_file_lock_prune(app_state: &Arc) { - let app_state = Arc::clone(app_state); +pub fn exec_file_lock_prune(app_config: &Arc) { + let app_config = Arc::clone(app_config); tokio::spawn({ async move { loop { tokio::time::sleep(Duration::from_mins(1)).await; - app_state.app_config.file_locks.prune_unused_locks().await; + app_config.file_locks.prune_unused_locks().await; } } }); diff --git a/backend/src/utils/file/file_utils.rs b/backend/core/src/utils/file/file_utils.rs similarity index 72% rename from backend/src/utils/file/file_utils.rs rename to backend/core/src/utils/file/file_utils.rs index 6aeccb179..2b6673c83 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/core/src/utils/file/file_utils.rs @@ -2,16 +2,21 @@ use crate::utils::debug_if_enabled; use log::{debug, error, trace, warn}; use path_clean::PathClean; use shared::{ + defaults::{ + API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, DEFAULT_WEB_DIR, MAPPING_FILE, SOURCE_FILE, TEMPLATE_FILE, USER_FILE, + USER_GROUP_FILE, + }, error::str_to_io_error, - utils::{DEFAULT_HOME_ENV_VAR, DEFAULT_WEB_ROOT_ENV_VAR, }, - defaults::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, DEFAULT_WEB_DIR, MAPPING_FILE, SOURCE_FILE, TEMPLATE_FILE, USER_FILE, USER_GROUP_FILE}, + utils::{DEFAULT_HOME_ENV_VAR, DEFAULT_WEB_ROOT_ENV_VAR}, +}; +use std::{ + borrow::Cow, + collections::HashSet, + env, fs, + fs::{File, OpenOptions}, + io::Read, + path::{Path, PathBuf}, }; -use std::borrow::Cow; -use std::collections::HashSet; -use std::fs::{File, OpenOptions}; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::{env, fs}; use tokio::fs as tokio_fs; pub const IO_BUFFER_SIZE: usize = 256 * 1024; // 256kb @@ -44,52 +49,35 @@ where tokio::io::BufReader::with_capacity(IO_BUFFER_SIZE, r) } - pub fn get_exe_path() -> PathBuf { let default_path = std::path::PathBuf::from("./"); let current_exe = std::env::current_exe(); match current_exe { - Ok(exe) => { - match fs::read_link(&exe) { - Ok(f) => f.parent().map_or(default_path, std::path::Path::to_path_buf), - Err(_) => exe.parent().map_or(default_path, std::path::Path::to_path_buf) - } - } - Err(_) => default_path + Ok(exe) => match fs::read_link(&exe) { + Ok(f) => f.parent().map_or(default_path, std::path::Path::to_path_buf), + Err(_) => exe.parent().map_or(default_path, std::path::Path::to_path_buf), + }, + Err(_) => default_path, } } pub fn get_home_path() -> PathBuf { - std::env::var(DEFAULT_HOME_ENV_VAR) - .ok() - .filter(|p| !p.trim().is_empty()) - .map_or_else(get_exe_path, PathBuf::from) + std::env::var(DEFAULT_HOME_ENV_VAR).ok().filter(|p| !p.trim().is_empty()).map_or_else(get_exe_path, PathBuf::from) } -pub fn get_default_web_root_path() -> PathBuf { - get_default_web_root_path_for_home(get_home_path().as_path()) -} +pub fn get_default_web_root_path() -> PathBuf { get_default_web_root_path_for_home(get_home_path().as_path()) } pub fn get_default_web_root_path_for_home(home_path: &Path) -> PathBuf { if let Some(web_root) = env::var(DEFAULT_WEB_ROOT_ENV_VAR).ok().filter(|p| !p.trim().is_empty()) { let configured = PathBuf::from(web_root); - return if configured.is_absolute() { - configured - } else { - home_path.join(configured) - }; + return if configured.is_absolute() { configured } else { home_path.join(configured) }; } get_default_path_for_home(home_path, DEFAULT_WEB_DIR) } +pub fn get_default_path(file: &str) -> PathBuf { get_default_path_for_home(get_home_path().as_path(), file) } -pub fn get_default_path(file: &str) -> PathBuf { - get_default_path_for_home(get_home_path().as_path(), file) -} - -pub fn get_default_path_for_home(home_path: &Path, file: &str) -> PathBuf { - home_path.join(file) -} +pub fn get_default_path_for_home(home_path: &Path, file: &str) -> PathBuf { home_path.join(file) } pub fn get_default_file_path(config_path: &str, file: &str) -> String { let path: PathBuf = PathBuf::from(config_path); @@ -97,9 +85,7 @@ pub fn get_default_file_path(config_path: &str, file: &str) -> String { } #[inline] -pub fn get_default_user_file_path(config_path: &str) -> String { - get_default_file_path(config_path, USER_FILE) -} +pub fn get_default_user_file_path(config_path: &str) -> String { get_default_file_path(config_path, USER_FILE) } #[inline] pub fn get_default_user_group_file_path(config_path: &str) -> String { @@ -107,29 +93,19 @@ pub fn get_default_user_group_file_path(config_path: &str) -> String { } #[inline] -pub fn get_default_config_path() -> String { - get_default_path(CONFIG_PATH).to_str().unwrap_or(CONFIG_PATH).to_string() -} +pub fn get_default_config_path() -> String { get_default_path(CONFIG_PATH).to_str().unwrap_or(CONFIG_PATH).to_string() } #[inline] -pub fn get_default_config_file_path(config_path: &str) -> String { - get_default_file_path(config_path, CONFIG_FILE) -} +pub fn get_default_config_file_path(config_path: &str) -> String { get_default_file_path(config_path, CONFIG_FILE) } #[inline] -pub fn get_default_sources_file_path(config_path: &str) -> String { - get_default_file_path(config_path, SOURCE_FILE) -} +pub fn get_default_sources_file_path(config_path: &str) -> String { get_default_file_path(config_path, SOURCE_FILE) } #[inline] -pub fn get_default_mappings_path(config_path: &str) -> String { - get_default_file_path(config_path, MAPPING_FILE) -} +pub fn get_default_mappings_path(config_path: &str) -> String { get_default_file_path(config_path, MAPPING_FILE) } #[inline] -pub fn get_default_templates_path(config_path: &str) -> String { - get_default_file_path(config_path, TEMPLATE_FILE) -} +pub fn get_default_templates_path(config_path: &str) -> String { get_default_file_path(config_path, TEMPLATE_FILE) } fn resolve_config_scoped_path(config_path: &str, candidate: &str) -> String { let candidate_path = PathBuf::from(candidate.trim()).clean(); @@ -226,30 +202,21 @@ pub fn resolve_directory_path(input: &str) -> String { let final_path = resolved_path.unwrap_or_else(|| current_dir.join(input)); - final_path - .canonicalize() - .map_or_else( - |_| { - error!("Path not found {}", final_path.display()); - String::from("./") - }, - |ap| String::from(ap.to_str().unwrap_or("./")), - ) + final_path.canonicalize().map_or_else( + |_| { + error!("Path not found {}", final_path.display()); + String::from("./") + }, + |ap| String::from(ap.to_str().unwrap_or("./")), + ) } -pub fn file_exists(path: &str) -> bool { - std::fs::metadata(path).is_ok_and(|metadata| metadata.is_file()) -} +pub fn file_exists(path: &str) -> bool { std::fs::metadata(path).is_ok_and(|metadata| metadata.is_file()) } - -pub async fn file_exists_async(path: &Path) -> bool { - tokio::fs::try_exists(path).await.unwrap_or(false) -} +pub async fn file_exists_async(path: &Path) -> bool { tokio::fs::try_exists(path).await.unwrap_or(false) } #[inline] -pub fn open_file(file_name: &Path) -> Result { - File::open(file_name) -} +pub fn open_file(file_name: &Path) -> Result { File::open(file_name) } pub async fn persist_file(persist_file: Option, text: &str) { if let Some(path_buf) = persist_file { @@ -269,18 +236,20 @@ pub fn prepare_persist_path(file_name: &str, date_prefix: &str) -> PathBuf { } pub fn get_file_path(wd: &str, path: Option) -> Option { - path.and_then(|p| if p.is_relative() { - let base = PathBuf::from(wd).clean(); - let joined = base.join(&p).clean(); - // Relative segments like ../ must not escape the working directory - if joined.starts_with(&base) { - Some(joined) + path.and_then(|p| { + if p.is_relative() { + let base = PathBuf::from(wd).clean(); + let joined = base.join(&p).clean(); + // Relative segments like ../ must not escape the working directory + if joined.starts_with(&base) { + Some(joined) + } else { + warn!("Relative path escapes working directory and is ignored: {}", p.display()); + None + } } else { - warn!("Relative path escapes working directory and is ignored: {}", p.display()); - None + Some(p) } - } else { - Some(p) }) } @@ -290,7 +259,7 @@ pub fn add_prefix_to_filename(path: &Path, prefix: &str, ext: Option<&str>) -> P let result = path.with_file_name(new_file_name); match ext { None => result, - Some(extension) => result.with_extension(extension) + Some(extension) => result.with_extension(extension), } } @@ -313,13 +282,9 @@ pub fn check_write(res: &std::io::Result<()>) -> Result<(), std::io::Error> { // path.with_extension(format!("{}{ext}", &extension.unwrap_or_default())) // } - #[inline] pub fn sanitize_filename(file_name: &str) -> String { - file_name - .chars() - .map(|c| if c.is_alphanumeric() || c == '_' || c == '-' { c } else { '_' }) - .collect() + file_name.chars().map(|c| if c.is_alphanumeric() || c == '_' || c == '-' { c } else { '_' }).collect() } #[inline] @@ -334,12 +299,7 @@ pub async fn async_append_or_create_file(path: &Path) -> std::io::Result tokio::io::Result { - tokio_fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(path) - .await + tokio_fs::OpenOptions::new().write(true).create(true).truncate(true).open(path).await } #[inline] @@ -362,6 +322,67 @@ pub async fn async_open_readonly_file(path: &Path) -> std::io::Result Result { + // Handle batch:// scheme: strip prefix and treat remainder as file path. + if let Some(path_str) = file_uri.strip_prefix(shared::utils::BATCH_SCHEME_PREFIX) { + let path = Path::new(path_str); + return if path.is_absolute() { Ok(path.to_path_buf()) } else { resolve_relative_path(path_str) }; + } + let raw_path = Path::new(file_uri); + if raw_path.is_absolute() { + return Ok(raw_path.to_path_buf()); + } + if file_uri.parse::().is_ok() { + Err(shared::error::string_to_io_error(format!( + "Unsupported URL scheme for batch CSV, use batch:// instead: {file_uri}" + ))) + } else { + resolve_relative_path(file_uri) + } +} + +/// The parent of `path`, or `.` when it has none. +/// +/// Used wherever a file has to be published and then its directory entry +/// synchronized: an empty parent means "the current directory", not "no parent". +pub fn parent_or_dot(path: &Path) -> &Path { + path.parent().filter(|parent| !parent.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")) +} + +pub fn same_parent_directory(left: &Path, right: &Path) -> bool { parent_or_dot(left) == parent_or_dot(right) } + +/// Rejects a staging/published pair that a rename cannot atomically publish. +pub fn require_same_parent_directory(staging: &Path, published: &Path) -> std::io::Result<()> { + if same_parent_directory(staging, published) { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "staging path {} and published path {} must share one parent directory", + staging.display(), + published.display() + ), + )) + } +} + +/// Removes `path`, treating "already gone" as success. +pub fn remove_file_if_exists(path: &Path) -> std::io::Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => { + Err(std::io::Error::new(error.kind(), format!("failed to remove file {}: {error}", path.display()))) + } + } +} + pub fn rename_or_copy(src: &Path, dest: &Path, remove_old: bool) -> std::io::Result<()> { // Try to rename the file if fs::rename(src, dest).is_err() { @@ -408,8 +429,7 @@ pub fn collect_yaml_files(path: &Path) -> std::io::Result> { } pub fn prepare_file_path(persist: Option<&str>, storage_dir: &str, action: &str) -> Option { - let persist_file: Option = - persist.map(|persist_path| prepare_persist_path(persist_path, action)); + let persist_file: Option = persist.map(|persist_path| prepare_persist_path(persist_path, action)); if persist_file.is_some() { let file_path = get_file_path(storage_dir, persist_file); debug_if_enabled!("persist to file: {}", file_path.as_ref().map_or(Cow::from("?"), |p| p.to_string_lossy())); @@ -457,15 +477,10 @@ pub fn resolve_relative_path(relative: &str) -> std::io::Result { Ok(current_dir.join(relative)) } -pub fn is_directory(path: &str) -> bool { - PathBuf::from(path).is_dir() -} +pub fn is_directory(path: &str) -> bool { PathBuf::from(path).is_dir() } // Cleans up the directories and deletes all files whic are not listed in the list -pub fn cleanup_unlisted_files_with_suffix( - keep_files: &Vec, - suffix: &str, -) -> std::io::Result<()> { +pub fn cleanup_unlisted_files_with_suffix(keep_files: &Vec, suffix: &str) -> std::io::Result<()> { let keep_set: HashSet<_> = keep_files.iter().collect(); let mut dirs: HashSet<&Path> = HashSet::new(); @@ -484,14 +499,9 @@ pub fn cleanup_unlisted_files_with_suffix( } let delete = { - let zero_size = entry - .metadata() - .is_ok_and(|m| m.len() == 0); + let zero_size = entry.metadata().is_ok_and(|m| m.len() == 0); - let suffix_match = path - .file_name() - .and_then(|n| n.to_str()) - .is_some_and(|name| name.ends_with(suffix)); + let suffix_match = path.file_name().and_then(|n| n.to_str()).is_some_and(|name| name.ends_with(suffix)); zero_size || suffix_match }; @@ -506,9 +516,7 @@ pub fn cleanup_unlisted_files_with_suffix( } pub fn truncate_filename(path: &Path, max_len: usize) -> PathBuf { - let file_name = path.file_name() - .and_then(|s| s.to_str()) - .unwrap_or_default(); + let file_name = path.file_name().and_then(|s| s.to_str()).unwrap_or_default(); let truncated_name = if file_name.chars().count() > max_len { // If a filename extension exists, keep it @@ -516,9 +524,7 @@ pub fn truncate_filename(path: &Path, max_len: usize) -> PathBuf { let ext_len = ext.len() + 1; // +1 for the dot if max_len > ext_len { let name_len = max_len - ext_len; - let name_without_ext = path.file_stem() - .and_then(|s| s.to_str()) - .unwrap_or_default(); + let name_without_ext = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default(); let truncated = name_without_ext.chars().take(name_len).collect::(); format!("{truncated}.{ext}") } else { @@ -536,18 +542,11 @@ pub fn truncate_filename(path: &Path, max_len: usize) -> PathBuf { } pub fn normalize_string_path(path: &str) -> String { - std::path::PathBuf::from(path) - .components() - .collect::() - .to_string_lossy() - .to_string() + std::path::PathBuf::from(path).components().collect::().to_string_lossy().to_string() } pub fn get_file_extension(path: &str) -> Option { - Path::new(path) - .extension() - .and_then(|s| s.to_str()) - .map(ToString::to_string) + Path::new(path).extension().and_then(|s| s.to_str()).map(ToString::to_string) } #[cfg(test)] @@ -596,10 +595,8 @@ mod tests { #[test] fn test_resolve_template_default_path_uses_config_dir() { let config_path = std::env::temp_dir().join("tuliprox").join("settings").join("config"); - let resolved = resolve_template_file_path( - config_path.to_string_lossy().as_ref(), - Some("./config/template.yml"), - ); + let resolved = + resolve_template_file_path(config_path.to_string_lossy().as_ref(), Some("./config/template.yml")); let expected = config_path.join("template.yml"); assert_eq!(PathBuf::from(resolved), expected); } @@ -623,8 +620,7 @@ mod tests { #[test] fn test_resolve_template_persist_file_for_directory_path() { let config_path = std::env::temp_dir().join("tuliprox").join("settings").join("config"); - let resolved = - resolve_template_persist_file_path(Some("template.d"), config_path.to_string_lossy().as_ref()); + let resolved = resolve_template_persist_file_path(Some("template.d"), config_path.to_string_lossy().as_ref()); let expected = config_path.join("template.d").join("template.yml"); assert_eq!(PathBuf::from(resolved), expected); } diff --git a/backend/src/utils/file/mapping_reader.rs b/backend/core/src/utils/file/mapping_reader.rs similarity index 82% rename from backend/src/utils/file/mapping_reader.rs rename to backend/core/src/utils/file/mapping_reader.rs index 7bdf2e353..35887551b 100644 --- a/backend/src/utils/file/mapping_reader.rs +++ b/backend/core/src/utils/file/mapping_reader.rs @@ -1,38 +1,42 @@ -use std::collections::HashMap; -use crate::model::{Mappings}; -use shared::error::TuliproxError; -use crate::utils::{config_file_reader, open_file}; -use log::{warn}; -use std::path::{Path, PathBuf}; -use shared::model::{MappingDefinitionDto, MappingDto, MappingsDto, PatternTemplate}; +use crate::{ + model::Mappings, + utils::{config_file_reader, open_file}, +}; +use log::warn; +use shared::{ + error::TuliproxError, + model::{MappingDefinitionDto, MappingDto, MappingsDto, PatternTemplate}, +}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; fn read_mapping(mapping_file: &Path, resolve_var: bool) -> Result, TuliproxError> { if let Ok(file) = open_file(mapping_file) { let maybe_mapping: Result = serde_saphyr::from_reader(config_file_reader(file, resolve_var)); return match maybe_mapping { Ok(mapping) => Ok(Some(mapping)), - Err(err) => { - Err(TuliproxError::Config(format!("{err}"))) - } + Err(err) => Err(TuliproxError::Config(format!("{err}"))), }; } warn!("Can't read mapping file: {}", mapping_file.to_str().unwrap_or("?")); Ok(None) } -fn read_mappings_from_file(mappings_file: &Path, resolve_env: bool) -> Result, MappingsDto)>, TuliproxError> { +fn read_mappings_from_file( + mappings_file: &Path, + resolve_env: bool, +) -> Result, MappingsDto)>, TuliproxError> { match read_mapping(mappings_file, resolve_env) { - Ok(mappings) => { - match mappings { - None => Ok(None), - Some(mappings_cfg) => Ok(Some((vec![mappings_file.to_path_buf()], mappings_cfg))), - } - } + Ok(mappings) => match mappings { + None => Ok(None), + Some(mappings_cfg) => Ok(Some((vec![mappings_file.to_path_buf()], mappings_cfg))), + }, Err(err) => Err(err), } } - fn merge_mappings(mappings: Vec<(PathBuf, MappingDto)>) -> Result, TuliproxError> { let mut result: Vec = Vec::with_capacity(mappings.len()); let mut index_by_id: HashMap = HashMap::with_capacity(mappings.len()); @@ -88,7 +92,10 @@ fn merge_mapping_definitions(mappings: Vec<(PathBuf, MappingsDto)>) -> Result Result, MappingsDto)>, TuliproxError> { +fn read_mappings_from_directory( + path: &Path, + resolve_env: bool, +) -> Result, MappingsDto)>, TuliproxError> { let mut files = crate::utils::collect_yaml_files(path) .map_err(|err| TuliproxError::Io(format!("Failed to read mappings {err}")))?; @@ -101,13 +108,10 @@ fn read_mappings_from_directory(path: &Path, resolve_env: bool) -> Result