diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7ba7b3172..96fbc0d15 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -150,14 +150,15 @@ jobs: (github.event_name == 'workflow_dispatch' && github.event.inputs.branch != 'master') id: version_commit run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config user.name "euzu" + git config user.email "euzu@proton.me" git add . if git diff --cached --quiet; then echo "did_commit=false" >> "$GITHUB_OUTPUT" else VERSION=$(grep '^version' backend/Cargo.toml | head -n1 | cut -d'"' -f2) git commit -m "ci: bump version v$VERSION" + git tag -a "v$VERSION" -m "ci: tag version v$VERSION" echo "did_commit=true" >> "$GITHUB_OUTPUT" fi @@ -221,8 +222,8 @@ jobs: ) && steps.version_commit.outputs.did_commit == 'true' run: | - # Use origin HEAD to push to the correct remote branch - git push origin HEAD + # Use origin HEAD to push to the correct remote branch, and push tags + git push origin HEAD --tags - name: Publish release bundles to GitHub Release if: >- diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index f2d729c0f..272cfb83b 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -11,7 +11,6 @@ use crate::{ }, auth::Fingerprint, model::{ConfigInput, ConfigTarget, ProxyUserCredentials}, - repository::{ConnectFailureReason, FailureStage}, utils::{ async_file_reader, async_file_writer, create_new_file_for_write, debug_if_enabled, get_file_extension, request, request::{content_type_from_ext, parse_range, send_with_retry_and_provider}, @@ -147,7 +146,7 @@ pub(crate) struct ConnectFailedAttempt<'a> { pub fingerprint: &'a Fingerprint, pub user: &'a ProxyUserCredentials, pub stream_channel: StreamChannel, - pub provider_name: &'a str, + pub provider_name: Arc, pub req_headers: &'a HeaderMap, pub reason: ConnectFailureReason, pub failure_stage: FailureStage, @@ -172,10 +171,24 @@ pub(crate) fn record_connect_failed_attempt(attempt: ConnectFailedAttempt<'_>) { None, None, ); + // Resolve target_name from target_id using the stable target config name. + let target_name = attempt + .app_state + .app_config + .get_target_by_id(info.channel.target_id) + .as_deref() + .map(|t| (&t.name).intern()); attempt .app_state .connection_manager - .record_connect_failed(&info, attempt.reason, attempt.failure_stage); + .record_connect_failed_with_provider_failure( + &info, + attempt.reason, + attempt.failure_stage, + None, + None, + target_name, + ); } fn admission_failure_video_type(reason: ConnectFailureReason) -> Option { @@ -192,7 +205,7 @@ pub(crate) fn admission_failure_response( fingerprint: &Fingerprint, user: &ProxyUserCredentials, stream_channel: StreamChannel, - provider_name: &str, + provider_name: Arc, req_headers: &HeaderMap, reason: ConnectFailureReason, ) -> axum::response::Response { @@ -333,7 +346,7 @@ macro_rules! try_result_not_found { use crate::api::panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}; pub use internal_server_error; use shared::error::TuliproxError; -use shared::model::AdmissionStrategy; +use shared::model::{AdmissionStrategy, ConnectFailureReason, FailureStage}; use shared::utils::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}; pub use try_option_bad_request; pub use try_option_forbidden; @@ -382,12 +395,12 @@ pub async fn serve_file(file_path: &Path, mime_type: String, cache_control: Opti match tokio::fs::try_exists(file_path).await { Ok(exists) => { if !exists { - return axum::http::StatusCode::NOT_FOUND.into_response(); + return StatusCode::NOT_FOUND.into_response(); } } Err(err) => { error!("Failed to open file {}, {err:?}", file_path.display()); - return axum::http::StatusCode::NOT_FOUND.into_response(); + return StatusCode::NOT_FOUND.into_response(); } } @@ -399,16 +412,16 @@ pub async fn serve_file(file_path: &Path, mime_type: String, cache_control: Opti }); let reader = async_file_reader(file); - let stream = tokio_util::io::ReaderStream::new(reader); - let body = axum::body::Body::from_stream(stream); + let stream = ReaderStream::new(reader); + let body = Body::from_stream(stream); let mut builder = axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, mime_type) - .header(axum::http::header::CACHE_CONTROL, cache_control.unwrap_or("no-cache")); + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime_type) + .header(header::CACHE_CONTROL, cache_control.unwrap_or("no-cache")); if let Some(lm) = last_modified { - builder = builder.header(axum::http::header::LAST_MODIFIED, lm); + builder = builder.header(header::LAST_MODIFIED, lm); } try_unwrap_body!(builder.body(body)) @@ -970,7 +983,7 @@ async fn create_stream_response_details( return Ok(create_panel_api_provisioning_stream_details( app_state, input, - guard_provider_name.clone(), + guard_provider_name.clone().or_else(|| Some(input.name.clone())), &grace_period_options, fingerprint.addr, virtual_id, @@ -981,10 +994,13 @@ async fn create_stream_response_details( // custom stream means we display our own stream like connection exhausted, channel-unavailable... ProviderStreamState::Custom(provider_stream) => { let (stream, stream_info) = provider_stream; + // When allocation is exhausted or no connection was acquired, guard_provider_name is None. + // Use input.name as fallback so the provider field is never empty. + let provider_name = guard_provider_name.clone().unwrap_or_else(|| input.name.clone()); Ok(StreamDetails { stream, stream_info, - provider_name: guard_provider_name.clone(), + provider_name: Some(provider_name), request_url: None, grace_period: grace_period_options, provider_grace_active: false, @@ -1489,7 +1505,7 @@ pub async fn stream_response( fingerprint, user, stream_channel: stream_channel.clone(), - provider_name: input.name.as_ref(), + provider_name: input.name.clone(), req_headers, reason: ConnectFailureReason::UserConnectionsExhausted, failure_stage: FailureStage::Admission, @@ -1950,7 +1966,7 @@ pub async fn local_stream_response( fingerprint, user, stream_channel: pli.clone(), - provider_name: input.name.as_ref(), + provider_name: input.name.clone(), req_headers, reason: ConnectFailureReason::UserConnectionsExhausted, failure_stage: FailureStage::Admission, @@ -3263,7 +3279,7 @@ mod tests { priority: 0, soft_priority: 10, fingerprint: &first_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &create_test_live_channel("http://provider-1.example/live/1.ts"), user_agent: std::borrow::Cow::Borrowed("ua"), session_token: Some("tok-first"), @@ -3283,7 +3299,7 @@ mod tests { priority: 0, soft_priority: 10, fingerprint: &second_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &create_test_live_channel("http://provider-1.example/live/2.ts"), user_agent: std::borrow::Cow::Borrowed("ua"), session_token: Some("tok-second"), @@ -3344,7 +3360,7 @@ mod tests { priority: 0, soft_priority: 10, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &create_test_live_channel("http://provider-1.example/movie/1.mkv"), user_agent: std::borrow::Cow::Borrowed("ua"), session_token: Some("vod-session"), @@ -3424,7 +3440,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &victim_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &victim_channel, user_agent: std::borrow::Cow::Borrowed("player/1.0"), session_token: Some("session-victim"), @@ -3441,7 +3457,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &winner_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &winner_channel, user_agent: std::borrow::Cow::Borrowed("winner/1.0"), session_token: Some("session-winner"), @@ -3517,7 +3533,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &victim_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &victim_channel, user_agent: std::borrow::Cow::Borrowed("player/1.0"), session_token: Some("session-victim"), @@ -3534,7 +3550,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &winner_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &winner_channel, user_agent: std::borrow::Cow::Borrowed("winner/1.0"), session_token: Some("session-winner"), @@ -3606,7 +3622,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &victim_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &channel, user_agent: std::borrow::Cow::Borrowed("player/1.0"), session_token: Some("session-victim"), @@ -3623,7 +3639,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &winner_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &channel, user_agent: std::borrow::Cow::Borrowed("player/1.0"), session_token: Some("session-winner"), @@ -3697,7 +3713,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &victim_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &victim_channel, user_agent: std::borrow::Cow::Borrowed("player/1.0"), session_token: Some("session-victim"), @@ -3714,7 +3730,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &winner_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &winner_channel, user_agent: std::borrow::Cow::Borrowed("winner/1.0"), session_token: Some("session-winner"), @@ -3939,7 +3955,7 @@ mod tests { priority: user.priority, soft_priority: user.soft_priority, fingerprint: &normal_fingerprint, - provider: input.name.as_ref(), + provider: input.name.clone(), stream_channel: &normal_channel, user_agent: Cow::Borrowed("ua"), session_token: Some("normal-session"), @@ -4364,7 +4380,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &first_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &create_test_live_channel("http://provider-1.example/vod/5001.ts"), user_agent: std::borrow::Cow::Borrowed("ua"), session_token: Some(&first_token), @@ -4427,7 +4443,7 @@ mod tests { priority: user.priority, soft_priority: user.soft_priority, fingerprint: &first_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &create_test_live_channel("http://provider-1.example/vod/6001.ts"), user_agent: std::borrow::Cow::Borrowed("ua"), session_token: Some(&first_token), @@ -4467,7 +4483,7 @@ mod tests { priority: user.priority, soft_priority: user.soft_priority, fingerprint: &second_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &create_test_live_channel("http://provider-1.example/vod/6002.ts"), user_agent: std::borrow::Cow::Borrowed("ua"), session_token: Some(&second_token), diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 64d51a51a..b3c030b82 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -25,6 +25,7 @@ use shared::{ }; use std::sync::Arc; use url::Url; +use shared::model::ConnectFailureReason; const PLAYLIST_TEMPLATE: &str = r"#EXTM3U #EXT-X-VERSION:3 @@ -349,10 +350,10 @@ async fn resolve_stream_channel( #[allow(clippy::too_many_lines)] async fn hls_api_stream( fingerprint: Fingerprint, - req_headers: axum::http::HeaderMap, + req_headers: HeaderMap, axum::extract::Path(params): axum::extract::Path, axum::extract::State(app_state): axum::extract::State>, -) -> impl axum::response::IntoResponse + Send { +) -> impl IntoResponse + Send { let (user, target) = try_option_bad_request!( app_state.app_config.get_target_for_user(¶ms.username, ¶ms.password), false, @@ -380,9 +381,9 @@ async fn hls_api_stream( &fingerprint, &user, denied_channel, - input.name.as_ref(), + input.name.clone(), &req_headers, - crate::repository::ConnectFailureReason::UserAccountExpired, + ConnectFailureReason::UserAccountExpired, ); } @@ -408,9 +409,9 @@ async fn hls_api_stream( &fingerprint, &user, stream_channel, - &session.provider, + session.provider.clone(), &req_headers, - crate::repository::ConnectFailureReason::UserConnectionsExhausted, + ConnectFailureReason::UserConnectionsExhausted, ); } @@ -421,9 +422,9 @@ async fn hls_api_stream( &fingerprint, &user, stream_channel, - &session.provider, + session.provider.clone(), &req_headers, - crate::repository::ConnectFailureReason::ProviderConnectionsExhausted, + ConnectFailureReason::ProviderConnectionsExhausted, ); } @@ -495,15 +496,20 @@ async fn hls_api_stream( session.permission = connection_permission; session.connection_kind = Some(connection_kind); if connection_permission == UserConnectionPermission::Exhausted { + let provider = if session.provider.is_empty() { + input.name.clone() + } else { + session.provider.clone() + }; let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, &session.stream_url).await; return admission_failure_response( &app_state, &fingerprint, &user, stream_channel, - input.name.as_ref(), + provider, &req_headers, - crate::repository::ConnectFailureReason::UserConnectionsExhausted, + ConnectFailureReason::UserConnectionsExhausted, ); } diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index 3fe083fb0..6d7bd1377 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -27,6 +27,7 @@ use shared::{ utils::{concat_path, extract_extension_from_url, sanitize_sensitive_info}, }; use std::sync::Arc; +use shared::model::ConnectFailureReason; async fn m3u_api(api_req: &UserApiRequest, app_state: &AppState) -> impl IntoResponse + Send { api_req.log_sanitized("m3u_api"); @@ -125,9 +126,9 @@ async fn m3u_api_stream( fingerprint, &user, pli.to_stream_channel(target.id), - pli.input_name.as_ref(), + pli.input_name.clone(), req_headers, - crate::repository::ConnectFailureReason::UserAccountExpired, + ConnectFailureReason::UserAccountExpired, ); } @@ -210,9 +211,9 @@ async fn m3u_api_stream( fingerprint, &user, pli.to_stream_channel(target.id), - &session.provider, + session.provider.clone(), req_headers, - crate::repository::ConnectFailureReason::UserConnectionsExhausted, + ConnectFailureReason::UserConnectionsExhausted, ); } @@ -222,9 +223,9 @@ async fn m3u_api_stream( fingerprint, &user, pli.to_stream_channel(target.id), - &session.provider, + session.provider.clone(), req_headers, - crate::repository::ConnectFailureReason::ProviderConnectionsExhausted, + ConnectFailureReason::ProviderConnectionsExhausted, ); } if session.virtual_id == virtual_id && is_seek_request(cluster, req_headers).await { @@ -292,9 +293,9 @@ async fn m3u_api_stream( fingerprint, &user, pli.to_stream_channel(target.id), - input.name.as_ref(), + input.name.clone(), req_headers, - crate::repository::ConnectFailureReason::UserConnectionsExhausted, + ConnectFailureReason::UserConnectionsExhausted, ); } diff --git a/backend/src/api/endpoints/stream_history_api.rs b/backend/src/api/endpoints/stream_history_api.rs index 4aaedc65a..7b3adc0dd 100644 --- a/backend/src/api/endpoints/stream_history_api.rs +++ b/backend/src/api/endpoints/stream_history_api.rs @@ -4,17 +4,18 @@ use std::path::Path; use std::sync::Arc; use axum::extract::{Path as AxumPath, Query, State}; -use axum::response::{IntoResponse, Response}; use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; use serde::{Deserialize, Serialize}; - -use crate::api::model::AppState; +use shared::model::StreamHistoryEventType; use crate::api::api_utils::json_or_bin_response; use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader; +use crate::api::model::AppState; +use crate::model::{StreamHistoryRecord}; use crate::repository::{QosSnapshotRecord, QosSnapshotRepository, StreamHistoryFileReader}; use crate::utils::stream_history_viewer::{ - CompiledFilter, StreamHistoryQuery, TimeRange, - discover_files, resolve_time_range, + discover_files, resolve_time_range, CompiledFilter, + StreamHistoryQuery, TimeRange, }; #[derive(Deserialize)] @@ -41,7 +42,7 @@ struct ErrorResponse { #[derive(Debug, Clone, Serialize, PartialEq, Eq)] pub(crate) struct ProviderSummary { - pub provider_name: String, + pub provider_name: Arc, pub session_count: u64, pub disconnect_count: u64, pub total_bytes_sent: u64, @@ -105,27 +106,29 @@ pub(crate) async fn stream_history_query( None => CompiledFilter::compile(&HashMap::new()).unwrap_or_else(|_| unreachable!()), }; - // Run blocking file I/O on the blocking thread pool - let result = tokio::task::spawn_blocking(move || { - collect_records(&history_dir, &time_range, &filters) - }) - .await; + // Collect file records and batch records on blocking thread, then combine. + // batch_records must be fetched on a real OS thread (not the async thread), + // so we include it inside spawn_blocking where blocking_recv() is safe. + let app_state_clone = Arc::clone(&app_state); + let mut file_records = collect_records(&history_dir, &time_range, &filters).await.unwrap_or_default(); + let batch_records = if let Some(hw) = app_state_clone + .connection_manager + .history_writer() + .load() + .as_ref() + { + hw.get_current_batch().await.unwrap_or_else(|_| Vec::new()) + } else { + Vec::new() + }; - match result { - Ok(Ok(records)) => axum::Json(records).into_response(), - Ok(Err(e)) => { - if e.kind() == io::ErrorKind::NotFound { - // Return empty array when no files found, not an error - axum::Json(Vec::::new()).into_response() - } else { - error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read history: {e}")) - } - } - Err(e) => error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("History query task failed: {e}")), - } + let mut all_records = batch_records; + all_records.append(&mut file_records); + axum::Json(all_records).into_response() } pub(crate) async fn stream_history_summary_query( + ExtractAcceptHeader(accept): ExtractAcceptHeader, State(app_state): State>, Query(params): Query, ) -> Response { @@ -153,23 +156,34 @@ pub(crate) async fn stream_history_summary_query( None => CompiledFilter::compile(&HashMap::new()).unwrap_or_else(|_| unreachable!()), }; - let result = tokio::task::spawn_blocking(move || { - collect_records(&history_dir, &time_range, &filters) - .map(|records| aggregate_provider_summaries(records.as_slice())) - }) - .await; + // Chain batch records (in-memory) with file records iterator. + // batch_records must be fetched on a real OS thread (not the async thread), + // so we include it inside spawn_blocking where blocking_recv() is safe. + let filters_arc = Arc::new(filters); + let app_state_clone = Arc::clone(&app_state); + // Fetch batch records from within the blocking thread so blocking_recv() is safe + let batch_records = if let Some(hw) = app_state_clone + .connection_manager + .history_writer() + .load() + .as_ref() + { + hw.get_current_batch().await.unwrap_or_else(|_| Vec::new()) + } else { + Vec::new() + }; - match result { - Ok(Ok(summaries)) => axum::Json(summaries).into_response(), - Ok(Err(e)) => { - if e.kind() == io::ErrorKind::NotFound { - axum::Json(Vec::::new()).into_response() - } else { - error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to summarize history: {e}")) - } + let file_iter = match collect_records_iter(&history_dir, time_range, filters_arc).await { + Ok(iter) => iter, + Err(e) => { + log::error!("Failed to discover stream history files for summary: {e}"); + return error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to discover history files: {e}")); } - Err(e) => error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("History summary task failed: {e}")), - } + }; + + let combined = batch_records.into_iter().chain(file_iter); + let summaries = aggregate_provider_summaries_from_iter(combined); + json_or_bin_response(accept.as_deref(), &summaries).into_response() } pub(crate) async fn qos_snapshot_query( @@ -190,7 +204,7 @@ pub(crate) async fn qos_snapshot_query( let result = tokio::task::spawn_blocking(move || { collect_filtered_qos_snapshots(Path::new(&storage_dir), &filters, limit) }) - .await; + .await; match result { Ok(Ok(records)) => json_or_bin_response(accept.as_deref(), &records).into_response(), @@ -214,7 +228,8 @@ pub(crate) async fn qos_snapshot_detail_query( return error_response(StatusCode::SERVICE_UNAVAILABLE, "QoS aggregation is not enabled"); }; - let result = tokio::task::spawn_blocking(move || load_qos_snapshot(&storage_dir, &stream_identity_key)).await; + let result = tokio::task::spawn_blocking(move || load_qos_snapshot(&storage_dir, &stream_identity_key)) + .await; match result { Ok(Ok(Some(record))) => json_or_bin_response(accept.as_deref(), &record).into_response(), @@ -230,25 +245,98 @@ pub(crate) async fn qos_snapshot_detail_query( } } -fn collect_records( +/// Iterator over all filtered `StreamHistoryRecord`s from history files in a directory. +/// Filters are applied lazily — records are never collected into a Vec. +/// Errors opening files or reading records are logged but do not stop iteration. +struct RecordFileIter { + files: std::vec::IntoIter, + current_iter: Option>>>, + time_range: TimeRange, + filters: Arc, +} + +impl RecordFileIter { + async fn new(dir: &str, time_range: TimeRange, filters: Arc) -> io::Result { + let files = discover_files(Path::new(dir), &time_range).await?; + Ok(Self { + files: files.into_iter(), + current_iter: None, + time_range, + filters, + }) + } +} + +impl Iterator for RecordFileIter { + type Item = StreamHistoryRecord; + + fn next(&mut self) -> Option { + let (range_start, range_end) = self.time_range; + + loop { + if self.current_iter.is_none() { + let file = self.files.next()?; + let iter: Box>> = if file.is_archive { + match StreamHistoryFileReader::from_archive(&file.path, Some(self.time_range)) { + Ok((r, _)) => Box::new(r), + Err(e) => { + log::warn!("Failed to open archive {}: {e}", file.path.display()); + continue; + } + } + } else { + match StreamHistoryFileReader::from_pending(&file.path, Some(self.time_range)) { + Ok((r, _)) => Box::new(r), + Err(e) => { + log::warn!("Failed to open pending file {}: {e}", file.path.display()); + continue; + } + } + }; + self.current_iter = Some(iter); + } + + match self.current_iter.as_mut().and_then(std::iter::Iterator::next) { + Some(Ok(record)) => { + if record.event_ts_utc < range_start || record.event_ts_utc > range_end { + continue; + } + if !self.filters.matches(&record) { + continue; + } + return Some(record); + } + Some(Err(e)) => { + log::warn!("Failed to read record: {e}"); + } + None => { + self.current_iter = None; + } + } + } + } +} + +/// Returns a Vec of filtered records from history files. +/// Used by `stream_history_query` to collect all records before responding. +async fn collect_records( dir: &str, time_range: &TimeRange, filters: &CompiledFilter, -) -> io::Result> { - let files = discover_files(Path::new(dir), time_range)?; +) -> io::Result> { + let files = discover_files(Path::new(dir), time_range).await?; let (range_start, range_end) = *time_range; let mut records = Vec::new(); for file in &files { - let iter: Box>> = - if file.is_archive { - let (reader, _) = StreamHistoryFileReader::from_archive(&file.path, Some(*time_range))?; - Box::new(reader) - } else { - let (reader, _) = StreamHistoryFileReader::from_pending(&file.path, Some(*time_range))?; - Box::new(reader) - }; + let iter: Box>> = if file.is_archive { + let (reader, _) = StreamHistoryFileReader::from_archive(&file.path, Some(*time_range))?; + Box::new(reader) + } else { + let (reader, _) = StreamHistoryFileReader::from_pending(&file.path, Some(*time_range))?; + Box::new(reader) + }; for result in iter { let record = result?; @@ -265,8 +353,18 @@ fn collect_records( Ok(records) } +/// Returns a lazy iterator over filtered records from history files. +async fn collect_records_iter( + dir: &str, + time_range: TimeRange, + filters: Arc, +) -> io::Result { + RecordFileIter::new(dir, time_range, filters).await +} + +#[allow(dead_code)] pub(crate) fn aggregate_provider_summaries( - records: &[crate::repository::StreamHistoryRecord], + records: &[StreamHistoryRecord], ) -> Vec { #[derive(Default)] struct Acc { @@ -279,13 +377,63 @@ pub(crate) fn aggregate_provider_summaries( first_byte_count: u64, } - let mut by_provider: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut by_provider: std::collections::BTreeMap, Acc> = std::collections::BTreeMap::new(); for record in records { - let provider_name = record.provider_name.clone().unwrap_or_else(|| String::from("unknown")); + let provider_name = record.provider_name.clone().unwrap_or_else(|| Arc::from("unknown")); let acc = by_provider.entry(provider_name).or_default(); acc.session_count = acc.session_count.saturating_add(1); - if matches!(record.event_type, crate::repository::EventType::Disconnect) { + if matches!(record.event_type, StreamHistoryEventType::Disconnect) { + acc.disconnect_count = acc.disconnect_count.saturating_add(1); + } + acc.total_bytes_sent = acc.total_bytes_sent.saturating_add(record.bytes_sent.unwrap_or(0)); + if let Some(duration) = record.session_duration { + acc.total_duration = acc.total_duration.saturating_add(duration); + acc.duration_count = acc.duration_count.saturating_add(1); + } + if let Some(latency) = record.first_byte_latency_ms { + acc.total_first_byte_latency = acc.total_first_byte_latency.saturating_add(latency); + acc.first_byte_count = acc.first_byte_count.saturating_add(1); + } + } + + by_provider + .into_iter() + .map(|(provider_name, acc)| ProviderSummary { + provider_name, + session_count: acc.session_count, + disconnect_count: acc.disconnect_count, + total_bytes_sent: acc.total_bytes_sent, + avg_session_duration_secs: acc.total_duration.checked_div(acc.duration_count), + avg_first_byte_latency_ms: acc.total_first_byte_latency.checked_div(acc.first_byte_count), + }) + .collect() +} + +/// Lazily aggregates provider summaries from a (possibly infinite) iterator of records. +/// Unlike `aggregate_provider_summaries` which collects into a Vec first, this processes +/// records one-by-one without any intermediate allocation. +pub fn aggregate_provider_summaries_from_iter>( + records: I, +) -> Vec { + #[derive(Default)] + struct Acc { + session_count: u64, + disconnect_count: u64, + total_bytes_sent: u64, + total_duration: u64, + duration_count: u64, + total_first_byte_latency: u64, + first_byte_count: u64, + } + + let mut by_provider: std::collections::BTreeMap, Acc> = std::collections::BTreeMap::new(); + + for record in records { + let provider_name = record.provider_name.unwrap_or_else(|| Arc::from("unknown")); + let acc = by_provider.entry(provider_name).or_default(); + acc.session_count = acc.session_count.saturating_add(1); + if matches!(record.event_type, StreamHistoryEventType::Disconnect) { acc.disconnect_count = acc.disconnect_count.saturating_add(1); } acc.total_bytes_sent = acc.total_bytes_sent.saturating_add(record.bytes_sent.unwrap_or(0)); @@ -318,20 +466,13 @@ struct CompiledQosSnapshotFilter { input_name: Option, provider_name: Option, item_type: Option, - target_id: Option, + target_name: Option, } impl CompiledQosSnapshotFilter { fn compile(raw: &HashMap) -> Result { - let parse_u16 = |key: &str| -> Result, String> { - raw.get(key) - .filter(|value| !value.trim().is_empty()) - .map(|value| { - value - .parse::() - .map_err(|_| format!("Invalid QoS snapshot filter value for {key}: {value}")) - }) - .transpose() + let parse_string = |key: &str| -> Result, String> { + Ok(raw.get(key).filter(|value| !value.trim().is_empty()).cloned()) }; Ok(Self { @@ -339,7 +480,7 @@ impl CompiledQosSnapshotFilter { input_name: raw.get("input_name").cloned().filter(|value| !value.trim().is_empty()), provider_name: raw.get("provider_name").cloned().filter(|value| !value.trim().is_empty()), item_type: raw.get("item_type").cloned().filter(|value| !value.trim().is_empty()), - target_id: parse_u16("target_id")?, + target_name: parse_string("target_name")?, }) } @@ -347,10 +488,10 @@ impl CompiledQosSnapshotFilter { self.stream_identity_key .as_ref() .is_none_or(|value| snapshot.stream_identity_key == *value) - && self.input_name.as_ref().is_none_or(|value| snapshot.input_name == *value) - && self.provider_name.as_ref().is_none_or(|value| snapshot.provider_name == *value) - && self.item_type.as_ref().is_none_or(|value| snapshot.item_type == *value) - && self.target_id.is_none_or(|value| snapshot.target_id == value) + && self.input_name.as_ref().is_none_or(|value| snapshot.input_name.as_ref() == value) + && self.provider_name.as_ref().is_none_or(|value| snapshot.provider_name.as_ref() == value) + && self.item_type.as_ref().is_none_or(|value| snapshot.item_type.as_str() == value) + && self.target_name.as_ref().is_none_or(|value| snapshot.target_name.as_ref() == value) } } @@ -387,11 +528,13 @@ fn load_qos_snapshot(storage_dir: &str, stream_identity_key: &str) -> io::Result #[cfg(test)] mod tests { + use shared::model::{DisconnectReason, PlaylistItemType}; use super::*; use crate::model::{QosAggregationConfig, ResourceRetryConfig, ReverseProxyConfig}; use crate::repository::{ - DisconnectReason, EventType, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow, StreamHistoryRecord, + QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow, }; + use shared::utils::Internable; fn make_record( provider_name: &str, @@ -402,17 +545,17 @@ mod tests { ) -> StreamHistoryRecord { StreamHistoryRecord { schema_version: 1, - event_type: EventType::Disconnect, + event_type: StreamHistoryEventType::Disconnect, event_ts_utc: 1, partition_day_utc: String::from("2026-03-22"), session_id: 1, source_addr: None, api_username: Some(String::from("alice")), - provider_name: Some(provider_name.to_string()), + provider_name: Some(provider_name.intern()), provider_username: None, - input_name: Some(String::from("input")), + input_name: Some("input".intern()), virtual_id: Some(1), - item_type: Some(String::from("live")), + item_type: Some(PlaylistItemType::Live), title: Some(String::from("Title")), group: None, country: None, @@ -442,7 +585,7 @@ mod tests { connect_failure_reason: None, disconnect_reason, previous_session_id: None, - target_id: Some(1), + target_name: None, } } @@ -455,7 +598,7 @@ mod tests { ]); assert_eq!(summaries.len(), 2); - assert_eq!(summaries[0].provider_name, "acme"); + assert_eq!(summaries[0].provider_name.as_ref(), "acme"); assert_eq!(summaries[0].session_count, 2); assert_eq!(summaries[0].total_bytes_sent, 400); assert_eq!(summaries[0].avg_session_duration_secs, Some(15)); @@ -467,17 +610,17 @@ mod tests { stream_identity_key: &str, input_name: &str, provider_name: &str, - target_id: u16, + target_name: &str, score_24h: u8, ) -> QosSnapshotRecord { QosSnapshotRecord { stream_identity_key: stream_identity_key.to_string(), - input_name: input_name.to_string(), - target_id, - provider_name: provider_name.to_string(), + input_name: input_name.intern(), + target_name: target_name.intern(), + provider_name: provider_name.intern(), provider_id: 1, virtual_id: 101, - item_type: "live".to_string(), + item_type: PlaylistItemType::Live, updated_at: 100, last_event_at: 99, window_24h: QosSnapshotWindow { @@ -494,13 +637,13 @@ mod tests { #[test] fn qos_snapshot_filter_matches_identity_and_provider_fields() { let snapshots = vec![ - make_qos_snapshot("stream-a", "input-a", "provider-a", 1, 81), - make_qos_snapshot("stream-b", "input-b", "provider-b", 2, 55), + make_qos_snapshot("stream-a", "input-a", "provider-a", "target-a", 81), + make_qos_snapshot("stream-b", "input-b", "provider-b", "target-b", 55), ]; let mut raw = HashMap::new(); raw.insert("provider_name".to_string(), "provider-b".to_string()); - raw.insert("target_id".to_string(), "2".to_string()); + raw.insert("target_name".to_string(), "target-b".to_string()); let filter = CompiledQosSnapshotFilter::compile(&raw).expect("qos snapshot filter should compile"); let mut filtered = snapshots @@ -526,7 +669,7 @@ mod tests { rate_limit: None, geoip: None, stream_history: None, - qos_aggregation: Some(crate::model::QosAggregationConfig { + qos_aggregation: Some(QosAggregationConfig { enabled: false, interval_secs: 300, }), @@ -550,9 +693,9 @@ mod tests { #[test] fn filter_qos_snapshots_orders_by_score_and_keeps_exact_match_filters() { let snapshots = vec![ - make_qos_snapshot("stream-a", "input-a", "provider-a", 1, 60), - make_qos_snapshot("stream-b", "input-b", "provider-a", 1, 80), - make_qos_snapshot("stream-c", "input-c", "provider-b", 2, 95), + make_qos_snapshot("stream-a", "input-a", "provider-a", "target-a", 60), + make_qos_snapshot("stream-b", "input-b", "provider-a", "target-a", 80), + make_qos_snapshot("stream-c", "input-c", "provider-b", "target-b", 95), ]; let mut raw = HashMap::new(); diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index eb689ce60..ba914849b 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -62,6 +62,7 @@ use std::{ str::FromStr, sync::Arc, }; +use shared::model::ConnectFailureReason; #[derive(Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq)] pub enum ApiStreamContext { @@ -277,9 +278,9 @@ async fn xtream_player_api_stream( fingerprint, &user, create_stream_channel_with_type(target.id, &pli, pli.item_type), - pli.input_name.as_ref(), + pli.input_name.clone(), req_headers, - crate::repository::ConnectFailureReason::UserAccountExpired, + ConnectFailureReason::UserAccountExpired, ); } @@ -382,9 +383,9 @@ async fn xtream_player_api_stream( fingerprint, &user, create_stream_channel_with_type(target.id, &pli, item_type), - &session.provider, + session.provider.clone(), req_headers, - crate::repository::ConnectFailureReason::UserConnectionsExhausted, + ConnectFailureReason::UserConnectionsExhausted, ); } @@ -394,9 +395,9 @@ async fn xtream_player_api_stream( fingerprint, &user, create_stream_channel_with_type(target.id, &pli, item_type), - &session.provider, + session.provider.clone(), req_headers, - crate::repository::ConnectFailureReason::ProviderConnectionsExhausted, + ConnectFailureReason::ProviderConnectionsExhausted, ); } @@ -409,7 +410,7 @@ async fn xtream_player_api_stream( app_state, session, stream_channel, - crate::api::api_utils::ForceStreamRequestContext { + api_utils::ForceStreamRequestContext { req_headers, input: &input, user: &user, @@ -468,9 +469,9 @@ async fn xtream_player_api_stream( fingerprint, &user, create_stream_channel_with_type(target.id, &pli, item_type), - input.name.as_ref(), + input.name.clone(), req_headers, - crate::repository::ConnectFailureReason::UserConnectionsExhausted, + ConnectFailureReason::UserConnectionsExhausted, ); } diff --git a/backend/src/api/model/active_user_manager.rs b/backend/src/api/model/active_user_manager.rs index 400d7f7eb..f5c224c17 100644 --- a/backend/src/api/model/active_user_manager.rs +++ b/backend/src/api/model/active_user_manager.rs @@ -2,7 +2,6 @@ use crate::{ api::model::{active_provider_manager::ConnectionKind, ActiveProviderManager, CustomVideoStreamType, EventManager, EventMessage}, auth::Fingerprint, model::{Config, ProxyUserCredentials}, - repository::utc_day_from_secs, utils::{debug_if_enabled, GeoIp}, }; use arc_swap::ArcSwapOption; @@ -32,6 +31,7 @@ use std::{ use tokio::sync::{mpsc, Mutex, Notify, RwLock}; use crate::api::model::connection_manager::CleanupEvent; use tokio_util::sync::CancellationToken; +use crate::utils::utc_day_from_secs; const USER_GC_TTL: u64 = 900; // 15 Min const USER_CON_TTL: u64 = 1_800; // 30 minutes @@ -320,7 +320,7 @@ pub struct ActiveUserConnectionParams<'a> { pub priority: i8, pub soft_priority: i8, pub fingerprint: &'a Fingerprint, - pub provider: &'a str, + pub provider: Arc, pub stream_channel: &'a StreamChannel, pub user_agent: Cow<'a, str>, pub session_token: Option<&'a str>, @@ -980,7 +980,7 @@ impl ActiveUserManager { // IMPORTANT: `resolve_disconnect_reason` in connection_manager.rs parses // `channel.title` back via `CustomVideoStreamType::from_str` to determine QoS // disconnect reasons. If these values change, update that function too. - stream.provider = String::from("tuliprox"); + stream.provider = "tuliprox".intern(); stream.channel.title = video_type.to_string().into(); stream.channel.group = "".intern(); stream.channel.technical = Some(Self::custom_stream_technical_info()); @@ -1084,7 +1084,7 @@ impl ActiveUserManager { stream_info.client_ip.clone_from(&client_ip); stream_info.country_code = self.lookup_country(&client_ip); stream_info.channel = stream_channel.clone(); - stream_info.provider = provider.to_string(); + stream_info.provider = provider.clone(); stream_info.user_agent.clone_from(&user_agent_string); if let Some(started_at) = session_started_at { @@ -2121,7 +2121,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1001), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-1"), @@ -2145,7 +2145,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-b", + provider: "provider-b".intern(), stream_channel: &test_channel(1002), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-2"), @@ -2185,7 +2185,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &at_limit_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1010), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-limit"), @@ -2206,7 +2206,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &over_limit_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1011), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-over-1"), @@ -2226,7 +2226,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &second_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1012), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-over-2"), @@ -2263,7 +2263,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &shared_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(channel_id), user_agent: Cow::Borrowed("ua"), session_token: Some(token), @@ -2282,7 +2282,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &unique_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1033), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-33"), @@ -2320,7 +2320,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &first_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1041), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-41"), @@ -2338,7 +2338,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &second_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(1042), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-42"), @@ -2389,7 +2389,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_adaptive_channel(2014), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-adaptive"), @@ -2458,7 +2458,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &kicked_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2015), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-kicked"), @@ -2475,7 +2475,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &survivor_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2016), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-survivor"), @@ -2523,7 +2523,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2017), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-grace"), @@ -2578,7 +2578,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &first, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2001), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-hls"), @@ -2602,7 +2602,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &second, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2001), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-hls"), @@ -2655,7 +2655,7 @@ mod tests { priority: 8, soft_priority: 8, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2002), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-prio"), @@ -2672,7 +2672,7 @@ mod tests { priority: -7, soft_priority: 8, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(2002), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-prio"), @@ -2706,7 +2706,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(3001), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-meter"), @@ -2727,7 +2727,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-b", + provider: "provider-b".intern(), stream_channel: &test_channel(3002), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-meter"), @@ -2742,7 +2742,7 @@ mod tests { assert_eq!(streams.len(), 1); assert_eq!(streams[0].uid, 11); assert_eq!(streams[0].meter_uid, 202); - assert_eq!(streams[0].provider, "provider-b"); + assert_eq!(streams[0].provider.as_ref(), "provider-b"); assert_eq!(streams[0].channel.virtual_id, 3002); } @@ -2786,7 +2786,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(4001) @@ -2820,7 +2820,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &next_fingerprint, - provider: "provider-b", + provider: "provider-b".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(4002) @@ -2880,7 +2880,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(5001) @@ -2934,7 +2934,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(6001) @@ -3010,7 +3010,7 @@ mod tests { priority: -1, soft_priority: 9, fingerprint: &normal_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(6002) @@ -3030,7 +3030,7 @@ mod tests { priority: -5, soft_priority: 9, fingerprint: &soft_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(6003), user_agent: Cow::Borrowed("ua-soft"), session_token: None, @@ -3101,7 +3101,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fp_a, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(7001) @@ -3125,7 +3125,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fp_b, - provider: "provider-b", + provider: "provider-b".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(7002) @@ -3185,7 +3185,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8001) @@ -3225,7 +3225,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8002) @@ -3278,7 +3278,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8003) @@ -3361,7 +3361,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8004) @@ -3433,7 +3433,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8004) @@ -3518,7 +3518,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8005) @@ -3558,7 +3558,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &next_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(8005) @@ -3571,7 +3571,7 @@ mod tests { assert_eq!(second.previous_session_id, Some((forced_old_ts << 32) | u64::from(first.uid))); assert!(second.ts > forced_old_ts); - assert_eq!(crate::repository::utc_day_from_secs(second.ts), crate::repository::utc_day_from_secs(current_time_secs())); + assert_eq!(utc_day_from_secs(second.ts), utc_day_from_secs(current_time_secs())); } #[tokio::test] @@ -3627,7 +3627,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &stale_fp, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(9201), user_agent: Cow::Borrowed("ua"), session_token: None, @@ -3645,7 +3645,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fresh_fp, - provider: "provider-b", + provider: "provider-b".intern(), stream_channel: &test_channel(9202), user_agent: Cow::Borrowed("ua"), session_token: None, @@ -3761,7 +3761,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_adaptive_channel(7777), user_agent: Cow::Borrowed("player/1.0"), session_token: Some("tok-hls-ts"), @@ -3835,7 +3835,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &old_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::Live, ..test_channel(9101) @@ -3903,7 +3903,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &base_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::Video, ..test_channel(9102) @@ -3926,7 +3926,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &range_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &StreamChannel { item_type: PlaylistItemType::Video, ..test_channel(9102) @@ -4000,7 +4000,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_channel(9001), user_agent: Cow::Borrowed("ua"), session_token: None, @@ -4163,7 +4163,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &first_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_adaptive_channel(9203), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-normal"), @@ -4188,7 +4188,7 @@ mod tests { priority: 0, soft_priority: 0, fingerprint: &second_fingerprint, - provider: "provider-a", + provider: "provider-a".intern(), stream_channel: &test_adaptive_channel(9204), user_agent: Cow::Borrowed("ua"), session_token: Some("tok-soft"), diff --git a/backend/src/api/model/connection_manager.rs b/backend/src/api/model/connection_manager.rs index 23f8d0aa0..b21c28880 100644 --- a/backend/src/api/model/connection_manager.rs +++ b/backend/src/api/model/connection_manager.rs @@ -4,7 +4,6 @@ use crate::{ EventMessage, ProviderHandle, SharedStreamManager, }, model::StreamHistoryConfig, - repository::{ConnectFailureReason, DisconnectQos, DisconnectReason, FailureStage, StreamHistoryRecord}, auth::Fingerprint, utils::debug_if_enabled, }; @@ -28,6 +27,8 @@ use std::{ time::Duration, }; use tokio::sync::{mpsc, Notify}; +use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage}; +use crate::model::{DisconnectQos, StreamHistoryRecord}; use crate::repository::{recover_pending_files, StreamHistoryWriter}; // Maximum number of deferred cleanup actions buffered before producers must wait/drop. @@ -272,13 +273,13 @@ pub(crate) enum CleanupEvent { } async fn handle_release_connection(deps: &CleanupWorkerDeps, addr: SocketAddr) { - release_connection_with_reason_from_deps(deps, &addr, &DisconnectReason::Cleanup, true).await; + release_connection_with_reason_from_deps(deps, &addr, DisconnectReason::Cleanup, true).await; } async fn release_connection_with_reason( connection_manager: &ConnectionManager, addr: &SocketAddr, - reason: &DisconnectReason, + reason: DisconnectReason, send_shared_stop_signal: bool, ) { release_connection_parts( @@ -298,7 +299,7 @@ async fn release_connection_with_reason( async fn release_connection_with_reason_from_deps( deps: &CleanupWorkerDeps, addr: &SocketAddr, - reason: &DisconnectReason, + reason: DisconnectReason, send_shared_stop_signal: bool, ) { release_connection_parts( @@ -324,7 +325,7 @@ async fn release_connection_parts( capacity_notify: &Arc, history_writer: &Arc>, addr: &SocketAddr, - reason: &DisconnectReason, + reason: DisconnectReason, send_shared_stop_signal: bool, ) { let removed = if matches!(reason, DisconnectReason::ClientKicked) { @@ -452,7 +453,7 @@ async fn handle_adaptive_session_expired(deps: &CleanupWorkerDeps, stream_info: emit_disconnect_record( &deps.history_writer, &stream_info, - &DisconnectReason::SessionExpired, + DisconnectReason::SessionExpired, &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, None, None, @@ -481,7 +482,7 @@ async fn release_stream_with_disconnect( emit_disconnect_record( &deps.history_writer, &stream_info, - &reason, + reason, &DisconnectQos { bytes_sent, first_byte_latency_ms, provider_reconnect_count }, provider_error_class, provider_http_status, @@ -537,7 +538,7 @@ pub struct ConnectionParams<'a> { pub priority: i8, pub soft_priority: i8, pub fingerprint: &'a Fingerprint, - pub provider: &'a str, + pub provider: Arc, pub stream_channel: &'a StreamChannel, pub user_agent: Cow<'a, str>, pub session_token: Option<&'a str>, @@ -601,6 +602,11 @@ impl ConnectionManager { self.history_writer.store(new_writer); } + /// Returns a reference to the history writer. + pub(crate) fn history_writer(&self) -> &Arc> { + &self.history_writer + } + fn spawn_socket_activity_worker( activity_tracker: SocketActivityTracker, user_manager: Arc, @@ -868,16 +874,16 @@ impl ConnectionManager { } pub async fn release_connection(&self, addr: &SocketAddr) { - release_connection_with_reason(self, addr, &DisconnectReason::ClientClosed, true).await; + release_connection_with_reason(self, addr, DisconnectReason::ClientClosed, true).await; } - pub async fn release_connection_with_reason(&self, addr: &SocketAddr, reason: &DisconnectReason) { + pub async fn release_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) { release_connection_with_reason(self, addr, reason, true).await; } pub async fn release_connection_as_kicked(&self, addr: &SocketAddr) { let _ = self.close_connection_with_reason(addr, DisconnectReason::ClientKicked); - release_connection_with_reason(self, addr, &DisconnectReason::ClientKicked, true).await; + release_connection_with_reason(self, addr, DisconnectReason::ClientKicked, true).await; } pub async fn release_provider_connection(&self, addr: &SocketAddr) { @@ -893,7 +899,7 @@ impl ConnectionManager { emit_disconnect_record( &self.history_writer, &stream_info, - &DisconnectReason::ClientClosed, + DisconnectReason::ClientClosed, &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, None, None, @@ -919,10 +925,6 @@ impl ConnectionManager { .unwrap_or(1) } - pub fn record_connect_failed(&self, info: &StreamInfo, reason: ConnectFailureReason, failure_stage: FailureStage) { - self.record_connect_failed_with_provider_failure(info, reason, failure_stage, None, None); - } - pub fn record_connect_failed_with_provider_failure( &self, info: &StreamInfo, @@ -930,6 +932,7 @@ impl ConnectionManager { failure_stage: FailureStage, provider_http_status: Option, provider_error_class: Option<&str>, + target_name: Option>, ) { let guard = self.history_writer.load(); let Some(writer) = guard.as_ref() else { return }; @@ -939,6 +942,7 @@ impl ConnectionManager { reason, attempt_uid, failure_stage, + target_name, ) .with_provider_failure(provider_http_status, provider_error_class)); } @@ -956,7 +960,7 @@ impl ConnectionManager { emit_disconnect_record( &self.history_writer, &stream_info, - &DisconnectReason::Shutdown, + DisconnectReason::Shutdown, &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, None, None, @@ -1042,7 +1046,7 @@ fn build_history_writer(config: Option<&StreamHistoryConfig>) -> Option DisconnectReason { - if stream_info.provider == "tuliprox" { + if stream_info.provider.as_ref() == "tuliprox" { if let Ok(video_type) = CustomVideoStreamType::from_str(&stream_info.channel.title) { match video_type { CustomVideoStreamType::LowPriorityPreempted => return DisconnectReason::Preempted, @@ -1075,7 +1079,7 @@ fn emit_connect_record(writer: &ArcSwapOption, info: &Strea fn emit_disconnect_record( writer: &ArcSwapOption, info: &StreamInfo, - reason: &DisconnectReason, + reason: DisconnectReason, qos: &DisconnectQos, provider_error_class: Option<&str>, provider_http_status: Option, @@ -1085,7 +1089,7 @@ fn emit_disconnect_record( w.send_record( StreamHistoryRecord::from_disconnect( info, - reason.clone(), + reason, qos, resolve_disconnect_failure_stage(info, reason, qos), ) @@ -1093,7 +1097,7 @@ fn emit_disconnect_record( ); } -fn resolve_disconnect_failure_stage(info: &StreamInfo, reason: &DisconnectReason, qos: &DisconnectQos) -> Option { +fn resolve_disconnect_failure_stage(info: &StreamInfo, reason: DisconnectReason, qos: &DisconnectQos) -> Option { match reason { DisconnectReason::ProviderError | DisconnectReason::ProviderClosed => { if !info.channel.shared && qos.first_byte_latency_ms.is_none() { @@ -1140,7 +1144,7 @@ mod tests { shared_stream_id: None, technical: None, }; - StreamInfo::new(0, 0, "test", &addr, "127.0.0.1", provider, channel, String::new(), None, None) + StreamInfo::new(0, 0, "test", &addr, "127.0.0.1", provider.intern(), channel, String::new(), None, None) } fn create_test_app_config() -> AppConfig { @@ -1401,7 +1405,7 @@ mod tests { assert_eq!( resolve_disconnect_failure_stage( &make_stream_info("some_provider", "Some Channel"), - &DisconnectReason::ClientKicked, + DisconnectReason::ClientKicked, &DisconnectQos::default(), ), None @@ -1413,7 +1417,7 @@ mod tests { assert_eq!( resolve_disconnect_failure_stage( &make_stream_info("some_provider", "Some Channel"), - &DisconnectReason::Provisioning, + DisconnectReason::Provisioning, &DisconnectQos::default(), ), None @@ -1488,7 +1492,7 @@ mod tests { assert_eq!( resolve_disconnect_failure_stage( &make_stream_info("some_provider", "Some Channel"), - &DisconnectReason::ProviderError, + DisconnectReason::ProviderError, &DisconnectQos { first_byte_latency_ms: Some(150), ..Default::default() }, ), Some(FailureStage::Streaming) @@ -1500,7 +1504,7 @@ mod tests { assert_eq!( resolve_disconnect_failure_stage( &make_stream_info("some_provider", "Some Channel"), - &DisconnectReason::SessionExpired, + DisconnectReason::SessionExpired, &DisconnectQos::default(), ), Some(FailureStage::SessionReconnect) @@ -1512,7 +1516,7 @@ mod tests { assert_eq!( resolve_disconnect_failure_stage( &make_stream_info("some_provider", "Some Channel"), - &DisconnectReason::ProviderError, + DisconnectReason::ProviderError, &DisconnectQos::default(), ), Some(FailureStage::FirstByte) @@ -1526,7 +1530,7 @@ mod tests { assert_eq!( resolve_disconnect_failure_stage( &info, - &DisconnectReason::ProviderError, + DisconnectReason::ProviderError, &DisconnectQos::default(), ), Some(FailureStage::Streaming) diff --git a/backend/src/api/model/qos_aggregation_manager.rs b/backend/src/api/model/qos_aggregation_manager.rs index ea644820a..e8078328f 100644 --- a/backend/src/api/model/qos_aggregation_manager.rs +++ b/backend/src/api/model/qos_aggregation_manager.rs @@ -2,6 +2,7 @@ use chrono::NaiveDate; use log::{error, info, warn}; +use shared::utils::Internable; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io; use std::path::{Path, PathBuf}; @@ -9,24 +10,26 @@ use std::sync::Arc; use std::time::Duration; use crate::api::model::AppState; -use crate::model::Config; +use crate::model::{Config, StreamHistoryRecord}; use crate::repository::{ - current_utc_day, extract_day_from_filename, now_utc_secs, ConnectFailureReason, DisconnectReason, EventType, - FailureStage, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow, - StreamHistoryFileReader, StreamHistoryRecord, + extract_day_from_filename, + QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow, + StreamHistoryFileReader, }; use tokio_util::sync::CancellationToken; +use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage, PlaylistItemType, StreamHistoryEventType}; +use crate::utils::{current_utc_day, now_utc_secs}; const MAX_COMPLETED_DAY_PARTITIONS_PER_RUN: usize = 3; #[derive(Debug, Clone)] struct AggregatedDayEntry { - input_name: String, - target_id: u16, - provider_name: String, + input_name: Arc, + target_name: Arc, + provider_name: Arc, provider_id: u32, virtual_id: u32, - item_type: String, + item_type: PlaylistItemType, bucket: QosSnapshotDailyBucket, } @@ -136,14 +139,14 @@ pub(crate) fn run_aggregation_once( pub(crate) fn fold_record_into_bucket(bucket: &mut QosSnapshotDailyBucket, record: &StreamHistoryRecord) { match record.event_type { - EventType::Connect => { + StreamHistoryEventType::Connect => { if record.shared_joined_existing.unwrap_or(false) { return; } bucket.connect_count = bucket.connect_count.saturating_add(1); bucket.last_success_ts = Some(bucket.last_success_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc))); } - EventType::ConnectFailed => { + StreamHistoryEventType::ConnectFailed => { bucket.connect_failed_count = bucket.connect_failed_count.saturating_add(1); if matches!( record.connect_failure_reason, @@ -156,7 +159,7 @@ pub(crate) fn fold_record_into_bucket(bucket: &mut QosSnapshotDailyBucket, recor } bucket.last_failure_ts = Some(bucket.last_failure_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc))); } - EventType::Disconnect => { + StreamHistoryEventType::Disconnect => { if matches!(record.failure_stage, Some(FailureStage::FirstByte)) { bucket.first_byte_failure_count = bucket.first_byte_failure_count.saturating_add(1); } @@ -194,6 +197,12 @@ pub(crate) fn fold_record_into_bucket(bucket: &mut QosSnapshotDailyBucket, recor bucket.last_failure_ts = Some(bucket.last_failure_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc))); } + }, + StreamHistoryEventType::Failure => { + // Failure events represent intermediate disconnects within an HLS session. + // Count them as runtime aborts since they reflect streaming failures. + bucket.runtime_abort_count = bucket.runtime_abort_count.saturating_add(1); + bucket.last_failure_ts = Some(bucket.last_failure_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc))); } } } @@ -382,20 +391,26 @@ fn aggregate_reader_entries( let Some(stream_identity_key) = record.stream_identity_key.clone() else { continue; }; - let (Some(input_name), Some(target_id), Some(provider_name), Some(provider_id), Some(virtual_id), Some(item_type)) = ( - record.input_name.clone(), - record.target_id, - record.provider_name.clone(), - record.provider_id, - record.virtual_id, - record.item_type.clone(), - ) else { + + // Required fields - skip record if missing + let Some(provider_name) = record.provider_name.clone() else { + continue; + }; + let Some(provider_id) = record.provider_id else { + continue; + }; + let Some(virtual_id) = record.virtual_id else { continue; }; + // Optional fields - use fallbacks for pre-migration records + let input_name = record.input_name.clone().unwrap_or_else(|| "unknown".intern()); + let target_name = record.target_name.clone().unwrap_or_else(|| "unknown".intern()); + let item_type = record.item_type.unwrap_or_default(); + let entry = entries.entry(stream_identity_key).or_insert_with(|| AggregatedDayEntry { input_name, - target_id, + target_name, provider_name, provider_id, virtual_id, @@ -472,11 +487,11 @@ fn apply_day_entries( day_entries.get(&key).map(|entry| QosSnapshotRecord { stream_identity_key: key.clone(), input_name: entry.input_name.clone(), - target_id: entry.target_id, + target_name: entry.target_name.clone(), provider_name: entry.provider_name.clone(), provider_id: entry.provider_id, virtual_id: entry.virtual_id, - item_type: entry.item_type.clone(), + item_type: entry.item_type, updated_at: now_utc_secs(), last_event_at: 0, window_24h: QosSnapshotWindow::default(), @@ -489,6 +504,12 @@ fn apply_day_entries( }; if let Some(entry) = day_entries.get(&key) { + snapshot.input_name = entry.input_name.clone(); + snapshot.target_name = entry.target_name.clone(); + snapshot.provider_name = entry.provider_name.clone(); + snapshot.provider_id = entry.provider_id; + snapshot.virtual_id = entry.virtual_id; + snapshot.item_type = entry.item_type; snapshot.daily_buckets.insert(day_utc.to_string(), entry.bucket.clone()); snapshot.last_event_at = max_opt(Some(snapshot.last_event_at), max_opt(entry.bucket.last_success_ts, entry.bucket.last_failure_ts)).unwrap_or(snapshot.last_event_at); } else { @@ -524,15 +545,16 @@ mod tests { use std::collections::BTreeMap; use tempfile::tempdir; - - use crate::model::{Config, QosAggregationConfig, ReverseProxyConfig, StreamHistoryConfig}; + use shared::model::{FailureStage, ConnectFailureReason, DisconnectReason, PlaylistItemType, StreamHistoryEventType}; + use shared::utils::Internable; + use crate::model::{Config, QosAggregationConfig, ReverseProxyConfig, StreamHistoryConfig, StreamHistoryRecord, RECORD_SCHEMA_VERSION}; use crate::repository::{ - current_utc_day, serialize_named, write_block_magic, write_file_magic, write_framed, BlockHeaderBody, - CompressionKind, ConnectFailureReason, CONTAINER_FORMAT_VERSION, DisconnectReason, EventType, FailureStage, + serialize_named, write_block_magic, write_file_magic, write_framed, BlockHeaderBody, + CompressionKind, CONTAINER_FORMAT_VERSION, FileHeaderBody, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow, - RecordEncodingKind, RECORD_SCHEMA_VERSION, SOURCE_KIND_STREAM_HISTORY, StreamHistoryRecord, + RecordEncodingKind, SOURCE_KIND_STREAM_HISTORY, }; - + use crate::utils::current_utc_day; use super::{fold_record_into_bucket, history_day_revision, qos_aggregation_is_enabled, rebuild_windows, run_aggregation_once}; fn write_pending_history_records( @@ -596,7 +618,7 @@ mod tests { file.flush().expect("pending file should flush"); } - fn base_record(event_type: EventType) -> StreamHistoryRecord { + fn base_record(event_type: StreamHistoryEventType) -> StreamHistoryRecord { StreamHistoryRecord { schema_version: 1, event_type, @@ -605,11 +627,11 @@ mod tests { session_id: 42, source_addr: None, api_username: None, - provider_name: Some("provider-a".to_string()), + provider_name: Some("provider-a".intern()), provider_username: None, - input_name: Some("input-a".to_string()), + input_name: Some("input-a".intern()), virtual_id: Some(33), - item_type: Some("live".to_string()), + item_type: Some(PlaylistItemType::Live), title: None, group: None, country: None, @@ -639,7 +661,7 @@ mod tests { connect_failure_reason: None, disconnect_reason: None, previous_session_id: None, - target_id: Some(11), + target_name: None, } } @@ -647,9 +669,9 @@ mod tests { fn fold_record_into_bucket_tracks_connect_and_connect_failed() { let mut bucket = QosSnapshotDailyBucket::default(); - fold_record_into_bucket(&mut bucket, &base_record(EventType::Connect)); + fold_record_into_bucket(&mut bucket, &base_record(StreamHistoryEventType::Connect)); - let mut failed = base_record(EventType::ConnectFailed); + let mut failed = base_record(StreamHistoryEventType::ConnectFailed); failed.connect_failure_reason = Some(ConnectFailureReason::ProviderConnectionsExhausted); fold_record_into_bucket(&mut bucket, &failed); @@ -663,13 +685,13 @@ mod tests { fn fold_record_into_bucket_tracks_disconnect_failure_stages() { let mut bucket = QosSnapshotDailyBucket::default(); - let mut first_byte = base_record(EventType::Disconnect); + let mut first_byte = base_record(StreamHistoryEventType::Disconnect); first_byte.failure_stage = Some(FailureStage::FirstByte); first_byte.disconnect_reason = Some(DisconnectReason::ProviderError); first_byte.first_byte_latency_ms = Some(250); fold_record_into_bucket(&mut bucket, &first_byte); - let mut streaming = base_record(EventType::Disconnect); + let mut streaming = base_record(StreamHistoryEventType::Disconnect); streaming.failure_stage = Some(FailureStage::Streaming); streaming.disconnect_reason = Some(DisconnectReason::ProviderClosed); streaming.session_duration = Some(900); @@ -684,11 +706,23 @@ mod tests { assert_eq!(bucket.total_provider_reconnect_count, 2); } + #[test] + fn fold_record_into_bucket_tracks_failure_event() { + let mut bucket = QosSnapshotDailyBucket::default(); + + // Failure events should increment runtime_abort_count and update last_failure_ts + let failure = base_record(StreamHistoryEventType::Failure); + fold_record_into_bucket(&mut bucket, &failure); + + assert_eq!(bucket.runtime_abort_count, 1); + assert_eq!(bucket.last_failure_ts, Some(failure.event_ts_utc)); + } + #[tokio::test] async fn history_day_revision_tracks_seconds_and_file_length() { let temp = tempdir().expect("tempdir should succeed"); let history_dir = temp.path(); - let record = base_record(EventType::Connect); + let record = base_record(StreamHistoryEventType::Connect); write_pending_history_records(history_dir, vec![record.clone(), record]); let history_dir_copy = history_dir.to_path_buf(); @@ -707,12 +741,12 @@ mod tests { fn rebuild_windows_uses_recent_daily_buckets() { let mut snapshot = QosSnapshotRecord { stream_identity_key: "stream-a".to_string(), - input_name: "input-a".to_string(), - target_id: 11, - provider_name: "provider-a".to_string(), + input_name: "input-a".intern(), + target_name: "target-a".intern(), + provider_name: "provider-a".intern(), provider_id: 22, virtual_id: 33, - item_type: "live".to_string(), + item_type: PlaylistItemType::Live, updated_at: 1_700_000_000, last_event_at: 1_700_000_123, window_24h: QosSnapshotWindow::default(), @@ -792,14 +826,14 @@ mod tests { let temp = tempdir().expect("tempdir should succeed"); let history_dir = temp.path().join("stream_history"); let today = current_utc_day(); - let mut connect = base_record(EventType::Connect); + let mut connect = base_record(StreamHistoryEventType::Connect); connect.partition_day_utc = today.clone(); - connect.input_name = Some("input-a".to_string()); - connect.provider_name = Some("provider-a".to_string()); + connect.input_name = Some("input-a".intern()); + connect.provider_name = Some("provider-a".intern()); connect.provider_id = Some(22); connect.virtual_id = Some(33); - connect.item_type = Some("live".to_string()); - connect.target_id = Some(11); + connect.item_type = Some(PlaylistItemType::Live); + connect.target_name = Some("target-a".intern()); connect.stream_identity_key = Some("stream-a".to_string()); connect.shared_joined_existing = Some(false); write_pending_history_records(&history_dir, vec![connect]); @@ -826,14 +860,14 @@ mod tests { let history_dir = temp.path().join("stream_history"); let today = current_utc_day(); - let mut connect = base_record(EventType::Connect); + let mut connect = base_record(StreamHistoryEventType::Connect); connect.partition_day_utc = today.clone(); - connect.input_name = Some("input-a".to_string()); - connect.provider_name = Some("provider-a".to_string()); + connect.input_name = Some("input-a".intern()); + connect.provider_name = Some("provider-a".intern()); connect.provider_id = Some(22); connect.virtual_id = Some(33); - connect.item_type = Some("live".to_string()); - connect.target_id = Some(11); + connect.item_type = Some(PlaylistItemType::Live); + connect.target_name = Some("target-a".intern()); connect.stream_identity_key = Some("stream-a".to_string()); connect.shared_joined_existing = Some(false); write_pending_history_records(&history_dir, vec![connect]); @@ -871,16 +905,16 @@ mod tests { ("2036-04-03", 3_u64), ("2036-04-04", 4_u64), ] { - let mut connect = base_record(EventType::Connect); + let mut connect = base_record(StreamHistoryEventType::Connect); connect.partition_day_utc = day.to_string(); connect.event_ts_utc = connect.event_ts_utc.saturating_add(session_id); connect.session_id = session_id; - connect.input_name = Some("input-a".to_string()); - connect.provider_name = Some("provider-a".to_string()); + connect.input_name = Some("input-a".intern()); + connect.provider_name = Some("provider-a".intern()); connect.provider_id = Some(22); connect.virtual_id = Some(33); - connect.item_type = Some("live".to_string()); - connect.target_id = Some(11); + connect.item_type = Some(PlaylistItemType::Live); + connect.target_name = Some("target-a".intern()); connect.stream_identity_key = Some("stream-a".to_string()); connect.shared_joined_existing = Some(false); write_pending_history_records(&history_dir, vec![connect]); diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/src/api/model/streams/active_client_stream.rs index ad72acaf0..20971c959 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/src/api/model/streams/active_client_stream.rs @@ -32,6 +32,8 @@ use std::{ }; use tokio::sync::Notify; use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use shared::utils::Internable; +use shared::model::FailureStage; /// Discriminates which byte-stream the client is consuming at any moment. /// Stored as `u8` in an `AtomicU8` for lock-free access inside `poll_next`. @@ -447,7 +449,7 @@ fn create_deferred_provider_open_future( username: None, client_ip: Some(&fingerprint.client_ip), stream_channel: Some(stream_channel), - connect_failure_stage: Some(crate::repository::FailureStage::ProviderOpen), + connect_failure_stage: Some(FailureStage::ProviderOpen), }, ); provider_stream_factory_options.set_provider(input.get_resolve_provider(stream_url.as_ref())); @@ -711,7 +713,7 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam } let grant_user_grace_period = connection_permission == UserConnectionPermission::GracePeriod; let username = user.username.as_str(); - let provider_name = stream_details.provider_name.as_deref().unwrap_or(""); + let provider_name = stream_details.provider_name.clone().unwrap_or_else(||"unknown".intern()); let user_agent = req_headers.get(USER_AGENT).map(|h| String::from_utf8_lossy(h.as_bytes())).unwrap_or_default(); diff --git a/backend/src/api/model/streams/provider_stream_factory.rs b/backend/src/api/model/streams/provider_stream_factory.rs index 069077f78..49f02f733 100644 --- a/backend/src/api/model/streams/provider_stream_factory.rs +++ b/backend/src/api/model/streams/provider_stream_factory.rs @@ -8,7 +8,6 @@ use crate::{ }, }, model::{ConfigProvider, ReverseProxyDisabledHeaderConfig}, - repository::{ConnectFailureReason, FailureStage}, utils::{ debug_if_enabled, request::{ @@ -42,7 +41,9 @@ use std::{ }; use tokio_util::sync::CancellationToken; use url::Url; -use shared::utils::DEFAULT_USER_AGENT; +use shared::model::ConnectFailureReason; +use shared::utils::{Internable, DEFAULT_USER_AGENT}; +use shared::model::{FailureStage}; const RETRY_SECONDS: u64 = 5; const ERR_MAX_RETRY_COUNT: u32 = 5; @@ -223,7 +224,7 @@ impl ProviderStreamFactoryOptions { std::borrow::Cow::Owned(preview_request_target_for_logging(&self.url, self.provider.as_ref())) } - fn build_connect_failed_stream_info(&self, provider_name: &str) -> Option { + fn build_connect_failed_stream_info(&self, provider_name: Arc) -> Option { let username = self.username.as_deref()?; let client_ip = self.client_ip.as_deref()?; let stream_channel = self.stream_channel.clone()?; @@ -256,8 +257,14 @@ fn record_provider_open_failure( let Some(failure_stage) = stream_options.get_connect_failure_stage() else { return }; let provider_name = stream_options .get_provider() - .map_or_else(|| "unknown".to_string(), |provider| provider.name.to_string()); - let Some(info) = stream_options.build_connect_failed_stream_info(&provider_name) else { return }; + .map_or_else(|| "unknown".intern(), |provider| provider.name.clone()); + let Some(info) = stream_options.build_connect_failed_stream_info(provider_name) else { return }; + // Resolve target_name from target_id using the stable target config name. + let target_name = app_state + .app_config + .get_target_by_id(info.channel.target_id) + .as_deref() + .map(|t| (&t.name).intern()); app_state .connection_manager .record_connect_failed_with_provider_failure( @@ -266,6 +273,7 @@ fn record_provider_open_failure( failure_stage, provider_http_status.map(|status| status.as_u16()), provider_error_class, + target_name, ); } @@ -890,6 +898,7 @@ pub async fn create_provider_stream( Ok(None) => None, Err(failure) => { let status = failure.status(); + app_state.connection_manager.release_provider_connection(&stream_options.addr).await; record_provider_open_failure( app_state, &stream_options, @@ -1120,11 +1129,11 @@ mod tests { connect_failure_stage: Some(FailureStage::ProviderOpen), }); - let info = options.build_connect_failed_stream_info("provider-a").expect("history context"); + let info = options.build_connect_failed_stream_info("provider-a".intern()).expect("history context"); assert_eq!(info.username, "alice"); assert_eq!(info.client_ip, "203.0.113.9"); - assert_eq!(info.provider, "provider-a"); + assert_eq!(info.provider.as_ref(), "provider-a"); assert_eq!(info.channel.input_name.as_ref(), "input-a"); assert_eq!(info.channel.virtual_id, 77); } diff --git a/backend/src/api/model/streams/timed_client_stream.rs b/backend/src/api/model/streams/timed_client_stream.rs index a525d5e98..91dbc2040 100644 --- a/backend/src/api/model/streams/timed_client_stream.rs +++ b/backend/src/api/model/streams/timed_client_stream.rs @@ -17,6 +17,7 @@ use std::{ time::Duration, }; use tokio::time::{sleep_until, Instant, Sleep}; +use shared::model::DisconnectReason; enum TimeoutAction { Kick { @@ -88,7 +89,7 @@ impl Stream for TimedClientStream { &addr, virtual_id, kick_secs, - crate::repository::DisconnectReason::Timeout, + DisconnectReason::Timeout, ) .await; }); diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index 533346729..bea0a97f8 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -46,6 +46,7 @@ use std::{ }; use tokio_util::sync::CancellationToken; use url::Url; +use shared::model::DisconnectReason; #[derive(Debug, Clone)] struct AccountCredentials { @@ -3500,7 +3501,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - crate::repository::DisconnectReason::Provisioning, + DisconnectReason::Provisioning, ) .await; return Ok(()); @@ -3517,7 +3518,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - crate::repository::DisconnectReason::Provisioning, + DisconnectReason::Provisioning, ) .await; return Ok(()); @@ -3534,7 +3535,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - crate::repository::DisconnectReason::Provisioning, + DisconnectReason::Provisioning, ) .await; return Ok(()); @@ -3595,7 +3596,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - crate::repository::DisconnectReason::Provisioning, + DisconnectReason::Provisioning, ) .await; return Ok(()); @@ -3621,7 +3622,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - crate::repository::DisconnectReason::Provisioning, + DisconnectReason::Provisioning, ) .await; return Ok(()); @@ -3704,7 +3705,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - crate::repository::DisconnectReason::Provisioning, + DisconnectReason::Provisioning, ) .await; Ok(()) diff --git a/backend/src/api/serve.rs b/backend/src/api/serve.rs index a1a98f188..6826ebc60 100644 --- a/backend/src/api/serve.rs +++ b/backend/src/api/serve.rs @@ -161,7 +161,7 @@ async fn handle_connection( Ok(signal) = addr_close_rx.recv() => { match signal { CloseConnectionSignal::WithReason(msg, reason) if msg == addr => { - connection_manager_clone.release_connection_with_reason(&addr, &reason).await; + connection_manager_clone.release_connection_with_reason(&addr, reason).await; debug!("Forced client close {msg} reason={reason:?}"); conn.as_mut().graceful_shutdown(); break; diff --git a/backend/src/main.rs b/backend/src/main.rs index 0b0bb4200..160b58f9e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -153,7 +153,7 @@ async fn main() { db_viewer(&args.db_viewer_args()); if let Some(ref sh_input) = args.stream_history { - std::process::exit(utils::stream_history_viewer(sh_input)); + std::process::exit(utils::stream_history_viewer(sh_input).await); } if args.genpwd { diff --git a/backend/src/model/mod.rs b/backend/src/model/mod.rs index 03802e036..b7a9e67ab 100644 --- a/backend/src/model/mod.rs +++ b/backend/src/model/mod.rs @@ -6,6 +6,7 @@ mod healthcheck; mod config; mod input_source; pub mod messaging; +mod stream_history; pub use self::playlist::*; pub use self::mapping::*; @@ -15,4 +16,5 @@ pub use self::healthcheck::*; pub use shared::model::xtream_const::*; pub use self::config::*; pub use self::input_source::*; -pub use self::messaging::*; \ No newline at end of file +pub use self::messaging::*; +pub use self::stream_history::*; \ No newline at end of file diff --git a/backend/src/model/stream_history.rs b/backend/src/model/stream_history.rs new file mode 100644 index 000000000..9bbf57b44 --- /dev/null +++ b/backend/src/model/stream_history.rs @@ -0,0 +1,250 @@ +use crate::utils::arc_str_option_serde; +use std::sync::Arc; +use serde::{Deserialize, Serialize}; +use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage, PlaylistItemType, StreamHistoryEventType, StreamHistoryRecordDto, StreamInfo}; +use crate::utils::{encode_base64_hash, now_utc_secs, utc_day_from_secs}; + +pub const RECORD_SCHEMA_VERSION: u8 = 1; + +/// A single stream lifecycle event record (connect or disconnect). +/// +/// Privacy: must never contain passwords, tokens, or credential-bearing URLs. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StreamHistoryRecord { + pub schema_version: u8, + pub event_type: StreamHistoryEventType, + /// Unix timestamp seconds UTC of when this event occurred. + pub event_ts_utc: u64, + /// UTC calendar day of this event, e.g. `"2026-03-22"`. + pub partition_day_utc: String, + /// Stable correlation id shared between the connect and disconnect events of the same session. + pub session_id: u64, + pub source_addr: Option, + // User and provider identity (no passwords, no tokens) + pub api_username: Option, + #[serde(default, with = "arc_str_option_serde")] + pub provider_name: Option>, + /// Provider-side username (e.g. Xtream account). Always `None` until `StreamInfo` + /// carries the provider credential — at which point populate from there. + pub provider_username: Option, + #[serde(default, with = "arc_str_option_serde")] + pub input_name: Option>, + // Stream metadata + pub virtual_id: Option, + pub item_type: Option, + pub title: Option, + pub group: Option, + pub country: Option, + // QoS metadata + pub user_agent: Option, + pub shared: Option, + pub shared_joined_existing: Option, + pub shared_stream_id: Option, + pub provider_id: Option, + pub cluster: Option, + pub container: Option, + pub stream_url_hash: Option, + pub stream_identity_key: Option, + pub video_codec: Option, + pub audio_codec: Option, + pub audio_channels: Option, + pub resolution: Option, + pub fps: Option, + // Session summary — populated on disconnect. + // bytes_sent and first_byte_latency_ms are None for shared streams (meter serves + // multiple clients; per-session totals are not meaningful in that case). + pub connect_ts_utc: Option, + pub disconnect_ts_utc: Option, + pub session_duration: Option, + pub bytes_sent: Option, + pub first_byte_latency_ms: Option, + pub provider_reconnect_count: Option, + pub failure_stage: Option, + pub provider_http_status: Option, + pub provider_error_class: Option, + pub connect_failure_reason: Option, + pub disconnect_reason: Option, + /// Set when this connect event continues a session that was split by a `DayRollover`. + /// Always `None` until the `DayRollover` mechanism is implemented. + pub previous_session_id: Option, + /// Target config name — stable identifier, does not change across restarts. + #[serde(default, with = "arc_str_option_serde")] + pub target_name: Option>, +} + +/// `QoS` metrics collected at disconnect time, passed as a bundle to avoid growing the signature. +#[derive(Debug, Default)] +pub struct DisconnectQos { + pub bytes_sent: Option, + pub first_byte_latency_ms: Option, + pub provider_reconnect_count: Option, +} + +impl StreamHistoryRecord { + fn build_stream_identity_key(info: &StreamInfo) -> Option { + if info.channel.input_name.is_empty() { + return None; + } + let mut raw = String::with_capacity(info.channel.input_name.len() + 32); + raw.push_str(info.channel.input_name.as_ref()); + raw.push('|'); + raw.push_str(&info.channel.target_id.to_string()); + raw.push('|'); + raw.push_str(&info.channel.provider_id.to_string()); + raw.push('|'); + raw.push_str(&info.channel.virtual_id.to_string()); + raw.push('|'); + raw.push_str(&info.channel.item_type.to_string()); + Some(encode_base64_hash(&raw)) + } + + /// Build a common base from a `StreamInfo`, leaving event-specific fields to the caller. + /// `event_ts` is the authoritative timestamp — `partition_day_utc` is derived from it + /// so both fields are always consistent. + fn base(info: &StreamInfo, event_ts: u64) -> Self { + Self { + schema_version: RECORD_SCHEMA_VERSION, + event_type: StreamHistoryEventType::Connect, // overridden by callers + event_ts_utc: event_ts, + partition_day_utc: utc_day_from_secs(event_ts), + // Combine connect-timestamp (upper 32 bits) with uid (lower 32 bits). + // This prevents session_id collision across server restarts and uid wrap-around. + // Both Connect and Disconnect derive this from the same StreamInfo, so they match. + session_id: (info.ts << 32) | u64::from(info.uid), + source_addr: Some(info.client_ip.clone()), + api_username: Some(info.username.clone()), + provider_name: Some(info.provider.clone()), + provider_username: None, + input_name: if info.channel.input_name.is_empty() { + None + } else { + Some(info.channel.input_name.clone()) + }, + virtual_id: Some(info.channel.virtual_id), + item_type: Some(info.channel.item_type), + title: Some(info.channel.title.to_string()), + group: Some(info.channel.group.to_string()), + country: info.country_code.clone(), + user_agent: if info.user_agent.is_empty() { None } else { Some(info.user_agent.clone()) }, + shared: Some(info.channel.shared), + shared_joined_existing: info.channel.shared_joined_existing, + shared_stream_id: info.channel.shared_stream_id, + provider_id: Some(info.channel.provider_id), + cluster: Some(info.channel.cluster.to_string()), + container: info.channel.technical.as_ref().and_then(|t| if t.container.is_empty() { None } else { Some(t.container.clone()) }), + stream_url_hash: if info.channel.url.is_empty() { + None + } else { + Some(encode_base64_hash(info.channel.url.as_ref())) + }, + stream_identity_key: Self::build_stream_identity_key(info), + video_codec: info.channel.technical.as_ref().and_then(|t| if t.video_codec.is_empty() { None } else { Some(t.video_codec.clone()) }), + audio_codec: info.channel.technical.as_ref().and_then(|t| if t.audio_codec.is_empty() { None } else { Some(t.audio_codec.clone()) }), + audio_channels: info.channel.technical.as_ref().and_then(|t| if t.audio_channels.is_empty() { None } else { Some(t.audio_channels.clone()) }), + resolution: info.channel.technical.as_ref().and_then(|t| if t.resolution.is_empty() { None } else { Some(t.resolution.clone()) }), + fps: info.channel.technical.as_ref().and_then(|t| if t.fps.is_empty() { None } else { Some(t.fps.clone()) }), + connect_ts_utc: None, + disconnect_ts_utc: None, + session_duration: None, + bytes_sent: None, + first_byte_latency_ms: None, + provider_reconnect_count: None, + failure_stage: None, + provider_http_status: None, + provider_error_class: None, + connect_failure_reason: None, + disconnect_reason: None, + previous_session_id: None, + target_name: None, + } + } + + pub fn from_connect(info: &StreamInfo) -> Self { + let mut record = Self::base(info, info.ts); + record.event_type = StreamHistoryEventType::Connect; + record.connect_ts_utc = Some(info.ts); + record.previous_session_id = info.previous_session_id; + record + } + + pub fn from_connect_failed( + info: &StreamInfo, + reason: ConnectFailureReason, + attempt_uid: u32, + failure_stage: FailureStage, + target_name: Option>, + ) -> Self { + let event_ts = now_utc_secs(); + let mut record = Self::base(info, event_ts); + record.event_type = StreamHistoryEventType::ConnectFailed; + record.session_id = (event_ts << 32) | u64::from(attempt_uid); + record.failure_stage = Some(failure_stage); + record.connect_failure_reason = Some(reason); + record.target_name = target_name; + record + } + + pub fn with_provider_failure(mut self, provider_http_status: Option, provider_error_class: Option<&str>) -> Self { + self.provider_http_status = provider_http_status; + self.provider_error_class = provider_error_class.map(ToString::to_string); + self + } + + /// Extra `QoS` fields carried as a struct to keep the signature stable. + pub fn from_disconnect( + info: &StreamInfo, + reason: DisconnectReason, + qos: &DisconnectQos, + failure_stage: Option, + ) -> Self { + let now_secs = now_utc_secs(); + let connect_secs = info.ts; + let mut record = Self::base(info, now_secs); + record.event_type = StreamHistoryEventType::Disconnect; + record.connect_ts_utc = Some(connect_secs); + record.disconnect_ts_utc = Some(now_secs); + record.session_duration = Some(now_secs.saturating_sub(connect_secs)); + record.bytes_sent = qos.bytes_sent; + record.first_byte_latency_ms = qos.first_byte_latency_ms; + record.provider_reconnect_count = qos.provider_reconnect_count; + record.failure_stage = failure_stage; + record.disconnect_reason = Some(reason); + record + } +} + +impl From<&StreamHistoryRecord> for StreamHistoryRecordDto { + fn from(record: &StreamHistoryRecord) -> Self { + Self { + event_type: record.event_type, + event_ts_utc: record.event_ts_utc, + partition_day_utc: record.partition_day_utc.clone(), + session_id: record.session_id, + source_addr: record.source_addr.clone(), + api_username: record.api_username.clone(), + provider_name: record.provider_name.clone(), + input_name: record.input_name.clone(), + virtual_id: record.virtual_id, + item_type: record.item_type, + title: record.title.clone(), + group: record.group.clone(), + country: record.country.clone(), + user_agent: record.user_agent.clone(), + shared: record.shared, + provider_id: record.provider_id, + cluster: record.cluster.clone(), + container: record.container.clone(), + video_codec: record.video_codec.clone(), + audio_codec: record.audio_codec.clone(), + resolution: record.resolution.clone(), + failure_stage: record.failure_stage, + connect_failure_reason: record.connect_failure_reason, + disconnect_reason: record.disconnect_reason, + session_duration: record.session_duration, + bytes_sent: record.bytes_sent, + first_byte_latency_ms: record.first_byte_latency_ms, + previous_session_id: record.previous_session_id, + target_name: record.target_name.clone(), + } + } +} \ No newline at end of file diff --git a/backend/src/repository/qos_snapshot_repository.rs b/backend/src/repository/qos_snapshot_repository.rs index 8deeccccc..e0cf09a76 100644 --- a/backend/src/repository/qos_snapshot_repository.rs +++ b/backend/src/repository/qos_snapshot_repository.rs @@ -1,9 +1,11 @@ +use crate::utils::arc_str_serde; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::io; use std::path::{Path, PathBuf}; - +use std::sync::Arc; +use shared::model::PlaylistItemType; use crate::repository::{BPlusTree, BPlusTreeUpdate, FlushPolicy}; const SNAPSHOT_FILE_NAME: &str = "qos_snapshot.db"; @@ -54,12 +56,15 @@ pub struct QosSnapshotDailyBucket { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct QosSnapshotRecord { pub stream_identity_key: String, - pub input_name: String, - pub target_id: u16, - pub provider_name: String, + #[serde(with = "arc_str_serde")] + pub input_name: Arc, + #[serde(with = "arc_str_serde")] + pub target_name: Arc, + #[serde(with = "arc_str_serde")] + pub provider_name: Arc, pub provider_id: u32, pub virtual_id: u32, - pub item_type: String, + pub item_type: PlaylistItemType, pub updated_at: u64, pub last_event_at: u64, pub window_24h: QosSnapshotWindow, @@ -206,7 +211,8 @@ mod tests { use std::path::Path; use tempfile::tempdir; - + use shared::model::PlaylistItemType; + use shared::utils::Internable; use super::{QosAggregationCheckpoint, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow}; #[test] @@ -216,12 +222,12 @@ mod tests { let snapshot = QosSnapshotRecord { stream_identity_key: "stream-a".to_string(), - input_name: "input-a".to_string(), - target_id: 11, - provider_name: "provider-a".to_string(), + input_name: "input-a".intern(), + target_name: "target-a".intern(), + provider_name: "provider-a".intern(), provider_id: 22, virtual_id: 33, - item_type: "live".to_string(), + item_type: PlaylistItemType::Live, updated_at: 1_700_000_000, last_event_at: 1_700_000_123, window_24h: QosSnapshotWindow { @@ -262,12 +268,12 @@ mod tests { let snapshot = QosSnapshotRecord { stream_identity_key: "stream-a".to_string(), - input_name: "input-a".to_string(), - target_id: 11, - provider_name: "provider-a".to_string(), + input_name: "input-a".intern(), + target_name: "target-a".intern(), + provider_name: "provider-a".intern(), provider_id: 22, virtual_id: 33, - item_type: "live".to_string(), + item_type: PlaylistItemType::Live, updated_at: 1_700_000_000, last_event_at: 1_700_000_123, window_24h: QosSnapshotWindow::default(), @@ -299,12 +305,12 @@ mod tests { let snapshot = QosSnapshotRecord { stream_identity_key: "stream-a".to_string(), - input_name: "input-a".to_string(), - target_id: 11, - provider_name: "provider-a".to_string(), + input_name: "input-a".intern(), + target_name: "target-a".intern(), + provider_name: "provider-a".intern(), provider_id: 22, virtual_id: 33, - item_type: "live".to_string(), + item_type: PlaylistItemType::Live, updated_at: 1_700_000_000, last_event_at: 1_700_000_123, window_24h: QosSnapshotWindow { diff --git a/backend/src/repository/stream_history/archive.rs b/backend/src/repository/stream_history/archive.rs index 750809dfa..d3ecafa65 100644 --- a/backend/src/repository/stream_history/archive.rs +++ b/backend/src/repository/stream_history/archive.rs @@ -1,9 +1,9 @@ use crate::repository::stream_history::{ BlockHeaderBody, CompressionKind, FileHeaderBody, RecordEncodingKind, BLOCK_MAGIC, CONTAINER_FORMAT_VERSION, - RECORD_SCHEMA_VERSION, SOURCE_KIND_STREAM_HISTORY, read_and_verify_file_magic, read_framed, + SOURCE_KIND_STREAM_HISTORY, read_and_verify_file_magic, read_framed, write_file_magic, write_framed, }; -use crate::repository::stream_history::writer::{apply_retention, current_utc_day, now_utc_secs}; +use crate::repository::stream_history::writer::apply_retention; use lz4_flex::frame::FrameEncoder; use log::{info, warn}; use std::{ @@ -11,6 +11,9 @@ use std::{ io::{self, BufReader, Read, Seek}, path::{Path, PathBuf}, }; +use tokio::task::spawn_blocking; +use crate::model::RECORD_SCHEMA_VERSION; +use crate::utils::{current_utc_day, now_utc_secs}; struct PendingSummary { header: FileHeaderBody, @@ -111,7 +114,7 @@ pub fn archive_pending_file(path: &Path) -> io::Result { // Stream the block data from the pending file let mut pending = File::open(path)?; - pending.seek(std::io::SeekFrom::Start(summary.blocks_start))?; + pending.seek(io::SeekFrom::Start(summary.blocks_start))?; io::copy(&mut pending, &mut lz4)?; lz4.finish().map_err(io::Error::other)?; @@ -187,15 +190,22 @@ pub fn recover_pending_files(directory: &str) -> io::Result<()> { } /// Called by the writer on day rollover or shutdown: flush, archive, apply retention. -pub fn finalize_and_archive(pending_path: &Path, directory: &str, retention_days: u16) { - if !pending_path.exists() { +/// `archive_pending_file` (lz4 compression + file I/O) runs in `spawn_blocking` since it's CPU-bound; +/// `apply_retention` is fully async. +pub async fn finalize_and_archive(pending_path: &Path, directory: &str, retention_days: u16) { + let path_buf = pending_path.to_path_buf(); + let path_buf_2 = path_buf.clone(); + let path_exists = tokio::fs::metadata(&path_buf).await.is_ok(); + if !path_exists { return; } - match archive_pending_file(pending_path) { - Ok(p) => info!("Finalized archive: {}", p.display()), - Err(e) => warn!("Failed to archive pending file {}: {e}", pending_path.display()), + let archive_result = spawn_blocking(move || archive_pending_file(&path_buf)).await; + match archive_result { + Ok(Ok(p)) => info!("Finalized archive: {}", p.display()), + Ok(Err(e)) => warn!("Failed to archive pending file {}: {e}", path_buf_2.display()), + Err(e) => warn!("Failed to archive pending file (join error): {e}"), } - if let Err(e) = apply_retention(directory, retention_days) { + if let Err(e) = apply_retention(directory, retention_days).await { warn!("Retention cleanup failed: {e}"); } } @@ -205,13 +215,15 @@ mod tests { use super::*; use std::io::Write; use crate::repository::stream_history::{ - CompressionKind, EventType, RECORD_SCHEMA_VERSION, StreamHistoryRecord, read_and_verify_file_magic, + CompressionKind, read_and_verify_file_magic, read_framed, serialize_named, write_block_magic, }; - use crate::repository::stream_history::writer::{StreamHistoryWriter, now_utc_secs, current_utc_day}; - use crate::model::StreamHistoryConfig; + use crate::repository::stream_history::writer::{StreamHistoryWriter}; + use crate::model::{StreamHistoryConfig, StreamHistoryRecord}; use lz4_flex::frame::FrameDecoder; use tempfile::TempDir; + use shared::model::{PlaylistItemType, StreamHistoryEventType}; + use shared::utils::Internable; fn test_config(dir: &str, batch_size: usize) -> StreamHistoryConfig { StreamHistoryConfig { @@ -225,17 +237,17 @@ mod tests { fn make_record(session_id: u64) -> StreamHistoryRecord { StreamHistoryRecord { schema_version: RECORD_SCHEMA_VERSION, - event_type: EventType::Connect, + event_type: StreamHistoryEventType::Connect, event_ts_utc: now_utc_secs(), partition_day_utc: current_utc_day(), session_id, source_addr: None, api_username: Some("alice".to_string()), - provider_name: Some("acme".to_string()), + provider_name: Some("acme".intern()), provider_username: None, - input_name: Some("input".to_string()), + input_name: Some("input".intern()), virtual_id: Some(1), - item_type: Some("live".to_string()), + item_type: Some(PlaylistItemType::Live), title: Some("Test".to_string()), group: None, country: None, @@ -265,14 +277,14 @@ mod tests { connect_failure_reason: None, disconnect_reason: None, previous_session_id: None, - target_id: None, + target_name: None, } } /// Write a minimal `.pending` file and return its path. fn write_test_pending(dir: &Path, day: &str, records: &[StreamHistoryRecord]) -> PathBuf { use crate::repository::stream_history::{ - BlockHeaderBody, CONTAINER_FORMAT_VERSION, RECORD_SCHEMA_VERSION, CompressionKind, + BlockHeaderBody, CONTAINER_FORMAT_VERSION, CompressionKind, RecordEncodingKind, SOURCE_KIND_STREAM_HISTORY, }; diff --git a/backend/src/repository/stream_history/async_iterator.rs b/backend/src/repository/stream_history/async_iterator.rs new file mode 100644 index 000000000..99a3715d3 --- /dev/null +++ b/backend/src/repository/stream_history/async_iterator.rs @@ -0,0 +1,157 @@ +use crate::repository::stream_history::{StreamHistoryFileReader, AsyncStreamHistoryPendingReader}; +use crate::utils::stream_history_viewer::{discover_files, CompiledFilter, TimeRange}; +use std::path::Path; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::Stream as TokioStream; +use crate::model::StreamHistoryRecord; + +/// Async stream of stream history records. +/// NOTE: The stream implementation is ready but not yet wired to the API. +/// Kept here for future use when we move back to streaming responses. +#[allow(dead_code)] +pub struct StreamHistoryStream { + inner: ReceiverStream, +} + +#[allow(dead_code)] +impl StreamHistoryStream { + /// Creates a new stream by reading files in a background task. + /// + /// Pending files (uncompressed) are read with fully async I/O using + /// `AsyncStreamHistoryPendingReader` (`tokio::fs::File`). + /// Archive files (LZ4-compressed) still require `spawn_blocking` because + /// `lz4_flex::frame::FrameDecoder` is sync-only. + #[allow(clippy::needless_pass_by_value)] + #[allow(dead_code)] + pub(crate) fn new(dir: String, time_range: TimeRange, filters: Arc) -> Self { + let (tx, rx) = mpsc::channel(1024); + let filters_clone = Arc::clone(&filters); + let dir_clone = dir.clone(); + + tokio::spawn(async move { + Self::read_files_async(&dir_clone, &time_range, &filters_clone, tx).await; + }); + + Self { + inner: ReceiverStream::new(rx), + } + } + + /// Reads files and sends records through the channel. + /// - Pending files: fully async (`tokio::fs::File` + `AsyncStreamHistoryPendingReader`) + /// - Archive files: sync via `spawn_blocking` (`lz4_flex` is sync-only) + #[allow(clippy::needless_pass_by_value)] + #[allow(dead_code)] + async fn read_files_async( + dir: &str, + time_range: &TimeRange, + filters: &Arc, + tx: mpsc::Sender, + ) { + let files = match discover_files(Path::new(dir), time_range).await { + Ok(f) => f, + Err(e) => { + log::error!("Failed to discover stream history files: {e}"); + return; + } + }; + let (range_start, range_end) = *time_range; + + for file in &files { + if file.is_archive { + // Archive files: LZ4 decompression is sync-only — use spawn_blocking. + // The blocking task sends directly into the async channel via blocking_send(), + // avoiding the deadlock that would occur with an intermediate std::sync::mpsc + // channel whose bounded sender blocks before the async side starts draining. + let file_path = file.path.clone(); + let time_range_val = *time_range; + let tx_clone = tx.clone(); + let filters_clone = Arc::clone(filters); + let result = tokio::task::spawn_blocking(move || { + Self::read_archive_file(&file_path, time_range_val, &filters_clone, &tx_clone); + }).await; + if let Err(e) = result { + log::error!("Archive read task failed: {e}"); + } + } else { + // Pending files: fully async using AsyncStreamHistoryPendingReader + match AsyncStreamHistoryPendingReader::open(&file.path, Some(*time_range)).await { + Ok((mut reader, _)) => { + while let Some(result) = reader.next_record().await { + match result { + Ok(record) => { + if record.event_ts_utc < range_start || record.event_ts_utc > range_end { + continue; + } + if !filters.matches(&record) { + continue; + } + if tx.send(record).await.is_err() { + return; // Receiver dropped + } + } + Err(e) => { + log::warn!("Failed to read pending record: {e}"); + } + } + } + } + Err(e) => { + log::warn!("Failed to open pending file {}: {e}", file.path.display()); + } + } + } + } + } + + /// Read a LZ4 archive file (runs in `spawn_blocking`). + /// Streams records incrementally via `blocking_send` into the async channel, + /// so memory usage stays bounded regardless of archive size. + #[allow(dead_code)] + fn read_archive_file( + path: &Path, + time_range: TimeRange, + filters: &CompiledFilter, + tx: &mpsc::Sender, + ) { + let (range_start, range_end) = time_range; + let (reader, _) = match StreamHistoryFileReader::from_archive(path, Some(time_range)) { + Ok(r) => r, + Err(e) => { + log::warn!("Failed to open archive {}: {e}", path.display()); + return; + } + }; + for result in reader { + match result { + Ok(record) => { + if record.event_ts_utc < range_start || record.event_ts_utc > range_end { + continue; + } + if !filters.matches(&record) { + continue; + } + if tx.blocking_send(record).is_err() { + // Receiver dropped, stop sending + return; + } + } + Err(e) => { + log::warn!("Failed to read archive record: {e}"); + } + } + } + } +} + +impl TokioStream for StreamHistoryStream { + type Item = StreamHistoryRecord; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_next(cx) + } +} diff --git a/backend/src/repository/stream_history/mod.rs b/backend/src/repository/stream_history/mod.rs index 189f932b3..8523bd007 100644 --- a/backend/src/repository/stream_history/mod.rs +++ b/backend/src/repository/stream_history/mod.rs @@ -2,9 +2,11 @@ mod archive; mod writer; mod storage; mod reader; +mod async_iterator; pub use writer::*; pub use archive::*; pub use storage::*; pub use reader::*; +pub use async_iterator::*; diff --git a/backend/src/repository/stream_history/reader.rs b/backend/src/repository/stream_history/reader.rs index 1cf46505b..2715220d2 100644 --- a/backend/src/repository/stream_history/reader.rs +++ b/backend/src/repository/stream_history/reader.rs @@ -3,11 +3,14 @@ use std::io::{self, BufReader, Read}; use std::path::Path; use lz4_flex::frame::FrameDecoder; - +use tokio::fs::File as TokioFile; +use tokio::io::{AsyncReadExt, AsyncSeekExt, BufReader as TokioBufReader}; +use crate::model::StreamHistoryRecord; use crate::repository::stream_history::storage::{ read_and_verify_file_magic, read_and_verify_block_magic, read_framed, - deserialize_named, FileHeaderBody, BlockHeaderBody, StreamHistoryRecord, + deserialize_named, FileHeaderBody, BlockHeaderBody, BLOCK_MAGIC, MAX_BLOCK_PAYLOAD_SIZE, + async_read_and_verify_file_magic, async_read_and_verify_block_magic, async_read_framed, }; pub struct StreamHistoryFileReader { @@ -160,7 +163,7 @@ impl StreamHistoryFileReader { // by the underlying reader yet — chain them before self.reader. let tail_start = (i + 4).saturating_sub(carry_len); let tail = &scan_buf[tail_start..n]; - let mut chained = io::Cursor::new(tail).chain(&mut self.reader); + let mut chained = std::io::Read::chain(io::Cursor::new(tail), &mut self.reader); let block_header: BlockHeaderBody = match read_framed::<_, BlockHeaderBody>(&mut chained) { Ok(h) if (h.payload_len as usize) <= MAX_BLOCK_PAYLOAD_SIZE => { @@ -266,17 +269,185 @@ impl Iterator for StreamHistoryFileReader { } } +// --------------------------------------------------------------------------- +// Async pending file reader — uses tokio for non-blocking I/O +// --------------------------------------------------------------------------- + +/// Async reader for `.pending` stream history files. +/// +/// Uses `tokio::fs::File` + `tokio::io::BufReader` for fully async I/O. +/// Exposes `next_record()` as a public async fn — call it in a loop in the +/// spawned reader task rather than trying to implement Stream (which would +/// require async-stream or a state machine in `poll_next`). +/// +/// NOTE: This does NOT support magic recovery — pending files are append-only +/// and blocks are read sequentially. If a block is corrupt the stream stops. +pub struct AsyncStreamHistoryPendingReader { + reader: TokioBufReader, + time_range: Option<(u64, u64)>, + current_block_remaining: u32, + payload_buf: Vec, + payload_offset: usize, + done: bool, + file_path: String, +} + +impl AsyncStreamHistoryPendingReader { + /// Open a pending file and verify its header. + pub async fn open(path: &Path, time_range: Option<(u64, u64)>) -> io::Result<(Self, FileHeaderBody)> { + let file = TokioFile::open(path).await?; + let mut reader = TokioBufReader::new(file); + async_read_and_verify_file_magic(&mut reader).await?; + let header: FileHeaderBody = async_read_framed(&mut reader).await?; + Ok((Self { + reader, + time_range, + current_block_remaining: 0, + payload_buf: Vec::new(), + payload_offset: 0, + done: false, + file_path: path.display().to_string(), + }, header)) + } + + /// Skip `len` bytes by seeking forward from current position. + async fn skip_payload(&mut self, len: u32) -> io::Result<()> { + self.reader.seek(tokio::io::SeekFrom::Current(i64::from(len))).await?; + Ok(()) + } + + /// Read and validate the next block header, then load the full payload into `payload_buf`. + async fn read_next_block(&mut self) -> io::Result> { + loop { + match async_read_and_verify_block_magic(&mut self.reader).await { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(e) => { + eprintln!("Warning: {}: corrupt block magic: {e}", self.file_path); + self.done = true; + return Ok(None); + } + } + + let block_header: BlockHeaderBody = match async_read_framed(&mut self.reader).await { + Ok(h) => h, + Err(e) => { + eprintln!("Warning: {}: corrupt block header: {e}", self.file_path); + self.done = true; + return Ok(None); + } + }; + + // Block-level time-range skip + if let Some((start, end)) = self.time_range { + if block_header.last_event_ts_utc < start || block_header.first_event_ts_utc > end { + if let Err(e) = self.skip_payload(block_header.payload_len).await { + if e.kind() == io::ErrorKind::UnexpectedEof { + self.done = true; + return Ok(None); + } + return Err(e); + } + continue; + } + } + + // OOM protection + if block_header.payload_len as usize > MAX_BLOCK_PAYLOAD_SIZE { + eprintln!("Warning: {}: block payload_len {} exceeds MAX_BLOCK_PAYLOAD_SIZE, skipping", + self.file_path, block_header.payload_len); + if let Err(e) = self.skip_payload(block_header.payload_len).await { + if e.kind() == io::ErrorKind::UnexpectedEof { + self.done = true; + return Ok(None); + } + return Err(e); + } + continue; + } + + self.payload_buf.resize(block_header.payload_len as usize, 0); + self.reader.read_exact(&mut self.payload_buf).await?; + + let actual_crc = crc32fast::hash(&self.payload_buf); + if actual_crc != block_header.payload_crc { + eprintln!("Warning: {}: payload CRC mismatch, stopping", self.file_path); + self.done = true; + return Ok(None); + } + + self.current_block_remaining = block_header.record_count; + self.payload_offset = 0; + return Ok(Some(())); + } + } + + /// Fetch the next record, or `None` on EOF / error. + pub async fn next_record(&mut self) -> Option> { + if self.done { + return None; + } + + loop { + if self.current_block_remaining > 0 { + if self.payload_offset + 4 > self.payload_buf.len() { + eprintln!("Warning: {}: truncated record length", self.file_path); + self.current_block_remaining = 0; + continue; + } + let mut record_len_bytes = [0u8; 4]; + record_len_bytes.copy_from_slice(&self.payload_buf[self.payload_offset..self.payload_offset + 4]); + let record_len = u32::from_be_bytes(record_len_bytes) as usize; + self.payload_offset += 4; + + if self.payload_offset + record_len > self.payload_buf.len() { + eprintln!("Warning: {}: truncated record data", self.file_path); + self.current_block_remaining = 0; + continue; + } + + let record_bytes = &self.payload_buf[self.payload_offset..self.payload_offset + record_len]; + self.payload_offset += record_len; + self.current_block_remaining -= 1; + + match deserialize_named(record_bytes) { + Ok(record) => return Some(Ok(record)), + Err(e) => { + eprintln!("Warning: {}: failed to deserialize record: {e}", self.file_path); + continue; + } + } + } + + match self.read_next_block().await { + Ok(Some(())) => {} // Got block, loop back + Ok(None) => { + self.done = true; + return None; + } + Err(e) => { + self.done = true; + return Some(Err(e)); + } + } + } + } +} + #[cfg(test)] mod tests { use super::*; use crate::repository::stream_history::storage::*; use std::io::Write; use tempfile::NamedTempFile; + use shared::model::StreamHistoryEventType; + use shared::utils::Internable; + use crate::model::{RECORD_SCHEMA_VERSION}; fn make_test_record(ts: u64, username: &str) -> StreamHistoryRecord { StreamHistoryRecord { schema_version: RECORD_SCHEMA_VERSION, - event_type: EventType::Connect, + event_type: StreamHistoryEventType::Connect, event_ts_utc: ts, partition_day_utc: "2026-03-22".to_string(), session_id: 1, @@ -284,7 +455,7 @@ mod tests { api_username: Some(username.to_string()), provider_name: None, provider_username: None, - input_name: Some("input".to_string()), + input_name: Some("input".intern()), virtual_id: None, item_type: None, title: None, @@ -316,7 +487,7 @@ mod tests { connect_failure_reason: None, disconnect_reason: None, previous_session_id: None, - target_id: None, + target_name: None, } } diff --git a/backend/src/repository/stream_history/storage.rs b/backend/src/repository/stream_history/storage.rs index 1b86051b8..256bbc9ab 100644 --- a/backend/src/repository/stream_history/storage.rs +++ b/backend/src/repository/stream_history/storage.rs @@ -1,14 +1,10 @@ use serde::{Deserialize, Serialize}; use shared::error::to_io_error; use std::io::{self, Read, Write}; -use shared::model::StreamInfo; -use crate::repository::{now_utc_secs, utc_day_from_secs}; -use crate::utils::encode_base64_hash; pub const FILE_MAGIC: [u8; 8] = *b"STRHIST\x01"; pub const BLOCK_MAGIC: [u8; 4] = *b"BLK\x01"; pub const CONTAINER_FORMAT_VERSION: u8 = 1; -pub const RECORD_SCHEMA_VERSION: u8 = 1; pub const SOURCE_KIND_STREAM_HISTORY: &str = "stream_history"; pub const MAX_FRAME_SIZE: usize = 8 * 1024 * 1024; // 8 MiB /// Maximum allowed block payload size when reading. Prevents memory exhaustion on corrupt/malicious input during magic-recovery. @@ -28,60 +24,6 @@ pub enum RecordEncodingKind { MessagePackNamed, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum EventType { - Connect, - ConnectFailed, - Disconnect, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ConnectFailureReason { - UserAccountExpired, - UserConnectionsExhausted, - ProviderConnectionsExhausted, - ProviderError, - ProviderClosed, - ChannelUnavailable, - Preempted, - SessionExpired, - Provisioning, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FailureStage { - Admission, - ProviderOpen, - FirstByte, - Streaming, - SessionReconnect, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DisconnectReason { - Cleanup, - ClientClosed, - ClientKicked, - Provisioning, - ServerError, - Timeout, - /// Reserved for future day-split logic: emitted at midnight for sessions that span - /// two calendar days so the previous day's file gets a closing record. - /// Not yet emitted — sessions crossing midnight have their disconnect in the new day's file. - DayRollover, - Shutdown, - Unknown, - ProviderError, - ProviderClosed, - Preempted, - SessionExpired, - UserConnectionsExhausted, - ProviderConnectionsExhausted, -} /// Serialized as `MessagePack` named (map encoding) for schema evolution safety. /// @@ -131,208 +73,6 @@ pub struct BlockHeaderBody { pub flags: u8, } -/// A single stream lifecycle event record (connect or disconnect). -/// -/// Privacy: must never contain passwords, tokens, or credential-bearing URLs. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StreamHistoryRecord { - pub schema_version: u8, - pub event_type: EventType, - /// Unix timestamp seconds UTC of when this event occurred. - pub event_ts_utc: u64, - /// UTC calendar day of this event, e.g. `"2026-03-22"`. - pub partition_day_utc: String, - /// Stable correlation id shared between the connect and disconnect events of the same session. - pub session_id: u64, - pub source_addr: Option, - // User and provider identity (no passwords, no tokens) - pub api_username: Option, - pub provider_name: Option, - /// Provider-side username (e.g. Xtream account). Always `None` until `StreamInfo` - /// carries the provider credential — at which point populate from there. - pub provider_username: Option, - pub input_name: Option, - // Stream metadata - pub virtual_id: Option, - pub item_type: Option, - pub title: Option, - pub group: Option, - pub country: Option, - // QoS metadata - pub user_agent: Option, - pub shared: Option, - pub shared_joined_existing: Option, - pub shared_stream_id: Option, - pub provider_id: Option, - pub cluster: Option, - pub container: Option, - pub stream_url_hash: Option, - pub stream_identity_key: Option, - pub video_codec: Option, - pub audio_codec: Option, - pub audio_channels: Option, - pub resolution: Option, - pub fps: Option, - // Session summary — populated on disconnect. - // bytes_sent and first_byte_latency_ms are None for shared streams (meter serves - // multiple clients; per-session totals are not meaningful in that case). - pub connect_ts_utc: Option, - pub disconnect_ts_utc: Option, - pub session_duration: Option, - pub bytes_sent: Option, - pub first_byte_latency_ms: Option, - pub provider_reconnect_count: Option, - pub failure_stage: Option, - pub provider_http_status: Option, - pub provider_error_class: Option, - pub connect_failure_reason: Option, - pub disconnect_reason: Option, - /// Set when this connect event continues a session that was split by a `DayRollover`. - /// Always `None` until the `DayRollover` mechanism is implemented. - pub previous_session_id: Option, - /// Target config id — identifies the failover bundle this stream belongs to. - pub target_id: Option, -} - -/// `QoS` metrics collected at disconnect time, passed as a bundle to avoid growing the signature. -#[derive(Debug, Default)] -pub struct DisconnectQos { - pub bytes_sent: Option, - pub first_byte_latency_ms: Option, - pub provider_reconnect_count: Option, -} - -impl StreamHistoryRecord { - fn build_stream_identity_key(info: &StreamInfo) -> Option { - if info.channel.input_name.is_empty() { - return None; - } - let mut raw = String::with_capacity(info.channel.input_name.len() + 32); - raw.push_str(info.channel.input_name.as_ref()); - raw.push('|'); - raw.push_str(&info.channel.target_id.to_string()); - raw.push('|'); - raw.push_str(&info.channel.provider_id.to_string()); - raw.push('|'); - raw.push_str(&info.channel.virtual_id.to_string()); - raw.push('|'); - raw.push_str(&info.channel.item_type.to_string()); - Some(encode_base64_hash(&raw)) - } - - /// Build a common base from a `StreamInfo`, leaving event-specific fields to the caller. - /// `event_ts` is the authoritative timestamp — `partition_day_utc` is derived from it - /// so both fields are always consistent. - fn base(info: &StreamInfo, event_ts: u64) -> Self { - Self { - schema_version: RECORD_SCHEMA_VERSION, - event_type: EventType::Connect, // overridden by callers - event_ts_utc: event_ts, - partition_day_utc: utc_day_from_secs(event_ts), - // Combine connect-timestamp (upper 32 bits) with uid (lower 32 bits). - // This prevents session_id collision across server restarts and uid wrap-around. - // Both Connect and Disconnect derive this from the same StreamInfo, so they match. - session_id: (info.ts << 32) | u64::from(info.uid), - source_addr: Some(info.client_ip.clone()), - api_username: Some(info.username.clone()), - provider_name: Some(info.provider.clone()), - provider_username: None, - input_name: if info.channel.input_name.is_empty() { - None - } else { - Some(info.channel.input_name.to_string()) - }, - virtual_id: Some(info.channel.virtual_id), - item_type: Some(info.channel.item_type.to_string()), - title: Some(info.channel.title.to_string()), - group: Some(info.channel.group.to_string()), - country: info.country_code.clone(), - user_agent: if info.user_agent.is_empty() { None } else { Some(info.user_agent.clone()) }, - shared: Some(info.channel.shared), - shared_joined_existing: info.channel.shared_joined_existing, - shared_stream_id: info.channel.shared_stream_id, - provider_id: Some(info.channel.provider_id), - cluster: Some(info.channel.cluster.to_string()), - container: info.channel.technical.as_ref().and_then(|t| if t.container.is_empty() { None } else { Some(t.container.clone()) }), - stream_url_hash: if info.channel.url.is_empty() { - None - } else { - Some(encode_base64_hash(info.channel.url.as_ref())) - }, - stream_identity_key: Self::build_stream_identity_key(info), - video_codec: info.channel.technical.as_ref().and_then(|t| if t.video_codec.is_empty() { None } else { Some(t.video_codec.clone()) }), - audio_codec: info.channel.technical.as_ref().and_then(|t| if t.audio_codec.is_empty() { None } else { Some(t.audio_codec.clone()) }), - audio_channels: info.channel.technical.as_ref().and_then(|t| if t.audio_channels.is_empty() { None } else { Some(t.audio_channels.clone()) }), - resolution: info.channel.technical.as_ref().and_then(|t| if t.resolution.is_empty() { None } else { Some(t.resolution.clone()) }), - fps: info.channel.technical.as_ref().and_then(|t| if t.fps.is_empty() { None } else { Some(t.fps.clone()) }), - connect_ts_utc: None, - disconnect_ts_utc: None, - session_duration: None, - bytes_sent: None, - first_byte_latency_ms: None, - provider_reconnect_count: None, - failure_stage: None, - provider_http_status: None, - provider_error_class: None, - connect_failure_reason: None, - disconnect_reason: None, - previous_session_id: None, - target_id: Some(info.channel.target_id), - } - } - - pub fn from_connect(info: &StreamInfo) -> Self { - let mut record = Self::base(info, info.ts); - record.event_type = EventType::Connect; - record.connect_ts_utc = Some(info.ts); - record.previous_session_id = info.previous_session_id; - record - } - - pub fn from_connect_failed( - info: &StreamInfo, - reason: ConnectFailureReason, - attempt_uid: u32, - failure_stage: FailureStage, - ) -> Self { - let event_ts = now_utc_secs(); - let mut record = Self::base(info, event_ts); - record.event_type = EventType::ConnectFailed; - record.session_id = (event_ts << 32) | u64::from(attempt_uid); - record.failure_stage = Some(failure_stage); - record.connect_failure_reason = Some(reason); - record - } - - pub fn with_provider_failure(mut self, provider_http_status: Option, provider_error_class: Option<&str>) -> Self { - self.provider_http_status = provider_http_status; - self.provider_error_class = provider_error_class.map(ToString::to_string); - self - } - - /// Extra `QoS` fields carried as a struct to keep the signature stable. - pub fn from_disconnect( - info: &StreamInfo, - reason: DisconnectReason, - qos: &DisconnectQos, - failure_stage: Option, - ) -> Self { - let now_secs = now_utc_secs(); - let connect_secs = info.ts; - let mut record = Self::base(info, now_secs); - record.event_type = EventType::Disconnect; - record.connect_ts_utc = Some(connect_secs); - record.disconnect_ts_utc = Some(now_secs); - record.session_duration = Some(now_secs.saturating_sub(connect_secs)); - record.bytes_sent = qos.bytes_sent; - record.first_byte_latency_ms = qos.first_byte_latency_ms; - record.provider_reconnect_count = qos.provider_reconnect_count; - record.failure_stage = failure_stage; - record.disconnect_reason = Some(reason); - record - } -} - /// Serialize a value to `MessagePack` using **named (map) encoding** for schema evolution safety. pub fn serialize_named(value: &T) -> io::Result> { rmp_serde::to_vec_named(value).map_err(to_io_error) @@ -425,6 +165,66 @@ pub fn read_and_verify_block_magic(reader: &mut R) -> io::Result<()> { Ok(()) } +// --------------------------------------------------------------------------- +// Async variants for use with `tokio::io::AsyncRead` +// Used by `AsyncStreamHistoryPendingReader` in `reader.rs` +// --------------------------------------------------------------------------- + +use tokio::io::{AsyncRead, AsyncReadExt}; + +/// Async version of `read_and_verify_file_magic` for `tokio::io::AsyncRead`. +pub async fn async_read_and_verify_file_magic(reader: &mut R) -> io::Result<()> { + let mut magic = [0u8; 8]; + reader.read_exact(&mut magic).await?; + if magic != FILE_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid file magic: {magic:02X?}"), + )); + } + Ok(()) +} + +/// Async version of `read_and_verify_block_magic` for `tokio::io::AsyncRead`. +pub async fn async_read_and_verify_block_magic(reader: &mut R) -> io::Result<()> { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic).await?; + if magic != BLOCK_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid block magic: {magic:02X?}"), + )); + } + Ok(()) +} + +/// Async version of `read_framed` for `tokio::io::AsyncRead`. +/// Reads: [len: u32 BE][payload][crc32: u32 BE], verifies CRC, deserializes. +pub async fn async_read_framed Deserialize<'de>>(reader: &mut R) -> io::Result { + let mut len_buf = [0u8; 4]; + reader.read_exact(&mut len_buf).await?; + let len = u32::from_be_bytes(len_buf) as usize; + if len == 0 || len > MAX_FRAME_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid frame size: {len} (max {MAX_FRAME_SIZE})"), + )); + } + let mut payload = vec![0u8; len]; + reader.read_exact(&mut payload).await?; + let mut crc_buf = [0u8; 4]; + reader.read_exact(&mut crc_buf).await?; + let expected_crc = u32::from_be_bytes(crc_buf); + let actual_crc = crc32fast::hash(&payload); + if actual_crc != expected_crc { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("CRC mismatch: expected {expected_crc:#010x}, got {actual_crc:#010x}"), + )); + } + deserialize_named(&payload) +} + #[cfg(test)] mod tests { use super::*; @@ -434,6 +234,9 @@ mod tests { }; use std::net::SocketAddr; use std::io::Cursor; + use shared::model::{ConnectFailureReason, FailureStage, DisconnectReason, StreamHistoryEventType}; + use crate::model::{DisconnectQos, StreamHistoryRecord, RECORD_SCHEMA_VERSION}; + use crate::utils::encode_base64_hash; fn sample_file_header() -> FileHeaderBody { FileHeaderBody { @@ -470,17 +273,17 @@ mod tests { fn sample_connect_record() -> StreamHistoryRecord { StreamHistoryRecord { schema_version: RECORD_SCHEMA_VERSION, - event_type: EventType::Connect, + event_type: StreamHistoryEventType::Connect, event_ts_utc: 1_742_600_001, partition_day_utc: "2026-03-22".to_string(), session_id: 999, source_addr: Some("192.0.2.1:12345".to_string()), api_username: Some("alice".to_string()), - provider_name: Some("acme-tv".to_string()), + provider_name: Some("acme-tv".intern()), provider_username: Some("acme_user".to_string()), - input_name: Some("provider-input".to_string()), + input_name: Some("provider-input".intern()), virtual_id: Some(1234), - item_type: Some("live".to_string()), + item_type: Some(PlaylistItemType::Live), title: Some("News Channel".to_string()), group: Some("News".to_string()), country: Some("DE".to_string()), @@ -510,24 +313,24 @@ mod tests { connect_failure_reason: None, disconnect_reason: None, previous_session_id: None, - target_id: Some(1), + target_name: None, } } fn sample_disconnect_record(session_id: u64) -> StreamHistoryRecord { StreamHistoryRecord { schema_version: RECORD_SCHEMA_VERSION, - event_type: EventType::Disconnect, + event_type: StreamHistoryEventType::Disconnect, event_ts_utc: 1_742_603_601, partition_day_utc: "2026-03-22".to_string(), session_id, source_addr: Some("192.0.2.1:12345".to_string()), api_username: Some("alice".to_string()), - provider_name: Some("acme-tv".to_string()), + provider_name: Some("acme-tv".intern()), provider_username: Some("acme_user".to_string()), - input_name: Some("provider-input".to_string()), + input_name: Some("provider-input".intern()), virtual_id: Some(1234), - item_type: Some("live".to_string()), + item_type: Some(PlaylistItemType::Live), title: Some("News Channel".to_string()), group: Some("News".to_string()), country: Some("DE".to_string()), @@ -557,7 +360,7 @@ mod tests { connect_failure_reason: None, disconnect_reason: Some(DisconnectReason::ClientClosed), previous_session_id: None, - target_id: Some(1), + target_name: None, } } @@ -569,7 +372,7 @@ mod tests { "alice", &addr, "192.0.2.1", - "acme-tv", + "acme-tv".intern(), StreamChannel { target_id: 1, virtual_id: 1234, @@ -631,7 +434,7 @@ mod tests { let original = sample_connect_record(); let bytes = serialize_named(&original).expect("serialize"); let decoded: StreamHistoryRecord = deserialize_named(&bytes).expect("deserialize"); - assert_eq!(decoded.event_type, EventType::Connect); + assert_eq!(decoded.event_type, StreamHistoryEventType::Connect); assert_eq!(decoded.session_id, 999); assert_eq!(decoded.api_username.as_deref(), Some("alice")); assert!(decoded.disconnect_ts_utc.is_none()); @@ -644,8 +447,8 @@ mod tests { connect.session_id = session_id; let disconnect = sample_disconnect_record(session_id); assert_eq!(connect.session_id, disconnect.session_id); - assert_eq!(connect.event_type, EventType::Connect); - assert_eq!(disconnect.event_type, EventType::Disconnect); + assert_eq!(connect.event_type, StreamHistoryEventType::Connect); + assert_eq!(disconnect.event_type, StreamHistoryEventType::Disconnect); assert_eq!(disconnect.session_duration, Some(3600)); } @@ -700,9 +503,10 @@ mod tests { ConnectFailureReason::ProviderConnectionsExhausted, 77, FailureStage::Admission, + None, ); - assert_eq!(record.event_type, EventType::ConnectFailed); + assert_eq!(record.event_type, StreamHistoryEventType::ConnectFailed); assert_eq!( record.connect_failure_reason, Some(ConnectFailureReason::ProviderConnectionsExhausted) @@ -726,6 +530,7 @@ mod tests { ConnectFailureReason::ProviderConnectionsExhausted, 77, FailureStage::Admission, + None, ); assert_eq!(record.failure_stage, Some(FailureStage::Admission)); @@ -740,6 +545,7 @@ mod tests { ConnectFailureReason::ChannelUnavailable, 77, FailureStage::ProviderOpen, + None, ) .with_provider_failure(Some(503), Some("http_5xx")); diff --git a/backend/src/repository/stream_history/writer.rs b/backend/src/repository/stream_history/writer.rs index b94328525..935e48bf0 100644 --- a/backend/src/repository/stream_history/writer.rs +++ b/backend/src/repository/stream_history/writer.rs @@ -1,47 +1,25 @@ use crate::repository::stream_history::{ BlockHeaderBody, CompressionKind, FileHeaderBody, RecordEncodingKind, CONTAINER_FORMAT_VERSION, - RECORD_SCHEMA_VERSION, SOURCE_KIND_STREAM_HISTORY, StreamHistoryRecord, - write_block_magic, write_file_magic, write_framed, + SOURCE_KIND_STREAM_HISTORY, write_framed, }; use crate::repository::stream_history::archive::finalize_and_archive; -use crate::model::StreamHistoryConfig; +use crate::model::{StreamHistoryConfig, StreamHistoryRecord, RECORD_SCHEMA_VERSION}; use log::{error, info, warn}; use std::{ - fs::{self, File, OpenOptions}, - io::{self, BufWriter, Write}, + io, path::PathBuf, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, }, - time::{SystemTime, UNIX_EPOCH}, }; +use tokio::fs::{self, OpenOptions}; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::{mpsc, oneshot}; - -const SECS_PER_DAY: u64 = 86_400; +use crate::utils::{current_utc_day, now_utc_secs, utc_day_from_secs, SECS_PER_DAY}; const QUEUE_CAPACITY: usize = 4096; -pub(crate) fn now_utc_secs() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - -pub(crate) fn utc_day_from_secs(ts_secs: u64) -> String { - chrono::DateTime::from_timestamp_secs(i64::try_from(ts_secs).unwrap_or(0)).map_or_else(|| "1970-01-01".to_string(), |dt| dt.format("%Y-%m-%d").to_string()) -} - -pub(crate) fn current_utc_day() -> String { - utc_day_from_secs(now_utc_secs()) -} - -/// Compute Unix seconds until the next UTC midnight after `from_ms`. -pub fn secs_until_next_utc_midnight(from_secs: u64) -> u64 { - let remaining = SECS_PER_DAY - (from_secs % SECS_PER_DAY); - if remaining == 0 { SECS_PER_DAY } else { remaining } -} fn pending_file_path(directory: &str, day: &str) -> PathBuf { PathBuf::from(directory).join(format!("stream-history-{day}.pending")) @@ -50,6 +28,7 @@ fn pending_file_path(directory: &str, day: &str) -> PathBuf { enum WriterCommand { Record(Box), Flush(oneshot::Sender>), + GetBatch(oneshot::Sender>), Shutdown(oneshot::Sender<()>), } @@ -129,6 +108,24 @@ impl StreamHistoryWriter { pub fn is_enabled(&self) -> bool { self.tx.is_some() } + + /// Returns a copy of the current in-memory batch records. + /// Async version — sends a command and waits for the response. + pub async fn get_current_batch(&self) -> io::Result> { + let Some(tx) = &self.tx else { + return Ok(Vec::new()); + }; + + let (resp_tx, resp_rx) = oneshot::channel(); + + tx.send(WriterCommand::GetBatch(resp_tx)) + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "stream history worker is closed"))?; + + resp_rx + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "stream history worker dropped GetBatch response")) + } } struct WriterWorker { @@ -143,7 +140,7 @@ impl WriterWorker { } async fn run(self, mut rx: mpsc::Receiver, dropped_events: Arc) { - let mut state = match WriterState::open(&self.config, self.writer_instance_id) { + let mut state = match WriterState::open(&self.config, self.writer_instance_id).await { Ok(s) => s, Err(e) => { error!("Stream history writer failed to initialize: {e}"); @@ -160,7 +157,7 @@ impl WriterWorker { WriterCommand::Record(record) => { // Day rollover check — use the record's partition day as the target if record.partition_day_utc != state.current_day { - if let Err(e) = state.flush_and_rollover_to(&record.partition_day_utc, &self.config, self.writer_instance_id) { + if let Err(e) = state.flush_and_rollover_to(&record.partition_day_utc, &self.config, self.writer_instance_id).await { error!("Stream history day rollover failed: {e}"); let dropped = dropped_events.fetch_add(1, Ordering::Relaxed) + 1; warn!("Stream history record dropped due to rollover failure (total dropped: {dropped})"); @@ -171,21 +168,24 @@ impl WriterWorker { state.push(*record); if state.batch.len() >= self.config.stream_history_batch_size { - if let Err(e) = state.flush_batch() { + if let Err(e) = state.flush_batch().await { error!("Stream history batch flush failed: {e}. Dropped events: {}", dropped_events.load(Ordering::Relaxed)); } } } WriterCommand::Flush(resp) => { - let result = state.flush_batch(); + let result = state.flush_batch().await; let _ = resp.send(result); } + WriterCommand::GetBatch(resp) => { + let _ = resp.send(state.batch.clone()); + } WriterCommand::Shutdown(resp) => { - if let Err(e) = state.flush_batch() { + if let Err(e) = state.flush_batch().await { error!("Stream history flush on shutdown failed: {e}"); } - if let Err(e) = state.finalize() { + if let Err(e) = state.finalize().await { error!("Stream history finalize on shutdown failed: {e}"); } info!("Stream history writer shut down. Total blocks: {}, records: {}", @@ -201,11 +201,9 @@ impl WriterWorker { struct WriterState { directory: String, current_day: String, - file: Option>, + file: Option, file_path: PathBuf, batch: Vec, - /// Reusable scratch buffer for building block payloads — retains capacity across flushes. - payload_buf: Vec, total_block_count: u64, total_record_count: u64, min_event_ts: Option, @@ -213,22 +211,21 @@ struct WriterState { } impl WriterState { - fn open(config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result { + async fn open(config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result { let day = current_utc_day(); let dir = &config.stream_history_directory; - fs::create_dir_all(dir)?; + fs::create_dir_all(dir).await?; let path = pending_file_path(dir, &day); - let file = open_or_create_pending_file(&path, &day, writer_instance_id)?; + let file = open_or_create_pending_file(&path, &day, writer_instance_id).await?; Ok(Self { directory: dir.clone(), current_day: day, - file: Some(BufWriter::new(file)), + file: Some(file), file_path: path, batch: Vec::with_capacity(config.stream_history_batch_size), - payload_buf: Vec::new(), total_block_count: 0, total_record_count: 0, min_event_ts: None, @@ -248,15 +245,13 @@ impl WriterState { } /// Flush buffered records to the pending file. - /// NOTE: This performs blocking file I/O on the tokio runtime. Acceptable because - /// the writer runs on its own task and stream history is best-effort — a slow disk - /// only delays history persistence, never the streaming hot path. - fn flush_batch(&mut self) -> io::Result<()> { + /// Serialization (CPU-bound) runs in `spawn_blocking`; file I/O is fully async. + async fn flush_batch(&mut self) -> io::Result<()> { if self.batch.is_empty() { return Ok(()); } - let Some(writer) = self.file.as_mut() else { + let Some(file) = self.file.as_mut() else { return Err(io::Error::new(io::ErrorKind::BrokenPipe, "pending file not open")); }; @@ -269,40 +264,72 @@ impl WriterState { let record_count = u32::try_from(self.batch.len()) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record count too large"))?; - // Serialize all records directly into the reusable payload buffer. - // Each record is prefixed with its length as a u32 BE. - // The length slot is reserved first and patched after serialization to avoid a - // temporary per-record Vec allocation. - self.payload_buf.clear(); - for record in &self.batch { - let len_offset = self.payload_buf.len(); - self.payload_buf.extend_from_slice(&[0u8; 4]); // placeholder for length - rmp_serde::encode::write_named(&mut self.payload_buf, record) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - let record_len = self.payload_buf.len() - len_offset - 4; - let record_len_u32 = u32::try_from(record_len) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record too large"))?; - self.payload_buf[len_offset..len_offset + 4].copy_from_slice(&record_len_u32.to_be_bytes()); - } + // Serialize all records into the reusable payload buffer, then write + // the whole block in one async I/O pass. + // The batch is cloned for serialization; self.batch is only cleared after all I/O succeeds. + let batch_snapshot = self.batch.clone(); + let (header_bytes, payload_bytes) = match tokio::task::spawn_blocking(move || { + let mut payload_buf = Vec::new(); + for record in &batch_snapshot { + let len_offset = payload_buf.len(); + payload_buf.extend_from_slice(&[0u8; 4]); // placeholder for length + if let Err(e) = rmp_serde::encode::write_named(&mut payload_buf, record) { + return Err(io::Error::new(io::ErrorKind::InvalidData, e)); + } + let record_len = payload_buf.len() - len_offset - 4; + let Ok(record_len_u32) = u32::try_from(record_len) else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "record too large")); + }; + payload_buf[len_offset..len_offset + 4].copy_from_slice(&record_len_u32.to_be_bytes()); + } - let payload_len = u32::try_from(self.payload_buf.len()) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "block payload too large"))?; - let payload_crc = crc32fast::hash(&self.payload_buf); + let Ok(payload_len) = u32::try_from(payload_buf.len()) else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "block payload too large")); + }; + let payload_crc = crc32fast::hash(&payload_buf); - let block_header = BlockHeaderBody { - block_version: 1, - record_count, - payload_len, - first_event_ts_utc: first_ts, - last_event_ts_utc: last_ts, - payload_crc, - flags: 0, + let block_header = BlockHeaderBody { + block_version: 1, + record_count, + payload_len, + first_event_ts_utc: first_ts, + last_event_ts_utc: last_ts, + payload_crc, + flags: 0, + }; + + // Serialize the framed block header + let mut header_buf = Vec::new(); + if let Err(e) = write_framed(&mut header_buf, &block_header) { + return Err(io::Error::new(io::ErrorKind::InvalidData, e)); + } + + Ok((header_buf, payload_buf)) + }).await { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(e), + Err(e) => return Err(io::Error::other(e)), }; - write_block_magic(writer)?; - write_framed(writer, &block_header)?; - writer.write_all(&self.payload_buf)?; - writer.flush()?; + // Now write the magic, header, and payload as three separate async write calls. + // This parks the task between calls if I/O would block, without blocking any thread. + // Capture starting position so we can truncate on failure, removing partial corrupt bytes. + let start_pos = file.stream_position().await?; + let block_magic = crate::repository::stream_history::BLOCK_MAGIC; + if let Err(e) = async { + file.write_all(&block_magic).await?; + file.write_all(&header_bytes).await?; + file.write_all(&payload_bytes).await?; + file.flush().await?; + io::Result::Ok(()) + }.await { + // Truncate back to the pre-write position to remove partial corrupt bytes, + // then seek so the file handle is ready for the next write attempt. + // self.batch stays intact so the caller can retry. + let _ = file.set_len(start_pos).await; + let _ = file.seek(std::io::SeekFrom::Start(start_pos)).await; + return Err(e); + } self.total_block_count += 1; self.total_record_count += u64::from(record_count); @@ -311,21 +338,21 @@ impl WriterState { Ok(()) } - fn flush_and_rollover_to(&mut self, target_day: &str, config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result<()> { - self.flush_batch()?; - self.finalize()?; // closes and drops the BufWriter; file_path is still valid + async fn flush_and_rollover_to(&mut self, target_day: &str, config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result<()> { + self.flush_batch().await?; + self.finalize().await?; // closes and drops the File; file_path is still valid // Archive the just-closed file and apply retention immediately. // At this point all pre-rollover records are safely on disk — the first record // with a new partition_day_utc triggered this rollover, so no pre-midnight record // can arrive after this point. The previous-day file is now complete and safe to archive. - finalize_and_archive(&self.file_path, &self.directory, config.stream_history_retention_days); + finalize_and_archive(&self.file_path, &self.directory, config.stream_history_retention_days).await; let new_path = pending_file_path(&self.directory, target_day); - let new_file = open_or_create_pending_file(&new_path, target_day, writer_instance_id)?; + let new_file = open_or_create_pending_file(&new_path, target_day, writer_instance_id).await?; self.current_day = target_day.to_string(); - self.file = Some(BufWriter::new(new_file)); + self.file = Some(new_file); self.file_path = new_path; self.total_block_count = 0; self.total_record_count = 0; @@ -335,54 +362,75 @@ impl WriterState { Ok(()) } - /// Flush writer buffer to OS. Does not compress or archive. - fn finalize(&mut self) -> io::Result<()> { - if let Some(mut writer) = self.file.take() { - writer.flush()?; + /// Flush file buffer to OS. Does not compress or archive. + async fn finalize(&mut self) -> io::Result<()> { + if let Some(mut file) = self.file.take() { + file.flush().await?; } Ok(()) } } /// Open an existing `.pending` file for appending, or create a new one with the file header. -fn open_or_create_pending_file(path: &PathBuf, day: &str, writer_instance_id: u64) -> io::Result { - if path.exists() { - // Continue appending to existing file - let file = OpenOptions::new().append(true).open(path)?; - return Ok(file); - } - - // Create new file with header - let mut file = File::create(path)?; - let header = FileHeaderBody { - container_format_version: CONTAINER_FORMAT_VERSION, - record_schema_version: RECORD_SCHEMA_VERSION, - source_kind: SOURCE_KIND_STREAM_HISTORY.to_string(), - created_at_ts_utc: now_utc_secs(), - partition_day_ts_utc: day.to_string(), - writer_instance_id, - host_id: std::env::var("HOSTNAME").ok(), - compression_kind: CompressionKind::None, - finalized: false, - record_encoding_kind: RecordEncodingKind::MessagePackNamed, - finalized_at_ts_utc: None, - total_block_count: None, - total_record_count: None, - min_event_ts_utc: None, - max_event_ts_utc: None, +async fn open_or_create_pending_file(path: &PathBuf, day: &str, writer_instance_id: u64) -> io::Result { + // Atomically try to create a new file first. If it already exists, + // open with append mode instead. This avoids TOCTOU races where + // finalize_and_archive might create/finalize the file between our + // metadata check and open call. + let file = match OpenOptions::new().write(true).create_new(true).open(path).await { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + // File already exists, open for appending + OpenOptions::new().append(true).open(path).await? + } + Err(e) => return Err(e), }; - write_file_magic(&mut file)?; - write_framed(&mut file, &header)?; - file.flush()?; + // If we created a new file (empty), write the header. + // For existing files the header was already written at creation time. + if file.metadata().await.map_or(0, |m| m.len()) == 0 { + let header = FileHeaderBody { + container_format_version: CONTAINER_FORMAT_VERSION, + record_schema_version: RECORD_SCHEMA_VERSION, + source_kind: SOURCE_KIND_STREAM_HISTORY.to_string(), + created_at_ts_utc: now_utc_secs(), + partition_day_ts_utc: day.to_string(), + writer_instance_id, + host_id: std::env::var("HOSTNAME").ok(), + compression_kind: CompressionKind::None, + finalized: false, + record_encoding_kind: RecordEncodingKind::MessagePackNamed, + finalized_at_ts_utc: None, + total_block_count: None, + total_record_count: None, + min_event_ts_utc: None, + max_event_ts_utc: None, + }; - Ok(file) + let mut header_buf = Vec::new(); + write_framed(&mut header_buf, &header) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + let file_magic = crate::repository::stream_history::FILE_MAGIC; + let mut f = file; + f.write_all(&file_magic).await?; + f.write_all(&header_buf).await?; + f.flush().await?; + Ok(f) + } else { + Ok(file) + } } /// Delete daily archive/pending files older than `retention_days`. -pub fn apply_retention(directory: &str, retention_days: u16) -> io::Result<()> { +pub async fn apply_retention(directory: &str, retention_days: u16) -> io::Result<()> { let dir = PathBuf::from(directory); - if !dir.exists() { + let metadata = match tokio::fs::metadata(&dir).await { + Ok(m) => m, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e), + }; + if !metadata.is_dir() { return Ok(()); } @@ -392,24 +440,26 @@ pub fn apply_retention(directory: &str, retention_days: u16) -> io::Result<()> { utc_day_from_secs(cutoff_secs) }; - for entry in fs::read_dir(&dir)? { - let entry = entry?; - let name = entry.file_name(); - let name = name.to_string_lossy(); + let mut entries = tokio::fs::read_dir(&dir).await?; + let mut to_delete = Vec::new(); - // Match "stream-history-YYYY-MM-DD.pending" or "stream-history-YYYY-MM-DD.archive.lz4" + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().to_string_lossy().into_owned(); if let Some(day) = extract_day_from_filename(&name) { if day < cutoff_day.as_str() { - let path = entry.path(); - if let Err(e) = fs::remove_file(&path) { - warn!("Failed to delete old history file {}: {e}", path.display()); - } else { - info!("Deleted expired history file: {}", path.display()); - } + to_delete.push(entry.path()); } } } + for path in to_delete { + if let Err(e) = tokio::fs::remove_file(&path).await { + warn!("Failed to delete old history file {}: {e}", path.display()); + } else { + info!("Deleted expired history file: {}", path.display()); + } + } + Ok(()) } @@ -426,11 +476,13 @@ pub fn extract_day_from_filename(name: &str) -> Option<&str> { #[cfg(test)] mod tests { use super::*; - use crate::repository::stream_history::{EventType}; use crate::repository::{BlockHeaderBody, FileHeaderBody, read_and_verify_block_magic, read_and_verify_file_magic, read_framed}; use std::io::BufReader; use std::sync::atomic::Ordering; use tempfile::TempDir; + use shared::model::{PlaylistItemType, StreamHistoryEventType}; + use shared::utils::Internable; + use crate::utils::secs_until_next_utc_midnight; fn test_config(dir: &str, batch_size: usize) -> StreamHistoryConfig { StreamHistoryConfig { @@ -441,7 +493,7 @@ mod tests { } } - fn make_record(session_id: u64, event_type: EventType) -> StreamHistoryRecord { + fn make_record(session_id: u64, event_type: StreamHistoryEventType) -> StreamHistoryRecord { StreamHistoryRecord { schema_version: RECORD_SCHEMA_VERSION, event_type, @@ -450,11 +502,11 @@ mod tests { session_id, source_addr: None, api_username: Some("user1".to_string()), - provider_name: Some("acme".to_string()), + provider_name: Some("acme".intern()), provider_username: None, - input_name: Some("input".to_string()), + input_name: Some("input".intern()), virtual_id: Some(1), - item_type: Some("live".to_string()), + item_type: Some(PlaylistItemType::Live), title: Some("Test Channel".to_string()), group: None, country: None, @@ -484,7 +536,7 @@ mod tests { connect_failure_reason: None, disconnect_reason: None, previous_session_id: None, - target_id: None, + target_name: None, } } @@ -492,7 +544,7 @@ mod tests { async fn stream_history_writer_disabled_accepts_records_silently() { let writer = StreamHistoryWriter::new_disabled(); // Must not panic - writer.send_record(make_record(1, EventType::Connect)); + writer.send_record(make_record(1, StreamHistoryEventType::Connect)); writer.flush().await.expect("flush on disabled writer"); writer.shutdown().await; assert_eq!(writer.dropped_events.load(Ordering::Relaxed), 0); @@ -515,7 +567,7 @@ mod tests { // Send 3 records (< batch_size=4), no file blocks should be written yet for i in 0..3_u64 { - writer.send_record(make_record(i, EventType::Connect)); + writer.send_record(make_record(i, StreamHistoryEventType::Connect)); } // Explicit flush — must not error even if no block written @@ -537,20 +589,20 @@ mod tests { // Send exactly batch_size records — triggers one block write for i in 0..batch_size as u64 { - writer.send_record(make_record(i, EventType::Connect)); + writer.send_record(make_record(i, StreamHistoryEventType::Connect)); } // Give the worker time to process tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; // Send one more and flush to ensure previous batch was processed - writer.send_record(make_record(99, EventType::Disconnect)); + writer.send_record(make_record(99, StreamHistoryEventType::Disconnect)); writer.flush().await.expect("flush ok"); writer.shutdown().await; let day = current_utc_day(); let pending = tmp.path().join(format!("stream-history-{day}.pending")); - let metadata = std::fs::metadata(&pending).unwrap(); + let metadata = tokio::fs::metadata(&pending).await.unwrap(); // File should be larger than just the header (a block was written) assert!(metadata.len() > 100, "file must contain at least one block (size={})", metadata.len()); } @@ -579,7 +631,7 @@ mod tests { // Send 3 records (well below batch_size=100) for i in 0..3_u64 { - writer.send_record(make_record(i, EventType::Connect)); + writer.send_record(make_record(i, StreamHistoryEventType::Connect)); } // Explicit flush must write the partial batch to disk @@ -588,7 +640,7 @@ mod tests { let day = current_utc_day(); let pending = tmp.path().join(format!("stream-history-{day}.pending")); - let metadata = std::fs::metadata(&pending).unwrap(); + let metadata = tokio::fs::metadata(&pending).await.unwrap(); assert!(metadata.len() > 100, "partial batch must be flushed to file (size={})", metadata.len()); } @@ -598,15 +650,15 @@ mod tests { let config = test_config(tmp.path().to_str().unwrap(), 100); let writer = StreamHistoryWriter::new(&config); - let mut late = make_record(1, EventType::Connect); + let mut late = make_record(1, StreamHistoryEventType::Connect); late.event_ts_utc = 3_000; late.partition_day_utc = "1970-01-01".to_string(); - let mut early = make_record(2, EventType::Connect); + let mut early = make_record(2, StreamHistoryEventType::Connect); early.event_ts_utc = 1_000; early.partition_day_utc = "1970-01-01".to_string(); - let mut middle = make_record(3, EventType::Connect); + let mut middle = make_record(3, StreamHistoryEventType::Connect); middle.event_ts_utc = 2_000; middle.partition_day_utc = "1970-01-01".to_string(); @@ -635,13 +687,6 @@ mod tests { assert_eq!(offset, 1); } - #[test] - fn stream_history_retention_extract_day_from_filename() { - assert_eq!(extract_day_from_filename("stream-history-2026-03-22.pending"), Some("2026-03-22")); - assert_eq!(extract_day_from_filename("stream-history-2026-03-22.archive.lz4"), Some("2026-03-22")); - assert_eq!(extract_day_from_filename("unrelated.txt"), None); - } - #[tokio::test] async fn stream_history_writer_creates_pending_file_on_init() { let tmp = TempDir::new().unwrap(); diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index b2b4d331e..3c95b08a7 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -1664,7 +1664,7 @@ mod tests { XtreamPlaylistItem { virtual_id: provider_id, provider_id, - name: "live".intern(), + name: "Live".intern(), logo: "".intern(), logo_small: "".intern(), group: "group".intern(), diff --git a/backend/src/utils/db_viewer.rs b/backend/src/utils/db_viewer.rs index 4c5439a9c..3feca1e1b 100644 --- a/backend/src/utils/db_viewer.rs +++ b/backend/src/utils/db_viewer.rs @@ -215,7 +215,8 @@ fn exit_app(code: i32) { mod tests { use super::dump_qos_snapshot_db; use tempfile::tempdir; - + use shared::model::PlaylistItemType; + use shared::utils::Internable; use crate::repository::{BPlusTree, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow}; #[test] @@ -226,12 +227,12 @@ mod tests { let mut tree = BPlusTree::::new(); let record = QosSnapshotRecord { stream_identity_key: "stream-a".to_string(), - input_name: "input-a".to_string(), - target_id: 11, - provider_name: "provider-a".to_string(), + input_name: "input-a".intern(), + target_name: "target-a".intern(), + provider_name: "provider-a".intern(), provider_id: 22, virtual_id: 33, - item_type: "live".to_string(), + item_type: PlaylistItemType::Live, updated_at: 1_700_000_000, last_event_at: 1_700_000_001, window_24h: QosSnapshotWindow { diff --git a/backend/src/utils/mod.rs b/backend/src/utils/mod.rs index db9deca53..a265c53ee 100644 --- a/backend/src/utils/mod.rs +++ b/backend/src/utils/mod.rs @@ -16,6 +16,7 @@ mod epg_parser; mod ordinal; pub mod ffmpeg; mod lru_cache; +mod time_utils; #[macro_export] macro_rules! debug_if_enabled { @@ -82,5 +83,6 @@ pub use self::json_utils::*; pub use self::network::*; pub use self::step_measure::*; pub use self::sys_utils::*; +pub use self::time_utils::*; pub use self::stream_history_viewer::stream_history_viewer; diff --git a/backend/src/utils/stream_history_viewer.rs b/backend/src/utils/stream_history_viewer.rs index 4d5809bb9..2c81aeb02 100644 --- a/backend/src/utils/stream_history_viewer.rs +++ b/backend/src/utils/stream_history_viewer.rs @@ -3,14 +3,15 @@ use std::collections::HashMap; use std::fs; use std::io::{self, Write}; use std::path::Path; - +use std::sync::Arc; use chrono; use regex::Regex; use serde::Deserialize; - +use shared::utils::Internable; +use crate::model::{StreamHistoryRecord}; use crate::repository::{ - ConnectFailureReason, DisconnectReason, EventType, FailureStage, FileHeaderBody, StreamHistoryFileReader, - StreamHistoryRecord, extract_day_from_filename, read_and_verify_file_magic, read_framed, + FileHeaderBody, StreamHistoryFileReader, + extract_day_from_filename, read_and_verify_file_magic, read_framed, }; #[derive(Deserialize)] @@ -98,7 +99,6 @@ const NUMERIC_FIELDS: &[&str] = &[ "session_id", "virtual_id", "provider_id", - "target_id", "provider_http_status", "shared_stream_id", ]; @@ -107,6 +107,7 @@ const STRING_FIELDS: &[&str] = &[ "api_username", "provider_name", "provider_username", + "target_name", "input_name", "item_type", "title", @@ -174,7 +175,12 @@ impl CompiledFilter { FilterValue::Regex(re) => re.is_match(s), FilterValue::NumericExact(_) => false, }, - RecordFieldValue::String(None) => false, + RecordFieldValue::ArcStr(Some(s)) => match value { + FilterValue::Exact(v) => s.as_ref().eq_ignore_ascii_case(v), + FilterValue::Regex(re) => re.is_match(s.as_ref()), + FilterValue::NumericExact(_) => false, + }, + RecordFieldValue::String(None) | RecordFieldValue::ArcStr(None) => false, RecordFieldValue::U64(n) => match value { FilterValue::NumericExact(v) => n == *v, _ => false, @@ -186,24 +192,18 @@ impl CompiledFilter { enum RecordFieldValue<'a> { String(Option<&'a str>), + ArcStr(Option>), U64(u64), } fn get_record_field<'a>(record: &'a StreamHistoryRecord, field: &str) -> RecordFieldValue<'a> { match field { - "event_type" => { - let s = match record.event_type { - EventType::Connect => "connect", - EventType::ConnectFailed => "connect_failed", - EventType::Disconnect => "disconnect", - }; - RecordFieldValue::String(Some(s)) - } + "event_type" => RecordFieldValue::ArcStr(Some(record.event_type.to_string().intern())), "api_username" => RecordFieldValue::String(record.api_username.as_deref()), "provider_name" => RecordFieldValue::String(record.provider_name.as_deref()), "provider_username" => RecordFieldValue::String(record.provider_username.as_deref()), "input_name" => RecordFieldValue::String(record.input_name.as_deref()), - "item_type" => RecordFieldValue::String(record.item_type.as_deref()), + "item_type" => RecordFieldValue::ArcStr(record.item_type.as_ref().map(ToString::to_string).map(Internable::intern)), "title" => RecordFieldValue::String(record.title.as_deref()), "group" => RecordFieldValue::String(record.group.as_deref()), "country" => RecordFieldValue::String(record.country.as_deref()), @@ -214,48 +214,9 @@ fn get_record_field<'a>(record: &'a StreamHistoryRecord, field: &str) -> RecordF .shared_joined_existing .map(|shared| if shared { "true" } else { "false" }), ), - "disconnect_reason" => { - // Match against serde rename_all = "snake_case" names - RecordFieldValue::String(record.disconnect_reason.as_ref().map(|r| match r { - DisconnectReason::Cleanup => "cleanup", - DisconnectReason::ClientClosed => "client_closed", - DisconnectReason::ClientKicked => "client_kicked", - DisconnectReason::Provisioning => "provisioning", - DisconnectReason::ServerError => "server_error", - DisconnectReason::Timeout => "timeout", - DisconnectReason::DayRollover => "day_rollover", - DisconnectReason::Shutdown => "shutdown", - DisconnectReason::Unknown => "unknown", - DisconnectReason::ProviderError => "provider_error", - DisconnectReason::ProviderClosed => "provider_closed", - DisconnectReason::Preempted => "preempted", - DisconnectReason::SessionExpired => "session_expired", - DisconnectReason::UserConnectionsExhausted => "user_connections_exhausted", - DisconnectReason::ProviderConnectionsExhausted => "provider_connections_exhausted", - })) - } - "connect_failure_reason" => { - RecordFieldValue::String(record.connect_failure_reason.as_ref().map(|r| match r { - ConnectFailureReason::UserAccountExpired => "user_account_expired", - ConnectFailureReason::UserConnectionsExhausted => "user_connections_exhausted", - ConnectFailureReason::ProviderConnectionsExhausted => "provider_connections_exhausted", - ConnectFailureReason::ProviderError => "provider_error", - ConnectFailureReason::ProviderClosed => "provider_closed", - ConnectFailureReason::ChannelUnavailable => "channel_unavailable", - ConnectFailureReason::Preempted => "preempted", - ConnectFailureReason::SessionExpired => "session_expired", - ConnectFailureReason::Provisioning => "provisioning", - })) - } - "failure_stage" => { - RecordFieldValue::String(record.failure_stage.as_ref().map(|stage| match stage { - FailureStage::Admission => "admission", - FailureStage::ProviderOpen => "provider_open", - FailureStage::FirstByte => "first_byte", - FailureStage::Streaming => "streaming", - FailureStage::SessionReconnect => "session_reconnect", - })) - } + "disconnect_reason" => RecordFieldValue::ArcStr(record.disconnect_reason.as_ref().map(|r| r.to_string().intern())), + "connect_failure_reason" => RecordFieldValue::ArcStr(record.connect_failure_reason.as_ref().map(|r| r.to_string().intern())), + "failure_stage" => RecordFieldValue::ArcStr(record.failure_stage.as_ref().map(|r| r.to_string().intern())), "provider_error_class" => RecordFieldValue::String(record.provider_error_class.as_deref()), "user_agent" => RecordFieldValue::String(record.user_agent.as_deref()), "cluster" => RecordFieldValue::String(record.cluster.as_deref()), @@ -270,7 +231,7 @@ fn get_record_field<'a>(record: &'a StreamHistoryRecord, field: &str) -> RecordF "session_id" => RecordFieldValue::U64(record.session_id), "virtual_id" => RecordFieldValue::U64(u64::from(record.virtual_id.unwrap_or(0))), "provider_id" => RecordFieldValue::U64(u64::from(record.provider_id.unwrap_or(0))), - "target_id" => RecordFieldValue::U64(u64::from(record.target_id.unwrap_or(0))), + "target_name" => RecordFieldValue::String(record.target_name.as_deref()), "provider_http_status" => RecordFieldValue::U64(u64::from(record.provider_http_status.unwrap_or(0))), "shared_stream_id" => RecordFieldValue::U64(record.shared_stream_id.unwrap_or(0)), _ => RecordFieldValue::String(None), // Unknown field: no match @@ -297,21 +258,39 @@ pub(crate) struct HistoryFile { pub is_archive: bool, } -pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result> { - if !dir.exists() { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("Stream history directory not found: {}", dir.display()), - )); +pub(crate) async fn discover_files( + dir: &Path, + time_range: &TimeRange, +) -> io::Result> { + match tokio::fs::metadata(dir).await { + Ok(meta) => { + if !meta.is_dir() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("Stream history directory not found: {}", dir.display()), + )); + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("Stream history directory not found: {}", dir.display()), + )); + } + Err(e) => return Err(e), } let (range_start, range_end) = *time_range; let mut files = Vec::new(); - for entry in fs::read_dir(dir)? { - let entry = entry?; + let mut entries = tokio::fs::read_dir(dir).await?; + while let Some(entry) = entries.next_entry().await? { let path = entry.path(); - let name = path.file_name().unwrap_or_default().to_string_lossy().to_string(); + let name = path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string(); let is_archive = name.ends_with(".archive.lz4"); let is_pending = name.ends_with(".pending"); @@ -332,17 +311,23 @@ pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result h, - Err(e) => { + let path_clone = path.clone(); + let header = match tokio::task::spawn_blocking(move || read_file_header(&path_clone, is_archive)).await { + Ok(Ok(h)) => h, + Ok(Err(e)) => { eprintln!("Warning: skipping {}: {e}", path.display()); continue; } + Err(e) => { + eprintln!("Warning: skipping {} (task failed): {e}", path.display()); + continue; + } }; // File-level skip for archives with known timestamp bounds if is_archive { - if let (Some(min_ts), Some(max_ts)) = (header.min_event_ts_utc, header.max_event_ts_utc) { + if let (Some(min_ts), Some(max_ts)) = (header.min_event_ts_utc, header.max_event_ts_utc) + { if max_ts < range_start || min_ts > range_end { continue; } @@ -358,14 +343,18 @@ pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result io::Result io::Result { if is_archive { - let file = std::fs::File::open(path)?; + let file = fs::File::open(path)?; let decoder = lz4_flex::frame::FrameDecoder::new(file); let mut reader = std::io::BufReader::new(decoder); read_and_verify_file_magic(&mut reader)?; @@ -452,7 +441,7 @@ fn stream_output( let _ = writeln!(out, "]"); } -fn run_stream_history_viewer(input: &str) -> Result<(), String> { +async fn run_stream_history_viewer(input: &str) -> Result<(), String> { eprintln!("[INFO] All timestamps interpreted as UTC"); let query = load_query(input)?; @@ -465,14 +454,14 @@ fn run_stream_history_viewer(input: &str) -> Result<(), String> { let dir = query.path.as_deref().unwrap_or("data/stream_history"); let dir_path = Path::new(dir); - let files = discover_files(dir_path, &time_range).map_err(|e| e.to_string())?; + let files = discover_files(dir_path, &time_range).await.map_err(|e| e.to_string())?; stream_output(files.as_slice(), &time_range, &filters); Ok(()) } -pub fn stream_history_viewer(input: &str) -> i32 { - match run_stream_history_viewer(input) { +pub async fn stream_history_viewer(input: &str) -> i32 { + match run_stream_history_viewer(input).await { Ok(()) => 0, Err(err) => { eprintln!("Error: {err}"); @@ -485,6 +474,8 @@ pub fn stream_history_viewer(input: &str) -> i32 { #[cfg(test)] mod tests { + use shared::model::{DisconnectReason, StreamHistoryEventType, FailureStage}; + use shared::utils::Internable; use super::*; #[test] @@ -588,10 +579,10 @@ mod tests { let filter = CompiledFilter::compile(&raw).unwrap(); let mut record = make_empty_record(); - record.provider_name = Some("acme-tv".to_string()); + record.provider_name = Some("acme-tv".intern()); assert!(filter.matches(&record)); - record.provider_name = Some("other-provider".to_string()); + record.provider_name = Some("other-provider".intern()); assert!(!filter.matches(&record)); } @@ -706,9 +697,9 @@ mod tests { assert_eq!(query.to.as_deref(), Some("2026-03-24")); } - #[test] - fn test_run_stream_history_viewer_returns_error_instead_of_exiting_on_invalid_query() { - let result = run_stream_history_viewer("{not-json"); + #[tokio::test] + async fn test_run_stream_history_viewer_returns_error_instead_of_exiting_on_invalid_query() { + let result = run_stream_history_viewer("{not-json").await; assert!(result.is_err()); } @@ -760,7 +751,7 @@ mod tests { fn make_empty_record() -> StreamHistoryRecord { StreamHistoryRecord { schema_version: 1, - event_type: EventType::Connect, + event_type: StreamHistoryEventType::Connect, event_ts_utc: 0, partition_day_utc: String::new(), session_id: 0, @@ -800,7 +791,7 @@ mod tests { connect_failure_reason: None, disconnect_reason: None, previous_session_id: None, - target_id: None, + target_name: None, } } } diff --git a/backend/src/utils/time_utils.rs b/backend/src/utils/time_utils.rs new file mode 100644 index 000000000..923fb8483 --- /dev/null +++ b/backend/src/utils/time_utils.rs @@ -0,0 +1,23 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +pub const SECS_PER_DAY: u64 = 86_400; + +pub(crate) fn now_utc_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +pub(crate) fn utc_day_from_secs(ts_secs: u64) -> String { + chrono::DateTime::from_timestamp_secs(i64::try_from(ts_secs).unwrap_or(0)).map_or_else(|| "1970-01-01".to_string(), |dt| dt.format("%Y-%m-%d").to_string()) +} + +pub(crate) fn current_utc_day() -> String { + utc_day_from_secs(now_utc_secs()) +} + +/// Compute Unix seconds until the next UTC midnight after `from_secs`. +pub fn secs_until_next_utc_midnight(from_secs: u64) -> u64 { + SECS_PER_DAY - (from_secs % SECS_PER_DAY) +} \ No newline at end of file diff --git a/frontend/scss/app/_component.scss b/frontend/scss/app/_component.scss index c579e678c..c0de78ad8 100644 --- a/frontend/scss/app/_component.scss +++ b/frontend/scss/app/_component.scss @@ -94,3 +94,4 @@ @forward "components/filter_editor"; @forward "components/field_explanation"; @forward "components/rbac/user_management"; +@forward "components/country"; diff --git a/frontend/scss/app/components/_country.scss b/frontend/scss/app/components/_country.scss new file mode 100644 index 000000000..88ff70651 --- /dev/null +++ b/frontend/scss/app/components/_country.scss @@ -0,0 +1,27 @@ +.tp__country { + display: inline-flex; + align-items: center; + gap: var(--gap-mini); + min-width: 0; + flex: 0 1 auto; + + &__name { + min-width: 0; + overflow-wrap: anywhere; + font-weight: 700; + } + + &__flag { + display: inline-flex; + width: 1rem; + height: 0.75rem; + overflow: hidden; + border-radius: 2px; + + svg { + display: block; + width: 100%; + height: 100%; + } + } +} \ No newline at end of file diff --git a/frontend/src/app/components/country.rs b/frontend/src/app/components/country.rs new file mode 100644 index 000000000..6162c9b06 --- /dev/null +++ b/frontend/src/app/components/country.rs @@ -0,0 +1,57 @@ +use crate::{hooks::use_service_context, i18n::use_translation, utils::t_safe}; +use yew::{classes, component, html, AttrValue, Html, Properties}; + +pub fn display_country_code(code: Option<&str>) -> Option { + let normalized = code?.trim().to_ascii_uppercase(); + let is_iso_country = normalized.len() == 2 && normalized.as_bytes().iter().all(|byte| byte.is_ascii_alphabetic()); + if is_iso_country { + Some(normalized) + } else { + None + } +} + +#[derive(Properties, Clone, PartialEq)] +pub struct CountryProps { + pub country_code: Option, + #[prop_or_default] + pub classes: Option, +} + +#[component] +pub fn Country(props: &CountryProps) -> Html { + let translate = use_translation(); + let services = use_service_context(); + + if let Some(code) = display_country_code(props.country_code.as_deref()).as_ref() { + let country = t_safe(&translate, &format!("COUNTRY.{code}")).unwrap_or_else(|| code.clone()); + let flag_svg = services.flags.get_flag(code); + return html! { + + if let Some(svg) = flag_svg.as_ref() { + + } + {&country} + + }; + } + + html! {} +} + +#[cfg(test)] +mod tests { + use super::display_country_code; + + #[test] + fn display_country_code_filters_special_network_labels() { + assert_eq!(display_country_code(Some("de")), Some("DE".to_string())); + assert_eq!(display_country_code(Some("LOOPBACK")), None); + assert_eq!(display_country_code(Some("LAN")), None); + assert_eq!(display_country_code(Some(" ")), None); + } +} diff --git a/frontend/src/app/components/dashboard/stream_display/helpers.rs b/frontend/src/app/components/dashboard/stream_display/helpers.rs index 988133fd7..387442426 100644 --- a/frontend/src/app/components/dashboard/stream_display/helpers.rs +++ b/frontend/src/app/components/dashboard/stream_display/helpers.rs @@ -46,7 +46,7 @@ pub fn is_adaptive_session_stream(stream: &StreamInfo) -> bool { } pub fn is_background_transfer_stream(stream: &StreamInfo) -> bool { - stream.client_ip == BACKGROUND_TRANSFER_CLIENT_IP && stream.provider == BACKGROUND_TRANSFER_PROVIDER + stream.client_ip == BACKGROUND_TRANSFER_CLIENT_IP && stream.provider.as_ref() == BACKGROUND_TRANSFER_PROVIDER } pub fn filter_visible_streams( @@ -226,7 +226,7 @@ mod tests { shared_stream_id: None, technical: None, }, - provider: "provider".to_string(), + provider: "provider".intern(), addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080), client_ip: "127.0.0.1:1234".to_string(), user_agent: String::new(), diff --git a/frontend/src/app/components/dashboard/stream_display/item.rs b/frontend/src/app/components/dashboard/stream_display/item.rs index 024ffe830..1cae345ca 100644 --- a/frontend/src/app/components/dashboard/stream_display/item.rs +++ b/frontend/src/app/components/dashboard/stream_display/item.rs @@ -3,10 +3,9 @@ use super::{ meter::{MeterDisplayKind, StreamMeterBadge}, }; use crate::{ - app::components::{AppIcon, Chip, RevealContent, ToggleSwitch}, - hooks::use_service_context, + app::components::{country::display_country_code, AppIcon, Chip, Country, RevealContent, ToggleSwitch}, i18n::use_translation, - utils::{format_duration, t_safe}, + utils::format_duration, }; use shared::{ model::StreamInfo, @@ -14,17 +13,7 @@ use shared::{ }; use std::rc::Rc; use web_sys::MouseEvent; -use yew::{prelude::*, virtual_dom::AttrValue}; - -fn display_country_code(country_code: Option<&str>) -> Option { - let normalized = country_code?.trim().to_ascii_uppercase(); - let is_iso_country = normalized.len() == 2 && normalized.as_bytes().iter().all(|byte| byte.is_ascii_alphabetic()); - if is_iso_country { - Some(normalized) - } else { - None - } -} +use yew::prelude::*; #[derive(Properties, PartialEq, Clone)] pub struct StreamDisplayItemProps { @@ -36,15 +25,9 @@ pub struct StreamDisplayItemProps { #[component] pub fn StreamDisplayItem(props: &StreamDisplayItemProps) -> Html { let translate = use_translation(); - let services = use_service_context(); let stream = props.stream.clone(); let is_background_transfer = is_background_transfer_stream(&stream); let chips = build_technical_chips(stream.channel.item_type, stream.channel.technical.as_ref()); - let country_code = display_country_code(stream.country_code.as_deref()); - let country = country_code.as_ref().map_or_else(String::new, |code| { - t_safe(&translate, &format!("COUNTRY.{code}")).unwrap_or_else(|| code.clone()) - }); - let flag_svg = country_code.as_ref().and_then(|code| services.flags.get_flag(code)); let client_ip = strip_port(&stream.client_ip).to_string(); let handle_popup_click = { @@ -87,24 +70,11 @@ pub fn StreamDisplayItem(props: &StreamDisplayItemProps) -> Html { {translate.t("LABEL.CLIENT_IP")} {client_ip.clone()} - if !country.is_empty() || country_code.is_some() { + if display_country_code(stream.country_code.as_deref()).is_some() {
{translate.t("LABEL.COUNTRY")} - - if let Some(svg) = flag_svg.as_ref() { - - } - if !country.is_empty() { - {country.clone()} - } else if let Some(code) = country_code.as_ref() { - {code.clone()} - } - +
} @@ -153,16 +123,3 @@ pub fn StreamDisplayItem(props: &StreamDisplayItemProps) -> Html { } } - -#[cfg(test)] -mod tests { - use super::display_country_code; - - #[test] - fn display_country_code_filters_special_network_labels() { - assert_eq!(display_country_code(Some("de")), Some("DE".to_string())); - assert_eq!(display_country_code(Some("LOOPBACK")), None); - assert_eq!(display_country_code(Some("LAN")), None); - assert_eq!(display_country_code(Some(" ")), None); - } -} diff --git a/frontend/src/app/components/dashboard/stream_display/mod.rs b/frontend/src/app/components/dashboard/stream_display/mod.rs index 8f7d867d2..e43554aaa 100644 --- a/frontend/src/app/components/dashboard/stream_display/mod.rs +++ b/frontend/src/app/components/dashboard/stream_display/mod.rs @@ -18,17 +18,16 @@ use crate::{ hooks::use_service_context, i18n::use_translation, model::EventMessage, - services::{DialogService, FlagsLoadState}, + services::DialogService, }; -use gloo_timers::{callback::Interval, future::TimeoutFuture}; -use log::error; +use gloo_timers::callback::Interval; use shared::{ error::TuliproxError, model::{PlaylistRequest, PlaylistUrlResolveRequest, ProtocolMessage, StreamInfo, UserCommand}, utils::default_kick_secs, }; -use std::{cell::Cell, collections::HashMap, fmt::Display, future, rc::Rc, str::FromStr}; -use yew::{platform::spawn_local, prelude::*, suspense::use_future}; +use std::{collections::HashMap, fmt::Display, rc::Rc, str::FromStr}; +use yew::{platform::spawn_local, prelude::*}; use yew_hooks::use_clipboard; const KICK: &str = "kick"; @@ -55,8 +54,6 @@ pub fn StreamDisplay(props: &StreamDisplayProps) -> Html { let cleanup_now_secs = use_state(shared::utils::current_time_secs); let adaptive_session_ttl_secs = get_adaptive_session_ttl_secs(&config_ctx); let metrics_enabled = is_stream_metrics_enabled(&config_ctx); - let geoip_enabled = config_ctx.config.as_ref().is_some_and(|cfg| cfg.config.is_geoip_enabled()); - let flags_ready = use_state(|| service_ctx.flags.is_loaded()); use_effect_with((), move |_| { let interval = Interval::new(1000, update_timestamps); @@ -107,45 +104,6 @@ pub fn StreamDisplay(props: &StreamDisplayProps) -> Html { }); } - { - let flags_service = service_ctx.flags.clone(); - let flags_ready = flags_ready.clone(); - let _ = use_future(|| async move { - flags_service - .loaded_subscribe(&mut |loaded| { - flags_ready.set(loaded); - future::ready(()) - }) - .await - }); - } - - { - let flags_service = service_ctx.flags.clone(); - use_effect_with(geoip_enabled, move |geoip_enabled| { - let cancelled = Rc::new(Cell::new(false)); - if *geoip_enabled { - let flags_service = flags_service.clone(); - let cancelled = cancelled.clone(); - spawn_local(async move { - while !cancelled.get() && !flags_service.is_loaded() { - match flags_service.ensure_loaded_from_assets().await { - Ok(FlagsLoadState::Loaded) => break, - Ok(FlagsLoadState::InProgress) => { - TimeoutFuture::new(250).await; - } - Err(err) => { - error!("Failed to load flags {err}"); - TimeoutFuture::new(5000).await; - } - } - } - }); - } - move || cancelled.set(true) - }); - } - let visible_streams = use_memo( (props.streams.clone(), (*adaptive_last_seen).clone(), *cleanup_now_secs, adaptive_session_ttl_secs), |(streams, adaptive_last_seen, cleanup_now_secs, adaptive_session_ttl_secs)| { @@ -298,9 +256,6 @@ pub fn StreamDisplay(props: &StreamDisplayProps) -> Html { }) }; - // Force a re-render after the flags service finishes loading so StreamDisplayItem can read flag SVGs. - let _flags_ready_marker = *flags_ready; - html! {
diff --git a/frontend/src/app/components/dashboard/stream_history_view.rs b/frontend/src/app/components/dashboard/stream_history_view.rs index 0b38b6131..c9eb2920d 100644 --- a/frontend/src/app/components/dashboard/stream_history_view.rs +++ b/frontend/src/app/components/dashboard/stream_history_view.rs @@ -1,21 +1,23 @@ use crate::{ - app::components::{DateInput, NoContent, Search, Table, TableDefinition, TextButton}, + app::components::{Country, DateInput, NoContent, RevealContent, Search, Table, TableDefinition, TextButton}, hooks::use_service_context, i18n::use_translation, - services::{StreamHistoryProviderSummary, StreamHistoryQosSnapshot, StreamHistoryRecord}, + services::{StreamHistoryProviderSummary, StreamHistoryQosSnapshot}, utils::{format_bytes, format_duration, format_ts}, }; use futures::join; -use shared::model::SearchRequest; -use std::rc::Rc; +use shared::model::{DisconnectReason, SearchRequest, StreamHistoryEventType, StreamHistoryRecordDto}; +use std::{borrow::Cow, rc::Rc}; use wasm_bindgen_futures::spawn_local; use yew::prelude::*; -const NUM_COLS: usize = 13; +const NUM_COLS: usize = 15; const SUMMARY_NUM_COLS: usize = 6; const QOS_SUMMARY_NUM_COLS: usize = 8; const QOS_DETAIL_NUM_COLS: usize = 8; +type HlsSessionKey<'a> = (Option<&'a str>, u64, Option<&'a str>, Option, Option<&'a str>); + #[derive(Clone, PartialEq)] struct ProviderSummaryRow { provider_name: String, @@ -32,7 +34,7 @@ struct QosSummaryRow { input_name: String, provider_name: String, provider_id: u32, - target_id: u16, + target_name: String, item_type: String, window_24h: crate::services::StreamHistoryQosSnapshotWindow, } @@ -53,45 +55,165 @@ fn ts_to_date_str(ts: i64) -> String { chrono::DateTime::from_timestamp(ts, 0).map_or_else(String::new, |dt| dt.format("%Y-%m-%d").to_string()) } -fn optional_record_text(value: Option<&str>) -> &str { value.unwrap_or("-") } +fn optional_record_text_str(value: Option<&str>) -> &str { value.unwrap_or("-") } +fn optional_record_text(value: Option) -> String { value.unwrap_or_else(|| String::from("-")) } -fn record_matches(record: &StreamHistoryRecord, filter: &SearchRequest) -> bool { +/// Returns true if the container indicates an HLS stream (HTTP-based segmented streaming). +fn is_hls_container(container: Option<&str>) -> bool { + matches!(container, Some("mpegts") | Some("fmp4") | Some("hls")) +} + +/// Aggregate HLS records by session key (source_addr + session_id + provider_name + virtual_id + input_name). +/// HLS streams create many rapid connect/disconnect events for each segment. This function groups them +/// into logical sessions, showing first connect, last disconnect, and summed bytes. +/// +/// For non-HLS records, they are passed through unchanged. +fn aggregate_hls_records(records: &[Rc]) -> Vec> { + use std::collections::HashMap; + + // Separate HLS and non-HLS records + let (hls_records, non_hls): (Vec<_>, Vec<_>) = + records.iter().partition(|r| is_hls_container(r.container.as_deref())); + + // Group HLS records by session key + let mut sessions: HashMap> = HashMap::new(); + for record in hls_records { + let key = ( + record.source_addr.as_deref(), + record.session_id, + record.provider_name.as_deref(), + record.virtual_id, + record.input_name.as_deref(), + ); + sessions.entry(key).or_default().push(record.as_ref()); + } + + // For each session, create aggregated records + let mut aggregated = Vec::new(); + + // First, add all non-HLS records unchanged + aggregated.extend(non_hls.into_iter().cloned()); + + // Then, process each HLS session + for (_, session_records) in sessions { + if session_records.is_empty() { + continue; + } + + // Sort by timestamp + let mut sorted = session_records.clone(); + sorted.sort_by_key(|r| r.event_ts_utc); + + // Find first connect and last disconnect + let first_connect = sorted.iter().find(|r| r.event_type == StreamHistoryEventType::Connect); + let last_disconnect = sorted.iter().rfind(|r| r.event_type == StreamHistoryEventType::Disconnect); + + // Calculate session duration from first connect to last disconnect + let session_duration = if let (Some(fc), Some(ld)) = (first_connect, last_disconnect) { + let fc_ts = fc.event_ts_utc; + let ld_ts = ld.event_ts_utc; + if ld_ts > fc_ts { + Some(ld_ts - fc_ts) + } else { + None + } + } else { + None + }; + + // Sum bytes from all records in session + let total_bytes: u64 = sorted.iter().filter_map(|r| r.bytes_sent).sum(); + + // Count intermediate failures (disconnects that are not the last one) + let intermediate_failure_count = sorted + .iter() + .filter(|r| { + r.event_type == StreamHistoryEventType::Disconnect + && !std::ptr::eq(*r, last_disconnect.unwrap_or(&sorted[0])) + }) + .count(); + + // Create aggregated connect record + if let Some(first) = first_connect { + let mut agg = (*first).clone(); + agg.event_ts_utc = first.event_ts_utc; + agg.session_duration = session_duration; + agg.bytes_sent = Some(total_bytes); + // Mark the disconnect reason from last disconnect if available + if let Some(last) = last_disconnect { + agg.disconnect_reason = last.disconnect_reason; + } + aggregated.push(Rc::new(agg)); + } + + // If there were intermediate failures, add a failure summary row + if intermediate_failure_count > 0 { + if let Some(first) = first_connect { + let mut failure_row = (*first).clone(); + failure_row.event_type = StreamHistoryEventType::Failure; + failure_row.event_ts_utc = first.event_ts_utc + 1; // Just after first connect + failure_row.session_duration = None; + failure_row.bytes_sent = Some(total_bytes); + failure_row.disconnect_reason = + Some(DisconnectReason::IntermediateFailures(intermediate_failure_count)); + aggregated.push(Rc::new(failure_row)); + } + } + + // Add disconnect record if it exists and is different from first connect + if let Some(last) = last_disconnect { + if !std::ptr::eq(last, first_connect.unwrap_or(&sorted[0])) { + let mut disc = (*last).clone(); + disc.event_ts_utc = last.event_ts_utc; + disc.bytes_sent = Some(total_bytes); + aggregated.push(Rc::new(disc)); + } + } + } + + // Sort by timestamp descending (most recent first) + aggregated.sort_by_key(|b| std::cmp::Reverse(b.event_ts_utc)); + aggregated +} + +fn record_matches(record: &StreamHistoryRecordDto, filter: &SearchRequest) -> bool { + if matches!(filter, SearchRequest::Clear) { + return true; + } + let fields = [ + // Cow::Owned(record.partition_day_utc.to_string()), + // Cow::Owned(record.virtual_id.as_ref().map_or_else(String::new, ToString::to_string)), + // Cow::Borrowed(record.input_name.as_deref().unwrap_or("")), + // Cow::Borrowed(record.video_codec.as_deref().unwrap_or("")), + // Cow::Borrowed(record.audio_codec.as_deref().unwrap_or("")), + // Cow::Borrowed(record.resolution.as_deref().unwrap_or("")), + // Cow::Borrowed(record.target_name.as_deref().unwrap_or("")), + // Cow::Borrowed(record.session_duration.as_deref().unwrap_or("")), + Cow::Owned(record.event_ts_utc.to_string()), + Cow::Owned(record.event_type.to_string()), + Cow::Borrowed(record.title.as_deref().unwrap_or("")), + Cow::Borrowed(record.group.as_deref().unwrap_or("")), + Cow::Borrowed(record.api_username.as_deref().unwrap_or("")), + Cow::Borrowed(record.provider_name.as_deref().unwrap_or("")), + Cow::Owned(record.provider_id.as_ref().map_or_else(String::new, ToString::to_string)), + Cow::Owned(record.bytes_sent.as_ref().map_or_else(String::new, ToString::to_string)), + Cow::Owned(record.first_byte_latency_ms.as_ref().map_or_else(String::new, ToString::to_string)), + Cow::Borrowed(record.user_agent.as_deref().unwrap_or("")), + Cow::Owned(record.item_type.as_ref().map_or_else(String::new, ToString::to_string)), + Cow::Borrowed(record.container.as_deref().unwrap_or("")), + Cow::Owned(record.disconnect_reason.as_ref().map_or_else(String::new, ToString::to_string)), + Cow::Borrowed(record.source_addr.as_deref().unwrap_or("")), + Cow::Borrowed(record.country.as_deref().unwrap_or("")), + Cow::Borrowed(record.cluster.as_deref().unwrap_or("")), + ]; match filter { SearchRequest::Clear => true, SearchRequest::Text(text, _) => { let text_lower = text.to_lowercase(); - let fields = [ - record.api_username.as_deref().unwrap_or(""), - record.title.as_deref().unwrap_or(""), - record.provider_name.as_deref().unwrap_or(""), - record.user_agent.as_deref().unwrap_or(""), - record.item_type.as_deref().unwrap_or(""), - record.container.as_deref().unwrap_or(""), - record.video_codec.as_deref().unwrap_or(""), - record.audio_codec.as_deref().unwrap_or(""), - record.resolution.as_deref().unwrap_or(""), - record.source_addr.as_deref().unwrap_or(""), - record.disconnect_reason.as_deref().unwrap_or(""), - record.group.as_deref().unwrap_or(""), - ]; fields.iter().any(|f| f.to_lowercase().contains(&text_lower)) } SearchRequest::Regexp(pattern, _) => { if let Ok(re) = shared::model::REGEX_CACHE.get_or_compile(pattern) { - let fields = [ - record.api_username.as_deref().unwrap_or(""), - record.title.as_deref().unwrap_or(""), - record.provider_name.as_deref().unwrap_or(""), - record.user_agent.as_deref().unwrap_or(""), - record.item_type.as_deref().unwrap_or(""), - record.container.as_deref().unwrap_or(""), - record.video_codec.as_deref().unwrap_or(""), - record.audio_codec.as_deref().unwrap_or(""), - record.resolution.as_deref().unwrap_or(""), - record.source_addr.as_deref().unwrap_or(""), - record.disconnect_reason.as_deref().unwrap_or(""), - record.group.as_deref().unwrap_or(""), - ]; fields.iter().any(|f| re.is_match(f)) } else { false @@ -113,7 +235,7 @@ fn qos_snapshot_matches(snapshot: &StreamHistoryQosSnapshot, filter: &SearchRequ ] .iter() .any(|field| field.to_lowercase().contains(&text_lower)) - || snapshot.target_id.to_string().contains(&text_lower) + || snapshot.target_name.to_lowercase().contains(&text_lower) || snapshot.provider_id.to_string().contains(&text_lower) || snapshot.virtual_id.to_string().contains(&text_lower) } @@ -123,7 +245,7 @@ fn qos_snapshot_matches(snapshot: &StreamHistoryQosSnapshot, filter: &SearchRequ || re.is_match(&snapshot.input_name) || re.is_match(&snapshot.provider_name) || re.is_match(&snapshot.item_type) - || re.is_match(&snapshot.target_id.to_string()) + || re.is_match(&snapshot.target_name) || re.is_match(&snapshot.provider_id.to_string()) || re.is_match(&snapshot.virtual_id.to_string()) } else { @@ -183,7 +305,7 @@ fn qos_summary_rows(snapshots: &[StreamHistoryQosSnapshot]) -> Vec Vec { } fn has_any_stream_history_content( - records: &[Rc], + records: &[Rc], summaries: &[StreamHistoryProviderSummary], qos_snapshots: &[StreamHistoryQosSnapshot], ) -> bool { @@ -211,57 +333,13 @@ pub fn StreamHistoryView() -> Html { let translate = use_translation(); let from_date = use_state(|| Some(today_start_ts())); let to_date = use_state(|| Some(today_start_ts())); - let all_records = use_state(Vec::>::new); + let all_records = use_state(Vec::>::new); let summaries = use_state(Vec::::new); let qos_snapshots = use_state(Vec::::new); let selected_qos_snapshot = use_state(|| None::); let search_filter = use_state(|| SearchRequest::Clear); let loading = use_state(|| false); - // Load on mount with default date range (today) - { - let services = services.clone(); - let all_records = all_records.clone(); - let summaries = summaries.clone(); - let qos_snapshots = qos_snapshots.clone(); - let selected_qos_snapshot = selected_qos_snapshot.clone(); - let loading = loading.clone(); - let from = *from_date; - let to = *to_date; - use_effect_with((), move |_| { - let from_str = from.map(ts_to_date_str); - let to_str = to.map(ts_to_date_str); - loading.set(true); - spawn_local(async move { - let (history_result, summary_result, qos_result) = join!( - services.stream_history.get_history(from_str.as_deref(), to_str.as_deref()), - services.stream_history.get_summary(from_str.as_deref(), to_str.as_deref()), - services.stream_history.get_qos_snapshots() - ); - match history_result { - Ok(Some(records)) => all_records.set(records.into_iter().map(Rc::new).collect()), - Ok(None) | Err(_) => all_records.set(Vec::new()), - } - match summary_result { - Ok(Some(items)) => summaries.set(items), - Ok(None) | Err(_) => summaries.set(Vec::new()), - } - match qos_result { - Ok(Some(items)) => { - selected_qos_snapshot.set(None); - qos_snapshots.set(items); - } - Ok(None) | Err(_) => { - selected_qos_snapshot.set(None); - qos_snapshots.set(Vec::new()); - } - } - loading.set(false); - }); - || () - }); - } - let handle_from_change = { let from_date = from_date.clone(); Callback::from(move |ts: Option| from_date.set(ts)) @@ -320,14 +398,27 @@ pub fn StreamHistoryView() -> Html { }) }; + // Load on mount with default date range (today) + { + let handle_load = handle_load.clone(); + use_effect_with((), move |_| { + handle_load.emit(String::new()); + || () + }); + } + let handle_search = { let search_filter = search_filter.clone(); Callback::from(move |req: SearchRequest| search_filter.set(req)) }; - let filtered: Rc>> = + let filtered: Rc>> = use_memo(((*all_records).clone(), (*search_filter).clone()), |(records, filter)| { - records.iter().filter(|r| record_matches(r, filter)).cloned().collect::>() + let mut sorted: Vec> = records.to_vec(); + sorted.sort_by_key(|r| r.event_ts_utc); + // Aggregate HLS records first on the full dataset so connect/disconnect pairs are complete + let aggregated = aggregate_hls_records(&sorted); + aggregated.into_iter().filter(|r| record_matches(r, filter)).collect() }); let has_any_content = has_any_stream_history_content(&all_records, &summaries, &qos_snapshots); @@ -359,7 +450,7 @@ pub fn StreamHistoryView() -> Html { }; let translate_for_table = translate.clone(); - let table_def: Rc> = use_memo(filtered.clone(), move |filtered| { + let table_def: Rc> = use_memo(filtered.clone(), move |filtered| { let translate = translate_for_table.clone(); TableDefinition { items: Some(filtered.clone()), @@ -371,89 +462,101 @@ pub fn StreamHistoryView() -> Html { 0 => translate.t("LABEL.STREAM_HISTORY_TIME"), 1 => translate.t("LABEL.STREAM_HISTORY_EVENT"), 2 => translate.t("LABEL.USERNAME"), - 3 => translate.t("LABEL.TITLE"), - 4 => translate.t("LABEL.PROVIDER"), - 5 => translate.t("LABEL.DURATION"), - 6 => translate.t("LABEL.STREAM_HISTORY_BYTES"), - 7 => translate.t("LABEL.STREAM_HISTORY_FIRST_BYTE"), - 8 => translate.t("LABEL.USER_AGENT"), - 9 => translate.t("LABEL.TYPE"), - 10 => translate.t("LABEL.CONTAINER"), - 11 => translate.t("LABEL.STREAM_HISTORY_REASON"), - 12 => translate.t("LABEL.STREAM_HISTORY_IP"), + 3 => translate.t("LABEL.GROUP"), + 4 => translate.t("LABEL.TITLE"), + 5 => translate.t("LABEL.PROVIDER"), + 6 => translate.t("LABEL.DURATION"), + 7 => translate.t("LABEL.STREAM_HISTORY_BYTES"), + 8 => translate.t("LABEL.STREAM_HISTORY_FIRST_BYTE"), + 9 => translate.t("LABEL.USER_AGENT"), + 10 => translate.t("LABEL.TYPE"), + 11 => translate.t("LABEL.CONTAINER"), + 12 => translate.t("LABEL.STREAM_HISTORY_REASON"), + 13 => translate.t("LABEL.STREAM_HISTORY_IP"), + 14 => translate.t("LABEL.COUNTRY"), _ => String::new(), }; html! { {label} } }), - render_data_cell: Callback::from(|(_, col, record): (usize, usize, Rc)| match col { - 0 => html! { {format_ts(record.event_ts_utc)} }, - 1 => { - let is_connect = record.event_type == "connect"; - let badge_class = if is_connect { - "tp__stream-history__badge tp__stream-history__badge--connect" - } else { - "tp__stream-history__badge tp__stream-history__badge--disconnect" - }; - html! { {record.event_type.clone()} } - } - 2 => html! { {record.api_username.as_deref().unwrap_or("-")} }, - 3 => { - html! { {record.title.as_deref().unwrap_or("-")} } - } - 4 => html! { - - { - match (record.provider_name.as_deref(), record.provider_id) { - (Some(name), Some(id)) => format!("{name} (#{id})"), - (Some(name), None) => name.to_string(), - (None, Some(id)) => format!("#{id}"), - (None, None) => String::from("-"), + render_data_cell: Callback::from( + |(_, col, record): (usize, usize, Rc)| match col { + 0 => html! { {format_ts(record.event_ts_utc)} }, + 1 => { + let is_connect = record.event_type == StreamHistoryEventType::Connect; + let badge_class = if is_connect { + "tp__stream-history__badge tp__stream-history__badge--connect" + } else { + "tp__stream-history__badge tp__stream-history__badge--disconnect" + }; + html! { {record.event_type.to_string()} } + } + 2 => html! { {record.api_username.as_deref().unwrap_or("-")} }, + 3 => { + html! { {record.group.as_deref().unwrap_or("-")} } + } + 4 => { + html! { {record.title.as_deref().unwrap_or("-")} } + } + 5 => html! { + + { + match (record.provider_name.as_deref(), record.provider_id) { + (Some(name), Some(id)) => format!("{name} (#{id})"), + (Some(name), None) => name.to_string(), + (None, Some(id)) => format!("#{id}"), + (None, None) => String::from("-"), + } } - } - + + }, + 6 => html! { + + {record.session_duration.map(format_duration).unwrap_or_default()} + + }, + 7 => html! { + + {record.bytes_sent.map(format_bytes).unwrap_or_default()} + + }, + 8 => html! { + + {record.first_byte_latency_ms.map(|v| v.to_string()).unwrap_or_default()} + + }, + 9 => html! { + + {record.user_agent.as_deref()} + + }, + 10 => html! { + + {optional_record_text(record.item_type.as_ref().map(ToString::to_string))} + + }, + 11 => html! { + + {optional_record_text_str(record.container.as_deref())} + + }, + 12 => html! { + + {record.disconnect_reason.as_ref().map(ToString::to_string).unwrap_or_else(|| String::from("-")).replace('_', " ")} + + }, + 13 => html! { + + {record.source_addr.as_deref().unwrap_or("-")} + + }, + 14 => html! { + + + + }, + _ => html! {}, }, - 5 => html! { - - {record.session_duration.map(format_duration).unwrap_or_default()} - - }, - 6 => html! { - - {record.bytes_sent.map(format_bytes).unwrap_or_default()} - - }, - 7 => html! { - - {record.first_byte_latency_ms.map(|v| v.to_string()).unwrap_or_default()} - - }, - 8 => html! { - - {optional_record_text(record.user_agent.as_deref())} - - }, - 9 => html! { - - {optional_record_text(record.item_type.as_deref())} - - }, - 10 => html! { - - {optional_record_text(record.container.as_deref())} - - }, - 11 => html! { - - {record.disconnect_reason.as_deref().unwrap_or("-").replace('_', " ")} - - }, - 12 => html! { - - {record.source_addr.as_deref().unwrap_or("-")} - - }, - _ => html! {}, - }), + ), } }); let translate_for_summary = translate.clone(); @@ -529,7 +632,7 @@ pub fn StreamHistoryView() -> Html { }, 1 => html! { {format!("{} (#{})", row.provider_name, row.provider_id)} }, - 2 => html! { {row.target_id} }, + 2 => html! { {row.target_name.clone()} }, 3 => html! { {row.item_type.clone()} }, 4 => { html! { {format!("{} ({})", row.window_24h.score, qos_score_label(row.window_24h.score))} } @@ -638,7 +741,7 @@ pub fn StreamHistoryView() -> Html { }

{translate.t("LABEL.STREAM_HISTORY")}

- definition={table_def} /> + definition={table_def} />
}
@@ -648,7 +751,7 @@ pub fn StreamHistoryView() -> Html { #[cfg(test)] mod tests { - use super::{optional_record_text, provider_summary_rows, qos_detail_rows, qos_score_label, top_qos_snapshots}; + use super::{optional_record_text_str, provider_summary_rows, qos_detail_rows, qos_score_label, top_qos_snapshots}; use crate::services::{StreamHistoryProviderSummary, StreamHistoryQosSnapshot, StreamHistoryQosSnapshotWindow}; use shared::model::SearchRequest; @@ -676,7 +779,7 @@ mod tests { StreamHistoryQosSnapshot { stream_identity_key: stream_identity_key.to_string(), input_name: "input-a".to_string(), - target_id: 1, + target_name: "target-a".to_string(), provider_name: provider_name.to_string(), provider_id: 10, virtual_id: 99, @@ -708,7 +811,7 @@ mod tests { let snapshot = StreamHistoryQosSnapshot { stream_identity_key: "stream-a".to_string(), input_name: "input-a".to_string(), - target_id: 1, + target_name: "target-a".to_string(), provider_name: "provider-a".to_string(), provider_id: 10, virtual_id: 99, @@ -759,8 +862,8 @@ mod tests { #[test] fn optional_record_text_preserves_value_or_dash_fallback() { - assert_eq!(optional_record_text(Some("Lavf53.32.100")), "Lavf53.32.100"); - assert_eq!(optional_record_text(Some("live")), "live"); - assert_eq!(optional_record_text(None), "-"); + assert_eq!(optional_record_text_str(Some("Lavf53.32.100")), "Lavf53.32.100"); + assert_eq!(optional_record_text_str(Some("live")), "live"); + assert_eq!(optional_record_text_str(None), "-"); } } diff --git a/frontend/src/app/components/home.rs b/frontend/src/app/components/home.rs index b8db136bf..856d787e7 100644 --- a/frontend/src/app/components/home.rs +++ b/frontend/src/app/components/home.rs @@ -13,8 +13,10 @@ use crate::{ i18n::use_translation, model::{EventMessage, ViewType}, provider::DialogProvider, - services::{ToastCloseMode, ToastOptions}, + services::{FlagsLoadState, ToastCloseMode, ToastOptions}, }; +use gloo_timers::future::TimeoutFuture; +use log::error; use shared::{ model::{ permission::{Permission, PERM_ALL}, @@ -22,8 +24,8 @@ use shared::{ }, utils::Internable, }; -use std::{future, rc::Rc}; -use yew::{prelude::*, suspense::use_future}; +use std::{cell::Cell, future, rc::Rc}; +use yew::{platform::spawn_local, prelude::*, suspense::use_future}; #[component] pub fn Home() -> Html { @@ -149,6 +151,34 @@ pub fn Home() -> Html { //
+ let geoip_enabled = config.as_ref().is_some_and(|cfg| cfg.config.is_geoip_enabled()); + + { + let flags_service = services.flags.clone(); + use_effect_with(geoip_enabled, move |geoip_enabled| { + let cancelled = Rc::new(Cell::new(false)); + if *geoip_enabled { + let flags_service = flags_service.clone(); + let cancelled = cancelled.clone(); + spawn_local(async move { + while !cancelled.get() && !flags_service.is_loaded() { + match flags_service.ensure_loaded_from_assets().await { + Ok(FlagsLoadState::Loaded) => break, + Ok(FlagsLoadState::InProgress) => { + TimeoutFuture::new(250).await; + } + Err(err) => { + error!("Failed to load flags {err}"); + TimeoutFuture::new(5000).await; + } + } + } + }); + } + move || cancelled.set(true) + }); + } + if config.is_none() { return html! {}; } diff --git a/frontend/src/app/components/mod.rs b/frontend/src/app/components/mod.rs index 1cdf7890d..9ea104543 100644 --- a/frontend/src/app/components/mod.rs +++ b/frontend/src/app/components/mod.rs @@ -60,6 +60,7 @@ mod particle_flow_background; mod source_editor; mod title_card; +mod country; mod setup; // pub use self::input::*; // pub use self::menu_item::*; @@ -69,7 +70,7 @@ mod setup; pub(crate) use self::{ accordion::*, accordion_panel::*, authentication::*, breadcrumbs::*, card::*, cell_value::*, chip::*, - cluster_flags_input::*, collapse_panel::*, csv_table::*, custom_dialog::*, dashboard::*, date_input::*, + cluster_flags_input::*, collapse_panel::*, country::*, csv_table::*, custom_dialog::*, dashboard::*, date_input::*, date_input_action::*, downloads::DownloadsView, drop_down_icon_button::*, field_explanation::*, field_id::*, filter::*, hide_content::*, home::*, icon_button::*, key_value_editor::*, loading_screen::*, login::*, no_access::*, no_content::*, panel::*, particle_flow_background::*, playlist::*, radio_button_group::*, rbac::*, diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index 4f3fe22af..084869cff 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -90,7 +90,7 @@ fn download_task_to_stream_with_ts(download: &FileDownloadDto, ts: u64) -> Strea shared_stream_id: None, technical: None, }, - provider: BACKGROUND_TRANSFER_PROVIDER.to_string(), + provider: BACKGROUND_TRANSFER_PROVIDER.intern(), addr: download_stream_addr(uid), client_ip: BACKGROUND_TRANSFER_CLIENT_IP.to_string(), user_agent: "Tuliprox download worker".to_string(), @@ -347,7 +347,7 @@ mod tests { shared_stream_id: None, technical: None, }, - provider: "provider".to_string(), + provider: "provider".intern(), addr: addr.parse::().unwrap_or_else(|_| unreachable!()), client_ip: "127.0.0.1".to_string(), user_agent: "ua".to_string(), @@ -440,7 +440,7 @@ mod tests { apply_downloads_snapshot(&mut streams, &response); assert_eq!(streams.len(), 1); - assert_eq!(streams[0].provider, "Download Manager"); + assert_eq!(streams[0].provider.as_ref(), "Download Manager"); assert_eq!(streams[0].client_ip, "background-task"); assert_eq!(streams[0].channel.item_type, PlaylistItemType::Video); } diff --git a/frontend/src/services/stream_history_service.rs b/frontend/src/services/stream_history_service.rs index a283fefa0..65348b6db 100644 --- a/frontend/src/services/stream_history_service.rs +++ b/frontend/src/services/stream_history_service.rs @@ -1,35 +1,6 @@ use crate::services::{get_base_href, request_get, Encoding}; use serde::Deserialize; -use shared::utils::concat_path_leading_slash; - -#[derive(Debug, Clone, Deserialize, PartialEq)] -pub struct StreamHistoryRecord { - pub event_type: String, - pub event_ts_utc: u64, - pub partition_day_utc: String, - pub session_id: u64, - pub source_addr: Option, - pub api_username: Option, - pub provider_name: Option, - pub virtual_id: Option, - pub item_type: Option, - pub title: Option, - pub group: Option, - pub user_agent: Option, - pub shared: Option, - pub provider_id: Option, - pub cluster: Option, - pub container: Option, - pub video_codec: Option, - pub audio_codec: Option, - pub resolution: Option, - pub disconnect_reason: Option, - pub session_duration: Option, - pub bytes_sent: Option, - pub first_byte_latency_ms: Option, - pub previous_session_id: Option, - pub target_id: Option, -} +use shared::{model::StreamHistoryRecordDto, utils::concat_path_leading_slash}; #[derive(Debug, Clone, Deserialize, PartialEq)] pub struct StreamHistoryProviderSummary { @@ -59,7 +30,7 @@ pub struct StreamHistoryQosSnapshotWindow { pub struct StreamHistoryQosSnapshot { pub stream_identity_key: String, pub input_name: String, - pub target_id: u16, + pub target_name: String, pub provider_name: String, pub provider_id: u32, pub virtual_id: u32, @@ -93,14 +64,14 @@ impl StreamHistoryService { &self, from: Option<&str>, to: Option<&str>, - ) -> Result>, crate::error::Error> { + ) -> Result>, crate::error::Error> { let url = match (from, to) { (Some(f), Some(t)) => format!("{}?from={}&to={}", self.path, f, t), (Some(f), None) => format!("{}?from={}", self.path, f), (None, Some(t)) => format!("{}?to={}", self.path, t), (None, None) => self.path.clone(), }; - request_get::>(&url, None, None).await + request_get::>(&url, None, Some(Encoding::Cbor)).await } pub async fn get_summary( diff --git a/shared/src/model/mod.rs b/shared/src/model/mod.rs index 9e869907d..4f21bda28 100644 --- a/shared/src/model/mod.rs +++ b/shared/src/model/mod.rs @@ -23,6 +23,7 @@ mod search_request; mod short_epg; mod stats; mod status_check; +mod stream_history_record; mod stream_info; mod stream_meter; mod stream_properties; @@ -42,7 +43,7 @@ pub use self::{ active_user_connection_change::*, auth::*, cluster_flags::*, config::*, download::*, epg::*, epg_request::*, ip_check::*, item_field::*, library_request::*, mapping::*, media_properties::*, messaging::*, playlist::*, playlist_categories::*, playlist_info_document::*, playlist_request::*, processing_order::*, regex_cache::*, - search_request::*, short_epg::*, stats::*, status_check::*, stream_info::*, stream_meter::*, stream_properties::*, - strm_export_style::*, system_info::*, target_type::*, transfer::*, ui_playlist_item::*, user_command::*, - uuidtype::*, web_socket::*, xtream::*, + search_request::*, short_epg::*, stats::*, status_check::*, stream_history_record::*, stream_info::*, + stream_meter::*, stream_properties::*, strm_export_style::*, system_info::*, target_type::*, transfer::*, + ui_playlist_item::*, user_command::*, uuidtype::*, web_socket::*, xtream::*, }; diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index 4ff58557c..a631aca80 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -85,16 +85,27 @@ impl TryFrom for XtreamCluster { #[repr(u8)] pub enum PlaylistItemType { #[default] + #[serde(alias = "live")] Live = 1, + #[serde(alias = "video")] Video = 2, - Series = 3, // xtream series description + #[serde(alias = "series")] + Series = 3, // xtream series description + #[serde(alias = "series_info")] SeriesInfo = 4, // xtream series info fetched for series description + #[serde(alias = "catchup")] Catchup = 5, + #[serde(alias = "live_unknown")] LiveUnknown = 6, // No Provider id - LiveHls = 7, // m3u8 entry - LiveDash = 8, // mpd + #[serde(alias = "live_hls")] + LiveHls = 7, // m3u8 entry + #[serde(alias = "live_dash")] + LiveDash = 8, // mpd + #[serde(alias = "local_video")] LocalVideo = 9, + #[serde(alias = "local_series")] LocalSeries = 10, + #[serde(alias = "local_series_info")] LocalSeriesInfo = 11, } diff --git a/shared/src/model/stream_history_record.rs b/shared/src/model/stream_history_record.rs new file mode 100644 index 000000000..d3c87206f --- /dev/null +++ b/shared/src/model/stream_history_record.rs @@ -0,0 +1,164 @@ +use crate::{model::PlaylistItemType, utils::arc_str_option_serde}; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::{Display, Formatter}, + sync::Arc, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StreamHistoryEventType { + Connect, + ConnectFailed, + Disconnect, + Failure, +} + +impl Display for StreamHistoryEventType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + StreamHistoryEventType::Connect => write!(f, "connect"), + StreamHistoryEventType::ConnectFailed => write!(f, "connect_failed"), + StreamHistoryEventType::Disconnect => write!(f, "disconnect"), + StreamHistoryEventType::Failure => write!(f, "failure"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DisconnectReason { + Cleanup, + ClientClosed, + ClientKicked, + Provisioning, + ServerError, + Timeout, + /// Reserved for future day-split logic: emitted at midnight for sessions that span + /// two calendar days so the previous day's file gets a closing record. + /// Not yet emitted — sessions crossing midnight have their disconnect in the new day's file. + DayRollover, + Shutdown, + Unknown, + ProviderError, + ProviderClosed, + Preempted, + SessionExpired, + UserConnectionsExhausted, + ProviderConnectionsExhausted, + IntermediateFailures(usize), +} + +impl Display for DisconnectReason { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + DisconnectReason::Cleanup => write!(f, "cleanup"), + DisconnectReason::ClientClosed => write!(f, "client_closed"), + DisconnectReason::ClientKicked => write!(f, "client_kicked"), + DisconnectReason::Provisioning => write!(f, "provisioning"), + DisconnectReason::ServerError => write!(f, "server_error"), + DisconnectReason::Timeout => write!(f, "timeout"), + DisconnectReason::DayRollover => write!(f, "day_rollover"), + DisconnectReason::Shutdown => write!(f, "shutdown"), + DisconnectReason::Unknown => write!(f, "unknown"), + DisconnectReason::ProviderError => write!(f, "provider_error"), + DisconnectReason::ProviderClosed => write!(f, "provider_closed"), + DisconnectReason::Preempted => write!(f, "preempted"), + DisconnectReason::SessionExpired => write!(f, "session_expired"), + DisconnectReason::UserConnectionsExhausted => write!(f, "user_connections_exhausted"), + DisconnectReason::ProviderConnectionsExhausted => write!(f, "provider_connections_exhausted"), + DisconnectReason::IntermediateFailures(count) => write!(f, "intermediate_failures({count})", count = count), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ConnectFailureReason { + UserAccountExpired, + UserConnectionsExhausted, + ProviderConnectionsExhausted, + ProviderError, + ProviderClosed, + ChannelUnavailable, + Preempted, + SessionExpired, + Provisioning, +} + +impl Display for ConnectFailureReason { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ConnectFailureReason::UserAccountExpired => write!(f, "user_account_expired"), + ConnectFailureReason::UserConnectionsExhausted => write!(f, "user_connections_exhausted"), + ConnectFailureReason::ProviderConnectionsExhausted => write!(f, "provider_connections_exhausted"), + ConnectFailureReason::ProviderError => write!(f, "provider_error"), + ConnectFailureReason::ProviderClosed => write!(f, "provider_closed"), + ConnectFailureReason::ChannelUnavailable => write!(f, "channel_unavailable"), + ConnectFailureReason::Preempted => write!(f, "preempted"), + ConnectFailureReason::SessionExpired => write!(f, "session_expired"), + ConnectFailureReason::Provisioning => write!(f, "provisioning"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FailureStage { + Admission, + ProviderOpen, + FirstByte, + Streaming, + SessionReconnect, +} + +impl Display for FailureStage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + FailureStage::Admission => write!(f, "admission"), + FailureStage::ProviderOpen => write!(f, "provider open"), + FailureStage::FirstByte => write!(f, "first byte"), + FailureStage::Streaming => write!(f, "streaming"), + FailureStage::SessionReconnect => write!(f, "session reconnect"), + } + } +} + +/// DTO for stream history records exchanged between backend and frontend. +/// Uses string-based enum variants for JSON compatibility. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub struct StreamHistoryRecordDto { + pub event_type: StreamHistoryEventType, + pub event_ts_utc: u64, + pub partition_day_utc: String, + pub session_id: u64, + pub source_addr: Option, + pub api_username: Option, + #[serde(default, with = "arc_str_option_serde")] + pub provider_name: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub input_name: Option>, + pub virtual_id: Option, + pub item_type: Option, + pub title: Option, + pub group: Option, + pub country: Option, + pub user_agent: Option, + pub shared: Option, + pub provider_id: Option, + pub cluster: Option, + pub container: Option, + pub video_codec: Option, + pub audio_codec: Option, + pub resolution: Option, + pub failure_stage: Option, + pub connect_failure_reason: Option, + pub disconnect_reason: Option, + pub session_duration: Option, + pub bytes_sent: Option, + pub first_byte_latency_ms: Option, + pub previous_session_id: Option, + #[serde(default, with = "arc_str_option_serde")] + pub target_name: Option>, +} diff --git a/shared/src/model/stream_info.rs b/shared/src/model/stream_info.rs index 9cddecf1d..bf0faefb7 100644 --- a/shared/src/model/stream_info.rs +++ b/shared/src/model/stream_info.rs @@ -275,7 +275,8 @@ pub struct StreamInfo { pub meter_uid: u32, pub username: String, pub channel: StreamChannel, - pub provider: String, + #[serde(default, with = "arc_str_serde")] + pub provider: Arc, pub addr: SocketAddr, pub client_ip: String, #[serde(default)] @@ -302,7 +303,7 @@ impl StreamInfo { username: &str, addr: &SocketAddr, client_ip: &str, - provider: &str, + provider: Arc, stream_channel: StreamChannel, user_agent: String, country_code: Option, @@ -314,7 +315,7 @@ impl StreamInfo { meter_uid, username: username.to_string(), channel: stream_channel, - provider: provider.to_string(), + provider, addr: *addr, client_ip: client_ip.to_string(), user_agent,