euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-24 00:32:30 +02:00
Code Issues Packages Projects Releases 5 Wiki Activity

Feature/stream history (#718)

Browse Source 
* stream history refactoring
* tag develop after build
This commit is contained in:
euzu
2026-04-16 14:40:38 +02:00
committed by GitHub
parent 6bd564be58
commit ba16a66c2b
44 changed files with 2153 additions and 1187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-build.yml
+5 -4
View File
@@ -150,14 +150,15 @@ jobs:
(github.event_name == 'workflow_dispatch' && github.event.inputs.branch != 'master')
id: version_commit
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config user.name "euzu"
git config user.email "euzu@proton.me"
git add .
if git diff --cached --quiet; then
echo "did_commit=false" >> "$GITHUB_OUTPUT"
else
VERSION=$(grep '^version' backend/Cargo.toml | head -n1 | cut -d'"' -f2)
git commit -m "ci: bump version v$VERSION"
git tag -a "v$VERSION" -m "ci: tag version v$VERSION"
echo "did_commit=true" >> "$GITHUB_OUTPUT"
fi
@@ -221,8 +222,8 @@ jobs:
) &&
steps.version_commit.outputs.did_commit == 'true'
run: |
# Use origin HEAD to push to the correct remote branch
git push origin HEAD
# Use origin HEAD to push to the correct remote branch, and push tags
git push origin HEAD --tags
- name: Publish release bundles to GitHub Release
if: >-
backend/src/api/api_utils.rs
+48 -32
View File
@@ -11,7 +11,6 @@ use crate::{
},
auth::Fingerprint,
model::{ConfigInput, ConfigTarget, ProxyUserCredentials},
repository::{ConnectFailureReason, FailureStage},
utils::{
async_file_reader, async_file_writer, create_new_file_for_write, debug_if_enabled, get_file_extension, request,
request::{content_type_from_ext, parse_range, send_with_retry_and_provider},
@@ -147,7 +146,7 @@ pub(crate) struct ConnectFailedAttempt<'a> {
pub fingerprint: &'a Fingerprint,
pub user: &'a ProxyUserCredentials,
pub stream_channel: StreamChannel,
pub provider_name: &'a str,
pub provider_name: Arc<str>,
pub req_headers: &'a HeaderMap,
pub reason: ConnectFailureReason,
pub failure_stage: FailureStage,
@@ -172,10 +171,24 @@ pub(crate) fn record_connect_failed_attempt(attempt: ConnectFailedAttempt<'_>) {
None,
None,
);
// Resolve target_name from target_id using the stable target config name.
let target_name = attempt
.app_state
.app_config
.get_target_by_id(info.channel.target_id)
.as_deref()
.map(|t| (&t.name).intern());
attempt
.app_state
.connection_manager
.record_connect_failed(&info, attempt.reason, attempt.failure_stage);
.record_connect_failed_with_provider_failure(
&info,
attempt.reason,
attempt.failure_stage,
None,
None,
target_name,
);
}
fn admission_failure_video_type(reason: ConnectFailureReason) -> Option<CustomVideoStreamType> {
@@ -192,7 +205,7 @@ pub(crate) fn admission_failure_response(
fingerprint: &Fingerprint,
user: &ProxyUserCredentials,
stream_channel: StreamChannel,
provider_name: &str,
provider_name: Arc<str>,
req_headers: &HeaderMap,
reason: ConnectFailureReason,
) -> axum::response::Response {
@@ -333,7 +346,7 @@ macro_rules! try_result_not_found {
use crate::api::panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details};
pub use internal_server_error;
use shared::error::TuliproxError;
use shared::model::AdmissionStrategy;
use shared::model::{AdmissionStrategy, ConnectFailureReason, FailureStage};
use shared::utils::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs};
pub use try_option_bad_request;
pub use try_option_forbidden;
@@ -382,12 +395,12 @@ pub async fn serve_file(file_path: &Path, mime_type: String, cache_control: Opti
match tokio::fs::try_exists(file_path).await {
Ok(exists) => {
if !exists {
return axum::http::StatusCode::NOT_FOUND.into_response();
return StatusCode::NOT_FOUND.into_response();
}
}
Err(err) => {
error!("Failed to open file {}, {err:?}", file_path.display());
return axum::http::StatusCode::NOT_FOUND.into_response();
return StatusCode::NOT_FOUND.into_response();
}
}
@@ -399,16 +412,16 @@ pub async fn serve_file(file_path: &Path, mime_type: String, cache_control: Opti
});
let reader = async_file_reader(file);
let stream = tokio_util::io::ReaderStream::new(reader);
let body = axum::body::Body::from_stream(stream);
let stream = ReaderStream::new(reader);
let body = Body::from_stream(stream);
let mut builder = axum::response::Response::builder()
.status(axum::http::StatusCode::OK)
.header(axum::http::header::CONTENT_TYPE, mime_type)
.header(axum::http::header::CACHE_CONTROL, cache_control.unwrap_or("no-cache"));
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, mime_type)
.header(header::CACHE_CONTROL, cache_control.unwrap_or("no-cache"));
if let Some(lm) = last_modified {
builder = builder.header(axum::http::header::LAST_MODIFIED, lm);
builder = builder.header(header::LAST_MODIFIED, lm);
}
try_unwrap_body!(builder.body(body))
@@ -970,7 +983,7 @@ async fn create_stream_response_details(
return Ok(create_panel_api_provisioning_stream_details(
app_state,
input,
guard_provider_name.clone(),
guard_provider_name.clone().or_else(|| Some(input.name.clone())),
&grace_period_options,
fingerprint.addr,
virtual_id,
@@ -981,10 +994,13 @@ async fn create_stream_response_details(
// custom stream means we display our own stream like connection exhausted, channel-unavailable...
ProviderStreamState::Custom(provider_stream) => {
let (stream, stream_info) = provider_stream;
// When allocation is exhausted or no connection was acquired, guard_provider_name is None.
// Use input.name as fallback so the provider field is never empty.
let provider_name = guard_provider_name.clone().unwrap_or_else(|| input.name.clone());
Ok(StreamDetails {
stream,
stream_info,
provider_name: guard_provider_name.clone(),
provider_name: Some(provider_name),
request_url: None,
grace_period: grace_period_options,
provider_grace_active: false,
@@ -1489,7 +1505,7 @@ pub async fn stream_response(
fingerprint,
user,
stream_channel: stream_channel.clone(),
provider_name: input.name.as_ref(),
provider_name: input.name.clone(),
req_headers,
reason: ConnectFailureReason::UserConnectionsExhausted,
failure_stage: FailureStage::Admission,
@@ -1950,7 +1966,7 @@ pub async fn local_stream_response(
fingerprint,
user,
stream_channel: pli.clone(),
provider_name: input.name.as_ref(),
provider_name: input.name.clone(),
req_headers,
reason: ConnectFailureReason::UserConnectionsExhausted,
failure_stage: FailureStage::Admission,
@@ -3263,7 +3279,7 @@ mod tests {
priority: 0,
soft_priority: 10,
fingerprint: &first_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &create_test_live_channel("http://provider-1.example/live/1.ts"),
user_agent: std::borrow::Cow::Borrowed("ua"),
session_token: Some("tok-first"),
@@ -3283,7 +3299,7 @@ mod tests {
priority: 0,
soft_priority: 10,
fingerprint: &second_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &create_test_live_channel("http://provider-1.example/live/2.ts"),
user_agent: std::borrow::Cow::Borrowed("ua"),
session_token: Some("tok-second"),
@@ -3344,7 +3360,7 @@ mod tests {
priority: 0,
soft_priority: 10,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &create_test_live_channel("http://provider-1.example/movie/1.mkv"),
user_agent: std::borrow::Cow::Borrowed("ua"),
session_token: Some("vod-session"),
@@ -3424,7 +3440,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &victim_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &victim_channel,
user_agent: std::borrow::Cow::Borrowed("player/1.0"),
session_token: Some("session-victim"),
@@ -3441,7 +3457,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &winner_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &winner_channel,
user_agent: std::borrow::Cow::Borrowed("winner/1.0"),
session_token: Some("session-winner"),
@@ -3517,7 +3533,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &victim_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &victim_channel,
user_agent: std::borrow::Cow::Borrowed("player/1.0"),
session_token: Some("session-victim"),
@@ -3534,7 +3550,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &winner_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &winner_channel,
user_agent: std::borrow::Cow::Borrowed("winner/1.0"),
session_token: Some("session-winner"),
@@ -3606,7 +3622,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &victim_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &channel,
user_agent: std::borrow::Cow::Borrowed("player/1.0"),
session_token: Some("session-victim"),
@@ -3623,7 +3639,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &winner_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &channel,
user_agent: std::borrow::Cow::Borrowed("player/1.0"),
session_token: Some("session-winner"),
@@ -3697,7 +3713,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &victim_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &victim_channel,
user_agent: std::borrow::Cow::Borrowed("player/1.0"),
session_token: Some("session-victim"),
@@ -3714,7 +3730,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &winner_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &winner_channel,
user_agent: std::borrow::Cow::Borrowed("winner/1.0"),
session_token: Some("session-winner"),
@@ -3939,7 +3955,7 @@ mod tests {
priority: user.priority,
soft_priority: user.soft_priority,
fingerprint: &normal_fingerprint,
provider: input.name.as_ref(),
provider: input.name.clone(),
stream_channel: &normal_channel,
user_agent: Cow::Borrowed("ua"),
session_token: Some("normal-session"),
@@ -4364,7 +4380,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &first_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &create_test_live_channel("http://provider-1.example/vod/5001.ts"),
user_agent: std::borrow::Cow::Borrowed("ua"),
session_token: Some(&first_token),
@@ -4427,7 +4443,7 @@ mod tests {
priority: user.priority,
soft_priority: user.soft_priority,
fingerprint: &first_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &create_test_live_channel("http://provider-1.example/vod/6001.ts"),
user_agent: std::borrow::Cow::Borrowed("ua"),
session_token: Some(&first_token),
@@ -4467,7 +4483,7 @@ mod tests {
priority: user.priority,
soft_priority: user.soft_priority,
fingerprint: &second_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &create_test_live_channel("http://provider-1.example/vod/6002.ts"),
user_agent: std::borrow::Cow::Borrowed("ua"),
session_token: Some(&second_token),
backend/src/api/endpoints/hls_api.rs
+16 -10
View File
@@ -25,6 +25,7 @@ use shared::{
};
use std::sync::Arc;
use url::Url;
use shared::model::ConnectFailureReason;
const PLAYLIST_TEMPLATE: &str = r"#EXTM3U
#EXT-X-VERSION:3
@@ -349,10 +350,10 @@ async fn resolve_stream_channel(
#[allow(clippy::too_many_lines)]
async fn hls_api_stream(
fingerprint: Fingerprint,
req_headers: axum::http::HeaderMap,
req_headers: HeaderMap,
axum::extract::Path(params): axum::extract::Path<HlsApiPathParams>,
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>,
) -> impl axum::response::IntoResponse + Send {
) -> impl IntoResponse + Send {
let (user, target) = try_option_bad_request!(
app_state.app_config.get_target_for_user(&params.username, &params.password),
false,
@@ -380,9 +381,9 @@ async fn hls_api_stream(
&fingerprint,
&user,
denied_channel,
input.name.as_ref(),
input.name.clone(),
&req_headers,
crate::repository::ConnectFailureReason::UserAccountExpired,
ConnectFailureReason::UserAccountExpired,
);
}
@@ -408,9 +409,9 @@ async fn hls_api_stream(
&fingerprint,
&user,
stream_channel,
&session.provider,
session.provider.clone(),
&req_headers,
crate::repository::ConnectFailureReason::UserConnectionsExhausted,
ConnectFailureReason::UserConnectionsExhausted,
);
}
@@ -421,9 +422,9 @@ async fn hls_api_stream(
&fingerprint,
&user,
stream_channel,
&session.provider,
session.provider.clone(),
&req_headers,
crate::repository::ConnectFailureReason::ProviderConnectionsExhausted,
ConnectFailureReason::ProviderConnectionsExhausted,
);
}
@@ -495,15 +496,20 @@ async fn hls_api_stream(
session.permission = connection_permission;
session.connection_kind = Some(connection_kind);
if connection_permission == UserConnectionPermission::Exhausted {
let provider = if session.provider.is_empty() {
input.name.clone()
} else {
session.provider.clone()
};
let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, &session.stream_url).await;
return admission_failure_response(
&app_state,
&fingerprint,
&user,
stream_channel,
input.name.as_ref(),
provider,
&req_headers,
crate::repository::ConnectFailureReason::UserConnectionsExhausted,
ConnectFailureReason::UserConnectionsExhausted,
);
}
backend/src/api/endpoints/m3u_api.rs
+9 -8
View File
@@ -27,6 +27,7 @@ use shared::{
utils::{concat_path, extract_extension_from_url, sanitize_sensitive_info},
};
use std::sync::Arc;
use shared::model::ConnectFailureReason;
async fn m3u_api(api_req: &UserApiRequest, app_state: &AppState) -> impl IntoResponse + Send {
api_req.log_sanitized("m3u_api");
@@ -125,9 +126,9 @@ async fn m3u_api_stream(
fingerprint,
&user,
pli.to_stream_channel(target.id),
pli.input_name.as_ref(),
pli.input_name.clone(),
req_headers,
crate::repository::ConnectFailureReason::UserAccountExpired,
ConnectFailureReason::UserAccountExpired,
);
}
@@ -210,9 +211,9 @@ async fn m3u_api_stream(
fingerprint,
&user,
pli.to_stream_channel(target.id),
&session.provider,
session.provider.clone(),
req_headers,
crate::repository::ConnectFailureReason::UserConnectionsExhausted,
ConnectFailureReason::UserConnectionsExhausted,
);
}
@@ -222,9 +223,9 @@ async fn m3u_api_stream(
fingerprint,
&user,
pli.to_stream_channel(target.id),
&session.provider,
session.provider.clone(),
req_headers,
crate::repository::ConnectFailureReason::ProviderConnectionsExhausted,
ConnectFailureReason::ProviderConnectionsExhausted,
);
}
if session.virtual_id == virtual_id && is_seek_request(cluster, req_headers).await {
@@ -292,9 +293,9 @@ async fn m3u_api_stream(
fingerprint,
&user,
pli.to_stream_channel(target.id),
input.name.as_ref(),
input.name.clone(),
req_headers,
crate::repository::ConnectFailureReason::UserConnectionsExhausted,
ConnectFailureReason::UserConnectionsExhausted,
);
}
backend/src/api/endpoints/stream_history_api.rs
+232 -89
View File
@@ -4,17 +4,18 @@ use std::path::Path;
use std::sync::Arc;
use axum::extract::{Path as AxumPath, Query, State};
use axum::response::{IntoResponse, Response};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use serde::{Deserialize, Serialize};
use crate::api::model::AppState;
use shared::model::StreamHistoryEventType;
use crate::api::api_utils::json_or_bin_response;
use crate::api::endpoints::extract_accept_header::ExtractAcceptHeader;
use crate::api::model::AppState;
use crate::model::{StreamHistoryRecord};
use crate::repository::{QosSnapshotRecord, QosSnapshotRepository, StreamHistoryFileReader};
use crate::utils::stream_history_viewer::{
CompiledFilter, StreamHistoryQuery, TimeRange,
discover_files, resolve_time_range,
discover_files, resolve_time_range, CompiledFilter,
StreamHistoryQuery, TimeRange,
};
#[derive(Deserialize)]
@@ -41,7 +42,7 @@ struct ErrorResponse {
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub(crate) struct ProviderSummary {
pub provider_name: String,
pub provider_name: Arc<str>,
pub session_count: u64,
pub disconnect_count: u64,
pub total_bytes_sent: u64,
@@ -105,27 +106,29 @@ pub(crate) async fn stream_history_query(
None => CompiledFilter::compile(&HashMap::new()).unwrap_or_else(|_| unreachable!()),
};
// Run blocking file I/O on the blocking thread pool
let result = tokio::task::spawn_blocking(move || {
collect_records(&history_dir, &time_range, &filters)
})
.await;
// Collect file records and batch records on blocking thread, then combine.
// batch_records must be fetched on a real OS thread (not the async thread),
// so we include it inside spawn_blocking where blocking_recv() is safe.
let app_state_clone = Arc::clone(&app_state);
let mut file_records = collect_records(&history_dir, &time_range, &filters).await.unwrap_or_default();
let batch_records = if let Some(hw) = app_state_clone
.connection_manager
.history_writer()
.load()
.as_ref()
{
hw.get_current_batch().await.unwrap_or_else(|_| Vec::new())
} else {
Vec::new()
};
match result {
Ok(Ok(records)) => axum::Json(records).into_response(),
Ok(Err(e)) => {
if e.kind() == io::ErrorKind::NotFound {
// Return empty array when no files found, not an error
axum::Json(Vec::<crate::repository::StreamHistoryRecord>::new()).into_response()
} else {
error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read history: {e}"))
}
}
Err(e) => error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("History query task failed: {e}")),
}
let mut all_records = batch_records;
all_records.append(&mut file_records);
axum::Json(all_records).into_response()
}
pub(crate) async fn stream_history_summary_query(
ExtractAcceptHeader(accept): ExtractAcceptHeader,
State(app_state): State<Arc<AppState>>,
Query(params): Query<HistoryQueryParams>,
) -> Response {
@@ -153,23 +156,34 @@ pub(crate) async fn stream_history_summary_query(
None => CompiledFilter::compile(&HashMap::new()).unwrap_or_else(|_| unreachable!()),
};
let result = tokio::task::spawn_blocking(move || {
collect_records(&history_dir, &time_range, &filters)
.map(|records| aggregate_provider_summaries(records.as_slice()))
})
.await;
// Chain batch records (in-memory) with file records iterator.
// batch_records must be fetched on a real OS thread (not the async thread),
// so we include it inside spawn_blocking where blocking_recv() is safe.
let filters_arc = Arc::new(filters);
let app_state_clone = Arc::clone(&app_state);
// Fetch batch records from within the blocking thread so blocking_recv() is safe
let batch_records = if let Some(hw) = app_state_clone
.connection_manager
.history_writer()
.load()
.as_ref()
{
hw.get_current_batch().await.unwrap_or_else(|_| Vec::new())
} else {
Vec::new()
};
match result {
Ok(Ok(summaries)) => axum::Json(summaries).into_response(),
Ok(Err(e)) => {
if e.kind() == io::ErrorKind::NotFound {
axum::Json(Vec::<ProviderSummary>::new()).into_response()
} else {
error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to summarize history: {e}"))
}
let file_iter = match collect_records_iter(&history_dir, time_range, filters_arc).await {
Ok(iter) => iter,
Err(e) => {
log::error!("Failed to discover stream history files for summary: {e}");
return error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to discover history files: {e}"));
}
Err(e) => error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("History summary task failed: {e}")),
}
};
let combined = batch_records.into_iter().chain(file_iter);
let summaries = aggregate_provider_summaries_from_iter(combined);
json_or_bin_response(accept.as_deref(), &summaries).into_response()
}
pub(crate) async fn qos_snapshot_query(
@@ -190,7 +204,7 @@ pub(crate) async fn qos_snapshot_query(
let result = tokio::task::spawn_blocking(move || {
collect_filtered_qos_snapshots(Path::new(&storage_dir), &filters, limit)
})
.await;
.await;
match result {
Ok(Ok(records)) => json_or_bin_response(accept.as_deref(), &records).into_response(),
@@ -214,7 +228,8 @@ pub(crate) async fn qos_snapshot_detail_query(
return error_response(StatusCode::SERVICE_UNAVAILABLE, "QoS aggregation is not enabled");
};
let result = tokio::task::spawn_blocking(move || load_qos_snapshot(&storage_dir, &stream_identity_key)).await;
let result = tokio::task::spawn_blocking(move || load_qos_snapshot(&storage_dir, &stream_identity_key))
.await;
match result {
Ok(Ok(Some(record))) => json_or_bin_response(accept.as_deref(), &record).into_response(),
@@ -230,25 +245,98 @@ pub(crate) async fn qos_snapshot_detail_query(
}
}
fn collect_records(
/// Iterator over all filtered `StreamHistoryRecord`s from history files in a directory.
/// Filters are applied lazily — records are never collected into a Vec.
/// Errors opening files or reading records are logged but do not stop iteration.
struct RecordFileIter {
files: std::vec::IntoIter<crate::utils::stream_history_viewer::HistoryFile>,
current_iter: Option<Box<dyn Iterator<Item=std::io::Result<StreamHistoryRecord>>>>,
time_range: TimeRange,
filters: Arc<CompiledFilter>,
}
impl RecordFileIter {
async fn new(dir: &str, time_range: TimeRange, filters: Arc<CompiledFilter>) -> io::Result<Self> {
let files = discover_files(Path::new(dir), &time_range).await?;
Ok(Self {
files: files.into_iter(),
current_iter: None,
time_range,
filters,
})
}
}
impl Iterator for RecordFileIter {
type Item = StreamHistoryRecord;
fn next(&mut self) -> Option<Self::Item> {
let (range_start, range_end) = self.time_range;
loop {
if self.current_iter.is_none() {
let file = self.files.next()?;
let iter: Box<dyn Iterator<Item = io::Result<StreamHistoryRecord>>> = if file.is_archive {
match StreamHistoryFileReader::from_archive(&file.path, Some(self.time_range)) {
Ok((r, _)) => Box::new(r),
Err(e) => {
log::warn!("Failed to open archive {}: {e}", file.path.display());
continue;
}
}
} else {
match StreamHistoryFileReader::from_pending(&file.path, Some(self.time_range)) {
Ok((r, _)) => Box::new(r),
Err(e) => {
log::warn!("Failed to open pending file {}: {e}", file.path.display());
continue;
}
}
};
self.current_iter = Some(iter);
}
match self.current_iter.as_mut().and_then(std::iter::Iterator::next) {
Some(Ok(record)) => {
if record.event_ts_utc < range_start || record.event_ts_utc > range_end {
continue;
}
if !self.filters.matches(&record) {
continue;
}
return Some(record);
}
Some(Err(e)) => {
log::warn!("Failed to read record: {e}");
}
None => {
self.current_iter = None;
}
}
}
}
}
/// Returns a Vec of filtered records from history files.
/// Used by `stream_history_query` to collect all records before responding.
async fn collect_records(
dir: &str,
time_range: &TimeRange,
filters: &CompiledFilter,
) -> io::Result<Vec<crate::repository::StreamHistoryRecord>> {
let files = discover_files(Path::new(dir), time_range)?;
) -> io::Result<Vec<StreamHistoryRecord>> {
let files = discover_files(Path::new(dir), time_range).await?;
let (range_start, range_end) = *time_range;
let mut records = Vec::new();
for file in &files {
let iter: Box<dyn Iterator<Item = io::Result<crate::repository::StreamHistoryRecord>>> =
if file.is_archive {
let (reader, _) = StreamHistoryFileReader::from_archive(&file.path, Some(*time_range))?;
Box::new(reader)
} else {
let (reader, _) = StreamHistoryFileReader::from_pending(&file.path, Some(*time_range))?;
Box::new(reader)
};
let iter: Box<dyn Iterator<Item=io::Result<StreamHistoryRecord>>> = if file.is_archive {
let (reader, _) = StreamHistoryFileReader::from_archive(&file.path, Some(*time_range))?;
Box::new(reader)
} else {
let (reader, _) = StreamHistoryFileReader::from_pending(&file.path, Some(*time_range))?;
Box::new(reader)
};
for result in iter {
let record = result?;
@@ -265,8 +353,18 @@ fn collect_records(
Ok(records)
}
/// Returns a lazy iterator over filtered records from history files.
async fn collect_records_iter(
dir: &str,
time_range: TimeRange,
filters: Arc<CompiledFilter>,
) -> io::Result<RecordFileIter> {
RecordFileIter::new(dir, time_range, filters).await
}
#[allow(dead_code)]
pub(crate) fn aggregate_provider_summaries(
records: &[crate::repository::StreamHistoryRecord],
records: &[StreamHistoryRecord],
) -> Vec<ProviderSummary> {
#[derive(Default)]
struct Acc {
@@ -279,13 +377,63 @@ pub(crate) fn aggregate_provider_summaries(
first_byte_count: u64,
}
let mut by_provider: std::collections::BTreeMap<String, Acc> = std::collections::BTreeMap::new();
let mut by_provider: std::collections::BTreeMap<Arc<str>, Acc> = std::collections::BTreeMap::new();
for record in records {
let provider_name = record.provider_name.clone().unwrap_or_else(|| String::from("unknown"));
let provider_name = record.provider_name.clone().unwrap_or_else(|| Arc::from("unknown"));
let acc = by_provider.entry(provider_name).or_default();
acc.session_count = acc.session_count.saturating_add(1);
if matches!(record.event_type, crate::repository::EventType::Disconnect) {
if matches!(record.event_type, StreamHistoryEventType::Disconnect) {
acc.disconnect_count = acc.disconnect_count.saturating_add(1);
}
acc.total_bytes_sent = acc.total_bytes_sent.saturating_add(record.bytes_sent.unwrap_or(0));
if let Some(duration) = record.session_duration {
acc.total_duration = acc.total_duration.saturating_add(duration);
acc.duration_count = acc.duration_count.saturating_add(1);
}
if let Some(latency) = record.first_byte_latency_ms {
acc.total_first_byte_latency = acc.total_first_byte_latency.saturating_add(latency);
acc.first_byte_count = acc.first_byte_count.saturating_add(1);
}
}
by_provider
.into_iter()
.map(|(provider_name, acc)| ProviderSummary {
provider_name,
session_count: acc.session_count,
disconnect_count: acc.disconnect_count,
total_bytes_sent: acc.total_bytes_sent,
avg_session_duration_secs: acc.total_duration.checked_div(acc.duration_count),
avg_first_byte_latency_ms: acc.total_first_byte_latency.checked_div(acc.first_byte_count),
})
.collect()
}
/// Lazily aggregates provider summaries from a (possibly infinite) iterator of records.
/// Unlike `aggregate_provider_summaries` which collects into a Vec first, this processes
/// records one-by-one without any intermediate allocation.
pub fn aggregate_provider_summaries_from_iter<I: Iterator<Item=StreamHistoryRecord>>(
records: I,
) -> Vec<ProviderSummary> {
#[derive(Default)]
struct Acc {
session_count: u64,
disconnect_count: u64,
total_bytes_sent: u64,
total_duration: u64,
duration_count: u64,
total_first_byte_latency: u64,
first_byte_count: u64,
}
let mut by_provider: std::collections::BTreeMap<Arc<str>, Acc> = std::collections::BTreeMap::new();
for record in records {
let provider_name = record.provider_name.unwrap_or_else(|| Arc::from("unknown"));
let acc = by_provider.entry(provider_name).or_default();
acc.session_count = acc.session_count.saturating_add(1);
if matches!(record.event_type, StreamHistoryEventType::Disconnect) {
acc.disconnect_count = acc.disconnect_count.saturating_add(1);
}
acc.total_bytes_sent = acc.total_bytes_sent.saturating_add(record.bytes_sent.unwrap_or(0));
@@ -318,20 +466,13 @@ struct CompiledQosSnapshotFilter {
input_name: Option<String>,
provider_name: Option<String>,
item_type: Option<String>,
target_id: Option<u16>,
target_name: Option<String>,
}
impl CompiledQosSnapshotFilter {
fn compile(raw: &HashMap<String, String>) -> Result<Self, String> {
let parse_u16 = |key: &str| -> Result<Option<u16>, String> {
raw.get(key)
.filter(|value| !value.trim().is_empty())
.map(|value| {
value
.parse::<u16>()
.map_err(|_| format!("Invalid QoS snapshot filter value for {key}: {value}"))
})
.transpose()
let parse_string = |key: &str| -> Result<Option<String>, String> {
Ok(raw.get(key).filter(|value| !value.trim().is_empty()).cloned())
};
Ok(Self {
@@ -339,7 +480,7 @@ impl CompiledQosSnapshotFilter {
input_name: raw.get("input_name").cloned().filter(|value| !value.trim().is_empty()),
provider_name: raw.get("provider_name").cloned().filter(|value| !value.trim().is_empty()),
item_type: raw.get("item_type").cloned().filter(|value| !value.trim().is_empty()),
target_id: parse_u16("target_id")?,
target_name: parse_string("target_name")?,
})
}
@@ -347,10 +488,10 @@ impl CompiledQosSnapshotFilter {
self.stream_identity_key
.as_ref()
.is_none_or(|value| snapshot.stream_identity_key == *value)
&& self.input_name.as_ref().is_none_or(|value| snapshot.input_name == *value)
&& self.provider_name.as_ref().is_none_or(|value| snapshot.provider_name == *value)
&& self.item_type.as_ref().is_none_or(|value| snapshot.item_type == *value)
&& self.target_id.is_none_or(|value| snapshot.target_id == value)
&& self.input_name.as_ref().is_none_or(|value| snapshot.input_name.as_ref() == value)
&& self.provider_name.as_ref().is_none_or(|value| snapshot.provider_name.as_ref() == value)
&& self.item_type.as_ref().is_none_or(|value| snapshot.item_type.as_str() == value)
&& self.target_name.as_ref().is_none_or(|value| snapshot.target_name.as_ref() == value)
}
}
@@ -387,11 +528,13 @@ fn load_qos_snapshot(storage_dir: &str, stream_identity_key: &str) -> io::Result
#[cfg(test)]
mod tests {
use shared::model::{DisconnectReason, PlaylistItemType};
use super::*;
use crate::model::{QosAggregationConfig, ResourceRetryConfig, ReverseProxyConfig};
use crate::repository::{
DisconnectReason, EventType, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow, StreamHistoryRecord,
QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow,
};
use shared::utils::Internable;
fn make_record(
provider_name: &str,
@@ -402,17 +545,17 @@ mod tests {
) -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: 1,
event_type: EventType::Disconnect,
event_type: StreamHistoryEventType::Disconnect,
event_ts_utc: 1,
partition_day_utc: String::from("2026-03-22"),
session_id: 1,
source_addr: None,
api_username: Some(String::from("alice")),
provider_name: Some(provider_name.to_string()),
provider_name: Some(provider_name.intern()),
provider_username: None,
input_name: Some(String::from("input")),
input_name: Some("input".intern()),
virtual_id: Some(1),
item_type: Some(String::from("live")),
item_type: Some(PlaylistItemType::Live),
title: Some(String::from("Title")),
group: None,
country: None,
@@ -442,7 +585,7 @@ mod tests {
connect_failure_reason: None,
disconnect_reason,
previous_session_id: None,
target_id: Some(1),
target_name: None,
}
}
@@ -455,7 +598,7 @@ mod tests {
]);
assert_eq!(summaries.len(), 2);
assert_eq!(summaries[0].provider_name, "acme");
assert_eq!(summaries[0].provider_name.as_ref(), "acme");
assert_eq!(summaries[0].session_count, 2);
assert_eq!(summaries[0].total_bytes_sent, 400);
assert_eq!(summaries[0].avg_session_duration_secs, Some(15));
@@ -467,17 +610,17 @@ mod tests {
stream_identity_key: &str,
input_name: &str,
provider_name: &str,
target_id: u16,
target_name: &str,
score_24h: u8,
) -> QosSnapshotRecord {
QosSnapshotRecord {
stream_identity_key: stream_identity_key.to_string(),
input_name: input_name.to_string(),
target_id,
provider_name: provider_name.to_string(),
input_name: input_name.intern(),
target_name: target_name.intern(),
provider_name: provider_name.intern(),
provider_id: 1,
virtual_id: 101,
item_type: "live".to_string(),
item_type: PlaylistItemType::Live,
updated_at: 100,
last_event_at: 99,
window_24h: QosSnapshotWindow {
@@ -494,13 +637,13 @@ mod tests {
#[test]
fn qos_snapshot_filter_matches_identity_and_provider_fields() {
let snapshots = vec![
make_qos_snapshot("stream-a", "input-a", "provider-a", 1, 81),
make_qos_snapshot("stream-b", "input-b", "provider-b", 2, 55),
make_qos_snapshot("stream-a", "input-a", "provider-a", "target-a", 81),
make_qos_snapshot("stream-b", "input-b", "provider-b", "target-b", 55),
];
let mut raw = HashMap::new();
raw.insert("provider_name".to_string(), "provider-b".to_string());
raw.insert("target_id".to_string(), "2".to_string());
raw.insert("target_name".to_string(), "target-b".to_string());
let filter = CompiledQosSnapshotFilter::compile(&raw).expect("qos snapshot filter should compile");
let mut filtered = snapshots
@@ -526,7 +669,7 @@ mod tests {
rate_limit: None,
geoip: None,
stream_history: None,
qos_aggregation: Some(crate::model::QosAggregationConfig {
qos_aggregation: Some(QosAggregationConfig {
enabled: false,
interval_secs: 300,
}),
@@ -550,9 +693,9 @@ mod tests {
#[test]
fn filter_qos_snapshots_orders_by_score_and_keeps_exact_match_filters() {
let snapshots = vec![
make_qos_snapshot("stream-a", "input-a", "provider-a", 1, 60),
make_qos_snapshot("stream-b", "input-b", "provider-a", 1, 80),
make_qos_snapshot("stream-c", "input-c", "provider-b", 2, 95),
make_qos_snapshot("stream-a", "input-a", "provider-a", "target-a", 60),
make_qos_snapshot("stream-b", "input-b", "provider-a", "target-a", 80),
make_qos_snapshot("stream-c", "input-c", "provider-b", "target-b", 95),
];
let mut raw = HashMap::new();
backend/src/api/endpoints/xtream_api.rs
+10 -9
View File
@@ -62,6 +62,7 @@ use std::{
str::FromStr,
sync::Arc,
};
use shared::model::ConnectFailureReason;
#[derive(Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq)]
pub enum ApiStreamContext {
@@ -277,9 +278,9 @@ async fn xtream_player_api_stream(
fingerprint,
&user,
create_stream_channel_with_type(target.id, &pli, pli.item_type),
pli.input_name.as_ref(),
pli.input_name.clone(),
req_headers,
crate::repository::ConnectFailureReason::UserAccountExpired,
ConnectFailureReason::UserAccountExpired,
);
}
@@ -382,9 +383,9 @@ async fn xtream_player_api_stream(
fingerprint,
&user,
create_stream_channel_with_type(target.id, &pli, item_type),
&session.provider,
session.provider.clone(),
req_headers,
crate::repository::ConnectFailureReason::UserConnectionsExhausted,
ConnectFailureReason::UserConnectionsExhausted,
);
}
@@ -394,9 +395,9 @@ async fn xtream_player_api_stream(
fingerprint,
&user,
create_stream_channel_with_type(target.id, &pli, item_type),
&session.provider,
session.provider.clone(),
req_headers,
crate::repository::ConnectFailureReason::ProviderConnectionsExhausted,
ConnectFailureReason::ProviderConnectionsExhausted,
);
}
@@ -409,7 +410,7 @@ async fn xtream_player_api_stream(
app_state,
session,
stream_channel,
crate::api::api_utils::ForceStreamRequestContext {
api_utils::ForceStreamRequestContext {
req_headers,
input: &input,
user: &user,
@@ -468,9 +469,9 @@ async fn xtream_player_api_stream(
fingerprint,
&user,
create_stream_channel_with_type(target.id, &pli, item_type),
input.name.as_ref(),
input.name.clone(),
req_headers,
crate::repository::ConnectFailureReason::UserConnectionsExhausted,
ConnectFailureReason::UserConnectionsExhausted,
);
}
backend/src/api/model/active_user_manager.rs
+49 -49
View File
@@ -2,7 +2,6 @@ use crate::{
api::model::{active_provider_manager::ConnectionKind, ActiveProviderManager, CustomVideoStreamType, EventManager, EventMessage},
auth::Fingerprint,
model::{Config, ProxyUserCredentials},
repository::utc_day_from_secs,
utils::{debug_if_enabled, GeoIp},
};
use arc_swap::ArcSwapOption;
@@ -32,6 +31,7 @@ use std::{
use tokio::sync::{mpsc, Mutex, Notify, RwLock};
use crate::api::model::connection_manager::CleanupEvent;
use tokio_util::sync::CancellationToken;
use crate::utils::utc_day_from_secs;
const USER_GC_TTL: u64 = 900; // 15 Min
const USER_CON_TTL: u64 = 1_800; // 30 minutes
@@ -320,7 +320,7 @@ pub struct ActiveUserConnectionParams<'a> {
pub priority: i8,
pub soft_priority: i8,
pub fingerprint: &'a Fingerprint,
pub provider: &'a str,
pub provider: Arc<str>,
pub stream_channel: &'a StreamChannel,
pub user_agent: Cow<'a, str>,
pub session_token: Option<&'a str>,
@@ -980,7 +980,7 @@ impl ActiveUserManager {
// IMPORTANT: `resolve_disconnect_reason` in connection_manager.rs parses
// `channel.title` back via `CustomVideoStreamType::from_str` to determine QoS
// disconnect reasons. If these values change, update that function too.
stream.provider = String::from("tuliprox");
stream.provider = "tuliprox".intern();
stream.channel.title = video_type.to_string().into();
stream.channel.group = "".intern();
stream.channel.technical = Some(Self::custom_stream_technical_info());
@@ -1084,7 +1084,7 @@ impl ActiveUserManager {
stream_info.client_ip.clone_from(&client_ip);
stream_info.country_code = self.lookup_country(&client_ip);
stream_info.channel = stream_channel.clone();
stream_info.provider = provider.to_string();
stream_info.provider = provider.clone();
stream_info.user_agent.clone_from(&user_agent_string);
if let Some(started_at) = session_started_at {
@@ -2121,7 +2121,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1001),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-1"),
@@ -2145,7 +2145,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-b",
provider: "provider-b".intern(),
stream_channel: &test_channel(1002),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-2"),
@@ -2185,7 +2185,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &at_limit_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1010),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-limit"),
@@ -2206,7 +2206,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &over_limit_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1011),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-over-1"),
@@ -2226,7 +2226,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &second_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1012),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-over-2"),
@@ -2263,7 +2263,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &shared_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(channel_id),
user_agent: Cow::Borrowed("ua"),
session_token: Some(token),
@@ -2282,7 +2282,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &unique_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1033),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-33"),
@@ -2320,7 +2320,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &first_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1041),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-41"),
@@ -2338,7 +2338,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &second_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(1042),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-42"),
@@ -2389,7 +2389,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_adaptive_channel(2014),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-adaptive"),
@@ -2458,7 +2458,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &kicked_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2015),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-kicked"),
@@ -2475,7 +2475,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &survivor_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2016),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-survivor"),
@@ -2523,7 +2523,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2017),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-grace"),
@@ -2578,7 +2578,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &first,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2001),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-hls"),
@@ -2602,7 +2602,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &second,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2001),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-hls"),
@@ -2655,7 +2655,7 @@ mod tests {
priority: 8,
soft_priority: 8,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2002),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-prio"),
@@ -2672,7 +2672,7 @@ mod tests {
priority: -7,
soft_priority: 8,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(2002),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-prio"),
@@ -2706,7 +2706,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(3001),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-meter"),
@@ -2727,7 +2727,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-b",
provider: "provider-b".intern(),
stream_channel: &test_channel(3002),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-meter"),
@@ -2742,7 +2742,7 @@ mod tests {
assert_eq!(streams.len(), 1);
assert_eq!(streams[0].uid, 11);
assert_eq!(streams[0].meter_uid, 202);
assert_eq!(streams[0].provider, "provider-b");
assert_eq!(streams[0].provider.as_ref(), "provider-b");
assert_eq!(streams[0].channel.virtual_id, 3002);
}
@@ -2786,7 +2786,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(4001)
@@ -2820,7 +2820,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &next_fingerprint,
provider: "provider-b",
provider: "provider-b".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveDash,
..test_channel(4002)
@@ -2880,7 +2880,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(5001)
@@ -2934,7 +2934,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(6001)
@@ -3010,7 +3010,7 @@ mod tests {
priority: -1,
soft_priority: 9,
fingerprint: &normal_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(6002)
@@ -3030,7 +3030,7 @@ mod tests {
priority: -5,
soft_priority: 9,
fingerprint: &soft_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(6003),
user_agent: Cow::Borrowed("ua-soft"),
session_token: None,
@@ -3101,7 +3101,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fp_a,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(7001)
@@ -3125,7 +3125,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fp_b,
provider: "provider-b",
provider: "provider-b".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveDash,
..test_channel(7002)
@@ -3185,7 +3185,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(8001)
@@ -3225,7 +3225,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(8002)
@@ -3278,7 +3278,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(8003)
@@ -3361,7 +3361,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(8004)
@@ -3433,7 +3433,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(8004)
@@ -3518,7 +3518,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveHls,
..test_channel(8005)
@@ -3558,7 +3558,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &next_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::LiveDash,
..test_channel(8005)
@@ -3571,7 +3571,7 @@ mod tests {
assert_eq!(second.previous_session_id, Some((forced_old_ts << 32) | u64::from(first.uid)));
assert!(second.ts > forced_old_ts);
assert_eq!(crate::repository::utc_day_from_secs(second.ts), crate::repository::utc_day_from_secs(current_time_secs()));
assert_eq!(utc_day_from_secs(second.ts), utc_day_from_secs(current_time_secs()));
}
#[tokio::test]
@@ -3627,7 +3627,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &stale_fp,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(9201),
user_agent: Cow::Borrowed("ua"),
session_token: None,
@@ -3645,7 +3645,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fresh_fp,
provider: "provider-b",
provider: "provider-b".intern(),
stream_channel: &test_channel(9202),
user_agent: Cow::Borrowed("ua"),
session_token: None,
@@ -3761,7 +3761,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_adaptive_channel(7777),
user_agent: Cow::Borrowed("player/1.0"),
session_token: Some("tok-hls-ts"),
@@ -3835,7 +3835,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &old_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::Live,
..test_channel(9101)
@@ -3903,7 +3903,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &base_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::Video,
..test_channel(9102)
@@ -3926,7 +3926,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &range_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &StreamChannel {
item_type: PlaylistItemType::Video,
..test_channel(9102)
@@ -4000,7 +4000,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_channel(9001),
user_agent: Cow::Borrowed("ua"),
session_token: None,
@@ -4163,7 +4163,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &first_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_adaptive_channel(9203),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-normal"),
@@ -4188,7 +4188,7 @@ mod tests {
priority: 0,
soft_priority: 0,
fingerprint: &second_fingerprint,
provider: "provider-a",
provider: "provider-a".intern(),
stream_channel: &test_adaptive_channel(9204),
user_agent: Cow::Borrowed("ua"),
session_token: Some("tok-soft"),
backend/src/api/model/connection_manager.rs
+32 -28
View File
@@ -4,7 +4,6 @@ use crate::{
EventMessage, ProviderHandle, SharedStreamManager,
},
model::StreamHistoryConfig,
repository::{ConnectFailureReason, DisconnectQos, DisconnectReason, FailureStage, StreamHistoryRecord},
auth::Fingerprint,
utils::debug_if_enabled,
};
@@ -28,6 +27,8 @@ use std::{
time::Duration,
};
use tokio::sync::{mpsc, Notify};
use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage};
use crate::model::{DisconnectQos, StreamHistoryRecord};
use crate::repository::{recover_pending_files, StreamHistoryWriter};
// Maximum number of deferred cleanup actions buffered before producers must wait/drop.
@@ -272,13 +273,13 @@ pub(crate) enum CleanupEvent {
}
async fn handle_release_connection(deps: &CleanupWorkerDeps, addr: SocketAddr) {
release_connection_with_reason_from_deps(deps, &addr, &DisconnectReason::Cleanup, true).await;
release_connection_with_reason_from_deps(deps, &addr, DisconnectReason::Cleanup, true).await;
}
async fn release_connection_with_reason(
connection_manager: &ConnectionManager,
addr: &SocketAddr,
reason: &DisconnectReason,
reason: DisconnectReason,
send_shared_stop_signal: bool,
) {
release_connection_parts(
@@ -298,7 +299,7 @@ async fn release_connection_with_reason(
async fn release_connection_with_reason_from_deps(
deps: &CleanupWorkerDeps,
addr: &SocketAddr,
reason: &DisconnectReason,
reason: DisconnectReason,
send_shared_stop_signal: bool,
) {
release_connection_parts(
@@ -324,7 +325,7 @@ async fn release_connection_parts(
capacity_notify: &Arc<Notify>,
history_writer: &Arc<ArcSwapOption<StreamHistoryWriter>>,
addr: &SocketAddr,
reason: &DisconnectReason,
reason: DisconnectReason,
send_shared_stop_signal: bool,
) {
let removed = if matches!(reason, DisconnectReason::ClientKicked) {
@@ -452,7 +453,7 @@ async fn handle_adaptive_session_expired(deps: &CleanupWorkerDeps, stream_info:
emit_disconnect_record(
&deps.history_writer,
&stream_info,
&DisconnectReason::SessionExpired,
DisconnectReason::SessionExpired,
&DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() },
None,
None,
@@ -481,7 +482,7 @@ async fn release_stream_with_disconnect(
emit_disconnect_record(
&deps.history_writer,
&stream_info,
&reason,
reason,
&DisconnectQos { bytes_sent, first_byte_latency_ms, provider_reconnect_count },
provider_error_class,
provider_http_status,
@@ -537,7 +538,7 @@ pub struct ConnectionParams<'a> {
pub priority: i8,
pub soft_priority: i8,
pub fingerprint: &'a Fingerprint,
pub provider: &'a str,
pub provider: Arc<str>,
pub stream_channel: &'a StreamChannel,
pub user_agent: Cow<'a, str>,
pub session_token: Option<&'a str>,
@@ -601,6 +602,11 @@ impl ConnectionManager {
self.history_writer.store(new_writer);
}
/// Returns a reference to the history writer.
pub(crate) fn history_writer(&self) -> &Arc<ArcSwapOption<StreamHistoryWriter>> {
&self.history_writer
}
fn spawn_socket_activity_worker(
activity_tracker: SocketActivityTracker,
user_manager: Arc<ActiveUserManager>,
@@ -868,16 +874,16 @@ impl ConnectionManager {
}
pub async fn release_connection(&self, addr: &SocketAddr) {
release_connection_with_reason(self, addr, &DisconnectReason::ClientClosed, true).await;
release_connection_with_reason(self, addr, DisconnectReason::ClientClosed, true).await;
}
pub async fn release_connection_with_reason(&self, addr: &SocketAddr, reason: &DisconnectReason) {
pub async fn release_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) {
release_connection_with_reason(self, addr, reason, true).await;
}
pub async fn release_connection_as_kicked(&self, addr: &SocketAddr) {
let _ = self.close_connection_with_reason(addr, DisconnectReason::ClientKicked);
release_connection_with_reason(self, addr, &DisconnectReason::ClientKicked, true).await;
release_connection_with_reason(self, addr, DisconnectReason::ClientKicked, true).await;
}
pub async fn release_provider_connection(&self, addr: &SocketAddr) {
@@ -893,7 +899,7 @@ impl ConnectionManager {
emit_disconnect_record(
&self.history_writer,
&stream_info,
&DisconnectReason::ClientClosed,
DisconnectReason::ClientClosed,
&DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() },
None,
None,
@@ -919,10 +925,6 @@ impl ConnectionManager {
.unwrap_or(1)
}
pub fn record_connect_failed(&self, info: &StreamInfo, reason: ConnectFailureReason, failure_stage: FailureStage) {
self.record_connect_failed_with_provider_failure(info, reason, failure_stage, None, None);
}
pub fn record_connect_failed_with_provider_failure(
&self,
info: &StreamInfo,
@@ -930,6 +932,7 @@ impl ConnectionManager {
failure_stage: FailureStage,
provider_http_status: Option<u16>,
provider_error_class: Option<&str>,
target_name: Option<Arc<str>>,
) {
let guard = self.history_writer.load();
let Some(writer) = guard.as_ref() else { return };
@@ -939,6 +942,7 @@ impl ConnectionManager {
reason,
attempt_uid,
failure_stage,
target_name,
)
.with_provider_failure(provider_http_status, provider_error_class));
}
@@ -956,7 +960,7 @@ impl ConnectionManager {
emit_disconnect_record(
&self.history_writer,
&stream_info,
&DisconnectReason::Shutdown,
DisconnectReason::Shutdown,
&DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() },
None,
None,
@@ -1042,7 +1046,7 @@ fn build_history_writer(config: Option<&StreamHistoryConfig>) -> Option<Arc<Stre
/// are wire-format identifiers shared with Serialize/Deserialize and the REST API.
/// If they ever change, update `CustomVideoStreamType::fmt`/`from_str` and this function together.
fn resolve_disconnect_reason(provider_end_reason: u8, stream_info: &StreamInfo) -> DisconnectReason {
if stream_info.provider == "tuliprox" {
if stream_info.provider.as_ref() == "tuliprox" {
if let Ok(video_type) = CustomVideoStreamType::from_str(&stream_info.channel.title) {
match video_type {
CustomVideoStreamType::LowPriorityPreempted => return DisconnectReason::Preempted,
@@ -1075,7 +1079,7 @@ fn emit_connect_record(writer: &ArcSwapOption<StreamHistoryWriter>, info: &Strea
fn emit_disconnect_record(
writer: &ArcSwapOption<StreamHistoryWriter>,
info: &StreamInfo,
reason: &DisconnectReason,
reason: DisconnectReason,
qos: &DisconnectQos,
provider_error_class: Option<&str>,
provider_http_status: Option<u16>,
@@ -1085,7 +1089,7 @@ fn emit_disconnect_record(
w.send_record(
StreamHistoryRecord::from_disconnect(
info,
reason.clone(),
reason,
qos,
resolve_disconnect_failure_stage(info, reason, qos),
)
@@ -1093,7 +1097,7 @@ fn emit_disconnect_record(
);
}
fn resolve_disconnect_failure_stage(info: &StreamInfo, reason: &DisconnectReason, qos: &DisconnectQos) -> Option<FailureStage> {
fn resolve_disconnect_failure_stage(info: &StreamInfo, reason: DisconnectReason, qos: &DisconnectQos) -> Option<FailureStage> {
match reason {
DisconnectReason::ProviderError | DisconnectReason::ProviderClosed => {
if !info.channel.shared && qos.first_byte_latency_ms.is_none() {
@@ -1140,7 +1144,7 @@ mod tests {
shared_stream_id: None,
technical: None,
};
StreamInfo::new(0, 0, "test", &addr, "127.0.0.1", provider, channel, String::new(), None, None)
StreamInfo::new(0, 0, "test", &addr, "127.0.0.1", provider.intern(), channel, String::new(), None, None)
}
fn create_test_app_config() -> AppConfig {
@@ -1401,7 +1405,7 @@ mod tests {
assert_eq!(
resolve_disconnect_failure_stage(
&make_stream_info("some_provider", "Some Channel"),
&DisconnectReason::ClientKicked,
DisconnectReason::ClientKicked,
&DisconnectQos::default(),
),
None
@@ -1413,7 +1417,7 @@ mod tests {
assert_eq!(
resolve_disconnect_failure_stage(
&make_stream_info("some_provider", "Some Channel"),
&DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
&DisconnectQos::default(),
),
None
@@ -1488,7 +1492,7 @@ mod tests {
assert_eq!(
resolve_disconnect_failure_stage(
&make_stream_info("some_provider", "Some Channel"),
&DisconnectReason::ProviderError,
DisconnectReason::ProviderError,
&DisconnectQos { first_byte_latency_ms: Some(150), ..Default::default() },
),
Some(FailureStage::Streaming)
@@ -1500,7 +1504,7 @@ mod tests {
assert_eq!(
resolve_disconnect_failure_stage(
&make_stream_info("some_provider", "Some Channel"),
&DisconnectReason::SessionExpired,
DisconnectReason::SessionExpired,
&DisconnectQos::default(),
),
Some(FailureStage::SessionReconnect)
@@ -1512,7 +1516,7 @@ mod tests {
assert_eq!(
resolve_disconnect_failure_stage(
&make_stream_info("some_provider", "Some Channel"),
&DisconnectReason::ProviderError,
DisconnectReason::ProviderError,
&DisconnectQos::default(),
),
Some(FailureStage::FirstByte)
@@ -1526,7 +1530,7 @@ mod tests {
assert_eq!(
resolve_disconnect_failure_stage(
&info,
&DisconnectReason::ProviderError,
DisconnectReason::ProviderError,
&DisconnectQos::default(),
),
Some(FailureStage::Streaming)
backend/src/api/model/qos_aggregation_manager.rs
+91 -57
View File
@@ -2,6 +2,7 @@
use chrono::NaiveDate;
use log::{error, info, warn};
use shared::utils::Internable;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io;
use std::path::{Path, PathBuf};
@@ -9,24 +10,26 @@ use std::sync::Arc;
use std::time::Duration;
use crate::api::model::AppState;
use crate::model::Config;
use crate::model::{Config, StreamHistoryRecord};
use crate::repository::{
current_utc_day, extract_day_from_filename, now_utc_secs, ConnectFailureReason, DisconnectReason, EventType,
FailureStage, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow,
StreamHistoryFileReader, StreamHistoryRecord,
extract_day_from_filename,
QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow,
StreamHistoryFileReader,
};
use tokio_util::sync::CancellationToken;
use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage, PlaylistItemType, StreamHistoryEventType};
use crate::utils::{current_utc_day, now_utc_secs};
const MAX_COMPLETED_DAY_PARTITIONS_PER_RUN: usize = 3;
#[derive(Debug, Clone)]
struct AggregatedDayEntry {
input_name: String,
target_id: u16,
provider_name: String,
input_name: Arc<str>,
target_name: Arc<str>,
provider_name: Arc<str>,
provider_id: u32,
virtual_id: u32,
item_type: String,
item_type: PlaylistItemType,
bucket: QosSnapshotDailyBucket,
}
@@ -136,14 +139,14 @@ pub(crate) fn run_aggregation_once(
pub(crate) fn fold_record_into_bucket(bucket: &mut QosSnapshotDailyBucket, record: &StreamHistoryRecord) {
match record.event_type {
EventType::Connect => {
StreamHistoryEventType::Connect => {
if record.shared_joined_existing.unwrap_or(false) {
return;
}
bucket.connect_count = bucket.connect_count.saturating_add(1);
bucket.last_success_ts = Some(bucket.last_success_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc)));
}
EventType::ConnectFailed => {
StreamHistoryEventType::ConnectFailed => {
bucket.connect_failed_count = bucket.connect_failed_count.saturating_add(1);
if matches!(
record.connect_failure_reason,
@@ -156,7 +159,7 @@ pub(crate) fn fold_record_into_bucket(bucket: &mut QosSnapshotDailyBucket, recor
}
bucket.last_failure_ts = Some(bucket.last_failure_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc)));
}
EventType::Disconnect => {
StreamHistoryEventType::Disconnect => {
if matches!(record.failure_stage, Some(FailureStage::FirstByte)) {
bucket.first_byte_failure_count = bucket.first_byte_failure_count.saturating_add(1);
}
@@ -194,6 +197,12 @@ pub(crate) fn fold_record_into_bucket(bucket: &mut QosSnapshotDailyBucket, recor
bucket.last_failure_ts =
Some(bucket.last_failure_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc)));
}
},
StreamHistoryEventType::Failure => {
// Failure events represent intermediate disconnects within an HLS session.
// Count them as runtime aborts since they reflect streaming failures.
bucket.runtime_abort_count = bucket.runtime_abort_count.saturating_add(1);
bucket.last_failure_ts = Some(bucket.last_failure_ts.map_or(record.event_ts_utc, |ts| ts.max(record.event_ts_utc)));
}
}
}
@@ -382,20 +391,26 @@ fn aggregate_reader_entries<R: io::Read>(
let Some(stream_identity_key) = record.stream_identity_key.clone() else {
continue;
};
let (Some(input_name), Some(target_id), Some(provider_name), Some(provider_id), Some(virtual_id), Some(item_type)) = (
record.input_name.clone(),
record.target_id,
record.provider_name.clone(),
record.provider_id,
record.virtual_id,
record.item_type.clone(),
) else {
// Required fields - skip record if missing
let Some(provider_name) = record.provider_name.clone() else {
continue;
};
let Some(provider_id) = record.provider_id else {
continue;
};
let Some(virtual_id) = record.virtual_id else {
continue;
};
// Optional fields - use fallbacks for pre-migration records
let input_name = record.input_name.clone().unwrap_or_else(|| "unknown".intern());
let target_name = record.target_name.clone().unwrap_or_else(|| "unknown".intern());
let item_type = record.item_type.unwrap_or_default();
let entry = entries.entry(stream_identity_key).or_insert_with(|| AggregatedDayEntry {
input_name,
target_id,
target_name,
provider_name,
provider_id,
virtual_id,
@@ -472,11 +487,11 @@ fn apply_day_entries(
day_entries.get(&key).map(|entry| QosSnapshotRecord {
stream_identity_key: key.clone(),
input_name: entry.input_name.clone(),
target_id: entry.target_id,
target_name: entry.target_name.clone(),
provider_name: entry.provider_name.clone(),
provider_id: entry.provider_id,
virtual_id: entry.virtual_id,
item_type: entry.item_type.clone(),
item_type: entry.item_type,
updated_at: now_utc_secs(),
last_event_at: 0,
window_24h: QosSnapshotWindow::default(),
@@ -489,6 +504,12 @@ fn apply_day_entries(
};
if let Some(entry) = day_entries.get(&key) {
snapshot.input_name = entry.input_name.clone();
snapshot.target_name = entry.target_name.clone();
snapshot.provider_name = entry.provider_name.clone();
snapshot.provider_id = entry.provider_id;
snapshot.virtual_id = entry.virtual_id;
snapshot.item_type = entry.item_type;
snapshot.daily_buckets.insert(day_utc.to_string(), entry.bucket.clone());
snapshot.last_event_at = max_opt(Some(snapshot.last_event_at), max_opt(entry.bucket.last_success_ts, entry.bucket.last_failure_ts)).unwrap_or(snapshot.last_event_at);
} else {
@@ -524,15 +545,16 @@ mod tests {
use std::collections::BTreeMap;
use tempfile::tempdir;
use crate::model::{Config, QosAggregationConfig, ReverseProxyConfig, StreamHistoryConfig};
use shared::model::{FailureStage, ConnectFailureReason, DisconnectReason, PlaylistItemType, StreamHistoryEventType};
use shared::utils::Internable;
use crate::model::{Config, QosAggregationConfig, ReverseProxyConfig, StreamHistoryConfig, StreamHistoryRecord, RECORD_SCHEMA_VERSION};
use crate::repository::{
current_utc_day, serialize_named, write_block_magic, write_file_magic, write_framed, BlockHeaderBody,
CompressionKind, ConnectFailureReason, CONTAINER_FORMAT_VERSION, DisconnectReason, EventType, FailureStage,
serialize_named, write_block_magic, write_file_magic, write_framed, BlockHeaderBody,
CompressionKind, CONTAINER_FORMAT_VERSION,
FileHeaderBody, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow,
RecordEncodingKind, RECORD_SCHEMA_VERSION, SOURCE_KIND_STREAM_HISTORY, StreamHistoryRecord,
RecordEncodingKind, SOURCE_KIND_STREAM_HISTORY,
};
use crate::utils::current_utc_day;
use super::{fold_record_into_bucket, history_day_revision, qos_aggregation_is_enabled, rebuild_windows, run_aggregation_once};
fn write_pending_history_records(
@@ -596,7 +618,7 @@ mod tests {
file.flush().expect("pending file should flush");
}
fn base_record(event_type: EventType) -> StreamHistoryRecord {
fn base_record(event_type: StreamHistoryEventType) -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: 1,
event_type,
@@ -605,11 +627,11 @@ mod tests {
session_id: 42,
source_addr: None,
api_username: None,
provider_name: Some("provider-a".to_string()),
provider_name: Some("provider-a".intern()),
provider_username: None,
input_name: Some("input-a".to_string()),
input_name: Some("input-a".intern()),
virtual_id: Some(33),
item_type: Some("live".to_string()),
item_type: Some(PlaylistItemType::Live),
title: None,
group: None,
country: None,
@@ -639,7 +661,7 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: Some(11),
target_name: None,
}
}
@@ -647,9 +669,9 @@ mod tests {
fn fold_record_into_bucket_tracks_connect_and_connect_failed() {
let mut bucket = QosSnapshotDailyBucket::default();
fold_record_into_bucket(&mut bucket, &base_record(EventType::Connect));
fold_record_into_bucket(&mut bucket, &base_record(StreamHistoryEventType::Connect));
let mut failed = base_record(EventType::ConnectFailed);
let mut failed = base_record(StreamHistoryEventType::ConnectFailed);
failed.connect_failure_reason = Some(ConnectFailureReason::ProviderConnectionsExhausted);
fold_record_into_bucket(&mut bucket, &failed);
@@ -663,13 +685,13 @@ mod tests {
fn fold_record_into_bucket_tracks_disconnect_failure_stages() {
let mut bucket = QosSnapshotDailyBucket::default();
let mut first_byte = base_record(EventType::Disconnect);
let mut first_byte = base_record(StreamHistoryEventType::Disconnect);
first_byte.failure_stage = Some(FailureStage::FirstByte);
first_byte.disconnect_reason = Some(DisconnectReason::ProviderError);
first_byte.first_byte_latency_ms = Some(250);
fold_record_into_bucket(&mut bucket, &first_byte);
let mut streaming = base_record(EventType::Disconnect);
let mut streaming = base_record(StreamHistoryEventType::Disconnect);
streaming.failure_stage = Some(FailureStage::Streaming);
streaming.disconnect_reason = Some(DisconnectReason::ProviderClosed);
streaming.session_duration = Some(900);
@@ -684,11 +706,23 @@ mod tests {
assert_eq!(bucket.total_provider_reconnect_count, 2);
}
#[test]
fn fold_record_into_bucket_tracks_failure_event() {
let mut bucket = QosSnapshotDailyBucket::default();
// Failure events should increment runtime_abort_count and update last_failure_ts
let failure = base_record(StreamHistoryEventType::Failure);
fold_record_into_bucket(&mut bucket, &failure);
assert_eq!(bucket.runtime_abort_count, 1);
assert_eq!(bucket.last_failure_ts, Some(failure.event_ts_utc));
}
#[tokio::test]
async fn history_day_revision_tracks_seconds_and_file_length() {
let temp = tempdir().expect("tempdir should succeed");
let history_dir = temp.path();
let record = base_record(EventType::Connect);
let record = base_record(StreamHistoryEventType::Connect);
write_pending_history_records(history_dir, vec![record.clone(), record]);
let history_dir_copy = history_dir.to_path_buf();
@@ -707,12 +741,12 @@ mod tests {
fn rebuild_windows_uses_recent_daily_buckets() {
let mut snapshot = QosSnapshotRecord {
stream_identity_key: "stream-a".to_string(),
input_name: "input-a".to_string(),
target_id: 11,
provider_name: "provider-a".to_string(),
input_name: "input-a".intern(),
target_name: "target-a".intern(),
provider_name: "provider-a".intern(),
provider_id: 22,
virtual_id: 33,
item_type: "live".to_string(),
item_type: PlaylistItemType::Live,
updated_at: 1_700_000_000,
last_event_at: 1_700_000_123,
window_24h: QosSnapshotWindow::default(),
@@ -792,14 +826,14 @@ mod tests {
let temp = tempdir().expect("tempdir should succeed");
let history_dir = temp.path().join("stream_history");
let today = current_utc_day();
let mut connect = base_record(EventType::Connect);
let mut connect = base_record(StreamHistoryEventType::Connect);
connect.partition_day_utc = today.clone();
connect.input_name = Some("input-a".to_string());
connect.provider_name = Some("provider-a".to_string());
connect.input_name = Some("input-a".intern());
connect.provider_name = Some("provider-a".intern());
connect.provider_id = Some(22);
connect.virtual_id = Some(33);
connect.item_type = Some("live".to_string());
connect.target_id = Some(11);
connect.item_type = Some(PlaylistItemType::Live);
connect.target_name = Some("target-a".intern());
connect.stream_identity_key = Some("stream-a".to_string());
connect.shared_joined_existing = Some(false);
write_pending_history_records(&history_dir, vec![connect]);
@@ -826,14 +860,14 @@ mod tests {
let history_dir = temp.path().join("stream_history");
let today = current_utc_day();
let mut connect = base_record(EventType::Connect);
let mut connect = base_record(StreamHistoryEventType::Connect);
connect.partition_day_utc = today.clone();
connect.input_name = Some("input-a".to_string());
connect.provider_name = Some("provider-a".to_string());
connect.input_name = Some("input-a".intern());
connect.provider_name = Some("provider-a".intern());
connect.provider_id = Some(22);
connect.virtual_id = Some(33);
connect.item_type = Some("live".to_string());
connect.target_id = Some(11);
connect.item_type = Some(PlaylistItemType::Live);
connect.target_name = Some("target-a".intern());
connect.stream_identity_key = Some("stream-a".to_string());
connect.shared_joined_existing = Some(false);
write_pending_history_records(&history_dir, vec![connect]);
@@ -871,16 +905,16 @@ mod tests {
("2036-04-03", 3_u64),
("2036-04-04", 4_u64),
] {
let mut connect = base_record(EventType::Connect);
let mut connect = base_record(StreamHistoryEventType::Connect);
connect.partition_day_utc = day.to_string();
connect.event_ts_utc = connect.event_ts_utc.saturating_add(session_id);
connect.session_id = session_id;
connect.input_name = Some("input-a".to_string());
connect.provider_name = Some("provider-a".to_string());
connect.input_name = Some("input-a".intern());
connect.provider_name = Some("provider-a".intern());
connect.provider_id = Some(22);
connect.virtual_id = Some(33);
connect.item_type = Some("live".to_string());
connect.target_id = Some(11);
connect.item_type = Some(PlaylistItemType::Live);
connect.target_name = Some("target-a".intern());
connect.stream_identity_key = Some("stream-a".to_string());
connect.shared_joined_existing = Some(false);
write_pending_history_records(&history_dir, vec![connect]);
backend/src/api/model/streams/active_client_stream.rs
+4 -2
View File
@@ -32,6 +32,8 @@ use std::{
};
use tokio::sync::Notify;
use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned};
use shared::utils::Internable;
use shared::model::FailureStage;
/// Discriminates which byte-stream the client is consuming at any moment.
/// Stored as `u8` in an `AtomicU8` for lock-free access inside `poll_next`.
@@ -447,7 +449,7 @@ fn create_deferred_provider_open_future(
username: None,
client_ip: Some(&fingerprint.client_ip),
stream_channel: Some(stream_channel),
connect_failure_stage: Some(crate::repository::FailureStage::ProviderOpen),
connect_failure_stage: Some(FailureStage::ProviderOpen),
},
);
provider_stream_factory_options.set_provider(input.get_resolve_provider(stream_url.as_ref()));
@@ -711,7 +713,7 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam
}
let grant_user_grace_period = connection_permission == UserConnectionPermission::GracePeriod;
let username = user.username.as_str();
let provider_name = stream_details.provider_name.as_deref().unwrap_or("");
let provider_name = stream_details.provider_name.clone().unwrap_or_else(||"unknown".intern());
let user_agent = req_headers.get(USER_AGENT).map(|h| String::from_utf8_lossy(h.as_bytes())).unwrap_or_default();
backend/src/api/model/streams/provider_stream_factory.rs
+16 -7
View File
@@ -8,7 +8,6 @@ use crate::{
},
},
model::{ConfigProvider, ReverseProxyDisabledHeaderConfig},
repository::{ConnectFailureReason, FailureStage},
utils::{
debug_if_enabled,
request::{
@@ -42,7 +41,9 @@ use std::{
};
use tokio_util::sync::CancellationToken;
use url::Url;
use shared::utils::DEFAULT_USER_AGENT;
use shared::model::ConnectFailureReason;
use shared::utils::{Internable, DEFAULT_USER_AGENT};
use shared::model::{FailureStage};
const RETRY_SECONDS: u64 = 5;
const ERR_MAX_RETRY_COUNT: u32 = 5;
@@ -223,7 +224,7 @@ impl ProviderStreamFactoryOptions {
std::borrow::Cow::Owned(preview_request_target_for_logging(&self.url, self.provider.as_ref()))
}
fn build_connect_failed_stream_info(&self, provider_name: &str) -> Option<StreamInfo> {
fn build_connect_failed_stream_info(&self, provider_name: Arc<str>) -> Option<StreamInfo> {
let username = self.username.as_deref()?;
let client_ip = self.client_ip.as_deref()?;
let stream_channel = self.stream_channel.clone()?;
@@ -256,8 +257,14 @@ fn record_provider_open_failure(
let Some(failure_stage) = stream_options.get_connect_failure_stage() else { return };
let provider_name = stream_options
.get_provider()
.map_or_else(|| "unknown".to_string(), |provider| provider.name.to_string());
let Some(info) = stream_options.build_connect_failed_stream_info(&provider_name) else { return };
.map_or_else(|| "unknown".intern(), |provider| provider.name.clone());
let Some(info) = stream_options.build_connect_failed_stream_info(provider_name) else { return };
// Resolve target_name from target_id using the stable target config name.
let target_name = app_state
.app_config
.get_target_by_id(info.channel.target_id)
.as_deref()
.map(|t| (&t.name).intern());
app_state
.connection_manager
.record_connect_failed_with_provider_failure(
@@ -266,6 +273,7 @@ fn record_provider_open_failure(
failure_stage,
provider_http_status.map(|status| status.as_u16()),
provider_error_class,
target_name,
);
}
@@ -890,6 +898,7 @@ pub async fn create_provider_stream(
Ok(None) => None,
Err(failure) => {
let status = failure.status();
app_state.connection_manager.release_provider_connection(&stream_options.addr).await;
record_provider_open_failure(
app_state,
&stream_options,
@@ -1120,11 +1129,11 @@ mod tests {
connect_failure_stage: Some(FailureStage::ProviderOpen),
});
let info = options.build_connect_failed_stream_info("provider-a").expect("history context");
let info = options.build_connect_failed_stream_info("provider-a".intern()).expect("history context");
assert_eq!(info.username, "alice");
assert_eq!(info.client_ip, "203.0.113.9");
assert_eq!(info.provider, "provider-a");
assert_eq!(info.provider.as_ref(), "provider-a");
assert_eq!(info.channel.input_name.as_ref(), "input-a");
assert_eq!(info.channel.virtual_id, 77);
}
backend/src/api/model/streams/timed_client_stream.rs
+2 -1
View File
@@ -17,6 +17,7 @@ use std::{
time::Duration,
};
use tokio::time::{sleep_until, Instant, Sleep};
use shared::model::DisconnectReason;
enum TimeoutAction {
Kick {
@@ -88,7 +89,7 @@ impl Stream for TimedClientStream {
&addr,
virtual_id,
kick_secs,
crate::repository::DisconnectReason::Timeout,
DisconnectReason::Timeout,
)
.await;
});
backend/src/api/panel_api.rs
+7 -6
View File
@@ -46,6 +46,7 @@ use std::{
};
use tokio_util::sync::CancellationToken;
use url::Url;
use shared::model::DisconnectReason;
#[derive(Debug, Clone)]
struct AccountCredentials {
@@ -3500,7 +3501,7 @@ pub(crate) async fn run_panel_api_provisioning_probe(
&addr,
virtual_id,
provisioning_kick_secs,
crate::repository::DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
)
.await;
return Ok(());
@@ -3517,7 +3518,7 @@ pub(crate) async fn run_panel_api_provisioning_probe(
&addr,
virtual_id,
provisioning_kick_secs,
crate::repository::DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
)
.await;
return Ok(());
@@ -3534,7 +3535,7 @@ pub(crate) async fn run_panel_api_provisioning_probe(
&addr,
virtual_id,
provisioning_kick_secs,
crate::repository::DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
)
.await;
return Ok(());
@@ -3595,7 +3596,7 @@ pub(crate) async fn run_panel_api_provisioning_probe(
&addr,
virtual_id,
provisioning_kick_secs,
crate::repository::DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
)
.await;
return Ok(());
@@ -3621,7 +3622,7 @@ pub(crate) async fn run_panel_api_provisioning_probe(
&addr,
virtual_id,
provisioning_kick_secs,
crate::repository::DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
)
.await;
return Ok(());
@@ -3704,7 +3705,7 @@ pub(crate) async fn run_panel_api_provisioning_probe(
&addr,
virtual_id,
provisioning_kick_secs,
crate::repository::DisconnectReason::Provisioning,
DisconnectReason::Provisioning,
)
.await;
Ok(())
backend/src/api/serve.rs
+1 -1
View File
@@ -161,7 +161,7 @@ async fn handle_connection<M, S>(
Ok(signal) = addr_close_rx.recv() => {
match signal {
CloseConnectionSignal::WithReason(msg, reason) if msg == addr => {
connection_manager_clone.release_connection_with_reason(&addr, &reason).await;
connection_manager_clone.release_connection_with_reason(&addr, reason).await;
debug!("Forced client close {msg} reason={reason:?}");
conn.as_mut().graceful_shutdown();
break;
backend/src/main.rs
+1 -1
View File
@@ -153,7 +153,7 @@ async fn main() {
db_viewer(&args.db_viewer_args());
if let Some(ref sh_input) = args.stream_history {
std::process::exit(utils::stream_history_viewer(sh_input));
std::process::exit(utils::stream_history_viewer(sh_input).await);
}
if args.genpwd {
backend/src/model/mod.rs
+3 -1
View File
@@ -6,6 +6,7 @@ mod healthcheck;
mod config;
mod input_source;
pub mod messaging;
mod stream_history;
pub use self::playlist::*;
pub use self::mapping::*;
@@ -15,4 +16,5 @@ pub use self::healthcheck::*;
pub use shared::model::xtream_const::*;
pub use self::config::*;
pub use self::input_source::*;
pub use self::messaging::*;
pub use self::messaging::*;
pub use self::stream_history::*;
backend/src/model/stream_history.rs
+250
View File
@@ -0,0 +1,250 @@
use crate::utils::arc_str_option_serde;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use shared::model::{ConnectFailureReason, DisconnectReason, FailureStage, PlaylistItemType, StreamHistoryEventType, StreamHistoryRecordDto, StreamInfo};
use crate::utils::{encode_base64_hash, now_utc_secs, utc_day_from_secs};
pub const RECORD_SCHEMA_VERSION: u8 = 1;
/// A single stream lifecycle event record (connect or disconnect).
///
/// Privacy: must never contain passwords, tokens, or credential-bearing URLs.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamHistoryRecord {
pub schema_version: u8,
pub event_type: StreamHistoryEventType,
/// Unix timestamp seconds UTC of when this event occurred.
pub event_ts_utc: u64,
/// UTC calendar day of this event, e.g. `"2026-03-22"`.
pub partition_day_utc: String,
/// Stable correlation id shared between the connect and disconnect events of the same session.
pub session_id: u64,
pub source_addr: Option<String>,
// User and provider identity (no passwords, no tokens)
pub api_username: Option<String>,
#[serde(default, with = "arc_str_option_serde")]
pub provider_name: Option<Arc<str>>,
/// Provider-side username (e.g. Xtream account). Always `None` until `StreamInfo`
/// carries the provider credential — at which point populate from there.
pub provider_username: Option<String>,
#[serde(default, with = "arc_str_option_serde")]
pub input_name: Option<Arc<str>>,
// Stream metadata
pub virtual_id: Option<u32>,
pub item_type: Option<PlaylistItemType>,
pub title: Option<String>,
pub group: Option<String>,
pub country: Option<String>,
// QoS metadata
pub user_agent: Option<String>,
pub shared: Option<bool>,
pub shared_joined_existing: Option<bool>,
pub shared_stream_id: Option<u64>,
pub provider_id: Option<u32>,
pub cluster: Option<String>,
pub container: Option<String>,
pub stream_url_hash: Option<String>,
pub stream_identity_key: Option<String>,
pub video_codec: Option<String>,
pub audio_codec: Option<String>,
pub audio_channels: Option<String>,
pub resolution: Option<String>,
pub fps: Option<String>,
// Session summary — populated on disconnect.
// bytes_sent and first_byte_latency_ms are None for shared streams (meter serves
// multiple clients; per-session totals are not meaningful in that case).
pub connect_ts_utc: Option<u64>,
pub disconnect_ts_utc: Option<u64>,
pub session_duration: Option<u64>,
pub bytes_sent: Option<u64>,
pub first_byte_latency_ms: Option<u64>,
pub provider_reconnect_count: Option<u8>,
pub failure_stage: Option<FailureStage>,
pub provider_http_status: Option<u16>,
pub provider_error_class: Option<String>,
pub connect_failure_reason: Option<ConnectFailureReason>,
pub disconnect_reason: Option<DisconnectReason>,
/// Set when this connect event continues a session that was split by a `DayRollover`.
/// Always `None` until the `DayRollover` mechanism is implemented.
pub previous_session_id: Option<u64>,
/// Target config name — stable identifier, does not change across restarts.
#[serde(default, with = "arc_str_option_serde")]
pub target_name: Option<Arc<str>>,
}
/// `QoS` metrics collected at disconnect time, passed as a bundle to avoid growing the signature.
#[derive(Debug, Default)]
pub struct DisconnectQos {
pub bytes_sent: Option<u64>,
pub first_byte_latency_ms: Option<u64>,
pub provider_reconnect_count: Option<u8>,
}
impl StreamHistoryRecord {
fn build_stream_identity_key(info: &StreamInfo) -> Option<String> {
if info.channel.input_name.is_empty() {
return None;
}
let mut raw = String::with_capacity(info.channel.input_name.len() + 32);
raw.push_str(info.channel.input_name.as_ref());
raw.push('|');
raw.push_str(&info.channel.target_id.to_string());
raw.push('|');
raw.push_str(&info.channel.provider_id.to_string());
raw.push('|');
raw.push_str(&info.channel.virtual_id.to_string());
raw.push('|');
raw.push_str(&info.channel.item_type.to_string());
Some(encode_base64_hash(&raw))
}
/// Build a common base from a `StreamInfo`, leaving event-specific fields to the caller.
/// `event_ts` is the authoritative timestamp — `partition_day_utc` is derived from it
/// so both fields are always consistent.
fn base(info: &StreamInfo, event_ts: u64) -> Self {
Self {
schema_version: RECORD_SCHEMA_VERSION,
event_type: StreamHistoryEventType::Connect, // overridden by callers
event_ts_utc: event_ts,
partition_day_utc: utc_day_from_secs(event_ts),
// Combine connect-timestamp (upper 32 bits) with uid (lower 32 bits).
// This prevents session_id collision across server restarts and uid wrap-around.
// Both Connect and Disconnect derive this from the same StreamInfo, so they match.
session_id: (info.ts << 32) | u64::from(info.uid),
source_addr: Some(info.client_ip.clone()),
api_username: Some(info.username.clone()),
provider_name: Some(info.provider.clone()),
provider_username: None,
input_name: if info.channel.input_name.is_empty() {
None
} else {
Some(info.channel.input_name.clone())
},
virtual_id: Some(info.channel.virtual_id),
item_type: Some(info.channel.item_type),
title: Some(info.channel.title.to_string()),
group: Some(info.channel.group.to_string()),
country: info.country_code.clone(),
user_agent: if info.user_agent.is_empty() { None } else { Some(info.user_agent.clone()) },
shared: Some(info.channel.shared),
shared_joined_existing: info.channel.shared_joined_existing,
shared_stream_id: info.channel.shared_stream_id,
provider_id: Some(info.channel.provider_id),
cluster: Some(info.channel.cluster.to_string()),
container: info.channel.technical.as_ref().and_then(|t| if t.container.is_empty() { None } else { Some(t.container.clone()) }),
stream_url_hash: if info.channel.url.is_empty() {
None
} else {
Some(encode_base64_hash(info.channel.url.as_ref()))
},
stream_identity_key: Self::build_stream_identity_key(info),
video_codec: info.channel.technical.as_ref().and_then(|t| if t.video_codec.is_empty() { None } else { Some(t.video_codec.clone()) }),
audio_codec: info.channel.technical.as_ref().and_then(|t| if t.audio_codec.is_empty() { None } else { Some(t.audio_codec.clone()) }),
audio_channels: info.channel.technical.as_ref().and_then(|t| if t.audio_channels.is_empty() { None } else { Some(t.audio_channels.clone()) }),
resolution: info.channel.technical.as_ref().and_then(|t| if t.resolution.is_empty() { None } else { Some(t.resolution.clone()) }),
fps: info.channel.technical.as_ref().and_then(|t| if t.fps.is_empty() { None } else { Some(t.fps.clone()) }),
connect_ts_utc: None,
disconnect_ts_utc: None,
session_duration: None,
bytes_sent: None,
first_byte_latency_ms: None,
provider_reconnect_count: None,
failure_stage: None,
provider_http_status: None,
provider_error_class: None,
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_name: None,
}
}
pub fn from_connect(info: &StreamInfo) -> Self {
let mut record = Self::base(info, info.ts);
record.event_type = StreamHistoryEventType::Connect;
record.connect_ts_utc = Some(info.ts);
record.previous_session_id = info.previous_session_id;
record
}
pub fn from_connect_failed(
info: &StreamInfo,
reason: ConnectFailureReason,
attempt_uid: u32,
failure_stage: FailureStage,
target_name: Option<Arc<str>>,
) -> Self {
let event_ts = now_utc_secs();
let mut record = Self::base(info, event_ts);
record.event_type = StreamHistoryEventType::ConnectFailed;
record.session_id = (event_ts << 32) | u64::from(attempt_uid);
record.failure_stage = Some(failure_stage);
record.connect_failure_reason = Some(reason);
record.target_name = target_name;
record
}
pub fn with_provider_failure(mut self, provider_http_status: Option<u16>, provider_error_class: Option<&str>) -> Self {
self.provider_http_status = provider_http_status;
self.provider_error_class = provider_error_class.map(ToString::to_string);
self
}
/// Extra `QoS` fields carried as a struct to keep the signature stable.
pub fn from_disconnect(
info: &StreamInfo,
reason: DisconnectReason,
qos: &DisconnectQos,
failure_stage: Option<FailureStage>,
) -> Self {
let now_secs = now_utc_secs();
let connect_secs = info.ts;
let mut record = Self::base(info, now_secs);
record.event_type = StreamHistoryEventType::Disconnect;
record.connect_ts_utc = Some(connect_secs);
record.disconnect_ts_utc = Some(now_secs);
record.session_duration = Some(now_secs.saturating_sub(connect_secs));
record.bytes_sent = qos.bytes_sent;
record.first_byte_latency_ms = qos.first_byte_latency_ms;
record.provider_reconnect_count = qos.provider_reconnect_count;
record.failure_stage = failure_stage;
record.disconnect_reason = Some(reason);
record
}
}
impl From<&StreamHistoryRecord> for StreamHistoryRecordDto {
fn from(record: &StreamHistoryRecord) -> Self {
Self {
event_type: record.event_type,
event_ts_utc: record.event_ts_utc,
partition_day_utc: record.partition_day_utc.clone(),
session_id: record.session_id,
source_addr: record.source_addr.clone(),
api_username: record.api_username.clone(),
provider_name: record.provider_name.clone(),
input_name: record.input_name.clone(),
virtual_id: record.virtual_id,
item_type: record.item_type,
title: record.title.clone(),
group: record.group.clone(),
country: record.country.clone(),
user_agent: record.user_agent.clone(),
shared: record.shared,
provider_id: record.provider_id,
cluster: record.cluster.clone(),
container: record.container.clone(),
video_codec: record.video_codec.clone(),
audio_codec: record.audio_codec.clone(),
resolution: record.resolution.clone(),
failure_stage: record.failure_stage,
connect_failure_reason: record.connect_failure_reason,
disconnect_reason: record.disconnect_reason,
session_duration: record.session_duration,
bytes_sent: record.bytes_sent,
first_byte_latency_ms: record.first_byte_latency_ms,
previous_session_id: record.previous_session_id,
target_name: record.target_name.clone(),
}
}
}
backend/src/repository/qos_snapshot_repository.rs
+24 -18
View File
@@ -1,9 +1,11 @@
use crate::utils::arc_str_serde;
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use shared::model::PlaylistItemType;
use crate::repository::{BPlusTree, BPlusTreeUpdate, FlushPolicy};
const SNAPSHOT_FILE_NAME: &str = "qos_snapshot.db";
@@ -54,12 +56,15 @@ pub struct QosSnapshotDailyBucket {
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct QosSnapshotRecord {
pub stream_identity_key: String,
pub input_name: String,
pub target_id: u16,
pub provider_name: String,
#[serde(with = "arc_str_serde")]
pub input_name: Arc<str>,
#[serde(with = "arc_str_serde")]
pub target_name: Arc<str>,
#[serde(with = "arc_str_serde")]
pub provider_name: Arc<str>,
pub provider_id: u32,
pub virtual_id: u32,
pub item_type: String,
pub item_type: PlaylistItemType,
pub updated_at: u64,
pub last_event_at: u64,
pub window_24h: QosSnapshotWindow,
@@ -206,7 +211,8 @@ mod tests {
use std::path::Path;
use tempfile::tempdir;
use shared::model::PlaylistItemType;
use shared::utils::Internable;
use super::{QosAggregationCheckpoint, QosSnapshotRecord, QosSnapshotRepository, QosSnapshotWindow};
#[test]
@@ -216,12 +222,12 @@ mod tests {
let snapshot = QosSnapshotRecord {
stream_identity_key: "stream-a".to_string(),
input_name: "input-a".to_string(),
target_id: 11,
provider_name: "provider-a".to_string(),
input_name: "input-a".intern(),
target_name: "target-a".intern(),
provider_name: "provider-a".intern(),
provider_id: 22,
virtual_id: 33,
item_type: "live".to_string(),
item_type: PlaylistItemType::Live,
updated_at: 1_700_000_000,
last_event_at: 1_700_000_123,
window_24h: QosSnapshotWindow {
@@ -262,12 +268,12 @@ mod tests {
let snapshot = QosSnapshotRecord {
stream_identity_key: "stream-a".to_string(),
input_name: "input-a".to_string(),
target_id: 11,
provider_name: "provider-a".to_string(),
input_name: "input-a".intern(),
target_name: "target-a".intern(),
provider_name: "provider-a".intern(),
provider_id: 22,
virtual_id: 33,
item_type: "live".to_string(),
item_type: PlaylistItemType::Live,
updated_at: 1_700_000_000,
last_event_at: 1_700_000_123,
window_24h: QosSnapshotWindow::default(),
@@ -299,12 +305,12 @@ mod tests {
let snapshot = QosSnapshotRecord {
stream_identity_key: "stream-a".to_string(),
input_name: "input-a".to_string(),
target_id: 11,
provider_name: "provider-a".to_string(),
input_name: "input-a".intern(),
target_name: "target-a".intern(),
provider_name: "provider-a".intern(),
provider_id: 22,
virtual_id: 33,
item_type: "live".to_string(),
item_type: PlaylistItemType::Live,
updated_at: 1_700_000_000,
last_event_at: 1_700_000_123,
window_24h: QosSnapshotWindow {
backend/src/repository/stream_history/archive.rs
+30 -18
View File
@@ -1,9 +1,9 @@
use crate::repository::stream_history::{
BlockHeaderBody, CompressionKind, FileHeaderBody, RecordEncodingKind, BLOCK_MAGIC, CONTAINER_FORMAT_VERSION,
RECORD_SCHEMA_VERSION, SOURCE_KIND_STREAM_HISTORY, read_and_verify_file_magic, read_framed,
SOURCE_KIND_STREAM_HISTORY, read_and_verify_file_magic, read_framed,
write_file_magic, write_framed,
};
use crate::repository::stream_history::writer::{apply_retention, current_utc_day, now_utc_secs};
use crate::repository::stream_history::writer::apply_retention;
use lz4_flex::frame::FrameEncoder;
use log::{info, warn};
use std::{
@@ -11,6 +11,9 @@ use std::{
io::{self, BufReader, Read, Seek},
path::{Path, PathBuf},
};
use tokio::task::spawn_blocking;
use crate::model::RECORD_SCHEMA_VERSION;
use crate::utils::{current_utc_day, now_utc_secs};
struct PendingSummary {
header: FileHeaderBody,
@@ -111,7 +114,7 @@ pub fn archive_pending_file(path: &Path) -> io::Result<PathBuf> {
// Stream the block data from the pending file
let mut pending = File::open(path)?;
pending.seek(std::io::SeekFrom::Start(summary.blocks_start))?;
pending.seek(io::SeekFrom::Start(summary.blocks_start))?;
io::copy(&mut pending, &mut lz4)?;
lz4.finish().map_err(io::Error::other)?;
@@ -187,15 +190,22 @@ pub fn recover_pending_files(directory: &str) -> io::Result<()> {
}
/// Called by the writer on day rollover or shutdown: flush, archive, apply retention.
pub fn finalize_and_archive(pending_path: &Path, directory: &str, retention_days: u16) {
if !pending_path.exists() {
/// `archive_pending_file` (lz4 compression + file I/O) runs in `spawn_blocking` since it's CPU-bound;
/// `apply_retention` is fully async.
pub async fn finalize_and_archive(pending_path: &Path, directory: &str, retention_days: u16) {
let path_buf = pending_path.to_path_buf();
let path_buf_2 = path_buf.clone();
let path_exists = tokio::fs::metadata(&path_buf).await.is_ok();
if !path_exists {
return;
}
match archive_pending_file(pending_path) {
Ok(p) => info!("Finalized archive: {}", p.display()),
Err(e) => warn!("Failed to archive pending file {}: {e}", pending_path.display()),
let archive_result = spawn_blocking(move || archive_pending_file(&path_buf)).await;
match archive_result {
Ok(Ok(p)) => info!("Finalized archive: {}", p.display()),
Ok(Err(e)) => warn!("Failed to archive pending file {}: {e}", path_buf_2.display()),
Err(e) => warn!("Failed to archive pending file (join error): {e}"),
}
if let Err(e) = apply_retention(directory, retention_days) {
if let Err(e) = apply_retention(directory, retention_days).await {
warn!("Retention cleanup failed: {e}");
}
}
@@ -205,13 +215,15 @@ mod tests {
use super::*;
use std::io::Write;
use crate::repository::stream_history::{
CompressionKind, EventType, RECORD_SCHEMA_VERSION, StreamHistoryRecord, read_and_verify_file_magic,
CompressionKind, read_and_verify_file_magic,
read_framed, serialize_named, write_block_magic,
};
use crate::repository::stream_history::writer::{StreamHistoryWriter, now_utc_secs, current_utc_day};
use crate::model::StreamHistoryConfig;
use crate::repository::stream_history::writer::{StreamHistoryWriter};
use crate::model::{StreamHistoryConfig, StreamHistoryRecord};
use lz4_flex::frame::FrameDecoder;
use tempfile::TempDir;
use shared::model::{PlaylistItemType, StreamHistoryEventType};
use shared::utils::Internable;
fn test_config(dir: &str, batch_size: usize) -> StreamHistoryConfig {
StreamHistoryConfig {
@@ -225,17 +237,17 @@ mod tests {
fn make_record(session_id: u64) -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: RECORD_SCHEMA_VERSION,
event_type: EventType::Connect,
event_type: StreamHistoryEventType::Connect,
event_ts_utc: now_utc_secs(),
partition_day_utc: current_utc_day(),
session_id,
source_addr: None,
api_username: Some("alice".to_string()),
provider_name: Some("acme".to_string()),
provider_name: Some("acme".intern()),
provider_username: None,
input_name: Some("input".to_string()),
input_name: Some("input".intern()),
virtual_id: Some(1),
item_type: Some("live".to_string()),
item_type: Some(PlaylistItemType::Live),
title: Some("Test".to_string()),
group: None,
country: None,
@@ -265,14 +277,14 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: None,
target_name: None,
}
}
/// Write a minimal `.pending` file and return its path.
fn write_test_pending(dir: &Path, day: &str, records: &[StreamHistoryRecord]) -> PathBuf {
use crate::repository::stream_history::{
BlockHeaderBody, CONTAINER_FORMAT_VERSION, RECORD_SCHEMA_VERSION, CompressionKind,
BlockHeaderBody, CONTAINER_FORMAT_VERSION, CompressionKind,
RecordEncodingKind, SOURCE_KIND_STREAM_HISTORY,
};
backend/src/repository/stream_history/async_iterator.rs
+157
View File
@@ -0,0 +1,157 @@
use crate::repository::stream_history::{StreamHistoryFileReader, AsyncStreamHistoryPendingReader};
use crate::utils::stream_history_viewer::{discover_files, CompiledFilter, TimeRange};
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tokio_stream::Stream as TokioStream;
use crate::model::StreamHistoryRecord;
/// Async stream of stream history records.
/// NOTE: The stream implementation is ready but not yet wired to the API.
/// Kept here for future use when we move back to streaming responses.
#[allow(dead_code)]
pub struct StreamHistoryStream {
inner: ReceiverStream<StreamHistoryRecord>,
}
#[allow(dead_code)]
impl StreamHistoryStream {
/// Creates a new stream by reading files in a background task.
///
/// Pending files (uncompressed) are read with fully async I/O using
/// `AsyncStreamHistoryPendingReader` (`tokio::fs::File`).
/// Archive files (LZ4-compressed) still require `spawn_blocking` because
/// `lz4_flex::frame::FrameDecoder` is sync-only.
#[allow(clippy::needless_pass_by_value)]
#[allow(dead_code)]
pub(crate) fn new(dir: String, time_range: TimeRange, filters: Arc<CompiledFilter>) -> Self {
let (tx, rx) = mpsc::channel(1024);
let filters_clone = Arc::clone(&filters);
let dir_clone = dir.clone();
tokio::spawn(async move {
Self::read_files_async(&dir_clone, &time_range, &filters_clone, tx).await;
});
Self {
inner: ReceiverStream::new(rx),
}
}
/// Reads files and sends records through the channel.
/// - Pending files: fully async (`tokio::fs::File` + `AsyncStreamHistoryPendingReader`)
/// - Archive files: sync via `spawn_blocking` (`lz4_flex` is sync-only)
#[allow(clippy::needless_pass_by_value)]
#[allow(dead_code)]
async fn read_files_async(
dir: &str,
time_range: &TimeRange,
filters: &Arc<CompiledFilter>,
tx: mpsc::Sender<StreamHistoryRecord>,
) {
let files = match discover_files(Path::new(dir), time_range).await {
Ok(f) => f,
Err(e) => {
log::error!("Failed to discover stream history files: {e}");
return;
}
};
let (range_start, range_end) = *time_range;
for file in &files {
if file.is_archive {
// Archive files: LZ4 decompression is sync-only — use spawn_blocking.
// The blocking task sends directly into the async channel via blocking_send(),
// avoiding the deadlock that would occur with an intermediate std::sync::mpsc
// channel whose bounded sender blocks before the async side starts draining.
let file_path = file.path.clone();
let time_range_val = *time_range;
let tx_clone = tx.clone();
let filters_clone = Arc::clone(filters);
let result = tokio::task::spawn_blocking(move || {
Self::read_archive_file(&file_path, time_range_val, &filters_clone, &tx_clone);
}).await;
if let Err(e) = result {
log::error!("Archive read task failed: {e}");
}
} else {
// Pending files: fully async using AsyncStreamHistoryPendingReader
match AsyncStreamHistoryPendingReader::open(&file.path, Some(*time_range)).await {
Ok((mut reader, _)) => {
while let Some(result) = reader.next_record().await {
match result {
Ok(record) => {
if record.event_ts_utc < range_start || record.event_ts_utc > range_end {
continue;
}
if !filters.matches(&record) {
continue;
}
if tx.send(record).await.is_err() {
return; // Receiver dropped
}
}
Err(e) => {
log::warn!("Failed to read pending record: {e}");
}
}
}
}
Err(e) => {
log::warn!("Failed to open pending file {}: {e}", file.path.display());
}
}
}
}
}
/// Read a LZ4 archive file (runs in `spawn_blocking`).
/// Streams records incrementally via `blocking_send` into the async channel,
/// so memory usage stays bounded regardless of archive size.
#[allow(dead_code)]
fn read_archive_file(
path: &Path,
time_range: TimeRange,
filters: &CompiledFilter,
tx: &mpsc::Sender<StreamHistoryRecord>,
) {
let (range_start, range_end) = time_range;
let (reader, _) = match StreamHistoryFileReader::from_archive(path, Some(time_range)) {
Ok(r) => r,
Err(e) => {
log::warn!("Failed to open archive {}: {e}", path.display());
return;
}
};
for result in reader {
match result {
Ok(record) => {
if record.event_ts_utc < range_start || record.event_ts_utc > range_end {
continue;
}
if !filters.matches(&record) {
continue;
}
if tx.blocking_send(record).is_err() {
// Receiver dropped, stop sending
return;
}
}
Err(e) => {
log::warn!("Failed to read archive record: {e}");
}
}
}
}
}
impl TokioStream for StreamHistoryStream {
type Item = StreamHistoryRecord;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.inner).poll_next(cx)
}
}
backend/src/repository/stream_history/mod.rs
+2
View File
@@ -2,9 +2,11 @@ mod archive;
mod writer;
mod storage;
mod reader;
mod async_iterator;
pub use writer::*;
pub use archive::*;
pub use storage::*;
pub use reader::*;
pub use async_iterator::*;
backend/src/repository/stream_history/reader.rs
+177 -6
View File
@@ -3,11 +3,14 @@ use std::io::{self, BufReader, Read};
use std::path::Path;
use lz4_flex::frame::FrameDecoder;
use tokio::fs::File as TokioFile;
use tokio::io::{AsyncReadExt, AsyncSeekExt, BufReader as TokioBufReader};
use crate::model::StreamHistoryRecord;
use crate::repository::stream_history::storage::{
read_and_verify_file_magic, read_and_verify_block_magic, read_framed,
deserialize_named, FileHeaderBody, BlockHeaderBody, StreamHistoryRecord,
deserialize_named, FileHeaderBody, BlockHeaderBody,
BLOCK_MAGIC, MAX_BLOCK_PAYLOAD_SIZE,
async_read_and_verify_file_magic, async_read_and_verify_block_magic, async_read_framed,
};
pub struct StreamHistoryFileReader<R: Read> {
@@ -160,7 +163,7 @@ impl<R: Read> StreamHistoryFileReader<R> {
// by the underlying reader yet — chain them before self.reader.
let tail_start = (i + 4).saturating_sub(carry_len);
let tail = &scan_buf[tail_start..n];
let mut chained = io::Cursor::new(tail).chain(&mut self.reader);
let mut chained = std::io::Read::chain(io::Cursor::new(tail), &mut self.reader);
let block_header: BlockHeaderBody = match read_framed::<_, BlockHeaderBody>(&mut chained) {
Ok(h) if (h.payload_len as usize) <= MAX_BLOCK_PAYLOAD_SIZE => {
@@ -266,17 +269,185 @@ impl<R: Read> Iterator for StreamHistoryFileReader<R> {
}
}
// ---------------------------------------------------------------------------
// Async pending file reader — uses tokio for non-blocking I/O
// ---------------------------------------------------------------------------
/// Async reader for `.pending` stream history files.
///
/// Uses `tokio::fs::File` + `tokio::io::BufReader` for fully async I/O.
/// Exposes `next_record()` as a public async fn — call it in a loop in the
/// spawned reader task rather than trying to implement Stream (which would
/// require async-stream or a state machine in `poll_next`).
///
/// NOTE: This does NOT support magic recovery — pending files are append-only
/// and blocks are read sequentially. If a block is corrupt the stream stops.
pub struct AsyncStreamHistoryPendingReader {
reader: TokioBufReader<TokioFile>,
time_range: Option<(u64, u64)>,
current_block_remaining: u32,
payload_buf: Vec<u8>,
payload_offset: usize,
done: bool,
file_path: String,
}
impl AsyncStreamHistoryPendingReader {
/// Open a pending file and verify its header.
pub async fn open(path: &Path, time_range: Option<(u64, u64)>) -> io::Result<(Self, FileHeaderBody)> {
let file = TokioFile::open(path).await?;
let mut reader = TokioBufReader::new(file);
async_read_and_verify_file_magic(&mut reader).await?;
let header: FileHeaderBody = async_read_framed(&mut reader).await?;
Ok((Self {
reader,
time_range,
current_block_remaining: 0,
payload_buf: Vec::new(),
payload_offset: 0,
done: false,
file_path: path.display().to_string(),
}, header))
}
/// Skip `len` bytes by seeking forward from current position.
async fn skip_payload(&mut self, len: u32) -> io::Result<()> {
self.reader.seek(tokio::io::SeekFrom::Current(i64::from(len))).await?;
Ok(())
}
/// Read and validate the next block header, then load the full payload into `payload_buf`.
async fn read_next_block(&mut self) -> io::Result<Option<()>> {
loop {
match async_read_and_verify_block_magic(&mut self.reader).await {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
Err(e) => {
eprintln!("Warning: {}: corrupt block magic: {e}", self.file_path);
self.done = true;
return Ok(None);
}
}
let block_header: BlockHeaderBody = match async_read_framed(&mut self.reader).await {
Ok(h) => h,
Err(e) => {
eprintln!("Warning: {}: corrupt block header: {e}", self.file_path);
self.done = true;
return Ok(None);
}
};
// Block-level time-range skip
if let Some((start, end)) = self.time_range {
if block_header.last_event_ts_utc < start || block_header.first_event_ts_utc > end {
if let Err(e) = self.skip_payload(block_header.payload_len).await {
if e.kind() == io::ErrorKind::UnexpectedEof {
self.done = true;
return Ok(None);
}
return Err(e);
}
continue;
}
}
// OOM protection
if block_header.payload_len as usize > MAX_BLOCK_PAYLOAD_SIZE {
eprintln!("Warning: {}: block payload_len {} exceeds MAX_BLOCK_PAYLOAD_SIZE, skipping",
self.file_path, block_header.payload_len);
if let Err(e) = self.skip_payload(block_header.payload_len).await {
if e.kind() == io::ErrorKind::UnexpectedEof {
self.done = true;
return Ok(None);
}
return Err(e);
}
continue;
}
self.payload_buf.resize(block_header.payload_len as usize, 0);
self.reader.read_exact(&mut self.payload_buf).await?;
let actual_crc = crc32fast::hash(&self.payload_buf);
if actual_crc != block_header.payload_crc {
eprintln!("Warning: {}: payload CRC mismatch, stopping", self.file_path);
self.done = true;
return Ok(None);
}
self.current_block_remaining = block_header.record_count;
self.payload_offset = 0;
return Ok(Some(()));
}
}
/// Fetch the next record, or `None` on EOF / error.
pub async fn next_record(&mut self) -> Option<io::Result<StreamHistoryRecord>> {
if self.done {
return None;
}
loop {
if self.current_block_remaining > 0 {
if self.payload_offset + 4 > self.payload_buf.len() {
eprintln!("Warning: {}: truncated record length", self.file_path);
self.current_block_remaining = 0;
continue;
}
let mut record_len_bytes = [0u8; 4];
record_len_bytes.copy_from_slice(&self.payload_buf[self.payload_offset..self.payload_offset + 4]);
let record_len = u32::from_be_bytes(record_len_bytes) as usize;
self.payload_offset += 4;
if self.payload_offset + record_len > self.payload_buf.len() {
eprintln!("Warning: {}: truncated record data", self.file_path);
self.current_block_remaining = 0;
continue;
}
let record_bytes = &self.payload_buf[self.payload_offset..self.payload_offset + record_len];
self.payload_offset += record_len;
self.current_block_remaining -= 1;
match deserialize_named(record_bytes) {
Ok(record) => return Some(Ok(record)),
Err(e) => {
eprintln!("Warning: {}: failed to deserialize record: {e}", self.file_path);
continue;
}
}
}
match self.read_next_block().await {
Ok(Some(())) => {} // Got block, loop back
Ok(None) => {
self.done = true;
return None;
}
Err(e) => {
self.done = true;
return Some(Err(e));
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::repository::stream_history::storage::*;
use std::io::Write;
use tempfile::NamedTempFile;
use shared::model::StreamHistoryEventType;
use shared::utils::Internable;
use crate::model::{RECORD_SCHEMA_VERSION};
fn make_test_record(ts: u64, username: &str) -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: RECORD_SCHEMA_VERSION,
event_type: EventType::Connect,
event_type: StreamHistoryEventType::Connect,
event_ts_utc: ts,
partition_day_utc: "2026-03-22".to_string(),
session_id: 1,
@@ -284,7 +455,7 @@ mod tests {
api_username: Some(username.to_string()),
provider_name: None,
provider_username: None,
input_name: Some("input".to_string()),
input_name: Some("input".intern()),
virtual_id: None,
item_type: None,
title: None,
@@ -316,7 +487,7 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: None,
target_name: None,
}
}
backend/src/repository/stream_history/storage.rs
+81 -275
View File
@@ -1,14 +1,10 @@
use serde::{Deserialize, Serialize};
use shared::error::to_io_error;
use std::io::{self, Read, Write};
use shared::model::StreamInfo;
use crate::repository::{now_utc_secs, utc_day_from_secs};
use crate::utils::encode_base64_hash;
pub const FILE_MAGIC: [u8; 8] = *b"STRHIST\x01";
pub const BLOCK_MAGIC: [u8; 4] = *b"BLK\x01";
pub const CONTAINER_FORMAT_VERSION: u8 = 1;
pub const RECORD_SCHEMA_VERSION: u8 = 1;
pub const SOURCE_KIND_STREAM_HISTORY: &str = "stream_history";
pub const MAX_FRAME_SIZE: usize = 8 * 1024 * 1024; // 8 MiB
/// Maximum allowed block payload size when reading. Prevents memory exhaustion on corrupt/malicious input during magic-recovery.
@@ -28,60 +24,6 @@ pub enum RecordEncodingKind {
MessagePackNamed,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EventType {
Connect,
ConnectFailed,
Disconnect,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ConnectFailureReason {
UserAccountExpired,
UserConnectionsExhausted,
ProviderConnectionsExhausted,
ProviderError,
ProviderClosed,
ChannelUnavailable,
Preempted,
SessionExpired,
Provisioning,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailureStage {
Admission,
ProviderOpen,
FirstByte,
Streaming,
SessionReconnect,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DisconnectReason {
Cleanup,
ClientClosed,
ClientKicked,
Provisioning,
ServerError,
Timeout,
/// Reserved for future day-split logic: emitted at midnight for sessions that span
/// two calendar days so the previous day's file gets a closing record.
/// Not yet emitted — sessions crossing midnight have their disconnect in the new day's file.
DayRollover,
Shutdown,
Unknown,
ProviderError,
ProviderClosed,
Preempted,
SessionExpired,
UserConnectionsExhausted,
ProviderConnectionsExhausted,
}
/// Serialized as `MessagePack` named (map encoding) for schema evolution safety.
///
@@ -131,208 +73,6 @@ pub struct BlockHeaderBody {
pub flags: u8,
}
/// A single stream lifecycle event record (connect or disconnect).
///
/// Privacy: must never contain passwords, tokens, or credential-bearing URLs.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamHistoryRecord {
pub schema_version: u8,
pub event_type: EventType,
/// Unix timestamp seconds UTC of when this event occurred.
pub event_ts_utc: u64,
/// UTC calendar day of this event, e.g. `"2026-03-22"`.
pub partition_day_utc: String,
/// Stable correlation id shared between the connect and disconnect events of the same session.
pub session_id: u64,
pub source_addr: Option<String>,
// User and provider identity (no passwords, no tokens)
pub api_username: Option<String>,
pub provider_name: Option<String>,
/// Provider-side username (e.g. Xtream account). Always `None` until `StreamInfo`
/// carries the provider credential — at which point populate from there.
pub provider_username: Option<String>,
pub input_name: Option<String>,
// Stream metadata
pub virtual_id: Option<u32>,
pub item_type: Option<String>,
pub title: Option<String>,
pub group: Option<String>,
pub country: Option<String>,
// QoS metadata
pub user_agent: Option<String>,
pub shared: Option<bool>,
pub shared_joined_existing: Option<bool>,
pub shared_stream_id: Option<u64>,
pub provider_id: Option<u32>,
pub cluster: Option<String>,
pub container: Option<String>,
pub stream_url_hash: Option<String>,
pub stream_identity_key: Option<String>,
pub video_codec: Option<String>,
pub audio_codec: Option<String>,
pub audio_channels: Option<String>,
pub resolution: Option<String>,
pub fps: Option<String>,
// Session summary — populated on disconnect.
// bytes_sent and first_byte_latency_ms are None for shared streams (meter serves
// multiple clients; per-session totals are not meaningful in that case).
pub connect_ts_utc: Option<u64>,
pub disconnect_ts_utc: Option<u64>,
pub session_duration: Option<u64>,
pub bytes_sent: Option<u64>,
pub first_byte_latency_ms: Option<u64>,
pub provider_reconnect_count: Option<u8>,
pub failure_stage: Option<FailureStage>,
pub provider_http_status: Option<u16>,
pub provider_error_class: Option<String>,
pub connect_failure_reason: Option<ConnectFailureReason>,
pub disconnect_reason: Option<DisconnectReason>,
/// Set when this connect event continues a session that was split by a `DayRollover`.
/// Always `None` until the `DayRollover` mechanism is implemented.
pub previous_session_id: Option<u64>,
/// Target config id — identifies the failover bundle this stream belongs to.
pub target_id: Option<u16>,
}
/// `QoS` metrics collected at disconnect time, passed as a bundle to avoid growing the signature.
#[derive(Debug, Default)]
pub struct DisconnectQos {
pub bytes_sent: Option<u64>,
pub first_byte_latency_ms: Option<u64>,
pub provider_reconnect_count: Option<u8>,
}
impl StreamHistoryRecord {
fn build_stream_identity_key(info: &StreamInfo) -> Option<String> {
if info.channel.input_name.is_empty() {
return None;
}
let mut raw = String::with_capacity(info.channel.input_name.len() + 32);
raw.push_str(info.channel.input_name.as_ref());
raw.push('|');
raw.push_str(&info.channel.target_id.to_string());
raw.push('|');
raw.push_str(&info.channel.provider_id.to_string());
raw.push('|');
raw.push_str(&info.channel.virtual_id.to_string());
raw.push('|');
raw.push_str(&info.channel.item_type.to_string());
Some(encode_base64_hash(&raw))
}
/// Build a common base from a `StreamInfo`, leaving event-specific fields to the caller.
/// `event_ts` is the authoritative timestamp — `partition_day_utc` is derived from it
/// so both fields are always consistent.
fn base(info: &StreamInfo, event_ts: u64) -> Self {
Self {
schema_version: RECORD_SCHEMA_VERSION,
event_type: EventType::Connect, // overridden by callers
event_ts_utc: event_ts,
partition_day_utc: utc_day_from_secs(event_ts),
// Combine connect-timestamp (upper 32 bits) with uid (lower 32 bits).
// This prevents session_id collision across server restarts and uid wrap-around.
// Both Connect and Disconnect derive this from the same StreamInfo, so they match.
session_id: (info.ts << 32) | u64::from(info.uid),
source_addr: Some(info.client_ip.clone()),
api_username: Some(info.username.clone()),
provider_name: Some(info.provider.clone()),
provider_username: None,
input_name: if info.channel.input_name.is_empty() {
None
} else {
Some(info.channel.input_name.to_string())
},
virtual_id: Some(info.channel.virtual_id),
item_type: Some(info.channel.item_type.to_string()),
title: Some(info.channel.title.to_string()),
group: Some(info.channel.group.to_string()),
country: info.country_code.clone(),
user_agent: if info.user_agent.is_empty() { None } else { Some(info.user_agent.clone()) },
shared: Some(info.channel.shared),
shared_joined_existing: info.channel.shared_joined_existing,
shared_stream_id: info.channel.shared_stream_id,
provider_id: Some(info.channel.provider_id),
cluster: Some(info.channel.cluster.to_string()),
container: info.channel.technical.as_ref().and_then(|t| if t.container.is_empty() { None } else { Some(t.container.clone()) }),
stream_url_hash: if info.channel.url.is_empty() {
None
} else {
Some(encode_base64_hash(info.channel.url.as_ref()))
},
stream_identity_key: Self::build_stream_identity_key(info),
video_codec: info.channel.technical.as_ref().and_then(|t| if t.video_codec.is_empty() { None } else { Some(t.video_codec.clone()) }),
audio_codec: info.channel.technical.as_ref().and_then(|t| if t.audio_codec.is_empty() { None } else { Some(t.audio_codec.clone()) }),
audio_channels: info.channel.technical.as_ref().and_then(|t| if t.audio_channels.is_empty() { None } else { Some(t.audio_channels.clone()) }),
resolution: info.channel.technical.as_ref().and_then(|t| if t.resolution.is_empty() { None } else { Some(t.resolution.clone()) }),
fps: info.channel.technical.as_ref().and_then(|t| if t.fps.is_empty() { None } else { Some(t.fps.clone()) }),
connect_ts_utc: None,
disconnect_ts_utc: None,
session_duration: None,
bytes_sent: None,
first_byte_latency_ms: None,
provider_reconnect_count: None,
failure_stage: None,
provider_http_status: None,
provider_error_class: None,
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: Some(info.channel.target_id),
}
}
pub fn from_connect(info: &StreamInfo) -> Self {
let mut record = Self::base(info, info.ts);
record.event_type = EventType::Connect;
record.connect_ts_utc = Some(info.ts);
record.previous_session_id = info.previous_session_id;
record
}
pub fn from_connect_failed(
info: &StreamInfo,
reason: ConnectFailureReason,
attempt_uid: u32,
failure_stage: FailureStage,
) -> Self {
let event_ts = now_utc_secs();
let mut record = Self::base(info, event_ts);
record.event_type = EventType::ConnectFailed;
record.session_id = (event_ts << 32) | u64::from(attempt_uid);
record.failure_stage = Some(failure_stage);
record.connect_failure_reason = Some(reason);
record
}
pub fn with_provider_failure(mut self, provider_http_status: Option<u16>, provider_error_class: Option<&str>) -> Self {
self.provider_http_status = provider_http_status;
self.provider_error_class = provider_error_class.map(ToString::to_string);
self
}
/// Extra `QoS` fields carried as a struct to keep the signature stable.
pub fn from_disconnect(
info: &StreamInfo,
reason: DisconnectReason,
qos: &DisconnectQos,
failure_stage: Option<FailureStage>,
) -> Self {
let now_secs = now_utc_secs();
let connect_secs = info.ts;
let mut record = Self::base(info, now_secs);
record.event_type = EventType::Disconnect;
record.connect_ts_utc = Some(connect_secs);
record.disconnect_ts_utc = Some(now_secs);
record.session_duration = Some(now_secs.saturating_sub(connect_secs));
record.bytes_sent = qos.bytes_sent;
record.first_byte_latency_ms = qos.first_byte_latency_ms;
record.provider_reconnect_count = qos.provider_reconnect_count;
record.failure_stage = failure_stage;
record.disconnect_reason = Some(reason);
record
}
}
/// Serialize a value to `MessagePack` using **named (map) encoding** for schema evolution safety.
pub fn serialize_named<T: Serialize>(value: &T) -> io::Result<Vec<u8>> {
rmp_serde::to_vec_named(value).map_err(to_io_error)
@@ -425,6 +165,66 @@ pub fn read_and_verify_block_magic<R: Read>(reader: &mut R) -> io::Result<()> {
Ok(())
}
// ---------------------------------------------------------------------------
// Async variants for use with `tokio::io::AsyncRead`
// Used by `AsyncStreamHistoryPendingReader` in `reader.rs`
// ---------------------------------------------------------------------------
use tokio::io::{AsyncRead, AsyncReadExt};
/// Async version of `read_and_verify_file_magic` for `tokio::io::AsyncRead`.
pub async fn async_read_and_verify_file_magic<R: AsyncRead + Unpin>(reader: &mut R) -> io::Result<()> {
let mut magic = [0u8; 8];
reader.read_exact(&mut magic).await?;
if magic != FILE_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid file magic: {magic:02X?}"),
));
}
Ok(())
}
/// Async version of `read_and_verify_block_magic` for `tokio::io::AsyncRead`.
pub async fn async_read_and_verify_block_magic<R: AsyncRead + Unpin>(reader: &mut R) -> io::Result<()> {
let mut magic = [0u8; 4];
reader.read_exact(&mut magic).await?;
if magic != BLOCK_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid block magic: {magic:02X?}"),
));
}
Ok(())
}
/// Async version of `read_framed` for `tokio::io::AsyncRead`.
/// Reads: [len: u32 BE][payload][crc32: u32 BE], verifies CRC, deserializes.
pub async fn async_read_framed<R: AsyncRead + Unpin, T: for<'de> Deserialize<'de>>(reader: &mut R) -> io::Result<T> {
let mut len_buf = [0u8; 4];
reader.read_exact(&mut len_buf).await?;
let len = u32::from_be_bytes(len_buf) as usize;
if len == 0 || len > MAX_FRAME_SIZE {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid frame size: {len} (max {MAX_FRAME_SIZE})"),
));
}
let mut payload = vec![0u8; len];
reader.read_exact(&mut payload).await?;
let mut crc_buf = [0u8; 4];
reader.read_exact(&mut crc_buf).await?;
let expected_crc = u32::from_be_bytes(crc_buf);
let actual_crc = crc32fast::hash(&payload);
if actual_crc != expected_crc {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("CRC mismatch: expected {expected_crc:#010x}, got {actual_crc:#010x}"),
));
}
deserialize_named(&payload)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -434,6 +234,9 @@ mod tests {
};
use std::net::SocketAddr;
use std::io::Cursor;
use shared::model::{ConnectFailureReason, FailureStage, DisconnectReason, StreamHistoryEventType};
use crate::model::{DisconnectQos, StreamHistoryRecord, RECORD_SCHEMA_VERSION};
use crate::utils::encode_base64_hash;
fn sample_file_header() -> FileHeaderBody {
FileHeaderBody {
@@ -470,17 +273,17 @@ mod tests {
fn sample_connect_record() -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: RECORD_SCHEMA_VERSION,
event_type: EventType::Connect,
event_type: StreamHistoryEventType::Connect,
event_ts_utc: 1_742_600_001,
partition_day_utc: "2026-03-22".to_string(),
session_id: 999,
source_addr: Some("192.0.2.1:12345".to_string()),
api_username: Some("alice".to_string()),
provider_name: Some("acme-tv".to_string()),
provider_name: Some("acme-tv".intern()),
provider_username: Some("acme_user".to_string()),
input_name: Some("provider-input".to_string()),
input_name: Some("provider-input".intern()),
virtual_id: Some(1234),
item_type: Some("live".to_string()),
item_type: Some(PlaylistItemType::Live),
title: Some("News Channel".to_string()),
group: Some("News".to_string()),
country: Some("DE".to_string()),
@@ -510,24 +313,24 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: Some(1),
target_name: None,
}
}
fn sample_disconnect_record(session_id: u64) -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: RECORD_SCHEMA_VERSION,
event_type: EventType::Disconnect,
event_type: StreamHistoryEventType::Disconnect,
event_ts_utc: 1_742_603_601,
partition_day_utc: "2026-03-22".to_string(),
session_id,
source_addr: Some("192.0.2.1:12345".to_string()),
api_username: Some("alice".to_string()),
provider_name: Some("acme-tv".to_string()),
provider_name: Some("acme-tv".intern()),
provider_username: Some("acme_user".to_string()),
input_name: Some("provider-input".to_string()),
input_name: Some("provider-input".intern()),
virtual_id: Some(1234),
item_type: Some("live".to_string()),
item_type: Some(PlaylistItemType::Live),
title: Some("News Channel".to_string()),
group: Some("News".to_string()),
country: Some("DE".to_string()),
@@ -557,7 +360,7 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: Some(DisconnectReason::ClientClosed),
previous_session_id: None,
target_id: Some(1),
target_name: None,
}
}
@@ -569,7 +372,7 @@ mod tests {
"alice",
&addr,
"192.0.2.1",
"acme-tv",
"acme-tv".intern(),
StreamChannel {
target_id: 1,
virtual_id: 1234,
@@ -631,7 +434,7 @@ mod tests {
let original = sample_connect_record();
let bytes = serialize_named(&original).expect("serialize");
let decoded: StreamHistoryRecord = deserialize_named(&bytes).expect("deserialize");
assert_eq!(decoded.event_type, EventType::Connect);
assert_eq!(decoded.event_type, StreamHistoryEventType::Connect);
assert_eq!(decoded.session_id, 999);
assert_eq!(decoded.api_username.as_deref(), Some("alice"));
assert!(decoded.disconnect_ts_utc.is_none());
@@ -644,8 +447,8 @@ mod tests {
connect.session_id = session_id;
let disconnect = sample_disconnect_record(session_id);
assert_eq!(connect.session_id, disconnect.session_id);
assert_eq!(connect.event_type, EventType::Connect);
assert_eq!(disconnect.event_type, EventType::Disconnect);
assert_eq!(connect.event_type, StreamHistoryEventType::Connect);
assert_eq!(disconnect.event_type, StreamHistoryEventType::Disconnect);
assert_eq!(disconnect.session_duration, Some(3600));
}
@@ -700,9 +503,10 @@ mod tests {
ConnectFailureReason::ProviderConnectionsExhausted,
77,
FailureStage::Admission,
None,
);
assert_eq!(record.event_type, EventType::ConnectFailed);
assert_eq!(record.event_type, StreamHistoryEventType::ConnectFailed);
assert_eq!(
record.connect_failure_reason,
Some(ConnectFailureReason::ProviderConnectionsExhausted)
@@ -726,6 +530,7 @@ mod tests {
ConnectFailureReason::ProviderConnectionsExhausted,
77,
FailureStage::Admission,
None,
);
assert_eq!(record.failure_stage, Some(FailureStage::Admission));
@@ -740,6 +545,7 @@ mod tests {
ConnectFailureReason::ChannelUnavailable,
77,
FailureStage::ProviderOpen,
None,
)
.with_provider_failure(Some(503), Some("http_5xx"));
backend/src/repository/stream_history/writer.rs
+197 -152
View File
@@ -1,47 +1,25 @@
use crate::repository::stream_history::{
BlockHeaderBody, CompressionKind, FileHeaderBody, RecordEncodingKind, CONTAINER_FORMAT_VERSION,
RECORD_SCHEMA_VERSION, SOURCE_KIND_STREAM_HISTORY, StreamHistoryRecord,
write_block_magic, write_file_magic, write_framed,
SOURCE_KIND_STREAM_HISTORY, write_framed,
};
use crate::repository::stream_history::archive::finalize_and_archive;
use crate::model::StreamHistoryConfig;
use crate::model::{StreamHistoryConfig, StreamHistoryRecord, RECORD_SCHEMA_VERSION};
use log::{error, info, warn};
use std::{
fs::{self, File, OpenOptions},
io::{self, BufWriter, Write},
io,
path::PathBuf,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
time::{SystemTime, UNIX_EPOCH},
};
use tokio::fs::{self, OpenOptions};
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
use tokio::sync::{mpsc, oneshot};
const SECS_PER_DAY: u64 = 86_400;
use crate::utils::{current_utc_day, now_utc_secs, utc_day_from_secs, SECS_PER_DAY};
const QUEUE_CAPACITY: usize = 4096;
pub(crate) fn now_utc_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
pub(crate) fn utc_day_from_secs(ts_secs: u64) -> String {
chrono::DateTime::from_timestamp_secs(i64::try_from(ts_secs).unwrap_or(0)).map_or_else(|| "1970-01-01".to_string(), |dt| dt.format("%Y-%m-%d").to_string())
}
pub(crate) fn current_utc_day() -> String {
utc_day_from_secs(now_utc_secs())
}
/// Compute Unix seconds until the next UTC midnight after `from_ms`.
pub fn secs_until_next_utc_midnight(from_secs: u64) -> u64 {
let remaining = SECS_PER_DAY - (from_secs % SECS_PER_DAY);
if remaining == 0 { SECS_PER_DAY } else { remaining }
}
fn pending_file_path(directory: &str, day: &str) -> PathBuf {
PathBuf::from(directory).join(format!("stream-history-{day}.pending"))
@@ -50,6 +28,7 @@ fn pending_file_path(directory: &str, day: &str) -> PathBuf {
enum WriterCommand {
Record(Box<StreamHistoryRecord>),
Flush(oneshot::Sender<io::Result<()>>),
GetBatch(oneshot::Sender<Vec<StreamHistoryRecord>>),
Shutdown(oneshot::Sender<()>),
}
@@ -129,6 +108,24 @@ impl StreamHistoryWriter {
pub fn is_enabled(&self) -> bool {
self.tx.is_some()
}
/// Returns a copy of the current in-memory batch records.
/// Async version — sends a command and waits for the response.
pub async fn get_current_batch(&self) -> io::Result<Vec<StreamHistoryRecord>> {
let Some(tx) = &self.tx else {
return Ok(Vec::new());
};
let (resp_tx, resp_rx) = oneshot::channel();
tx.send(WriterCommand::GetBatch(resp_tx))
.await
.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "stream history worker is closed"))?;
resp_rx
.await
.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "stream history worker dropped GetBatch response"))
}
}
struct WriterWorker {
@@ -143,7 +140,7 @@ impl WriterWorker {
}
async fn run(self, mut rx: mpsc::Receiver<WriterCommand>, dropped_events: Arc<AtomicU64>) {
let mut state = match WriterState::open(&self.config, self.writer_instance_id) {
let mut state = match WriterState::open(&self.config, self.writer_instance_id).await {
Ok(s) => s,
Err(e) => {
error!("Stream history writer failed to initialize: {e}");
@@ -160,7 +157,7 @@ impl WriterWorker {
WriterCommand::Record(record) => {
// Day rollover check — use the record's partition day as the target
if record.partition_day_utc != state.current_day {
if let Err(e) = state.flush_and_rollover_to(&record.partition_day_utc, &self.config, self.writer_instance_id) {
if let Err(e) = state.flush_and_rollover_to(&record.partition_day_utc, &self.config, self.writer_instance_id).await {
error!("Stream history day rollover failed: {e}");
let dropped = dropped_events.fetch_add(1, Ordering::Relaxed) + 1;
warn!("Stream history record dropped due to rollover failure (total dropped: {dropped})");
@@ -171,21 +168,24 @@ impl WriterWorker {
state.push(*record);
if state.batch.len() >= self.config.stream_history_batch_size {
if let Err(e) = state.flush_batch() {
if let Err(e) = state.flush_batch().await {
error!("Stream history batch flush failed: {e}. Dropped events: {}",
dropped_events.load(Ordering::Relaxed));
}
}
}
WriterCommand::Flush(resp) => {
let result = state.flush_batch();
let result = state.flush_batch().await;
let _ = resp.send(result);
}
WriterCommand::GetBatch(resp) => {
let _ = resp.send(state.batch.clone());
}
WriterCommand::Shutdown(resp) => {
if let Err(e) = state.flush_batch() {
if let Err(e) = state.flush_batch().await {
error!("Stream history flush on shutdown failed: {e}");
}
if let Err(e) = state.finalize() {
if let Err(e) = state.finalize().await {
error!("Stream history finalize on shutdown failed: {e}");
}
info!("Stream history writer shut down. Total blocks: {}, records: {}",
@@ -201,11 +201,9 @@ impl WriterWorker {
struct WriterState {
directory: String,
current_day: String,
file: Option<BufWriter<File>>,
file: Option<fs::File>,
file_path: PathBuf,
batch: Vec<StreamHistoryRecord>,
/// Reusable scratch buffer for building block payloads — retains capacity across flushes.
payload_buf: Vec<u8>,
total_block_count: u64,
total_record_count: u64,
min_event_ts: Option<u64>,
@@ -213,22 +211,21 @@ struct WriterState {
}
impl WriterState {
fn open(config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result<Self> {
async fn open(config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result<Self> {
let day = current_utc_day();
let dir = &config.stream_history_directory;
fs::create_dir_all(dir)?;
fs::create_dir_all(dir).await?;
let path = pending_file_path(dir, &day);
let file = open_or_create_pending_file(&path, &day, writer_instance_id)?;
let file = open_or_create_pending_file(&path, &day, writer_instance_id).await?;
Ok(Self {
directory: dir.clone(),
current_day: day,
file: Some(BufWriter::new(file)),
file: Some(file),
file_path: path,
batch: Vec::with_capacity(config.stream_history_batch_size),
payload_buf: Vec::new(),
total_block_count: 0,
total_record_count: 0,
min_event_ts: None,
@@ -248,15 +245,13 @@ impl WriterState {
}
/// Flush buffered records to the pending file.
/// NOTE: This performs blocking file I/O on the tokio runtime. Acceptable because
/// the writer runs on its own task and stream history is best-effort — a slow disk
/// only delays history persistence, never the streaming hot path.
fn flush_batch(&mut self) -> io::Result<()> {
/// Serialization (CPU-bound) runs in `spawn_blocking`; file I/O is fully async.
async fn flush_batch(&mut self) -> io::Result<()> {
if self.batch.is_empty() {
return Ok(());
}
let Some(writer) = self.file.as_mut() else {
let Some(file) = self.file.as_mut() else {
return Err(io::Error::new(io::ErrorKind::BrokenPipe, "pending file not open"));
};
@@ -269,40 +264,72 @@ impl WriterState {
let record_count = u32::try_from(self.batch.len())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record count too large"))?;
// Serialize all records directly into the reusable payload buffer.
// Each record is prefixed with its length as a u32 BE.
// The length slot is reserved first and patched after serialization to avoid a
// temporary per-record Vec allocation.
self.payload_buf.clear();
for record in &self.batch {
let len_offset = self.payload_buf.len();
self.payload_buf.extend_from_slice(&[0u8; 4]); // placeholder for length
rmp_serde::encode::write_named(&mut self.payload_buf, record)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let record_len = self.payload_buf.len() - len_offset - 4;
let record_len_u32 = u32::try_from(record_len)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record too large"))?;
self.payload_buf[len_offset..len_offset + 4].copy_from_slice(&record_len_u32.to_be_bytes());
}
// Serialize all records into the reusable payload buffer, then write
// the whole block in one async I/O pass.
// The batch is cloned for serialization; self.batch is only cleared after all I/O succeeds.
let batch_snapshot = self.batch.clone();
let (header_bytes, payload_bytes) = match tokio::task::spawn_blocking(move || {
let mut payload_buf = Vec::new();
for record in &batch_snapshot {
let len_offset = payload_buf.len();
payload_buf.extend_from_slice(&[0u8; 4]); // placeholder for length
if let Err(e) = rmp_serde::encode::write_named(&mut payload_buf, record) {
return Err(io::Error::new(io::ErrorKind::InvalidData, e));
}
let record_len = payload_buf.len() - len_offset - 4;
let Ok(record_len_u32) = u32::try_from(record_len) else {
return Err(io::Error::new(io::ErrorKind::InvalidData, "record too large"));
};
payload_buf[len_offset..len_offset + 4].copy_from_slice(&record_len_u32.to_be_bytes());
}
let payload_len = u32::try_from(self.payload_buf.len())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "block payload too large"))?;
let payload_crc = crc32fast::hash(&self.payload_buf);
let Ok(payload_len) = u32::try_from(payload_buf.len()) else {
return Err(io::Error::new(io::ErrorKind::InvalidData, "block payload too large"));
};
let payload_crc = crc32fast::hash(&payload_buf);
let block_header = BlockHeaderBody {
block_version: 1,
record_count,
payload_len,
first_event_ts_utc: first_ts,
last_event_ts_utc: last_ts,
payload_crc,
flags: 0,
let block_header = BlockHeaderBody {
block_version: 1,
record_count,
payload_len,
first_event_ts_utc: first_ts,
last_event_ts_utc: last_ts,
payload_crc,
flags: 0,
};
// Serialize the framed block header
let mut header_buf = Vec::new();
if let Err(e) = write_framed(&mut header_buf, &block_header) {
return Err(io::Error::new(io::ErrorKind::InvalidData, e));
}
Ok((header_buf, payload_buf))
}).await {
Ok(Ok(v)) => v,
Ok(Err(e)) => return Err(e),
Err(e) => return Err(io::Error::other(e)),
};
write_block_magic(writer)?;
write_framed(writer, &block_header)?;
writer.write_all(&self.payload_buf)?;
writer.flush()?;
// Now write the magic, header, and payload as three separate async write calls.
// This parks the task between calls if I/O would block, without blocking any thread.
// Capture starting position so we can truncate on failure, removing partial corrupt bytes.
let start_pos = file.stream_position().await?;
let block_magic = crate::repository::stream_history::BLOCK_MAGIC;
if let Err(e) = async {
file.write_all(&block_magic).await?;
file.write_all(&header_bytes).await?;
file.write_all(&payload_bytes).await?;
file.flush().await?;
io::Result::Ok(())
}.await {
// Truncate back to the pre-write position to remove partial corrupt bytes,
// then seek so the file handle is ready for the next write attempt.
// self.batch stays intact so the caller can retry.
let _ = file.set_len(start_pos).await;
let _ = file.seek(std::io::SeekFrom::Start(start_pos)).await;
return Err(e);
}
self.total_block_count += 1;
self.total_record_count += u64::from(record_count);
@@ -311,21 +338,21 @@ impl WriterState {
Ok(())
}
fn flush_and_rollover_to(&mut self, target_day: &str, config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result<()> {
self.flush_batch()?;
self.finalize()?; // closes and drops the BufWriter; file_path is still valid
async fn flush_and_rollover_to(&mut self, target_day: &str, config: &StreamHistoryConfig, writer_instance_id: u64) -> io::Result<()> {
self.flush_batch().await?;
self.finalize().await?; // closes and drops the File; file_path is still valid
// Archive the just-closed file and apply retention immediately.
// At this point all pre-rollover records are safely on disk — the first record
// with a new partition_day_utc triggered this rollover, so no pre-midnight record
// can arrive after this point. The previous-day file is now complete and safe to archive.
finalize_and_archive(&self.file_path, &self.directory, config.stream_history_retention_days);
finalize_and_archive(&self.file_path, &self.directory, config.stream_history_retention_days).await;
let new_path = pending_file_path(&self.directory, target_day);
let new_file = open_or_create_pending_file(&new_path, target_day, writer_instance_id)?;
let new_file = open_or_create_pending_file(&new_path, target_day, writer_instance_id).await?;
self.current_day = target_day.to_string();
self.file = Some(BufWriter::new(new_file));
self.file = Some(new_file);
self.file_path = new_path;
self.total_block_count = 0;
self.total_record_count = 0;
@@ -335,54 +362,75 @@ impl WriterState {
Ok(())
}
/// Flush writer buffer to OS. Does not compress or archive.
fn finalize(&mut self) -> io::Result<()> {
if let Some(mut writer) = self.file.take() {
writer.flush()?;
/// Flush file buffer to OS. Does not compress or archive.
async fn finalize(&mut self) -> io::Result<()> {
if let Some(mut file) = self.file.take() {
file.flush().await?;
}
Ok(())
}
}
/// Open an existing `.pending` file for appending, or create a new one with the file header.
fn open_or_create_pending_file(path: &PathBuf, day: &str, writer_instance_id: u64) -> io::Result<File> {
if path.exists() {
// Continue appending to existing file
let file = OpenOptions::new().append(true).open(path)?;
return Ok(file);
}
// Create new file with header
let mut file = File::create(path)?;
let header = FileHeaderBody {
container_format_version: CONTAINER_FORMAT_VERSION,
record_schema_version: RECORD_SCHEMA_VERSION,
source_kind: SOURCE_KIND_STREAM_HISTORY.to_string(),
created_at_ts_utc: now_utc_secs(),
partition_day_ts_utc: day.to_string(),
writer_instance_id,
host_id: std::env::var("HOSTNAME").ok(),
compression_kind: CompressionKind::None,
finalized: false,
record_encoding_kind: RecordEncodingKind::MessagePackNamed,
finalized_at_ts_utc: None,
total_block_count: None,
total_record_count: None,
min_event_ts_utc: None,
max_event_ts_utc: None,
async fn open_or_create_pending_file(path: &PathBuf, day: &str, writer_instance_id: u64) -> io::Result<fs::File> {
// Atomically try to create a new file first. If it already exists,
// open with append mode instead. This avoids TOCTOU races where
// finalize_and_archive might create/finalize the file between our
// metadata check and open call.
let file = match OpenOptions::new().write(true).create_new(true).open(path).await {
Ok(f) => f,
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
// File already exists, open for appending
OpenOptions::new().append(true).open(path).await?
}
Err(e) => return Err(e),
};
write_file_magic(&mut file)?;
write_framed(&mut file, &header)?;
file.flush()?;
// If we created a new file (empty), write the header.
// For existing files the header was already written at creation time.
if file.metadata().await.map_or(0, |m| m.len()) == 0 {
let header = FileHeaderBody {
container_format_version: CONTAINER_FORMAT_VERSION,
record_schema_version: RECORD_SCHEMA_VERSION,
source_kind: SOURCE_KIND_STREAM_HISTORY.to_string(),
created_at_ts_utc: now_utc_secs(),
partition_day_ts_utc: day.to_string(),
writer_instance_id,
host_id: std::env::var("HOSTNAME").ok(),
compression_kind: CompressionKind::None,
finalized: false,
record_encoding_kind: RecordEncodingKind::MessagePackNamed,
finalized_at_ts_utc: None,
total_block_count: None,
total_record_count: None,
min_event_ts_utc: None,
max_event_ts_utc: None,
};
Ok(file)
let mut header_buf = Vec::new();
write_framed(&mut header_buf, &header)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let file_magic = crate::repository::stream_history::FILE_MAGIC;
let mut f = file;
f.write_all(&file_magic).await?;
f.write_all(&header_buf).await?;
f.flush().await?;
Ok(f)
} else {
Ok(file)
}
}
/// Delete daily archive/pending files older than `retention_days`.
pub fn apply_retention(directory: &str, retention_days: u16) -> io::Result<()> {
pub async fn apply_retention(directory: &str, retention_days: u16) -> io::Result<()> {
let dir = PathBuf::from(directory);
if !dir.exists() {
let metadata = match tokio::fs::metadata(&dir).await {
Ok(m) => m,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(e),
};
if !metadata.is_dir() {
return Ok(());
}
@@ -392,24 +440,26 @@ pub fn apply_retention(directory: &str, retention_days: u16) -> io::Result<()> {
utc_day_from_secs(cutoff_secs)
};
for entry in fs::read_dir(&dir)? {
let entry = entry?;
let name = entry.file_name();
let name = name.to_string_lossy();
let mut entries = tokio::fs::read_dir(&dir).await?;
let mut to_delete = Vec::new();
// Match "stream-history-YYYY-MM-DD.pending" or "stream-history-YYYY-MM-DD.archive.lz4"
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name().to_string_lossy().into_owned();
if let Some(day) = extract_day_from_filename(&name) {
if day < cutoff_day.as_str() {
let path = entry.path();
if let Err(e) = fs::remove_file(&path) {
warn!("Failed to delete old history file {}: {e}", path.display());
} else {
info!("Deleted expired history file: {}", path.display());
}
to_delete.push(entry.path());
}
}
}
for path in to_delete {
if let Err(e) = tokio::fs::remove_file(&path).await {
warn!("Failed to delete old history file {}: {e}", path.display());
} else {
info!("Deleted expired history file: {}", path.display());
}
}
Ok(())
}
@@ -426,11 +476,13 @@ pub fn extract_day_from_filename(name: &str) -> Option<&str> {
#[cfg(test)]
mod tests {
use super::*;
use crate::repository::stream_history::{EventType};
use crate::repository::{BlockHeaderBody, FileHeaderBody, read_and_verify_block_magic, read_and_verify_file_magic, read_framed};
use std::io::BufReader;
use std::sync::atomic::Ordering;
use tempfile::TempDir;
use shared::model::{PlaylistItemType, StreamHistoryEventType};
use shared::utils::Internable;
use crate::utils::secs_until_next_utc_midnight;
fn test_config(dir: &str, batch_size: usize) -> StreamHistoryConfig {
StreamHistoryConfig {
@@ -441,7 +493,7 @@ mod tests {
}
}
fn make_record(session_id: u64, event_type: EventType) -> StreamHistoryRecord {
fn make_record(session_id: u64, event_type: StreamHistoryEventType) -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: RECORD_SCHEMA_VERSION,
event_type,
@@ -450,11 +502,11 @@ mod tests {
session_id,
source_addr: None,
api_username: Some("user1".to_string()),
provider_name: Some("acme".to_string()),
provider_name: Some("acme".intern()),
provider_username: None,
input_name: Some("input".to_string()),
input_name: Some("input".intern()),
virtual_id: Some(1),
item_type: Some("live".to_string()),
item_type: Some(PlaylistItemType::Live),
title: Some("Test Channel".to_string()),
group: None,
country: None,
@@ -484,7 +536,7 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: None,
target_name: None,
}
}
@@ -492,7 +544,7 @@ mod tests {
async fn stream_history_writer_disabled_accepts_records_silently() {
let writer = StreamHistoryWriter::new_disabled();
// Must not panic
writer.send_record(make_record(1, EventType::Connect));
writer.send_record(make_record(1, StreamHistoryEventType::Connect));
writer.flush().await.expect("flush on disabled writer");
writer.shutdown().await;
assert_eq!(writer.dropped_events.load(Ordering::Relaxed), 0);
@@ -515,7 +567,7 @@ mod tests {
// Send 3 records (< batch_size=4), no file blocks should be written yet
for i in 0..3_u64 {
writer.send_record(make_record(i, EventType::Connect));
writer.send_record(make_record(i, StreamHistoryEventType::Connect));
}
// Explicit flush — must not error even if no block written
@@ -537,20 +589,20 @@ mod tests {
// Send exactly batch_size records — triggers one block write
for i in 0..batch_size as u64 {
writer.send_record(make_record(i, EventType::Connect));
writer.send_record(make_record(i, StreamHistoryEventType::Connect));
}
// Give the worker time to process
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
// Send one more and flush to ensure previous batch was processed
writer.send_record(make_record(99, EventType::Disconnect));
writer.send_record(make_record(99, StreamHistoryEventType::Disconnect));
writer.flush().await.expect("flush ok");
writer.shutdown().await;
let day = current_utc_day();
let pending = tmp.path().join(format!("stream-history-{day}.pending"));
let metadata = std::fs::metadata(&pending).unwrap();
let metadata = tokio::fs::metadata(&pending).await.unwrap();
// File should be larger than just the header (a block was written)
assert!(metadata.len() > 100, "file must contain at least one block (size={})", metadata.len());
}
@@ -579,7 +631,7 @@ mod tests {
// Send 3 records (well below batch_size=100)
for i in 0..3_u64 {
writer.send_record(make_record(i, EventType::Connect));
writer.send_record(make_record(i, StreamHistoryEventType::Connect));
}
// Explicit flush must write the partial batch to disk
@@ -588,7 +640,7 @@ mod tests {
let day = current_utc_day();
let pending = tmp.path().join(format!("stream-history-{day}.pending"));
let metadata = std::fs::metadata(&pending).unwrap();
let metadata = tokio::fs::metadata(&pending).await.unwrap();
assert!(metadata.len() > 100, "partial batch must be flushed to file (size={})", metadata.len());
}
@@ -598,15 +650,15 @@ mod tests {
let config = test_config(tmp.path().to_str().unwrap(), 100);
let writer = StreamHistoryWriter::new(&config);
let mut late = make_record(1, EventType::Connect);
let mut late = make_record(1, StreamHistoryEventType::Connect);
late.event_ts_utc = 3_000;
late.partition_day_utc = "1970-01-01".to_string();
let mut early = make_record(2, EventType::Connect);
let mut early = make_record(2, StreamHistoryEventType::Connect);
early.event_ts_utc = 1_000;
early.partition_day_utc = "1970-01-01".to_string();
let mut middle = make_record(3, EventType::Connect);
let mut middle = make_record(3, StreamHistoryEventType::Connect);
middle.event_ts_utc = 2_000;
middle.partition_day_utc = "1970-01-01".to_string();
@@ -635,13 +687,6 @@ mod tests {
assert_eq!(offset, 1);
}
#[test]
fn stream_history_retention_extract_day_from_filename() {
assert_eq!(extract_day_from_filename("stream-history-2026-03-22.pending"), Some("2026-03-22"));
assert_eq!(extract_day_from_filename("stream-history-2026-03-22.archive.lz4"), Some("2026-03-22"));
assert_eq!(extract_day_from_filename("unrelated.txt"), None);
}
#[tokio::test]
async fn stream_history_writer_creates_pending_file_on_init() {
let tmp = TempDir::new().unwrap();
backend/src/repository/xtream_repository.rs
+1 -1
View File
@@ -1664,7 +1664,7 @@ mod tests {
XtreamPlaylistItem {
virtual_id: provider_id,
provider_id,
name: "live".intern(),
name: "Live".intern(),
logo: "".intern(),
logo_small: "".intern(),
group: "group".intern(),
backend/src/utils/db_viewer.rs
+6 -5
View File
@@ -215,7 +215,8 @@ fn exit_app(code: i32) {
mod tests {
use super::dump_qos_snapshot_db;
use tempfile::tempdir;
use shared::model::PlaylistItemType;
use shared::utils::Internable;
use crate::repository::{BPlusTree, QosSnapshotDailyBucket, QosSnapshotRecord, QosSnapshotWindow};
#[test]
@@ -226,12 +227,12 @@ mod tests {
let mut tree = BPlusTree::<String, QosSnapshotRecord>::new();
let record = QosSnapshotRecord {
stream_identity_key: "stream-a".to_string(),
input_name: "input-a".to_string(),
target_id: 11,
provider_name: "provider-a".to_string(),
input_name: "input-a".intern(),
target_name: "target-a".intern(),
provider_name: "provider-a".intern(),
provider_id: 22,
virtual_id: 33,
item_type: "live".to_string(),
item_type: PlaylistItemType::Live,
updated_at: 1_700_000_000,
last_event_at: 1_700_000_001,
window_24h: QosSnapshotWindow {
backend/src/utils/mod.rs
+2
View File
@@ -16,6 +16,7 @@ mod epg_parser;
mod ordinal;
pub mod ffmpeg;
mod lru_cache;
mod time_utils;
#[macro_export]
macro_rules! debug_if_enabled {
@@ -82,5 +83,6 @@ pub use self::json_utils::*;
pub use self::network::*;
pub use self::step_measure::*;
pub use self::sys_utils::*;
pub use self::time_utils::*;
pub use self::stream_history_viewer::stream_history_viewer;
backend/src/utils/stream_history_viewer.rs
+77 -86
View File
@@ -3,14 +3,15 @@ use std::collections::HashMap;
use std::fs;
use std::io::{self, Write};
use std::path::Path;
use std::sync::Arc;
use chrono;
use regex::Regex;
use serde::Deserialize;
use shared::utils::Internable;
use crate::model::{StreamHistoryRecord};
use crate::repository::{
ConnectFailureReason, DisconnectReason, EventType, FailureStage, FileHeaderBody, StreamHistoryFileReader,
StreamHistoryRecord, extract_day_from_filename, read_and_verify_file_magic, read_framed,
FileHeaderBody, StreamHistoryFileReader,
extract_day_from_filename, read_and_verify_file_magic, read_framed,
};
#[derive(Deserialize)]
@@ -98,7 +99,6 @@ const NUMERIC_FIELDS: &[&str] = &[
"session_id",
"virtual_id",
"provider_id",
"target_id",
"provider_http_status",
"shared_stream_id",
];
@@ -107,6 +107,7 @@ const STRING_FIELDS: &[&str] = &[
"api_username",
"provider_name",
"provider_username",
"target_name",
"input_name",
"item_type",
"title",
@@ -174,7 +175,12 @@ impl CompiledFilter {
FilterValue::Regex(re) => re.is_match(s),
FilterValue::NumericExact(_) => false,
},
RecordFieldValue::String(None) => false,
RecordFieldValue::ArcStr(Some(s)) => match value {
FilterValue::Exact(v) => s.as_ref().eq_ignore_ascii_case(v),
FilterValue::Regex(re) => re.is_match(s.as_ref()),
FilterValue::NumericExact(_) => false,
},
RecordFieldValue::String(None) | RecordFieldValue::ArcStr(None) => false,
RecordFieldValue::U64(n) => match value {
FilterValue::NumericExact(v) => n == *v,
_ => false,
@@ -186,24 +192,18 @@ impl CompiledFilter {
enum RecordFieldValue<'a> {
String(Option<&'a str>),
ArcStr(Option<Arc<str>>),
U64(u64),
}
fn get_record_field<'a>(record: &'a StreamHistoryRecord, field: &str) -> RecordFieldValue<'a> {
match field {
"event_type" => {
let s = match record.event_type {
EventType::Connect => "connect",
EventType::ConnectFailed => "connect_failed",
EventType::Disconnect => "disconnect",
};
RecordFieldValue::String(Some(s))
}
"event_type" => RecordFieldValue::ArcStr(Some(record.event_type.to_string().intern())),
"api_username" => RecordFieldValue::String(record.api_username.as_deref()),
"provider_name" => RecordFieldValue::String(record.provider_name.as_deref()),
"provider_username" => RecordFieldValue::String(record.provider_username.as_deref()),
"input_name" => RecordFieldValue::String(record.input_name.as_deref()),
"item_type" => RecordFieldValue::String(record.item_type.as_deref()),
"item_type" => RecordFieldValue::ArcStr(record.item_type.as_ref().map(ToString::to_string).map(Internable::intern)),
"title" => RecordFieldValue::String(record.title.as_deref()),
"group" => RecordFieldValue::String(record.group.as_deref()),
"country" => RecordFieldValue::String(record.country.as_deref()),
@@ -214,48 +214,9 @@ fn get_record_field<'a>(record: &'a StreamHistoryRecord, field: &str) -> RecordF
.shared_joined_existing
.map(|shared| if shared { "true" } else { "false" }),
),
"disconnect_reason" => {
// Match against serde rename_all = "snake_case" names
RecordFieldValue::String(record.disconnect_reason.as_ref().map(|r| match r {
DisconnectReason::Cleanup => "cleanup",
DisconnectReason::ClientClosed => "client_closed",
DisconnectReason::ClientKicked => "client_kicked",
DisconnectReason::Provisioning => "provisioning",
DisconnectReason::ServerError => "server_error",
DisconnectReason::Timeout => "timeout",
DisconnectReason::DayRollover => "day_rollover",
DisconnectReason::Shutdown => "shutdown",
DisconnectReason::Unknown => "unknown",
DisconnectReason::ProviderError => "provider_error",
DisconnectReason::ProviderClosed => "provider_closed",
DisconnectReason::Preempted => "preempted",
DisconnectReason::SessionExpired => "session_expired",
DisconnectReason::UserConnectionsExhausted => "user_connections_exhausted",
DisconnectReason::ProviderConnectionsExhausted => "provider_connections_exhausted",
}))
}
"connect_failure_reason" => {
RecordFieldValue::String(record.connect_failure_reason.as_ref().map(|r| match r {
ConnectFailureReason::UserAccountExpired => "user_account_expired",
ConnectFailureReason::UserConnectionsExhausted => "user_connections_exhausted",
ConnectFailureReason::ProviderConnectionsExhausted => "provider_connections_exhausted",
ConnectFailureReason::ProviderError => "provider_error",
ConnectFailureReason::ProviderClosed => "provider_closed",
ConnectFailureReason::ChannelUnavailable => "channel_unavailable",
ConnectFailureReason::Preempted => "preempted",
ConnectFailureReason::SessionExpired => "session_expired",
ConnectFailureReason::Provisioning => "provisioning",
}))
}
"failure_stage" => {
RecordFieldValue::String(record.failure_stage.as_ref().map(|stage| match stage {
FailureStage::Admission => "admission",
FailureStage::ProviderOpen => "provider_open",
FailureStage::FirstByte => "first_byte",
FailureStage::Streaming => "streaming",
FailureStage::SessionReconnect => "session_reconnect",
}))
}
"disconnect_reason" => RecordFieldValue::ArcStr(record.disconnect_reason.as_ref().map(|r| r.to_string().intern())),
"connect_failure_reason" => RecordFieldValue::ArcStr(record.connect_failure_reason.as_ref().map(|r| r.to_string().intern())),
"failure_stage" => RecordFieldValue::ArcStr(record.failure_stage.as_ref().map(|r| r.to_string().intern())),
"provider_error_class" => RecordFieldValue::String(record.provider_error_class.as_deref()),
"user_agent" => RecordFieldValue::String(record.user_agent.as_deref()),
"cluster" => RecordFieldValue::String(record.cluster.as_deref()),
@@ -270,7 +231,7 @@ fn get_record_field<'a>(record: &'a StreamHistoryRecord, field: &str) -> RecordF
"session_id" => RecordFieldValue::U64(record.session_id),
"virtual_id" => RecordFieldValue::U64(u64::from(record.virtual_id.unwrap_or(0))),
"provider_id" => RecordFieldValue::U64(u64::from(record.provider_id.unwrap_or(0))),
"target_id" => RecordFieldValue::U64(u64::from(record.target_id.unwrap_or(0))),
"target_name" => RecordFieldValue::String(record.target_name.as_deref()),
"provider_http_status" => RecordFieldValue::U64(u64::from(record.provider_http_status.unwrap_or(0))),
"shared_stream_id" => RecordFieldValue::U64(record.shared_stream_id.unwrap_or(0)),
_ => RecordFieldValue::String(None), // Unknown field: no match
@@ -297,21 +258,39 @@ pub(crate) struct HistoryFile {
pub is_archive: bool,
}
pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result<Vec<HistoryFile>> {
if !dir.exists() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("Stream history directory not found: {}", dir.display()),
));
pub(crate) async fn discover_files(
dir: &Path,
time_range: &TimeRange,
) -> io::Result<Vec<HistoryFile>> {
match tokio::fs::metadata(dir).await {
Ok(meta) => {
if !meta.is_dir() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("Stream history directory not found: {}", dir.display()),
));
}
}
Err(e) if e.kind() == io::ErrorKind::NotFound => {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("Stream history directory not found: {}", dir.display()),
));
}
Err(e) => return Err(e),
}
let (range_start, range_end) = *time_range;
let mut files = Vec::new();
for entry in fs::read_dir(dir)? {
let entry = entry?;
let mut entries = tokio::fs::read_dir(dir).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let name = path.file_name().unwrap_or_default().to_string_lossy().to_string();
let name = path
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_string();
let is_archive = name.ends_with(".archive.lz4");
let is_pending = name.ends_with(".pending");
@@ -332,17 +311,23 @@ pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result<V
}
// Read file header for archive-level timestamp bounds
let header = match read_file_header(&path, is_archive) {
Ok(h) => h,
Err(e) => {
let path_clone = path.clone();
let header = match tokio::task::spawn_blocking(move || read_file_header(&path_clone, is_archive)).await {
Ok(Ok(h)) => h,
Ok(Err(e)) => {
eprintln!("Warning: skipping {}: {e}", path.display());
continue;
}
Err(e) => {
eprintln!("Warning: skipping {} (task failed): {e}", path.display());
continue;
}
};
// File-level skip for archives with known timestamp bounds
if is_archive {
if let (Some(min_ts), Some(max_ts)) = (header.min_event_ts_utc, header.max_event_ts_utc) {
if let (Some(min_ts), Some(max_ts)) = (header.min_event_ts_utc, header.max_event_ts_utc)
{
if max_ts < range_start || min_ts > range_end {
continue;
}
@@ -358,14 +343,18 @@ pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result<V
// Sort: primary by partition_day, secondary archive before pending
files.sort_by(|a, b| {
a.partition_day.cmp(&b.partition_day)
.then(b.is_archive.cmp(&a.is_archive)) // true (archive) before false (pending)
a.partition_day
.cmp(&b.partition_day)
.then(b.is_archive.cmp(&a.is_archive))
});
if files.is_empty() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("No stream history files found in range. Check directory: {}", dir.display()),
format!(
"No stream history files found in range. Check directory: {}",
dir.display()
),
));
}
@@ -374,7 +363,7 @@ pub(crate) fn discover_files(dir: &Path, time_range: &TimeRange) -> io::Result<V
fn read_file_header(path: &Path, is_archive: bool) -> io::Result<FileHeaderBody> {
if is_archive {
let file = std::fs::File::open(path)?;
let file = fs::File::open(path)?;
let decoder = lz4_flex::frame::FrameDecoder::new(file);
let mut reader = std::io::BufReader::new(decoder);
read_and_verify_file_magic(&mut reader)?;
@@ -452,7 +441,7 @@ fn stream_output(
let _ = writeln!(out, "]");
}
fn run_stream_history_viewer(input: &str) -> Result<(), String> {
async fn run_stream_history_viewer(input: &str) -> Result<(), String> {
eprintln!("[INFO] All timestamps interpreted as UTC");
let query = load_query(input)?;
@@ -465,14 +454,14 @@ fn run_stream_history_viewer(input: &str) -> Result<(), String> {
let dir = query.path.as_deref().unwrap_or("data/stream_history");
let dir_path = Path::new(dir);
let files = discover_files(dir_path, &time_range).map_err(|e| e.to_string())?;
let files = discover_files(dir_path, &time_range).await.map_err(|e| e.to_string())?;
stream_output(files.as_slice(), &time_range, &filters);
Ok(())
}
pub fn stream_history_viewer(input: &str) -> i32 {
match run_stream_history_viewer(input) {
pub async fn stream_history_viewer(input: &str) -> i32 {
match run_stream_history_viewer(input).await {
Ok(()) => 0,
Err(err) => {
eprintln!("Error: {err}");
@@ -485,6 +474,8 @@ pub fn stream_history_viewer(input: &str) -> i32 {
#[cfg(test)]
mod tests {
use shared::model::{DisconnectReason, StreamHistoryEventType, FailureStage};
use shared::utils::Internable;
use super::*;
#[test]
@@ -588,10 +579,10 @@ mod tests {
let filter = CompiledFilter::compile(&raw).unwrap();
let mut record = make_empty_record();
record.provider_name = Some("acme-tv".to_string());
record.provider_name = Some("acme-tv".intern());
assert!(filter.matches(&record));
record.provider_name = Some("other-provider".to_string());
record.provider_name = Some("other-provider".intern());
assert!(!filter.matches(&record));
}
@@ -706,9 +697,9 @@ mod tests {
assert_eq!(query.to.as_deref(), Some("2026-03-24"));
}
#[test]
fn test_run_stream_history_viewer_returns_error_instead_of_exiting_on_invalid_query() {
let result = run_stream_history_viewer("{not-json");
#[tokio::test]
async fn test_run_stream_history_viewer_returns_error_instead_of_exiting_on_invalid_query() {
let result = run_stream_history_viewer("{not-json").await;
assert!(result.is_err());
}
@@ -760,7 +751,7 @@ mod tests {
fn make_empty_record() -> StreamHistoryRecord {
StreamHistoryRecord {
schema_version: 1,
event_type: EventType::Connect,
event_type: StreamHistoryEventType::Connect,
event_ts_utc: 0,
partition_day_utc: String::new(),
session_id: 0,
@@ -800,7 +791,7 @@ mod tests {
connect_failure_reason: None,
disconnect_reason: None,
previous_session_id: None,
target_id: None,
target_name: None,
}
}
}
backend/src/utils/time_utils.rs
+23
View File
@@ -0,0 +1,23 @@
use std::time::{SystemTime, UNIX_EPOCH};
pub const SECS_PER_DAY: u64 = 86_400;
pub(crate) fn now_utc_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
pub(crate) fn utc_day_from_secs(ts_secs: u64) -> String {
chrono::DateTime::from_timestamp_secs(i64::try_from(ts_secs).unwrap_or(0)).map_or_else(|| "1970-01-01".to_string(), |dt| dt.format("%Y-%m-%d").to_string())
}
pub(crate) fn current_utc_day() -> String {
utc_day_from_secs(now_utc_secs())
}
/// Compute Unix seconds until the next UTC midnight after `from_secs`.
pub fn secs_until_next_utc_midnight(from_secs: u64) -> u64 {
SECS_PER_DAY - (from_secs % SECS_PER_DAY)
}
frontend/scss/app/_component.scss
+1
View File
@@ -94,3 +94,4 @@
@forward "components/filter_editor";
@forward "components/field_explanation";
@forward "components/rbac/user_management";
@forward "components/country";
frontend/scss/app/components/_country.scss
+27
View File
@@ -0,0 +1,27 @@
.tp__country {
display: inline-flex;
align-items: center;
gap: var(--gap-mini);
min-width: 0;
flex: 0 1 auto;
&__name {
min-width: 0;
overflow-wrap: anywhere;
font-weight: 700;
}
&__flag {
display: inline-flex;
width: 1rem;
height: 0.75rem;
overflow: hidden;
border-radius: 2px;
svg {
display: block;
width: 100%;
height: 100%;
}
}
}
frontend/src/app/components/country.rs
+57
View File
@@ -0,0 +1,57 @@
use crate::{hooks::use_service_context, i18n::use_translation, utils::t_safe};
use yew::{classes, component, html, AttrValue, Html, Properties};
pub fn display_country_code(code: Option<&str>) -> Option<String> {
let normalized = code?.trim().to_ascii_uppercase();
let is_iso_country = normalized.len() == 2 && normalized.as_bytes().iter().all(|byte| byte.is_ascii_alphabetic());
if is_iso_country {
Some(normalized)
} else {
None
}
}
#[derive(Properties, Clone, PartialEq)]
pub struct CountryProps {
pub country_code: Option<String>,
#[prop_or_default]
pub classes: Option<String>,
}
#[component]
pub fn Country(props: &CountryProps) -> Html {
let translate = use_translation();
let services = use_service_context();
if let Some(code) = display_country_code(props.country_code.as_deref()).as_ref() {
let country = t_safe(&translate, &format!("COUNTRY.{code}")).unwrap_or_else(|| code.clone());
let flag_svg = services.flags.get_flag(code);
return html! {
<span class={classes!("tp__country", props.classes.as_ref())}>
if let Some(svg) = flag_svg.as_ref() {
<span class="tp__country__flag" aria-hidden="true">
// SAFETY: flags.dat is built offline by flags_builder from a trusted flag directory.
// If this source ever becomes user-controlled, replace this with sanitized SVG rendering.
{Html::from_html_unchecked(AttrValue::from(svg.clone()))}
</span>
}
<span class="tp__country__name">{&country}</span>
</span>
};
}
html! {}
}
#[cfg(test)]
mod tests {
use super::display_country_code;
#[test]
fn display_country_code_filters_special_network_labels() {
assert_eq!(display_country_code(Some("de")), Some("DE".to_string()));
assert_eq!(display_country_code(Some("LOOPBACK")), None);
assert_eq!(display_country_code(Some("LAN")), None);
assert_eq!(display_country_code(Some(" ")), None);
}
}
frontend/src/app/components/dashboard/stream_display/helpers.rs
+2 -2
View File
@@ -46,7 +46,7 @@ pub fn is_adaptive_session_stream(stream: &StreamInfo) -> bool {
}
pub fn is_background_transfer_stream(stream: &StreamInfo) -> bool {
stream.client_ip == BACKGROUND_TRANSFER_CLIENT_IP && stream.provider == BACKGROUND_TRANSFER_PROVIDER
stream.client_ip == BACKGROUND_TRANSFER_CLIENT_IP && stream.provider.as_ref() == BACKGROUND_TRANSFER_PROVIDER
}
pub fn filter_visible_streams(
@@ -226,7 +226,7 @@ mod tests {
shared_stream_id: None,
technical: None,
},
provider: "provider".to_string(),
provider: "provider".intern(),
addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080),
client_ip: "127.0.0.1:1234".to_string(),
user_agent: String::new(),
frontend/src/app/components/dashboard/stream_display/item.rs
+5 -48
View File
@@ -3,10 +3,9 @@ use super::{
meter::{MeterDisplayKind, StreamMeterBadge},
};
use crate::{
app::components::{AppIcon, Chip, RevealContent, ToggleSwitch},
hooks::use_service_context,
app::components::{country::display_country_code, AppIcon, Chip, Country, RevealContent, ToggleSwitch},
i18n::use_translation,
utils::{format_duration, t_safe},
utils::format_duration,
};
use shared::{
model::StreamInfo,
@@ -14,17 +13,7 @@ use shared::{
};
use std::rc::Rc;
use web_sys::MouseEvent;
use yew::{prelude::*, virtual_dom::AttrValue};
fn display_country_code(country_code: Option<&str>) -> Option<String> {
let normalized = country_code?.trim().to_ascii_uppercase();
let is_iso_country = normalized.len() == 2 && normalized.as_bytes().iter().all(|byte| byte.is_ascii_alphabetic());
if is_iso_country {
Some(normalized)
} else {
None
}
}
use yew::prelude::*;
#[derive(Properties, PartialEq, Clone)]
pub struct StreamDisplayItemProps {
@@ -36,15 +25,9 @@ pub struct StreamDisplayItemProps {
#[component]
pub fn StreamDisplayItem(props: &StreamDisplayItemProps) -> Html {
let translate = use_translation();
let services = use_service_context();
let stream = props.stream.clone();
let is_background_transfer = is_background_transfer_stream(&stream);
let chips = build_technical_chips(stream.channel.item_type, stream.channel.technical.as_ref());
let country_code = display_country_code(stream.country_code.as_deref());
let country = country_code.as_ref().map_or_else(String::new, |code| {
t_safe(&translate, &format!("COUNTRY.{code}")).unwrap_or_else(|| code.clone())
});
let flag_svg = country_code.as_ref().and_then(|code| services.flags.get_flag(code));
let client_ip = strip_port(&stream.client_ip).to_string();
let handle_popup_click = {
@@ -87,24 +70,11 @@ pub fn StreamDisplayItem(props: &StreamDisplayItemProps) -> Html {
<span class="tp__stream-display__stat-label">{translate.t("LABEL.CLIENT_IP")}</span>
<span class="tp__stream-display__stat-value tp__stream-display__stat-value--ip">{client_ip.clone()}</span>
</div>
if !country.is_empty() || country_code.is_some() {
if display_country_code(stream.country_code.as_deref()).is_some() {
<div class="tp__stream-display__stat tp__stream-display__stat--country">
<span class="tp__stream-display__stat-label">{translate.t("LABEL.COUNTRY")}</span>
<span class="tp__stream-display__stat-value">
<span class="tp__stream-display__country tp__stream-display__country--stat">
if let Some(svg) = flag_svg.as_ref() {
<span class="tp__stream-display__flag" aria-hidden="true">
// SAFETY: flags.dat is built offline by flags_builder from a trusted flag directory.
// If this source ever becomes user-controlled, replace this with sanitized SVG rendering.
{Html::from_html_unchecked(AttrValue::from(svg.clone()))}
</span>
}
if !country.is_empty() {
<span class="tp__stream-display__country-name">{country.clone()}</span>
} else if let Some(code) = country_code.as_ref() {
<span class="tp__stream-display__country-code">{code.clone()}</span>
}
</span>
<Country country_code={stream.country_code.clone()}/>
</span>
</div>
}
@@ -153,16 +123,3 @@ pub fn StreamDisplayItem(props: &StreamDisplayItemProps) -> Html {
</article>
}
}
#[cfg(test)]
mod tests {
use super::display_country_code;
#[test]
fn display_country_code_filters_special_network_labels() {
assert_eq!(display_country_code(Some("de")), Some("DE".to_string()));
assert_eq!(display_country_code(Some("LOOPBACK")), None);
assert_eq!(display_country_code(Some("LAN")), None);
assert_eq!(display_country_code(Some(" ")), None);
}
}
frontend/src/app/components/dashboard/stream_display/mod.rs
+4 -49
View File
@@ -18,17 +18,16 @@ use crate::{
hooks::use_service_context,
i18n::use_translation,
model::EventMessage,
services::{DialogService, FlagsLoadState},
services::DialogService,
};
use gloo_timers::{callback::Interval, future::TimeoutFuture};
use log::error;
use gloo_timers::callback::Interval;
use shared::{
error::TuliproxError,
model::{PlaylistRequest, PlaylistUrlResolveRequest, ProtocolMessage, StreamInfo, UserCommand},
utils::default_kick_secs,
};
use std::{cell::Cell, collections::HashMap, fmt::Display, future, rc::Rc, str::FromStr};
use yew::{platform::spawn_local, prelude::*, suspense::use_future};
use std::{collections::HashMap, fmt::Display, rc::Rc, str::FromStr};
use yew::{platform::spawn_local, prelude::*};
use yew_hooks::use_clipboard;
const KICK: &str = "kick";
@@ -55,8 +54,6 @@ pub fn StreamDisplay(props: &StreamDisplayProps) -> Html {
let cleanup_now_secs = use_state(shared::utils::current_time_secs);
let adaptive_session_ttl_secs = get_adaptive_session_ttl_secs(&config_ctx);
let metrics_enabled = is_stream_metrics_enabled(&config_ctx);
let geoip_enabled = config_ctx.config.as_ref().is_some_and(|cfg| cfg.config.is_geoip_enabled());
let flags_ready = use_state(|| service_ctx.flags.is_loaded());
use_effect_with((), move |_| {
let interval = Interval::new(1000, update_timestamps);
@@ -107,45 +104,6 @@ pub fn StreamDisplay(props: &StreamDisplayProps) -> Html {
});
}
{
let flags_service = service_ctx.flags.clone();
let flags_ready = flags_ready.clone();
let _ = use_future(|| async move {
flags_service
.loaded_subscribe(&mut |loaded| {
flags_ready.set(loaded);
future::ready(())
})
.await
});
}
{
let flags_service = service_ctx.flags.clone();
use_effect_with(geoip_enabled, move |geoip_enabled| {
let cancelled = Rc::new(Cell::new(false));
if *geoip_enabled {
let flags_service = flags_service.clone();
let cancelled = cancelled.clone();
spawn_local(async move {
while !cancelled.get() && !flags_service.is_loaded() {
match flags_service.ensure_loaded_from_assets().await {
Ok(FlagsLoadState::Loaded) => break,
Ok(FlagsLoadState::InProgress) => {
TimeoutFuture::new(250).await;
}
Err(err) => {
error!("Failed to load flags {err}");
TimeoutFuture::new(5000).await;
}
}
}
});
}
move || cancelled.set(true)
});
}
let visible_streams = use_memo(
(props.streams.clone(), (*adaptive_last_seen).clone(), *cleanup_now_secs, adaptive_session_ttl_secs),
|(streams, adaptive_last_seen, cleanup_now_secs, adaptive_session_ttl_secs)| {
@@ -298,9 +256,6 @@ pub fn StreamDisplay(props: &StreamDisplayProps) -> Html {
})
};
// Force a re-render after the flags service finishes loading so StreamDisplayItem can read flag SVGs.
let _flags_ready_marker = *flags_ready;
html! {
<div class="tp__stream-display">
<div class="tp__stream-display__header">
frontend/src/app/components/dashboard/stream_history_view.rs
+276 -173
View File
@@ -1,21 +1,23 @@
use crate::{
app::components::{DateInput, NoContent, Search, Table, TableDefinition, TextButton},
app::components::{Country, DateInput, NoContent, RevealContent, Search, Table, TableDefinition, TextButton},
hooks::use_service_context,
i18n::use_translation,
services::{StreamHistoryProviderSummary, StreamHistoryQosSnapshot, StreamHistoryRecord},
services::{StreamHistoryProviderSummary, StreamHistoryQosSnapshot},
utils::{format_bytes, format_duration, format_ts},
};
use futures::join;
use shared::model::SearchRequest;
use std::rc::Rc;
use shared::model::{DisconnectReason, SearchRequest, StreamHistoryEventType, StreamHistoryRecordDto};
use std::{borrow::Cow, rc::Rc};
use wasm_bindgen_futures::spawn_local;
use yew::prelude::*;
const NUM_COLS: usize = 13;
const NUM_COLS: usize = 15;
const SUMMARY_NUM_COLS: usize = 6;
const QOS_SUMMARY_NUM_COLS: usize = 8;
const QOS_DETAIL_NUM_COLS: usize = 8;
type HlsSessionKey<'a> = (Option<&'a str>, u64, Option<&'a str>, Option<u32>, Option<&'a str>);
#[derive(Clone, PartialEq)]
struct ProviderSummaryRow {
provider_name: String,
@@ -32,7 +34,7 @@ struct QosSummaryRow {
input_name: String,
provider_name: String,
provider_id: u32,
target_id: u16,
target_name: String,
item_type: String,
window_24h: crate::services::StreamHistoryQosSnapshotWindow,
}
@@ -53,45 +55,165 @@ fn ts_to_date_str(ts: i64) -> String {
chrono::DateTime::from_timestamp(ts, 0).map_or_else(String::new, |dt| dt.format("%Y-%m-%d").to_string())
}
fn optional_record_text(value: Option<&str>) -> &str { value.unwrap_or("-") }
fn optional_record_text_str(value: Option<&str>) -> &str { value.unwrap_or("-") }
fn optional_record_text(value: Option<String>) -> String { value.unwrap_or_else(|| String::from("-")) }
fn record_matches(record: &StreamHistoryRecord, filter: &SearchRequest) -> bool {
/// Returns true if the container indicates an HLS stream (HTTP-based segmented streaming).
fn is_hls_container(container: Option<&str>) -> bool {
matches!(container, Some("mpegts") | Some("fmp4") | Some("hls"))
}
/// Aggregate HLS records by session key (source_addr + session_id + provider_name + virtual_id + input_name).
/// HLS streams create many rapid connect/disconnect events for each segment. This function groups them
/// into logical sessions, showing first connect, last disconnect, and summed bytes.
///
/// For non-HLS records, they are passed through unchanged.
fn aggregate_hls_records(records: &[Rc<StreamHistoryRecordDto>]) -> Vec<Rc<StreamHistoryRecordDto>> {
use std::collections::HashMap;
// Separate HLS and non-HLS records
let (hls_records, non_hls): (Vec<_>, Vec<_>) =
records.iter().partition(|r| is_hls_container(r.container.as_deref()));
// Group HLS records by session key
let mut sessions: HashMap<HlsSessionKey, Vec<&StreamHistoryRecordDto>> = HashMap::new();
for record in hls_records {
let key = (
record.source_addr.as_deref(),
record.session_id,
record.provider_name.as_deref(),
record.virtual_id,
record.input_name.as_deref(),
);
sessions.entry(key).or_default().push(record.as_ref());
}
// For each session, create aggregated records
let mut aggregated = Vec::new();
// First, add all non-HLS records unchanged
aggregated.extend(non_hls.into_iter().cloned());
// Then, process each HLS session
for (_, session_records) in sessions {
if session_records.is_empty() {
continue;
}
// Sort by timestamp
let mut sorted = session_records.clone();
sorted.sort_by_key(|r| r.event_ts_utc);
// Find first connect and last disconnect
let first_connect = sorted.iter().find(|r| r.event_type == StreamHistoryEventType::Connect);
let last_disconnect = sorted.iter().rfind(|r| r.event_type == StreamHistoryEventType::Disconnect);
// Calculate session duration from first connect to last disconnect
let session_duration = if let (Some(fc), Some(ld)) = (first_connect, last_disconnect) {
let fc_ts = fc.event_ts_utc;
let ld_ts = ld.event_ts_utc;
if ld_ts > fc_ts {
Some(ld_ts - fc_ts)
} else {
None
}
} else {
None
};
// Sum bytes from all records in session
let total_bytes: u64 = sorted.iter().filter_map(|r| r.bytes_sent).sum();
// Count intermediate failures (disconnects that are not the last one)
let intermediate_failure_count = sorted
.iter()
.filter(|r| {
r.event_type == StreamHistoryEventType::Disconnect
&& !std::ptr::eq(*r, last_disconnect.unwrap_or(&sorted[0]))
})
.count();
// Create aggregated connect record
if let Some(first) = first_connect {
let mut agg = (*first).clone();
agg.event_ts_utc = first.event_ts_utc;
agg.session_duration = session_duration;
agg.bytes_sent = Some(total_bytes);
// Mark the disconnect reason from last disconnect if available
if let Some(last) = last_disconnect {
agg.disconnect_reason = last.disconnect_reason;
}
aggregated.push(Rc::new(agg));
}
// If there were intermediate failures, add a failure summary row
if intermediate_failure_count > 0 {
if let Some(first) = first_connect {
let mut failure_row = (*first).clone();
failure_row.event_type = StreamHistoryEventType::Failure;
failure_row.event_ts_utc = first.event_ts_utc + 1; // Just after first connect
failure_row.session_duration = None;
failure_row.bytes_sent = Some(total_bytes);
failure_row.disconnect_reason =
Some(DisconnectReason::IntermediateFailures(intermediate_failure_count));
aggregated.push(Rc::new(failure_row));
}
}
// Add disconnect record if it exists and is different from first connect
if let Some(last) = last_disconnect {
if !std::ptr::eq(last, first_connect.unwrap_or(&sorted[0])) {
let mut disc = (*last).clone();
disc.event_ts_utc = last.event_ts_utc;
disc.bytes_sent = Some(total_bytes);
aggregated.push(Rc::new(disc));
}
}
}
// Sort by timestamp descending (most recent first)
aggregated.sort_by_key(|b| std::cmp::Reverse(b.event_ts_utc));
aggregated
}
fn record_matches(record: &StreamHistoryRecordDto, filter: &SearchRequest) -> bool {
if matches!(filter, SearchRequest::Clear) {
return true;
}
let fields = [
// Cow::Owned(record.partition_day_utc.to_string()),
// Cow::Owned(record.virtual_id.as_ref().map_or_else(String::new, ToString::to_string)),
// Cow::Borrowed(record.input_name.as_deref().unwrap_or("")),
// Cow::Borrowed(record.video_codec.as_deref().unwrap_or("")),
// Cow::Borrowed(record.audio_codec.as_deref().unwrap_or("")),
// Cow::Borrowed(record.resolution.as_deref().unwrap_or("")),
// Cow::Borrowed(record.target_name.as_deref().unwrap_or("")),
// Cow::Borrowed(record.session_duration.as_deref().unwrap_or("")),
Cow::Owned(record.event_ts_utc.to_string()),
Cow::Owned(record.event_type.to_string()),
Cow::Borrowed(record.title.as_deref().unwrap_or("")),
Cow::Borrowed(record.group.as_deref().unwrap_or("")),
Cow::Borrowed(record.api_username.as_deref().unwrap_or("")),
Cow::Borrowed(record.provider_name.as_deref().unwrap_or("")),
Cow::Owned(record.provider_id.as_ref().map_or_else(String::new, ToString::to_string)),
Cow::Owned(record.bytes_sent.as_ref().map_or_else(String::new, ToString::to_string)),
Cow::Owned(record.first_byte_latency_ms.as_ref().map_or_else(String::new, ToString::to_string)),
Cow::Borrowed(record.user_agent.as_deref().unwrap_or("")),
Cow::Owned(record.item_type.as_ref().map_or_else(String::new, ToString::to_string)),
Cow::Borrowed(record.container.as_deref().unwrap_or("")),
Cow::Owned(record.disconnect_reason.as_ref().map_or_else(String::new, ToString::to_string)),
Cow::Borrowed(record.source_addr.as_deref().unwrap_or("")),
Cow::Borrowed(record.country.as_deref().unwrap_or("")),
Cow::Borrowed(record.cluster.as_deref().unwrap_or("")),
];
match filter {
SearchRequest::Clear => true,
SearchRequest::Text(text, _) => {
let text_lower = text.to_lowercase();
let fields = [
record.api_username.as_deref().unwrap_or(""),
record.title.as_deref().unwrap_or(""),
record.provider_name.as_deref().unwrap_or(""),
record.user_agent.as_deref().unwrap_or(""),
record.item_type.as_deref().unwrap_or(""),
record.container.as_deref().unwrap_or(""),
record.video_codec.as_deref().unwrap_or(""),
record.audio_codec.as_deref().unwrap_or(""),
record.resolution.as_deref().unwrap_or(""),
record.source_addr.as_deref().unwrap_or(""),
record.disconnect_reason.as_deref().unwrap_or(""),
record.group.as_deref().unwrap_or(""),
];
fields.iter().any(|f| f.to_lowercase().contains(&text_lower))
}
SearchRequest::Regexp(pattern, _) => {
if let Ok(re) = shared::model::REGEX_CACHE.get_or_compile(pattern) {
let fields = [
record.api_username.as_deref().unwrap_or(""),
record.title.as_deref().unwrap_or(""),
record.provider_name.as_deref().unwrap_or(""),
record.user_agent.as_deref().unwrap_or(""),
record.item_type.as_deref().unwrap_or(""),
record.container.as_deref().unwrap_or(""),
record.video_codec.as_deref().unwrap_or(""),
record.audio_codec.as_deref().unwrap_or(""),
record.resolution.as_deref().unwrap_or(""),
record.source_addr.as_deref().unwrap_or(""),
record.disconnect_reason.as_deref().unwrap_or(""),
record.group.as_deref().unwrap_or(""),
];
fields.iter().any(|f| re.is_match(f))
} else {
false
@@ -113,7 +235,7 @@ fn qos_snapshot_matches(snapshot: &StreamHistoryQosSnapshot, filter: &SearchRequ
]
.iter()
.any(|field| field.to_lowercase().contains(&text_lower))
|| snapshot.target_id.to_string().contains(&text_lower)
|| snapshot.target_name.to_lowercase().contains(&text_lower)
|| snapshot.provider_id.to_string().contains(&text_lower)
|| snapshot.virtual_id.to_string().contains(&text_lower)
}
@@ -123,7 +245,7 @@ fn qos_snapshot_matches(snapshot: &StreamHistoryQosSnapshot, filter: &SearchRequ
|| re.is_match(&snapshot.input_name)
|| re.is_match(&snapshot.provider_name)
|| re.is_match(&snapshot.item_type)
|| re.is_match(&snapshot.target_id.to_string())
|| re.is_match(&snapshot.target_name)
|| re.is_match(&snapshot.provider_id.to_string())
|| re.is_match(&snapshot.virtual_id.to_string())
} else {
@@ -183,7 +305,7 @@ fn qos_summary_rows(snapshots: &[StreamHistoryQosSnapshot]) -> Vec<QosSummaryRow
input_name: snapshot.input_name.clone(),
provider_name: snapshot.provider_name.clone(),
provider_id: snapshot.provider_id,
target_id: snapshot.target_id,
target_name: snapshot.target_name.clone(),
item_type: snapshot.item_type.clone(),
window_24h: snapshot.window_24h.clone(),
})
@@ -198,7 +320,7 @@ fn qos_detail_rows(snapshot: &StreamHistoryQosSnapshot) -> Vec<QosDetailRow> {
}
fn has_any_stream_history_content(
records: &[Rc<StreamHistoryRecord>],
records: &[Rc<StreamHistoryRecordDto>],
summaries: &[StreamHistoryProviderSummary],
qos_snapshots: &[StreamHistoryQosSnapshot],
) -> bool {
@@ -211,57 +333,13 @@ pub fn StreamHistoryView() -> Html {
let translate = use_translation();
let from_date = use_state(|| Some(today_start_ts()));
let to_date = use_state(|| Some(today_start_ts()));
let all_records = use_state(Vec::<Rc<StreamHistoryRecord>>::new);
let all_records = use_state(Vec::<Rc<StreamHistoryRecordDto>>::new);
let summaries = use_state(Vec::<StreamHistoryProviderSummary>::new);
let qos_snapshots = use_state(Vec::<StreamHistoryQosSnapshot>::new);
let selected_qos_snapshot = use_state(|| None::<StreamHistoryQosSnapshot>);
let search_filter = use_state(|| SearchRequest::Clear);
let loading = use_state(|| false);
// Load on mount with default date range (today)
{
let services = services.clone();
let all_records = all_records.clone();
let summaries = summaries.clone();
let qos_snapshots = qos_snapshots.clone();
let selected_qos_snapshot = selected_qos_snapshot.clone();
let loading = loading.clone();
let from = *from_date;
let to = *to_date;
use_effect_with((), move |_| {
let from_str = from.map(ts_to_date_str);
let to_str = to.map(ts_to_date_str);
loading.set(true);
spawn_local(async move {
let (history_result, summary_result, qos_result) = join!(
services.stream_history.get_history(from_str.as_deref(), to_str.as_deref()),
services.stream_history.get_summary(from_str.as_deref(), to_str.as_deref()),
services.stream_history.get_qos_snapshots()
);
match history_result {
Ok(Some(records)) => all_records.set(records.into_iter().map(Rc::new).collect()),
Ok(None) | Err(_) => all_records.set(Vec::new()),
}
match summary_result {
Ok(Some(items)) => summaries.set(items),
Ok(None) | Err(_) => summaries.set(Vec::new()),
}
match qos_result {
Ok(Some(items)) => {
selected_qos_snapshot.set(None);
qos_snapshots.set(items);
}
Ok(None) | Err(_) => {
selected_qos_snapshot.set(None);
qos_snapshots.set(Vec::new());
}
}
loading.set(false);
});
|| ()
});
}
let handle_from_change = {
let from_date = from_date.clone();
Callback::from(move |ts: Option<i64>| from_date.set(ts))
@@ -320,14 +398,27 @@ pub fn StreamHistoryView() -> Html {
})
};
// Load on mount with default date range (today)
{
let handle_load = handle_load.clone();
use_effect_with((), move |_| {
handle_load.emit(String::new());
|| ()
});
}
let handle_search = {
let search_filter = search_filter.clone();
Callback::from(move |req: SearchRequest| search_filter.set(req))
};
let filtered: Rc<Vec<Rc<StreamHistoryRecord>>> =
let filtered: Rc<Vec<Rc<StreamHistoryRecordDto>>> =
use_memo(((*all_records).clone(), (*search_filter).clone()), |(records, filter)| {
records.iter().filter(|r| record_matches(r, filter)).cloned().collect::<Vec<_>>()
let mut sorted: Vec<Rc<StreamHistoryRecordDto>> = records.to_vec();
sorted.sort_by_key(|r| r.event_ts_utc);
// Aggregate HLS records first on the full dataset so connect/disconnect pairs are complete
let aggregated = aggregate_hls_records(&sorted);
aggregated.into_iter().filter(|r| record_matches(r, filter)).collect()
});
let has_any_content = has_any_stream_history_content(&all_records, &summaries, &qos_snapshots);
@@ -359,7 +450,7 @@ pub fn StreamHistoryView() -> Html {
};
let translate_for_table = translate.clone();
let table_def: Rc<TableDefinition<StreamHistoryRecord>> = use_memo(filtered.clone(), move |filtered| {
let table_def: Rc<TableDefinition<StreamHistoryRecordDto>> = use_memo(filtered.clone(), move |filtered| {
let translate = translate_for_table.clone();
TableDefinition {
items: Some(filtered.clone()),
@@ -371,89 +462,101 @@ pub fn StreamHistoryView() -> Html {
0 => translate.t("LABEL.STREAM_HISTORY_TIME"),
1 => translate.t("LABEL.STREAM_HISTORY_EVENT"),
2 => translate.t("LABEL.USERNAME"),
3 => translate.t("LABEL.TITLE"),
4 => translate.t("LABEL.PROVIDER"),
5 => translate.t("LABEL.DURATION"),
6 => translate.t("LABEL.STREAM_HISTORY_BYTES"),
7 => translate.t("LABEL.STREAM_HISTORY_FIRST_BYTE"),
8 => translate.t("LABEL.USER_AGENT"),
9 => translate.t("LABEL.TYPE"),
10 => translate.t("LABEL.CONTAINER"),
11 => translate.t("LABEL.STREAM_HISTORY_REASON"),
12 => translate.t("LABEL.STREAM_HISTORY_IP"),
3 => translate.t("LABEL.GROUP"),
4 => translate.t("LABEL.TITLE"),
5 => translate.t("LABEL.PROVIDER"),
6 => translate.t("LABEL.DURATION"),
7 => translate.t("LABEL.STREAM_HISTORY_BYTES"),
8 => translate.t("LABEL.STREAM_HISTORY_FIRST_BYTE"),
9 => translate.t("LABEL.USER_AGENT"),
10 => translate.t("LABEL.TYPE"),
11 => translate.t("LABEL.CONTAINER"),
12 => translate.t("LABEL.STREAM_HISTORY_REASON"),
13 => translate.t("LABEL.STREAM_HISTORY_IP"),
14 => translate.t("LABEL.COUNTRY"),
_ => String::new(),
};
html! { <span>{label}</span> }
}),
render_data_cell: Callback::from(|(_, col, record): (usize, usize, Rc<StreamHistoryRecord>)| match col {
0 => html! { <span class="tp__stream-history__cell--time">{format_ts(record.event_ts_utc)}</span> },
1 => {
let is_connect = record.event_type == "connect";
let badge_class = if is_connect {
"tp__stream-history__badge tp__stream-history__badge--connect"
} else {
"tp__stream-history__badge tp__stream-history__badge--disconnect"
};
html! { <span class={badge_class}>{record.event_type.clone()}</span> }
}
2 => html! { <span>{record.api_username.as_deref().unwrap_or("-")}</span> },
3 => {
html! { <span class="tp__stream-history__cell--title">{record.title.as_deref().unwrap_or("-")}</span> }
}
4 => html! {
<span>
{
match (record.provider_name.as_deref(), record.provider_id) {
(Some(name), Some(id)) => format!("{name} (#{id})"),
(Some(name), None) => name.to_string(),
(None, Some(id)) => format!("#{id}"),
(None, None) => String::from("-"),
render_data_cell: Callback::from(
|(_, col, record): (usize, usize, Rc<StreamHistoryRecordDto>)| match col {
0 => html! { <span class="tp__stream-history__cell--time">{format_ts(record.event_ts_utc)}</span> },
1 => {
let is_connect = record.event_type == StreamHistoryEventType::Connect;
let badge_class = if is_connect {
"tp__stream-history__badge tp__stream-history__badge--connect"
} else {
"tp__stream-history__badge tp__stream-history__badge--disconnect"
};
html! { <span class={badge_class}>{record.event_type.to_string()}</span> }
}
2 => html! { <span>{record.api_username.as_deref().unwrap_or("-")}</span> },
3 => {
html! { <span class="tp__stream-history__cell--title">{record.group.as_deref().unwrap_or("-")}</span> }
}
4 => {
html! { <span class="tp__stream-history__cell--title">{record.title.as_deref().unwrap_or("-")}</span> }
}
5 => html! {
<span>
{
match (record.provider_name.as_deref(), record.provider_id) {
(Some(name), Some(id)) => format!("{name} (#{id})"),
(Some(name), None) => name.to_string(),
(None, Some(id)) => format!("#{id}"),
(None, None) => String::from("-"),
}
}
}
</span>
</span>
},
6 => html! {
<span class="tp__stream-history__cell--mono">
{record.session_duration.map(format_duration).unwrap_or_default()}
</span>
},
7 => html! {
<span class="tp__stream-history__cell--mono">
{record.bytes_sent.map(format_bytes).unwrap_or_default()}
</span>
},
8 => html! {
<span class="tp__stream-history__cell--mono">
{record.first_byte_latency_ms.map(|v| v.to_string()).unwrap_or_default()}
</span>
},
9 => html! {
<span class="tp__stream-history__cell--title">
<RevealContent preview={record.user_agent.as_deref().map(|ua| html! {ua})}>{record.user_agent.as_deref()}</RevealContent>
</span>
},
10 => html! {
<span>
{optional_record_text(record.item_type.as_ref().map(ToString::to_string))}
</span>
},
11 => html! {
<span>
{optional_record_text_str(record.container.as_deref())}
</span>
},
12 => html! {
<span>
{record.disconnect_reason.as_ref().map(ToString::to_string).unwrap_or_else(|| String::from("-")).replace('_', " ")}
</span>
},
13 => html! {
<span class="tp__stream-history__cell--ip">
{record.source_addr.as_deref().unwrap_or("-")}
</span>
},
14 => html! {
<span class="tp__stream-history__cell--country">
<Country country_code={record.country.clone()} />
</span>
},
_ => html! {},
},
5 => html! {
<span class="tp__stream-history__cell--mono">
{record.session_duration.map(format_duration).unwrap_or_default()}
</span>
},
6 => html! {
<span class="tp__stream-history__cell--mono">
{record.bytes_sent.map(format_bytes).unwrap_or_default()}
</span>
},
7 => html! {
<span class="tp__stream-history__cell--mono">
{record.first_byte_latency_ms.map(|v| v.to_string()).unwrap_or_default()}
</span>
},
8 => html! {
<span class="tp__stream-history__cell--title">
{optional_record_text(record.user_agent.as_deref())}
</span>
},
9 => html! {
<span>
{optional_record_text(record.item_type.as_deref())}
</span>
},
10 => html! {
<span>
{optional_record_text(record.container.as_deref())}
</span>
},
11 => html! {
<span>
{record.disconnect_reason.as_deref().unwrap_or("-").replace('_', " ")}
</span>
},
12 => html! {
<span class="tp__stream-history__cell--ip">
{record.source_addr.as_deref().unwrap_or("-")}
</span>
},
_ => html! {},
}),
),
}
});
let translate_for_summary = translate.clone();
@@ -529,7 +632,7 @@ pub fn StreamHistoryView() -> Html {
</button>
},
1 => html! { <span>{format!("{} (#{})", row.provider_name, row.provider_id)}</span> },
2 => html! { <span>{row.target_id}</span> },
2 => html! { <span>{row.target_name.clone()}</span> },
3 => html! { <span>{row.item_type.clone()}</span> },
4 => {
html! { <span>{format!("{} ({})", row.window_24h.score, qos_score_label(row.window_24h.score))}</span> }
@@ -638,7 +741,7 @@ pub fn StreamHistoryView() -> Html {
}
<div class="tp__stream-history__summary">
<h2>{translate.t("LABEL.STREAM_HISTORY")}</h2>
<Table::<StreamHistoryRecord> definition={table_def} />
<Table::<StreamHistoryRecordDto> definition={table_def} />
</div>
}
</div>
@@ -648,7 +751,7 @@ pub fn StreamHistoryView() -> Html {
#[cfg(test)]
mod tests {
use super::{optional_record_text, provider_summary_rows, qos_detail_rows, qos_score_label, top_qos_snapshots};
use super::{optional_record_text_str, provider_summary_rows, qos_detail_rows, qos_score_label, top_qos_snapshots};
use crate::services::{StreamHistoryProviderSummary, StreamHistoryQosSnapshot, StreamHistoryQosSnapshotWindow};
use shared::model::SearchRequest;
@@ -676,7 +779,7 @@ mod tests {
StreamHistoryQosSnapshot {
stream_identity_key: stream_identity_key.to_string(),
input_name: "input-a".to_string(),
target_id: 1,
target_name: "target-a".to_string(),
provider_name: provider_name.to_string(),
provider_id: 10,
virtual_id: 99,
@@ -708,7 +811,7 @@ mod tests {
let snapshot = StreamHistoryQosSnapshot {
stream_identity_key: "stream-a".to_string(),
input_name: "input-a".to_string(),
target_id: 1,
target_name: "target-a".to_string(),
provider_name: "provider-a".to_string(),
provider_id: 10,
virtual_id: 99,
@@ -759,8 +862,8 @@ mod tests {
#[test]
fn optional_record_text_preserves_value_or_dash_fallback() {
assert_eq!(optional_record_text(Some("Lavf53.32.100")), "Lavf53.32.100");
assert_eq!(optional_record_text(Some("live")), "live");
assert_eq!(optional_record_text(None), "-");
assert_eq!(optional_record_text_str(Some("Lavf53.32.100")), "Lavf53.32.100");
assert_eq!(optional_record_text_str(Some("live")), "live");
assert_eq!(optional_record_text_str(None), "-");
}
}
frontend/src/app/components/home.rs
+33 -3
View File
@@ -13,8 +13,10 @@ use crate::{
i18n::use_translation,
model::{EventMessage, ViewType},
provider::DialogProvider,
services::{ToastCloseMode, ToastOptions},
services::{FlagsLoadState, ToastCloseMode, ToastOptions},
};
use gloo_timers::future::TimeoutFuture;
use log::error;
use shared::{
model::{
permission::{Permission, PERM_ALL},
@@ -22,8 +24,8 @@ use shared::{
},
utils::Internable,
};
use std::{future, rc::Rc};
use yew::{prelude::*, suspense::use_future};
use std::{cell::Cell, future, rc::Rc};
use yew::{platform::spawn_local, prelude::*, suspense::use_future};
#[component]
pub fn Home() -> Html {
@@ -149,6 +151,34 @@ pub fn Home() -> Html {
//<div class={"app-header__toolbar"}><select onchange={handle_language} defaultValue={i18next.language}>{services.config().getUiConfig().languages.map(l => <option key={l} value={l}>{l}</option>)}</select></div>
let geoip_enabled = config.as_ref().is_some_and(|cfg| cfg.config.is_geoip_enabled());
{
let flags_service = services.flags.clone();
use_effect_with(geoip_enabled, move |geoip_enabled| {
let cancelled = Rc::new(Cell::new(false));
if *geoip_enabled {
let flags_service = flags_service.clone();
let cancelled = cancelled.clone();
spawn_local(async move {
while !cancelled.get() && !flags_service.is_loaded() {
match flags_service.ensure_loaded_from_assets().await {
Ok(FlagsLoadState::Loaded) => break,
Ok(FlagsLoadState::InProgress) => {
TimeoutFuture::new(250).await;
}
Err(err) => {
error!("Failed to load flags {err}");
TimeoutFuture::new(5000).await;
}
}
}
});
}
move || cancelled.set(true)
});
}
if config.is_none() {
return html! {};
}
frontend/src/app/components/mod.rs
+2 -1
View File
@@ -60,6 +60,7 @@ mod particle_flow_background;
mod source_editor;
mod title_card;
mod country;
mod setup;
// pub use self::input::*;
// pub use self::menu_item::*;
@@ -69,7 +70,7 @@ mod setup;
pub(crate) use self::{
accordion::*, accordion_panel::*, authentication::*, breadcrumbs::*, card::*, cell_value::*, chip::*,
cluster_flags_input::*, collapse_panel::*, csv_table::*, custom_dialog::*, dashboard::*, date_input::*,
cluster_flags_input::*, collapse_panel::*, country::*, csv_table::*, custom_dialog::*, dashboard::*, date_input::*,
date_input_action::*, downloads::DownloadsView, drop_down_icon_button::*, field_explanation::*, field_id::*,
filter::*, hide_content::*, home::*, icon_button::*, key_value_editor::*, loading_screen::*, login::*,
no_access::*, no_content::*, panel::*, particle_flow_background::*, playlist::*, radio_button_group::*, rbac::*,
frontend/src/hooks/use_server_status.rs
+3 -3
View File
@@ -90,7 +90,7 @@ fn download_task_to_stream_with_ts(download: &FileDownloadDto, ts: u64) -> Strea
shared_stream_id: None,
technical: None,
},
provider: BACKGROUND_TRANSFER_PROVIDER.to_string(),
provider: BACKGROUND_TRANSFER_PROVIDER.intern(),
addr: download_stream_addr(uid),
client_ip: BACKGROUND_TRANSFER_CLIENT_IP.to_string(),
user_agent: "Tuliprox download worker".to_string(),
@@ -347,7 +347,7 @@ mod tests {
shared_stream_id: None,
technical: None,
},
provider: "provider".to_string(),
provider: "provider".intern(),
addr: addr.parse::<SocketAddr>().unwrap_or_else(|_| unreachable!()),
client_ip: "127.0.0.1".to_string(),
user_agent: "ua".to_string(),
@@ -440,7 +440,7 @@ mod tests {
apply_downloads_snapshot(&mut streams, &response);
assert_eq!(streams.len(), 1);
assert_eq!(streams[0].provider, "Download Manager");
assert_eq!(streams[0].provider.as_ref(), "Download Manager");
assert_eq!(streams[0].client_ip, "background-task");
assert_eq!(streams[0].channel.item_type, PlaylistItemType::Video);
}
frontend/src/services/stream_history_service.rs
+4 -33
View File
@@ -1,35 +1,6 @@
use crate::services::{get_base_href, request_get, Encoding};
use serde::Deserialize;
use shared::utils::concat_path_leading_slash;
#[derive(Debug, Clone, Deserialize, PartialEq)]
pub struct StreamHistoryRecord {
pub event_type: String,
pub event_ts_utc: u64,
pub partition_day_utc: String,
pub session_id: u64,
pub source_addr: Option<String>,
pub api_username: Option<String>,
pub provider_name: Option<String>,
pub virtual_id: Option<u32>,
pub item_type: Option<String>,
pub title: Option<String>,
pub group: Option<String>,
pub user_agent: Option<String>,
pub shared: Option<bool>,
pub provider_id: Option<u32>,
pub cluster: Option<String>,
pub container: Option<String>,
pub video_codec: Option<String>,
pub audio_codec: Option<String>,
pub resolution: Option<String>,
pub disconnect_reason: Option<String>,
pub session_duration: Option<u64>,
pub bytes_sent: Option<u64>,
pub first_byte_latency_ms: Option<u64>,
pub previous_session_id: Option<u64>,
pub target_id: Option<u16>,
}
use shared::{model::StreamHistoryRecordDto, utils::concat_path_leading_slash};
#[derive(Debug, Clone, Deserialize, PartialEq)]
pub struct StreamHistoryProviderSummary {
@@ -59,7 +30,7 @@ pub struct StreamHistoryQosSnapshotWindow {
pub struct StreamHistoryQosSnapshot {
pub stream_identity_key: String,
pub input_name: String,
pub target_id: u16,
pub target_name: String,
pub provider_name: String,
pub provider_id: u32,
pub virtual_id: u32,
@@ -93,14 +64,14 @@ impl StreamHistoryService {
&self,
from: Option<&str>,
to: Option<&str>,
) -> Result<Option<Vec<StreamHistoryRecord>>, crate::error::Error> {
) -> Result<Option<Vec<StreamHistoryRecordDto>>, crate::error::Error> {
let url = match (from, to) {
(Some(f), Some(t)) => format!("{}?from={}&to={}", self.path, f, t),
(Some(f), None) => format!("{}?from={}", self.path, f),
(None, Some(t)) => format!("{}?to={}", self.path, t),
(None, None) => self.path.clone(),
};
request_get::<Vec<StreamHistoryRecord>>(&url, None, None).await
request_get::<Vec<StreamHistoryRecordDto>>(&url, None, Some(Encoding::Cbor)).await
}
pub async fn get_summary(
shared/src/model/mod.rs
+4 -3
View File
@@ -23,6 +23,7 @@ mod search_request;
mod short_epg;
mod stats;
mod status_check;
mod stream_history_record;
mod stream_info;
mod stream_meter;
mod stream_properties;
@@ -42,7 +43,7 @@ pub use self::{
active_user_connection_change::*, auth::*, cluster_flags::*, config::*, download::*, epg::*, epg_request::*,
ip_check::*, item_field::*, library_request::*, mapping::*, media_properties::*, messaging::*, playlist::*,
playlist_categories::*, playlist_info_document::*, playlist_request::*, processing_order::*, regex_cache::*,
search_request::*, short_epg::*, stats::*, status_check::*, stream_info::*, stream_meter::*, stream_properties::*,
strm_export_style::*, system_info::*, target_type::*, transfer::*, ui_playlist_item::*, user_command::*,
uuidtype::*, web_socket::*, xtream::*,
search_request::*, short_epg::*, stats::*, status_check::*, stream_history_record::*, stream_info::*,
stream_meter::*, stream_properties::*, strm_export_style::*, system_info::*, target_type::*, transfer::*,
ui_playlist_item::*, user_command::*, uuidtype::*, web_socket::*, xtream::*,
};
shared/src/model/playlist.rs
+14 -3
View File
@@ -85,16 +85,27 @@ impl TryFrom<PlaylistItemType> for XtreamCluster {
#[repr(u8)]
pub enum PlaylistItemType {
#[default]
#[serde(alias = "live")]
Live = 1,
#[serde(alias = "video")]
Video = 2,
Series = 3, // xtream series description
#[serde(alias = "series")]
Series = 3, // xtream series description
#[serde(alias = "series_info")]
SeriesInfo = 4, // xtream series info fetched for series description
#[serde(alias = "catchup")]
Catchup = 5,
#[serde(alias = "live_unknown")]
LiveUnknown = 6, // No Provider id
LiveHls = 7, // m3u8 entry
LiveDash = 8, // mpd
#[serde(alias = "live_hls")]
LiveHls = 7, // m3u8 entry
#[serde(alias = "live_dash")]
LiveDash = 8, // mpd
#[serde(alias = "local_video")]
LocalVideo = 9,
#[serde(alias = "local_series")]
LocalSeries = 10,
#[serde(alias = "local_series_info")]
LocalSeriesInfo = 11,
}
shared/src/model/stream_history_record.rs
+164
View File
@@ -0,0 +1,164 @@
use crate::{model::PlaylistItemType, utils::arc_str_option_serde};
use serde::{Deserialize, Serialize};
use std::{
fmt::{Display, Formatter},
sync::Arc,
};
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StreamHistoryEventType {
Connect,
ConnectFailed,
Disconnect,
Failure,
}
impl Display for StreamHistoryEventType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
StreamHistoryEventType::Connect => write!(f, "connect"),
StreamHistoryEventType::ConnectFailed => write!(f, "connect_failed"),
StreamHistoryEventType::Disconnect => write!(f, "disconnect"),
StreamHistoryEventType::Failure => write!(f, "failure"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DisconnectReason {
Cleanup,
ClientClosed,
ClientKicked,
Provisioning,
ServerError,
Timeout,
/// Reserved for future day-split logic: emitted at midnight for sessions that span
/// two calendar days so the previous day's file gets a closing record.
/// Not yet emitted — sessions crossing midnight have their disconnect in the new day's file.
DayRollover,
Shutdown,
Unknown,
ProviderError,
ProviderClosed,
Preempted,
SessionExpired,
UserConnectionsExhausted,
ProviderConnectionsExhausted,
IntermediateFailures(usize),
}
impl Display for DisconnectReason {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
DisconnectReason::Cleanup => write!(f, "cleanup"),
DisconnectReason::ClientClosed => write!(f, "client_closed"),
DisconnectReason::ClientKicked => write!(f, "client_kicked"),
DisconnectReason::Provisioning => write!(f, "provisioning"),
DisconnectReason::ServerError => write!(f, "server_error"),
DisconnectReason::Timeout => write!(f, "timeout"),
DisconnectReason::DayRollover => write!(f, "day_rollover"),
DisconnectReason::Shutdown => write!(f, "shutdown"),
DisconnectReason::Unknown => write!(f, "unknown"),
DisconnectReason::ProviderError => write!(f, "provider_error"),
DisconnectReason::ProviderClosed => write!(f, "provider_closed"),
DisconnectReason::Preempted => write!(f, "preempted"),
DisconnectReason::SessionExpired => write!(f, "session_expired"),
DisconnectReason::UserConnectionsExhausted => write!(f, "user_connections_exhausted"),
DisconnectReason::ProviderConnectionsExhausted => write!(f, "provider_connections_exhausted"),
DisconnectReason::IntermediateFailures(count) => write!(f, "intermediate_failures({count})", count = count),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ConnectFailureReason {
UserAccountExpired,
UserConnectionsExhausted,
ProviderConnectionsExhausted,
ProviderError,
ProviderClosed,
ChannelUnavailable,
Preempted,
SessionExpired,
Provisioning,
}
impl Display for ConnectFailureReason {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
ConnectFailureReason::UserAccountExpired => write!(f, "user_account_expired"),
ConnectFailureReason::UserConnectionsExhausted => write!(f, "user_connections_exhausted"),
ConnectFailureReason::ProviderConnectionsExhausted => write!(f, "provider_connections_exhausted"),
ConnectFailureReason::ProviderError => write!(f, "provider_error"),
ConnectFailureReason::ProviderClosed => write!(f, "provider_closed"),
ConnectFailureReason::ChannelUnavailable => write!(f, "channel_unavailable"),
ConnectFailureReason::Preempted => write!(f, "preempted"),
ConnectFailureReason::SessionExpired => write!(f, "session_expired"),
ConnectFailureReason::Provisioning => write!(f, "provisioning"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailureStage {
Admission,
ProviderOpen,
FirstByte,
Streaming,
SessionReconnect,
}
impl Display for FailureStage {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
FailureStage::Admission => write!(f, "admission"),
FailureStage::ProviderOpen => write!(f, "provider open"),
FailureStage::FirstByte => write!(f, "first byte"),
FailureStage::Streaming => write!(f, "streaming"),
FailureStage::SessionReconnect => write!(f, "session reconnect"),
}
}
}
/// DTO for stream history records exchanged between backend and frontend.
/// Uses string-based enum variants for JSON compatibility.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub struct StreamHistoryRecordDto {
pub event_type: StreamHistoryEventType,
pub event_ts_utc: u64,
pub partition_day_utc: String,
pub session_id: u64,
pub source_addr: Option<String>,
pub api_username: Option<String>,
#[serde(default, with = "arc_str_option_serde")]
pub provider_name: Option<Arc<str>>,
#[serde(default, with = "arc_str_option_serde")]
pub input_name: Option<Arc<str>>,
pub virtual_id: Option<u32>,
pub item_type: Option<PlaylistItemType>,
pub title: Option<String>,
pub group: Option<String>,
pub country: Option<String>,
pub user_agent: Option<String>,
pub shared: Option<bool>,
pub provider_id: Option<u32>,
pub cluster: Option<String>,
pub container: Option<String>,
pub video_codec: Option<String>,
pub audio_codec: Option<String>,
pub resolution: Option<String>,
pub failure_stage: Option<FailureStage>,
pub connect_failure_reason: Option<ConnectFailureReason>,
pub disconnect_reason: Option<DisconnectReason>,
pub session_duration: Option<u64>,
pub bytes_sent: Option<u64>,
pub first_byte_latency_ms: Option<u64>,
pub previous_session_id: Option<u64>,
#[serde(default, with = "arc_str_option_serde")]
pub target_name: Option<Arc<str>>,
}
shared/src/model/stream_info.rs
+4 -3
View File
@@ -275,7 +275,8 @@ pub struct StreamInfo {
pub meter_uid: u32,
pub username: String,
pub channel: StreamChannel,
pub provider: String,
#[serde(default, with = "arc_str_serde")]
pub provider: Arc<str>,
pub addr: SocketAddr,
pub client_ip: String,
#[serde(default)]
@@ -302,7 +303,7 @@ impl StreamInfo {
username: &str,
addr: &SocketAddr,
client_ip: &str,
provider: &str,
provider: Arc<str>,
stream_channel: StreamChannel,
user_agent: String,
country_code: Option<String>,
@@ -314,7 +315,7 @@ impl StreamInfo {
meter_uid,
username: username.to_string(),
channel: stream_channel,
provider: provider.to_string(),
provider,
addr: *addr,
client_ip: client_ip.to_string(),
user_agent,