mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-17 05:12:34 +02:00
feat(watch): group added/removed contexts per watch entry (#912)
* feat(watch): group added/removed contexts per watch entry
- watch entries: regex string or {pattern, on:
[channels|group_added|group_removed]}
- one playlist.watch.changed per refresh with WatchChange list;
playlist.groups.changed aliased
- Telegram/ntfy message length limits
- web UI: Mapping and Watch sub pages, new Mapping icon
- ClusterFlags on create_bitset!, bitflags removed
BREAKING CHANGE: playlist.watch.changed templates must use the new
watch.* fields
* feat(watch): group added/removed contexts per watch entry
- watch entries: regex string or {pattern, on:
[channels|group_added|group_removed]}
- one playlist.watch.changed per refresh with WatchChange list;
playlist.groups.changed aliased
- Telegram/ntfy message length limits
- web UI: Mapping and Watch sub pages, new Mapping icon
- ClusterFlags on create_bitset!, bitflags removed
BREAKING CHANGE: playlist.watch.changed templates must use the new
watch.* fields
This commit is contained in:
euzu
@@ -1,10 +1,11 @@
|
||||
use log::error;
|
||||
use shared::model::{
|
||||
EventMessage, EventSink, PlaylistGroup, PlaylistGroupsChanged, WatchChanges, WatchDisabled, WatchDisabledReason,
|
||||
EventMessage, EventSink, PlaylistGroup, WatchChange, WatchChanges, WatchContext, WatchDisabled,
|
||||
WatchDisabledReason, WatchGroupItems, WatchItems,
|
||||
};
|
||||
use std::{collections::BTreeSet, path::Path, sync::Arc};
|
||||
use tuliprox_core::{
|
||||
model::AppConfig,
|
||||
model::ConfigWatch,
|
||||
utils,
|
||||
utils::{binary_deserialize, binary_serialize, file_exists_async},
|
||||
};
|
||||
@@ -12,45 +13,79 @@ use tuliprox_core::{
|
||||
const WATCH_NOTIFICATION_LIST_LIMIT: usize = 120;
|
||||
const WATCH_NOTIFICATION_SUMMARY_THRESHOLD: usize = 500;
|
||||
|
||||
/// Track which *groups* a target has, as opposed to which channels a group has.
|
||||
/// Groups that appeared or vanished, kept only where a `group_added` /
|
||||
/// `group_removed` entry matches.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct GroupChanges {
|
||||
pub added: Option<WatchItems>,
|
||||
pub removed: Option<WatchItems>,
|
||||
}
|
||||
|
||||
/// One watched group's channel delta. An unchanged group yields no diff at
|
||||
/// all, so at least one side is always set.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct GroupChannelDiff {
|
||||
pub group: Arc<str>,
|
||||
pub added: Option<WatchItems>,
|
||||
pub removed: Option<WatchItems>,
|
||||
}
|
||||
|
||||
/// Sample sorted titles: the list is capped, the total is not.
|
||||
///
|
||||
/// `process_group_watch` below is blind to the group set in both directions.
|
||||
/// A group appearing is silent - it finds no baseline, writes one and emits
|
||||
/// nothing, so the whole channel list reads as "not new" from then on. A
|
||||
/// group vanishing is never even looked at: it is absent from the refreshed
|
||||
/// playlist, so nothing iterates it.
|
||||
/// Takes an iterator so a set difference is sampled as it is walked, without
|
||||
/// collecting it first.
|
||||
fn sample<'a>(titles: impl Iterator<Item = &'a Arc<str>>) -> WatchItems {
|
||||
let mut items = Vec::new();
|
||||
let mut total = 0usize;
|
||||
for title in titles {
|
||||
if items.len() < WATCH_NOTIFICATION_LIST_LIMIT {
|
||||
items.push(Arc::clone(title));
|
||||
}
|
||||
total += 1;
|
||||
}
|
||||
WatchItems { truncated: items.len() < total, items, total }
|
||||
}
|
||||
|
||||
fn sample_non_empty<'a>(titles: impl Iterator<Item = &'a Arc<str>>) -> Option<WatchItems> {
|
||||
let items = sample(titles);
|
||||
(items.total > 0).then_some(items)
|
||||
}
|
||||
|
||||
/// Diff the target's group titles against the persisted index.
|
||||
///
|
||||
/// The index lives beside the per-group directory rather than inside it
|
||||
/// The index holds *every* group, whatever the watch entries say, so editing
|
||||
/// a pattern never reads as a change and a newly added `group_added` entry
|
||||
/// already has a baseline. It lives beside the per-group directory
|
||||
/// (`<target>.groups.bin`, not `<target>/__groups.bin`) so it cannot collide
|
||||
/// with a group whose sanitized title happens to match.
|
||||
pub async fn process_target_groups_watch<E: EventSink>(
|
||||
app_config: &Arc<AppConfig>,
|
||||
/// with a group whose sanitized title matches.
|
||||
///
|
||||
/// First sight writes the baseline and reports nothing: announcing every
|
||||
/// group as new on the first refresh would be noise, not news. An unreadable
|
||||
/// index is reported and replaced - a corrupt file never becomes readable
|
||||
/// again, and keeping it would switch group tracking off for good.
|
||||
pub async fn diff_target_groups<E: EventSink>(
|
||||
storage_dir: &str,
|
||||
events: &E,
|
||||
target_name: &str,
|
||||
playlist: &[PlaylistGroup],
|
||||
) {
|
||||
let new_tree: BTreeSet<Arc<str>> = playlist.iter().map(|group| group.title.clone()).collect();
|
||||
watches: &[ConfigWatch],
|
||||
) -> GroupChanges {
|
||||
let new_tree: BTreeSet<Arc<str>> = playlist.iter().map(|group| Arc::clone(&group.title)).collect();
|
||||
|
||||
let index_filename = format!("{}.groups.bin", utils::sanitize_filename(target_name));
|
||||
let cfg = app_config.config.load();
|
||||
let Some(path) = utils::get_file_path(&cfg.storage_dir, Some(std::path::PathBuf::from(&index_filename))) else {
|
||||
let Some(path) = utils::get_file_path(storage_dir, Some(std::path::PathBuf::from(&index_filename))) else {
|
||||
error!("failed to resolve a path for group index {index_filename}");
|
||||
emit_watch_disabled(events, target_name, "", format!("could not resolve a path for {index_filename}"));
|
||||
return;
|
||||
return GroupChanges::default();
|
||||
};
|
||||
|
||||
// First sight writes the baseline and says nothing. Announcing every
|
||||
// group as new on the first refresh after an upgrade would be noise, not
|
||||
// news.
|
||||
let mut reported = GroupChanges::default();
|
||||
let mut changed = true;
|
||||
if file_exists_async(&path).await {
|
||||
if let Some(loaded_tree) = load_watch_tree(&path).await {
|
||||
let added: BTreeSet<Arc<str>> = new_tree.difference(&loaded_tree).cloned().collect();
|
||||
let removed: BTreeSet<Arc<str>> = loaded_tree.difference(&new_tree).cloned().collect();
|
||||
if added.is_empty() && removed.is_empty() {
|
||||
changed = false;
|
||||
} else {
|
||||
emit_groups_changed(events, target_name, &added, &removed);
|
||||
changed = new_tree != loaded_tree;
|
||||
if changed {
|
||||
reported = group_changes(&new_tree, &loaded_tree, watches);
|
||||
}
|
||||
} else {
|
||||
error!("failed to load group index {}", path.to_str().unwrap_or_default());
|
||||
@@ -74,55 +109,97 @@ pub async fn process_target_groups_watch<E: EventSink>(
|
||||
);
|
||||
}
|
||||
}
|
||||
reported
|
||||
}
|
||||
|
||||
/// Same sampling discipline as the per-group watch: titles only in the lists,
|
||||
/// counts that stay true whatever the lists carry.
|
||||
fn emit_groups_changed<E: EventSink>(
|
||||
events: &E,
|
||||
/// The groups in `current` but not `previous` that a `group_added` entry
|
||||
/// matches, and the reverse for `group_removed`.
|
||||
///
|
||||
/// The set differences are filtered and sampled while they are walked, so no
|
||||
/// intermediate set is built, and a direction no entry asks for is never
|
||||
/// walked at all.
|
||||
pub fn group_changes(
|
||||
current: &BTreeSet<Arc<str>>,
|
||||
previous: &BTreeSet<Arc<str>>,
|
||||
watches: &[ConfigWatch],
|
||||
) -> GroupChanges {
|
||||
let kept = |from: &BTreeSet<Arc<str>>, without: &BTreeSet<Arc<str>>, context: WatchContext| {
|
||||
if !watches.iter().any(|watch| watch.contexts.contains(context)) {
|
||||
return None;
|
||||
}
|
||||
sample_non_empty(from.difference(without).filter(|title| {
|
||||
watches.iter().any(|watch| watch.contexts.contains(context) && watch.pattern.is_match(title))
|
||||
}))
|
||||
};
|
||||
GroupChanges {
|
||||
added: kept(current, previous, WatchContext::GroupAdded),
|
||||
removed: kept(previous, current, WatchContext::GroupRemoved),
|
||||
}
|
||||
}
|
||||
|
||||
/// Turn one refresh's group and channel changes into a single event payload.
|
||||
///
|
||||
/// Order: group additions, group removals, then channel changes by group
|
||||
/// title (added before removed), so notifications and tests do not depend on
|
||||
/// which concurrent group diff finished first. Above the summary threshold,
|
||||
/// summed over every group, channel lists are dropped and only counts remain:
|
||||
/// nobody reads thousands of titles, and channels such as Telegram reject a
|
||||
/// body that large outright.
|
||||
pub fn assemble_watch_changes(
|
||||
target_name: &str,
|
||||
added: &BTreeSet<Arc<str>>,
|
||||
removed: &BTreeSet<Arc<str>>,
|
||||
) {
|
||||
let added_total = added.len();
|
||||
let removed_total = removed.len();
|
||||
let mut added: Vec<String> = added.iter().map(std::string::ToString::to_string).collect();
|
||||
let mut removed: Vec<String> = removed.iter().map(std::string::ToString::to_string).collect();
|
||||
let mut truncated = false;
|
||||
for list in [&mut added, &mut removed] {
|
||||
if list.len() > WATCH_NOTIFICATION_LIST_LIMIT {
|
||||
list.truncate(WATCH_NOTIFICATION_LIST_LIMIT);
|
||||
truncated = true;
|
||||
groups: GroupChanges,
|
||||
mut channel_diffs: Vec<GroupChannelDiff>,
|
||||
) -> Option<WatchChanges> {
|
||||
channel_diffs.sort_unstable_by(|a, b| a.group.cmp(&b.group));
|
||||
let channel_total: usize = channel_diffs
|
||||
.iter()
|
||||
.flat_map(|diff| diff.added.iter().chain(diff.removed.iter()))
|
||||
.map(|items| items.total)
|
||||
.sum();
|
||||
let counts_only = channel_total > WATCH_NOTIFICATION_SUMMARY_THRESHOLD;
|
||||
|
||||
let mut changes = Vec::with_capacity(2 + channel_diffs.len() * 2);
|
||||
changes.extend(groups.added.map(WatchChange::GroupsAdded));
|
||||
changes.extend(groups.removed.map(WatchChange::GroupsRemoved));
|
||||
let listed = |mut items: WatchItems| {
|
||||
if counts_only {
|
||||
items.items = Vec::new();
|
||||
items.truncated = true;
|
||||
}
|
||||
items
|
||||
};
|
||||
for GroupChannelDiff { group, added, removed } in channel_diffs {
|
||||
if let Some(items) = added {
|
||||
changes
|
||||
.push(WatchChange::ChannelsAdded(WatchGroupItems { group: Arc::clone(&group), items: listed(items) }));
|
||||
}
|
||||
if let Some(items) = removed {
|
||||
changes.push(WatchChange::ChannelsRemoved(WatchGroupItems { group, items: listed(items) }));
|
||||
}
|
||||
}
|
||||
|
||||
events.emit(EventMessage::PlaylistGroupsChanged(PlaylistGroupsChanged {
|
||||
target: target_name.to_string(),
|
||||
added,
|
||||
removed,
|
||||
added_total,
|
||||
removed_total,
|
||||
truncated,
|
||||
}));
|
||||
(!changes.is_empty()).then(|| WatchChanges::new(target_name.to_string(), changes))
|
||||
}
|
||||
|
||||
/// Diff one watched group's channels against its baseline.
|
||||
pub async fn process_group_watch<E: EventSink>(
|
||||
app_config: &Arc<AppConfig>,
|
||||
storage_dir: &str,
|
||||
events: &E,
|
||||
target_name: &str,
|
||||
pl: &PlaylistGroup,
|
||||
) {
|
||||
let mut new_tree = BTreeSet::new();
|
||||
pl.channels.iter().for_each(|chan| {
|
||||
let header = &chan.header;
|
||||
let title = if header.title.is_empty() { header.name.clone() } else { header.title.clone() };
|
||||
new_tree.insert(title);
|
||||
});
|
||||
) -> Option<GroupChannelDiff> {
|
||||
let new_tree: BTreeSet<Arc<str>> = pl
|
||||
.channels
|
||||
.iter()
|
||||
.map(|chan| {
|
||||
let header = &chan.header;
|
||||
Arc::clone(if header.title.is_empty() { &header.name } else { &header.title })
|
||||
})
|
||||
.collect();
|
||||
|
||||
let watch_filename =
|
||||
format!("{}/{}.bin", utils::sanitize_filename(target_name), utils::sanitize_filename(&pl.title));
|
||||
let cfg = app_config.config.load();
|
||||
let Some(path) = utils::get_file_path(&cfg.storage_dir, Some(std::path::PathBuf::from(&watch_filename))) else {
|
||||
let Some(path) = utils::get_file_path(storage_dir, Some(std::path::PathBuf::from(&watch_filename))) else {
|
||||
error!("failed to write watch_file {watch_filename}");
|
||||
emit_watch_disabled(
|
||||
events,
|
||||
@@ -130,19 +207,18 @@ pub async fn process_group_watch<E: EventSink>(
|
||||
&pl.title,
|
||||
format!("could not resolve a storage path for {watch_filename}"),
|
||||
);
|
||||
return;
|
||||
return None;
|
||||
};
|
||||
|
||||
let save_path = path.as_path();
|
||||
let mut diff = None;
|
||||
let mut changed = true;
|
||||
if file_exists_async(&path).await {
|
||||
if let Some(loaded_tree) = load_watch_tree(&path).await {
|
||||
// Find elements in set2 but not in set1
|
||||
let added_difference: BTreeSet<Arc<str>> = new_tree.difference(&loaded_tree).cloned().collect();
|
||||
let removed_difference: BTreeSet<Arc<str>> = loaded_tree.difference(&new_tree).cloned().collect();
|
||||
changed = !added_difference.is_empty() || !removed_difference.is_empty();
|
||||
let added = sample_non_empty(new_tree.difference(&loaded_tree));
|
||||
let removed = sample_non_empty(loaded_tree.difference(&new_tree));
|
||||
changed = added.is_some() || removed.is_some();
|
||||
if changed {
|
||||
handle_watch_notification(events, &added_difference, &removed_difference, target_name, &pl.title);
|
||||
diff = Some(GroupChannelDiff { group: Arc::clone(&pl.title), added, removed });
|
||||
}
|
||||
} else {
|
||||
error!("failed to load watch_file {}", path.to_str().unwrap_or_default());
|
||||
@@ -159,16 +235,17 @@ pub async fn process_group_watch<E: EventSink>(
|
||||
}
|
||||
|
||||
if changed {
|
||||
if let Err(err) = save_watch_tree(save_path, &new_tree).await {
|
||||
error!("failed to write watch_file {}: {err}", save_path.to_str().unwrap_or_default());
|
||||
if let Err(err) = save_watch_tree(path.as_path(), &new_tree).await {
|
||||
error!("failed to write watch_file {}: {err}", path.to_str().unwrap_or_default());
|
||||
emit_watch_disabled(
|
||||
events,
|
||||
target_name,
|
||||
&pl.title,
|
||||
format!("could not write watch state at {}: {err}", save_path.to_str().unwrap_or_default()),
|
||||
format!("could not write watch state at {}: {err}", path.to_str().unwrap_or_default()),
|
||||
);
|
||||
}
|
||||
}
|
||||
diff
|
||||
}
|
||||
|
||||
/// Report that a group's watch state could not be read or written.
|
||||
@@ -187,56 +264,6 @@ fn emit_watch_disabled<E: EventSink>(events: &E, target_name: &str, group_name:
|
||||
events.emit(EventMessage::PlaylistWatchDisabled(disabled));
|
||||
}
|
||||
|
||||
/// Turn a group's membership delta into an event.
|
||||
///
|
||||
/// Large changes are sampled rather than listed in full, but the sampling is
|
||||
/// reported in [`WatchChanges::added_total`] / [`WatchChanges::removed_total`]
|
||||
/// and [`WatchChanges::truncated`] rather than written into the lists as prose.
|
||||
fn handle_watch_notification<E: EventSink>(
|
||||
events: &E,
|
||||
added: &BTreeSet<Arc<str>>,
|
||||
removed: &BTreeSet<Arc<str>>,
|
||||
target_name: &str,
|
||||
group_name: &str,
|
||||
) {
|
||||
let added_total = added.len();
|
||||
let removed_total = removed.len();
|
||||
let total_changed = added_total.saturating_add(removed_total);
|
||||
if total_changed == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut added: Vec<String> = added.iter().map(std::string::ToString::to_string).collect();
|
||||
let mut removed: Vec<String> = removed.iter().map(std::string::ToString::to_string).collect();
|
||||
let mut truncated = false;
|
||||
|
||||
if total_changed > WATCH_NOTIFICATION_SUMMARY_THRESHOLD {
|
||||
// A whole-provider reshuffle. Nobody reads ten thousand titles, and
|
||||
// every channel routing this pays for the bytes, so send the counts
|
||||
// alone.
|
||||
added.clear();
|
||||
removed.clear();
|
||||
truncated = true;
|
||||
} else {
|
||||
for list in [&mut added, &mut removed] {
|
||||
if list.len() > WATCH_NOTIFICATION_LIST_LIMIT {
|
||||
list.truncate(WATCH_NOTIFICATION_LIST_LIMIT);
|
||||
truncated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
events.emit(EventMessage::PlaylistWatchChanged(WatchChanges {
|
||||
target: target_name.to_string(),
|
||||
group: group_name.to_string(),
|
||||
added,
|
||||
removed,
|
||||
added_total,
|
||||
removed_total,
|
||||
truncated,
|
||||
}));
|
||||
}
|
||||
|
||||
async fn load_watch_tree(path: &Path) -> Option<BTreeSet<Arc<str>>> {
|
||||
let encoded = tokio::fs::read(path).await.ok()?;
|
||||
binary_deserialize(&encoded[..]).ok()
|
||||
@@ -253,79 +280,115 @@ async fn save_watch_tree(path: &Path, tree: &BTreeSet<Arc<str>>) -> std::io::Res
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{handle_watch_notification, WATCH_NOTIFICATION_LIST_LIMIT, WATCH_NOTIFICATION_SUMMARY_THRESHOLD};
|
||||
use shared::model::{EventMessage, EventSink, WatchChanges};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
sync::{Arc, Mutex},
|
||||
use super::{
|
||||
assemble_watch_changes, group_changes, sample, GroupChanges, GroupChannelDiff, WATCH_NOTIFICATION_LIST_LIMIT,
|
||||
};
|
||||
use shared::model::{ConfigWatchDto, WatchChange, WatchContext, WatchItems};
|
||||
use std::{collections::BTreeSet, sync::Arc};
|
||||
use tuliprox_core::model::ConfigWatch;
|
||||
|
||||
#[derive(Default)]
|
||||
struct CollectSink(Mutex<Vec<WatchChanges>>);
|
||||
fn set(names: &[&str]) -> BTreeSet<Arc<str>> { names.iter().map(|n| Arc::from(*n)).collect() }
|
||||
|
||||
impl EventSink for CollectSink {
|
||||
fn emit(&self, event: EventMessage) {
|
||||
if let EventMessage::PlaylistWatchChanged(changes) = event {
|
||||
self.0.lock().unwrap().push(changes);
|
||||
fn titles(count: usize) -> BTreeSet<Arc<str>> {
|
||||
// Zero-padded so the `BTreeSet` order matches the numeric order.
|
||||
(0..count).map(|i| Arc::from(format!("Channel {i:05}").as_str())).collect()
|
||||
}
|
||||
|
||||
fn watch(pattern: &str, contexts: &[WatchContext]) -> ConfigWatch {
|
||||
ConfigWatch {
|
||||
pattern: Arc::new(regex::Regex::new(pattern).expect("regex")),
|
||||
contexts: ConfigWatchDto::new(pattern, contexts.iter().copied()).on,
|
||||
}
|
||||
}
|
||||
|
||||
fn diff(group: &str, added: usize, removed: usize) -> GroupChannelDiff {
|
||||
GroupChannelDiff {
|
||||
group: Arc::from(group),
|
||||
added: (added > 0).then(|| sample(titles(added).iter())),
|
||||
removed: (removed > 0).then(|| sample(titles(removed).iter())),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_small_list_is_complete() {
|
||||
let items = sample(titles(3).iter());
|
||||
assert_eq!((items.items.len(), items.total, items.truncated), (3, 3, false));
|
||||
}
|
||||
|
||||
/// The lists used to carry a synthesised "... N more entries omitted"
|
||||
/// string beside real channel titles, which a plugin reading the JSON
|
||||
/// payload could not tell from a channel actually named that.
|
||||
#[test]
|
||||
fn a_long_list_keeps_titles_only_and_the_true_total() {
|
||||
let items = sample(titles(WATCH_NOTIFICATION_LIST_LIMIT + 40).iter());
|
||||
assert_eq!(items.items.len(), WATCH_NOTIFICATION_LIST_LIMIT);
|
||||
assert_eq!(items.total, WATCH_NOTIFICATION_LIST_LIMIT + 40);
|
||||
assert!(items.truncated);
|
||||
assert!(items.items.iter().all(|t| t.starts_with("Channel ")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn group_changes_are_filtered_per_context() {
|
||||
let previous = set(&["DE Kids", "Event B", "Stays"]);
|
||||
let current = set(&["Event A", "News 2", "Stays"]);
|
||||
let watches = [watch("^Event", &[WatchContext::GroupAdded]), watch(".*", &[WatchContext::GroupRemoved])];
|
||||
let changes = group_changes(¤t, &previous, &watches);
|
||||
assert_eq!(changes.added.map(|i| i.items), Some(vec![Arc::from("Event A")]));
|
||||
assert_eq!(changes.removed.map(|i| i.items), Some(vec![Arc::from("DE Kids"), Arc::from("Event B")]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn channel_only_watches_report_no_group_changes() {
|
||||
let changes = group_changes(&set(&["X"]), &set(&["Y"]), &[watch(".*", &[WatchContext::Channels])]);
|
||||
assert_eq!(changes, GroupChanges::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_changed_assembles_nothing() {
|
||||
assert!(assemble_watch_changes("t", GroupChanges::default(), Vec::new()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changes_are_ordered_groups_first_then_channels_by_group() {
|
||||
let groups =
|
||||
GroupChanges { added: Some(sample(set(&["New"]).iter())), removed: Some(sample(set(&["Old"]).iter())) };
|
||||
let changes = assemble_watch_changes("t", groups, vec![diff("B", 1, 1), diff("A", 1, 0)]).expect("changes");
|
||||
let kinds: Vec<String> = changes
|
||||
.changes
|
||||
.iter()
|
||||
.map(|c| match c {
|
||||
WatchChange::GroupsAdded(_) => "ga".to_string(),
|
||||
WatchChange::GroupsRemoved(_) => "gr".to_string(),
|
||||
WatchChange::ChannelsAdded(g) => format!("ca:{}", g.group),
|
||||
WatchChange::ChannelsRemoved(g) => format!("cr:{}", g.group),
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(kinds, ["ga", "gr", "ca:A", "ca:B", "cr:B"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_large_aggregate_reports_counts_only_for_channels() {
|
||||
// 20 groups x 30 changes = 600 > WATCH_NOTIFICATION_SUMMARY_THRESHOLD.
|
||||
let diffs = (0..20).map(|i| diff(&format!("G{i:02}"), 30, 0)).collect();
|
||||
let groups = GroupChanges { added: Some(sample(set(&["New"]).iter())), removed: None };
|
||||
let changes = assemble_watch_changes("t", groups, diffs).expect("changes");
|
||||
for change in &changes.changes {
|
||||
match change {
|
||||
WatchChange::ChannelsAdded(g) => {
|
||||
assert!(g.items.items.is_empty() && g.items.truncated && g.items.total == 30);
|
||||
}
|
||||
WatchChange::GroupsAdded(items) => assert_eq!(items, &WatchItems::complete(vec![Arc::from("New")])),
|
||||
other => panic!("unexpected {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn titles(count: usize) -> BTreeSet<Arc<str>> {
|
||||
// Zero-padded so the `BTreeSet` order matches the numeric order, which
|
||||
// keeps the truncation assertions readable.
|
||||
(0..count).map(|i| Arc::from(format!("Channel {i:05}").as_str())).collect()
|
||||
}
|
||||
|
||||
fn emit(added: usize, removed: usize) -> WatchChanges {
|
||||
let sink = CollectSink::default();
|
||||
handle_watch_notification(&sink, &titles(added), &titles(removed), "target", "group");
|
||||
let mut captured = sink.0.lock().unwrap();
|
||||
assert_eq!(captured.len(), 1, "expected exactly one event");
|
||||
captured.pop().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unchanged_group_emits_nothing() {
|
||||
let sink = CollectSink::default();
|
||||
handle_watch_notification(&sink, &BTreeSet::new(), &BTreeSet::new(), "target", "group");
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_small_change_lists_every_title() {
|
||||
let changes = emit(3, 2);
|
||||
assert_eq!(changes.added.len(), 3);
|
||||
assert_eq!(changes.removed.len(), 2);
|
||||
assert_eq!(changes.added_total, 3);
|
||||
assert_eq!(changes.removed_total, 2);
|
||||
assert!(!changes.truncated);
|
||||
}
|
||||
|
||||
/// The regression this whole field set exists for: the lists used to carry
|
||||
/// a synthesised "... N more entries omitted" string beside real channel
|
||||
/// titles, which a plugin reading the JSON payload could not distinguish
|
||||
/// from a channel actually named that.
|
||||
#[test]
|
||||
fn a_truncated_list_carries_only_channel_titles() {
|
||||
let added = WATCH_NOTIFICATION_LIST_LIMIT + 40;
|
||||
let changes = emit(added, 0);
|
||||
|
||||
assert_eq!(changes.added.len(), WATCH_NOTIFICATION_LIST_LIMIT);
|
||||
assert_eq!(changes.added_total, added, "the total must survive the truncation");
|
||||
assert!(changes.truncated);
|
||||
for title in &changes.added {
|
||||
assert!(title.starts_with("Channel "), "list carries prose, not a channel title: {title}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_very_large_change_reports_counts_only() {
|
||||
let added = WATCH_NOTIFICATION_SUMMARY_THRESHOLD + 100;
|
||||
let changes = emit(added, 0);
|
||||
|
||||
assert!(changes.added.is_empty(), "a change this size should not list titles at all");
|
||||
assert_eq!(changes.added_total, added);
|
||||
assert!(changes.truncated);
|
||||
fn a_small_aggregate_keeps_channel_lists() {
|
||||
let changes = assemble_watch_changes("t", GroupChanges::default(), vec![diff("G", 3, 2)]).expect("changes");
|
||||
assert!(changes.changes.iter().all(|c| match c {
|
||||
WatchChange::ChannelsAdded(g) | WatchChange::ChannelsRemoved(g) => !g.items.truncated,
|
||||
_ => false,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::{
|
||||
input_cache,
|
||||
metadata_sink::{MetadataUpdateSink, NoopMetadataSink},
|
||||
parser::xmltv::{flatten_tvguide, merge_epg_trees, EpgMergeAccumulator},
|
||||
playlist_watch::{process_group_watch, process_target_groups_watch},
|
||||
playlist_watch::{assemble_watch_changes, diff_target_groups, process_group_watch},
|
||||
processor::{
|
||||
epg::{
|
||||
adopt_guide_live_epg_id_case, clear_invalid_live_epg_ids, process_playlist_epg,
|
||||
@@ -29,8 +29,8 @@ use shared::{
|
||||
InputRefreshOverride, InputRefreshPolicy, InputStats, InputType, InputUpdateAction, InputUpdateRequest,
|
||||
MappingStage, PipelineStats, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType,
|
||||
PlaylistUpdateInputTelemetry, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder,
|
||||
PlaylistUpdateState, PlaylistUpdateSummary, StreamProperties, TargetStats, UUIDType, WatchDisabled,
|
||||
WatchDisabledReason, WatchUnmatched, XtreamCluster,
|
||||
PlaylistUpdateState, PlaylistUpdateSummary, StreamProperties, TargetStats, UUIDType, WatchContext,
|
||||
WatchDisabled, WatchDisabledReason, WatchUnmatched, XtreamCluster,
|
||||
},
|
||||
utils::{create_alias_uuid, interner_gc, Internable},
|
||||
};
|
||||
|
||||
@@ -1105,34 +1105,48 @@ pub(crate) async fn process_watch<E: EventSink>(
|
||||
return false;
|
||||
}
|
||||
|
||||
// One snapshot for the whole refresh rather than a config load per group.
|
||||
let config = app_config.config.load_full();
|
||||
let storage_dir = config.storage_dir.as_str();
|
||||
|
||||
// Before the per-group fan-out: this is about which groups exist, not
|
||||
// what is inside the ones the patterns name, so it must see every group
|
||||
// rather than only the watched ones.
|
||||
process_target_groups_watch(app_config, events, &target.name, new_playlist).await;
|
||||
let group_changes = diff_target_groups(storage_dir, events, &target.name, new_playlist, watches).await;
|
||||
|
||||
// Only `channels` entries are matched against the current groups: they
|
||||
// pick the groups to diff, and they are the only ones the unmatched
|
||||
// warning reports on (`ConfigWatch::reports_unmatched`). Once a group is
|
||||
// watched, an entry that already matched elsewhere needs no regex run.
|
||||
let mut matched = vec![false; watches.len()];
|
||||
let mut watched_groups = Vec::new();
|
||||
let mut group_watches = Vec::new();
|
||||
for group in new_playlist {
|
||||
let mut any = false;
|
||||
for (index, pattern) in watches.iter().enumerate() {
|
||||
if pattern.is_match(&group.title) {
|
||||
let mut watch_channels = false;
|
||||
for (index, watch) in watches.iter().enumerate() {
|
||||
if !watch.contexts.contains(WatchContext::Channels) || (watch_channels && matched[index]) {
|
||||
continue;
|
||||
}
|
||||
if watch.pattern.is_match(&group.title) {
|
||||
matched[index] = true;
|
||||
any = true;
|
||||
watch_channels = true;
|
||||
}
|
||||
}
|
||||
if any {
|
||||
watched_groups.push(group);
|
||||
if watch_channels {
|
||||
// Futures built up front: a lazy `map` closure over borrowed state
|
||||
// trips the higher-ranked `Send` check of the spawned pipeline task.
|
||||
group_watches.push(process_group_watch(storage_dir, events, &target.name, group));
|
||||
}
|
||||
}
|
||||
|
||||
// A pattern that matches nothing looks exactly like a group that has not
|
||||
// changed. `EventKindMask::from_wire_names` already reports unmatched
|
||||
// subscription names for the same reason: a typo must surface.
|
||||
// A channel pattern that matches nothing looks exactly like a group that
|
||||
// has not changed, so a typo must surface. Entries waiting for a group to
|
||||
// appear, and removal-only entries, match nothing by design.
|
||||
// `EventKindMask::from_wire_names` reports unmatched names for the same reason.
|
||||
let unmatched: Vec<String> = watches
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(index, _)| !matched[*index])
|
||||
.map(|(_, pattern)| pattern.as_str().to_string())
|
||||
.zip(&matched)
|
||||
.filter(|(watch, matched)| !**matched && watch.reports_unmatched())
|
||||
.map(|(watch, _)| watch.pattern.as_str().to_string())
|
||||
.collect();
|
||||
if !unmatched.is_empty() {
|
||||
warn!("target '{}' has {} watch pattern(s) matching no group", target.name, unmatched.len());
|
||||
@@ -1143,11 +1157,12 @@ pub(crate) async fn process_watch<E: EventSink>(
|
||||
)));
|
||||
}
|
||||
|
||||
futures::stream::iter(
|
||||
watched_groups.into_iter().map(|pl| process_group_watch(app_config, events, &target.name, pl)),
|
||||
)
|
||||
.for_each_concurrent(16, |f| f)
|
||||
.await;
|
||||
let channel_diffs: Vec<_> =
|
||||
futures::stream::iter(group_watches).buffer_unordered(16).filter_map(futures::future::ready).collect().await;
|
||||
|
||||
if let Some(changes) = assemble_watch_changes(&target.name, group_changes, channel_diffs) {
|
||||
events.emit(EventMessage::PlaylistWatchChanged(changes));
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ mod mapping_stage;
|
||||
|
||||
mod curation_effect_gate;
|
||||
|
||||
mod watch_contexts;
|
||||
|
||||
#[cfg(test)]
|
||||
mod quality_rejection_fallback;
|
||||
|
||||
|
||||
@@ -181,7 +181,7 @@ async fn assert_curation_preserves_empty_mixed_output_guard(canonical: Option<sh
|
||||
..M3uTargetOutputDto::default()
|
||||
}),
|
||||
],
|
||||
watch: Some(vec![".*".to_string()]),
|
||||
watch: Some(vec![shared::model::ConfigWatchDto::channels(".*")]),
|
||||
use_memory_cache: true,
|
||||
..ConfigTargetDto::default()
|
||||
});
|
||||
@@ -282,7 +282,7 @@ async fn complete_empty_curation_publishes_empty_watch_group_state() {
|
||||
}),
|
||||
..XtreamTargetOutputDto::default()
|
||||
})],
|
||||
watch: Some(vec![".*".to_string()]),
|
||||
watch: Some(vec![shared::model::ConfigWatchDto::channels(".*")]),
|
||||
..ConfigTargetDto::default()
|
||||
});
|
||||
let mut seeded = vec![PlaylistGroup {
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
// `use super::*` brings `process_watch` into scope the same way `curation_effect_gate.rs` gets it.
|
||||
use super::{curation_effect_gate::app_config, support::PlaylistRunCollectSink, *};
|
||||
use shared::model::{ConfigTargetDto, ConfigWatchDto, WatchChange, WatchContext};
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn group(title: &str, channels: &[&str]) -> PlaylistGroup {
|
||||
PlaylistGroup {
|
||||
id: 1,
|
||||
title: title.intern(),
|
||||
xtream_cluster: XtreamCluster::Live,
|
||||
channels: channels
|
||||
.iter()
|
||||
.map(|name| PlaylistItem {
|
||||
header: PlaylistItemHeader { name: name.intern(), title: name.intern(), ..Default::default() },
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn target(watch: Vec<ConfigWatchDto>) -> ConfigTarget {
|
||||
ConfigTarget::from(&ConfigTargetDto { name: "sports".to_string(), watch: Some(watch), ..Default::default() })
|
||||
}
|
||||
|
||||
fn watch_events(sink: &PlaylistRunCollectSink) -> Vec<EventMessage> {
|
||||
sink.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.drain(..)
|
||||
.filter(|e| {
|
||||
matches!(
|
||||
e,
|
||||
EventMessage::PlaylistWatchChanged(_)
|
||||
| EventMessage::PlaylistWatchUnmatched(_)
|
||||
| EventMessage::PlaylistWatchDisabled(_)
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_run_is_silent_then_one_event_carries_groups_and_channels() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let config = app_config(dir.path());
|
||||
let target = target(vec![
|
||||
ConfigWatchDto::channels("^DE Sport"),
|
||||
ConfigWatchDto::new("^Event", [WatchContext::GroupAdded]),
|
||||
ConfigWatchDto::new(".*", [WatchContext::GroupRemoved]),
|
||||
]);
|
||||
let sink = PlaylistRunCollectSink::default();
|
||||
|
||||
let first = vec![group("DE Sport HD", &["Sky 1"]), group("DE Kids", &["KiKa"])];
|
||||
assert!(process_watch(&config, &sink, &target, &first).await);
|
||||
assert!(watch_events(&sink).is_empty(), "first run must only write baselines");
|
||||
|
||||
let second =
|
||||
vec![group("DE Sport HD", &["Sky 1", "Sky 5"]), group("Event Bundesliga", &["E1"]), group("News 2", &[])];
|
||||
assert!(process_watch(&config, &sink, &target, &second).await);
|
||||
let events = watch_events(&sink);
|
||||
assert_eq!(events.len(), 1, "{events:?}");
|
||||
let EventMessage::PlaylistWatchChanged(changes) = &events[0] else { panic!("{events:?}") };
|
||||
assert_eq!(changes.target, "sports");
|
||||
assert!(
|
||||
matches!(&changes.changes[0], WatchChange::GroupsAdded(i) if i.items == vec![Arc::from("Event Bundesliga")])
|
||||
);
|
||||
assert!(matches!(&changes.changes[1], WatchChange::GroupsRemoved(i) if i.items == vec![Arc::from("DE Kids")]));
|
||||
assert!(matches!(&changes.changes[2], WatchChange::ChannelsAdded(g)
|
||||
if g.group.as_ref() == "DE Sport HD" && g.items.items == vec![Arc::from("Sky 5")]));
|
||||
assert_eq!(changes.changes.len(), 3, "News 2 matches no group_added entry");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn channel_only_watch_reports_no_group_changes() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let config = app_config(dir.path());
|
||||
let target = target(vec![ConfigWatchDto::channels(".*")]);
|
||||
let sink = PlaylistRunCollectSink::default();
|
||||
assert!(process_watch(&config, &sink, &target, &[group("A", &["a"])]).await);
|
||||
assert!(process_watch(&config, &sink, &target, &[group("A", &["a"]), group("B", &["b"])]).await);
|
||||
assert!(watch_events(&sink).is_empty(), "a new group is only a new channel baseline here");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unmatched_warning_only_for_channel_entries_without_group_added() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let config = app_config(dir.path());
|
||||
let target = target(vec![
|
||||
ConfigWatchDto::channels("^Nope1"),
|
||||
ConfigWatchDto::new("^Nope2", [WatchContext::Channels, WatchContext::GroupRemoved]),
|
||||
ConfigWatchDto::new("^Nope3", [WatchContext::Channels, WatchContext::GroupAdded]),
|
||||
ConfigWatchDto::new("^Nope4", [WatchContext::GroupAdded]),
|
||||
ConfigWatchDto::new("^Nope5", [WatchContext::GroupRemoved]),
|
||||
]);
|
||||
let sink = PlaylistRunCollectSink::default();
|
||||
assert!(process_watch(&config, &sink, &target, &[group("A", &["a"])]).await);
|
||||
let events = watch_events(&sink);
|
||||
let EventMessage::PlaylistWatchUnmatched(unmatched) = &events[0] else { panic!("{events:?}") };
|
||||
assert_eq!(unmatched.patterns, vec!["^Nope1".to_string(), "^Nope2".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_group_index_still_reports_channel_changes() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let config = app_config(dir.path());
|
||||
let target = target(vec![ConfigWatchDto::new(".*", [WatchContext::Channels, WatchContext::GroupAdded])]);
|
||||
let sink = PlaylistRunCollectSink::default();
|
||||
assert!(process_watch(&config, &sink, &target, &[group("A", &["a"])]).await);
|
||||
std::fs::write(dir.path().join("sports.groups.bin"), [0xff_u8; 3]).expect("corrupt index");
|
||||
|
||||
assert!(process_watch(&config, &sink, &target, &[group("A", &["a", "b"]), group("B", &["x"])]).await);
|
||||
let events = watch_events(&sink);
|
||||
assert!(events.iter().any(|e| matches!(e, EventMessage::PlaylistWatchDisabled(_))), "{events:?}");
|
||||
let changes = events
|
||||
.iter()
|
||||
.find_map(|e| if let EventMessage::PlaylistWatchChanged(c) = e { Some(c) } else { None })
|
||||
.expect("channel change still reported");
|
||||
assert!(changes.changes.iter().all(|c| matches!(c, WatchChange::ChannelsAdded(_))), "no group change: {changes:?}");
|
||||
|
||||
// The index was rewritten, so the next refresh diffs normally.
|
||||
let third = [group("A", &["a", "b"]), group("B", &["x"]), group("C", &[])];
|
||||
assert!(process_watch(&config, &sink, &target, &third).await);
|
||||
let events = watch_events(&sink);
|
||||
assert!(
|
||||
events.iter().any(|e| matches!(e, EventMessage::PlaylistWatchChanged(c)
|
||||
if matches!(&c.changes[0], WatchChange::GroupsAdded(i) if i.items == vec![Arc::from("C")]))),
|
||||
"{events:?}"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user