diff --git a/CHANGELOG.md b/CHANGELOG.md index 370d3a755..018bb79e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ epg_url: ['file:///${env:M3U_FILTER_HOME}/epg.xml', 'file:///${env:M3U_FILTER_HO # multi url epg epg_url: ['http://localhost:3001/xmltv.php?epg_id=1', 'http://localhost:3001/xmltv.php?epg_id=2'] ``` +- added `epg_strip` to input for auto epg matching, if not given `["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]` is default +When no matching epg_id is found, the display name is used to match a channel name. The given strings are stripped to get a better match. # 2.2.5 (2025-03-27) - fixed web ui playlist regexp search diff --git a/Cargo.lock b/Cargo.lock index 37a2d0dcb..898923d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,9 +165,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ "axum-core", "axum-macros", @@ -200,12 +200,12 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", - "futures-util", + "futures-core", "http 1.3.1", "http-body", "http-body-util", @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" +checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" dependencies = [ "arrayref", "arrayvec", @@ -362,9 +362,9 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" [[package]] name = "cc" -version = "1.2.17" +version = "1.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" dependencies = [ "jobserver", "libc", @@ -399,9 +399,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.32" +version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" +checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" dependencies = [ "clap_builder", "clap_derive", @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.32" +version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" +checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" dependencies = [ "anstream", "anstyle", @@ -552,9 +552,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -562,9 +562,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", @@ -576,9 +576,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", @@ -601,9 +601,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] @@ -707,9 +707,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -726,9 +726,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -769,9 +769,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", "miniz_oxide", @@ -1150,9 +1150,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", @@ -1160,6 +1160,7 @@ dependencies = [ "http 1.3.1", "http-body", "hyper", + "libc", "pin-project-lite", "socket2", "tokio", @@ -1169,9 +1170,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.62" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1232,9 +1233,9 @@ dependencies = [ [[package]] name = "icu_locid_transform_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" [[package]] name = "icu_normalizer" @@ -1256,9 +1257,9 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" [[package]] name = "icu_properties" @@ -1277,9 +1278,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" [[package]] name = "icu_provider" @@ -1338,9 +1339,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1404,9 +1405,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" +checksum = "1f33145a5cbea837164362c7bd596106eb7c5198f97d1ba6f6ebb3223952e488" dependencies = [ "jiff-static", "log", @@ -1417,9 +1418,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" +checksum = "43ce13c40ec6956157a3635d97a1ee2df323b263f09ea14165131289cb0f5c19" dependencies = [ "proc-macro2", "quote", @@ -1428,10 +1429,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -1501,9 +1503,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" @@ -1564,6 +1566,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "strsim", "tempfile", "tokio", "tokio-stream", @@ -1607,9 +1610,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -1714,15 +1717,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.1" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -1752,18 +1755,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.4.2+3.4.1" +version = "300.5.0+3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2" +checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" dependencies = [ "cc", "libc", @@ -1984,9 +1987,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.3" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf763ab1c7a3aa408be466efc86efe35ed1bd3dd74173ed39d6b0d0a6f0ba148" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" dependencies = [ "memchr", ] @@ -2101,9 +2104,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags 2.9.0", ] @@ -2262,9 +2265,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags 2.9.0", "errno", @@ -2275,9 +2278,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ "once_cell", "ring", @@ -2493,15 +2496,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2699,9 +2702,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -2975,9 +2978,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.0.4" +version = "9.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d2f179f8075b805a43a2a21728a46f0cc2921b3c58695b28fa8817e103cd9a" +checksum = "6b2bf58be11fc9414104c6d3a2e464163db5ef74b12296bda593cac37b6e4777" dependencies = [ "anyhow", "derive_builder", @@ -3176,11 +3179,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.52.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-targets 0.52.6", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -3196,7 +3225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -3218,6 +3247,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 7ab0c59bb..4b5ee2c51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,8 @@ paste = "1.0" tempfile = "3.19" ruzstd = "0" filetime = "0.2" -zeroize = "1.8.1" +zeroize = "1.8" +strsim = "0.11" #[cfg(target_os = "macos")] libc = "0" #[cfg(target_os = "windows")] diff --git a/src/model/config.rs b/src/model/config.rs index d443f6757..d156e93fd 100644 --- a/src/model/config.rs +++ b/src/model/config.rs @@ -777,6 +777,8 @@ pub struct ConfigInput { #[serde(default, skip_serializing_if = "Option::is_none")] pub epg_url: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub epg_strip: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub username: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub password: Option, diff --git a/src/processing/parser/xmltv.rs b/src/processing/parser/xmltv.rs index 075dbbe8f..354a1bee1 100644 --- a/src/processing/parser/xmltv.rs +++ b/src/processing/parser/xmltv.rs @@ -8,20 +8,18 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, LazyLock}; use deunicode::deunicode; +use crate::processing::processor::playlist::EpgIdCache; static NORMALIZE_CHANNEL: LazyLock = LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9\-]").unwrap()); -// TODO into config -const TERMS_TO_REMOVE: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]; - -pub fn normalize_channel_name(name: &str) -> String { +pub fn normalize_channel_name(name: &str, terms_to_remove: &[&str]) -> String { let normalized = deunicode(name).to_lowercase(); // Remove all non-alphanumeric characters (except dashes and underscores). let cleaned_name = NORMALIZE_CHANNEL.replace_all(&normalized, ""); // Remove terms like resolution - let result = TERMS_TO_REMOVE.iter().fold(cleaned_name.to_string(), |acc, term| { + let result = terms_to_remove.iter().fold(cleaned_name.to_string(), |acc, term| { acc.replace(*term, "") }); @@ -41,8 +39,8 @@ impl TVGuide { }) } - fn process_epg_file(epg_channel_ids: &mut HashSet>, normalized_epg_channel_ids: &mut HashMap, Option>>, - processed_epg_channel_ids: &mut HashSet, epg_file: &Path) -> Option { + fn process_epg_file(id_cache: &mut EpgIdCache, epg_file: &Path) -> Option { + let strip = id_cache.strip.clone(); match CompressedFileReader::new(epg_file) { Ok(mut reader) => { let mut children: Vec = vec![]; @@ -51,19 +49,19 @@ impl TVGuide { match tag.name.as_str() { EPG_TAG_CHANNEL => { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { - if !processed_epg_channel_ids.contains(epg_id) { + if !id_cache.processed.contains(epg_id) { let id: Cow = Cow::Owned(epg_id.to_string()); for normalized_epg_id in &tag.normalized_epg_ids { let key = Cow::Owned(normalized_epg_id.to_string()); - match normalized_epg_channel_ids.entry(key) { + match id_cache.normalized.entry(key) { std::collections::hash_map::Entry::Occupied(mut entry) => { entry.insert(Some(id.clone())); - epg_channel_ids.insert(id.clone()); + id_cache.channel.insert(id.clone()); } std::collections::hash_map::Entry::Vacant(_entry) => {} } } - if epg_channel_ids.contains(&id) { + if id_cache.channel.contains(&id) { children.push(tag); } } @@ -71,9 +69,9 @@ impl TVGuide { } EPG_TAG_PROGRAMME => { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_CHANNEL) { - if !processed_epg_channel_ids.contains(epg_id) { + if !id_cache.processed.contains(epg_id) { let borrowed_epg_id = Cow::Borrowed(epg_id.as_str()); - if epg_channel_ids.contains(&borrowed_epg_id) { + if id_cache.channel.contains(&borrowed_epg_id) { children.push(tag); } } @@ -86,7 +84,7 @@ impl TVGuide { } }; - parse_tvguide(&mut reader, &mut filter_tags); + parse_tvguide(&mut reader, &mut filter_tags, &strip); if children.is_empty() { return None; @@ -94,7 +92,7 @@ impl TVGuide { children.iter().filter(|tag| tag.name == EPG_TAG_CHANNEL).for_each(|tag| { if let Some(epg_id) = tag.get_attribute_value(EPG_ATTRIB_ID) { - processed_epg_channel_ids.insert(epg_id.to_string()); + id_cache.processed.insert(epg_id.to_string()); } }); @@ -107,13 +105,12 @@ impl TVGuide { } } - pub fn filter(&self, epg_channel_ids: &mut HashSet>, normalized_epg_channel_ids: &mut HashMap, Option>>) -> Option { - if epg_channel_ids.is_empty() && normalized_epg_channel_ids.is_empty() { + pub fn filter(&self, id_cache: &mut EpgIdCache) -> Option { + if id_cache.channel.is_empty() && id_cache.normalized.is_empty() { return None; } - let mut processed_epg_ids: HashSet = HashSet::new(); let epgs: Vec = self.file_paths.iter() - .filter_map(|path| Self::process_epg_file(epg_channel_ids, normalized_epg_channel_ids, &mut processed_epg_ids, path)) + .filter_map(|path| Self::process_epg_file(id_cache, path)) .collect(); if epgs.len() == 1 { epgs.into_iter().next() @@ -123,7 +120,7 @@ impl TVGuide { } } -pub fn parse_tvguide(content: R, callback: &mut F) +pub fn parse_tvguide(content: R, callback: &mut F, strip: &[&str]) where R: std::io::BufRead, F: FnMut(XmlTag), @@ -163,7 +160,7 @@ where match child.name.as_str() { EPG_TAG_DISPLAY_NAME => { if let Some(name) = &child.value { - tag.normalized_epg_ids.insert(normalize_channel_name(name)); + tag.normalized_epg_ids.insert(normalize_channel_name(name, strip)); } } EPG_TAG_ICON => { diff --git a/src/processing/processor/playlist.rs b/src/processing/processor/playlist.rs index 7c3a718cb..51aab48a3 100644 --- a/src/processing/processor/playlist.rs +++ b/src/processing/processor/playlist.rs @@ -500,6 +500,35 @@ fn flatten_groups(playlistgroups: Vec) -> Vec { sort_order } +pub struct EpgIdCache<'a > { + pub channel: HashSet>, + pub normalized: HashMap, Option>>, + pub processed: HashSet, + pub strip: Vec<&'a str>, +} + + +const TERMS_TO_REMOVE: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"]; + +impl<'a> EpgIdCache<'a> { + pub fn new(strip: Option<&'a Vec>) -> Self { + let str_refs: Vec<&'a str> = match strip { + Some(vec) => vec.iter().map(std::string::String::as_str).collect(), + None => TERMS_TO_REMOVE.to_vec(), + }; + EpgIdCache { + channel: HashSet::new(), + normalized: HashMap::new(), + processed: HashSet::new(), + strip: str_refs, + } + } + + pub fn normalize(&self, name: &str) -> String { + normalize_channel_name(name, &self.strip) + } +} + async fn process_playlist_for_target(client: Arc, playlists: &mut [FetchedPlaylist<'_>], target: &ConfigTarget, @@ -535,29 +564,28 @@ async fn process_playlist_for_target(client: Arc, // we need to process each input epg. for mut fp in processed_fetched_playlists { // collect all epg_channel ids - let mut epg_channel_ids: HashSet> = HashSet::new(); - let mut normalized_epg_channel_ids: HashMap, Option>> = HashMap::new(); + let mut id_cache = EpgIdCache::new(fp.input.epg_strip.as_ref()); for channel in fp.playlistgroups.iter().flat_map(|g| &g.channels) { match channel.header.epg_channel_id.as_ref() { None => { - normalized_epg_channel_ids.insert(Cow::Owned(normalize_channel_name(&channel.header.name)), None); + id_cache.normalized.insert(Cow::Owned(id_cache.normalize(&channel.header.name)), None); }, Some(epg_id) => { if epg_id.is_empty() { - normalized_epg_channel_ids.insert(Cow::Owned(normalize_channel_name(&channel.header.name)), None); + id_cache.normalized.insert(Cow::Owned(id_cache.normalize(&channel.header.name)), None); } else { - epg_channel_ids.insert(Cow::Owned(epg_id.to_string())); + id_cache.channel.insert(Cow::Owned(epg_id.to_string())); } }, } } // let epg_channel_ids: HashSet<_> = fp.playlistgroups.iter().flat_map(|g| &g.channels) // .filter_map(|c| c.header.epg_channel_id.as_ref()).map(|a| a.as_str()).collect(); - if epg_channel_ids.is_empty() && normalized_epg_channel_ids.is_empty() { + if id_cache.channel.is_empty() && id_cache.normalized.is_empty() { debug!("channel ids are empty"); } else if let Some(tv_guide) = fp.epg { debug_if_enabled!("found epg information for {}", &target.name); - if let Some(epg) = tv_guide.filter(&mut epg_channel_ids, &mut normalized_epg_channel_ids) { + if let Some(epg) = tv_guide.filter(&mut id_cache) { let epg_icons: HashMap<&String, &String> = epg.children.iter() .filter(|tag| tag.icon.is_some() && tag.get_attribute_value(EPG_ATTRIB_ID).is_some()) .map(|t| (t.get_attribute_value(EPG_ATTRIB_ID).unwrap(), t.icon.as_ref().unwrap())).collect(); @@ -567,8 +595,8 @@ async fn process_playlist_for_target(client: Arc, .filter(|c| c.header.epg_channel_id.is_none() || c.header.logo.is_empty() || c.header.logo_small.is_empty()) .for_each(|c| { if c.header.epg_channel_id.as_ref().is_none() { - let normalized = normalize_channel_name(&c.header.name); - if let Some(Some(epg_id)) = normalized_epg_channel_ids.get(&Cow::Borrowed(normalized.as_str())) { + let normalized = id_cache.normalize(&c.header.name); + if let Some(Some(epg_id)) = id_cache.normalized.get(&Cow::Borrowed(normalized.as_str())) { c.header.epg_channel_id = Some(epg_id.to_string()); } }