diff --git a/README.md b/README.md index 01e432694..a43d274da 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,6 @@ services: image: ghcr.io/euzu/tuliprox-alpine:latest working_dir: /app volumes: - - /home/tuliprox/tuliprox:/app/tuliprox - /home/tuliprox/config:/app/config - /home/tuliprox/data:/app/data - /home/tuliprox/cache:/app/cache @@ -108,12 +107,31 @@ The detailed documentation lives in Markdown under `docs/` and is meant to be re Main entry points: -- [`Getting Started`](docs/src/getting-started.md) -- [`Core Features`](docs/src/features.md) -- [`Config Reference`](docs/src/configuration/main-config.md) -- [`Sources And Targets`](docs/src/configuration/sources-and-targets.md) -- [`API Proxy`](docs/src/configuration/api-proxy.md) -- [`Streaming And Proxy Behavior`](docs/src/streaming-and-proxy.md) -- [`Mapping And Templates`](docs/src/mapping-and-templates.md) -- [`Deployment`](docs/src/deployment.md) -- [`Examples And Recipes`](docs/src/examples-and-recipes.md) +- **[Getting Started](docs/src/getting-started.md)** +- [Core Features](docs/src/features.md) +- [Build & Deploy](docs/src/build-and-deploy.md) +- **[Installation](docs/src/installation.md)** +- **[Configuration Overview](docs/src/configuration/overview.md)** + - [Main Config](docs/src/configuration/config.md) + - [Sources & Targets](docs/src/configuration/source.md) + - [API Proxy](docs/src/configuration/api-proxy.md) + - [Streaming & Proxy Behavior](docs/src/configuration/reverse-proxy.md) + - [Mapping & Templates](docs/src/configuration/template.md) +- [Examples & Recipes](docs/src/examples-recipes.md) +- [Operations & Debugging](docs/src/operations-debugging.md) +- [Troubleshooting & Resilience](docs/src/troubleshooting.md) + +## Documentation strategy + +The recommended format is: + +- source in Markdown +- generated as static HTML +- shipped together with the frontend/web root + +For this repository, `mdBook` is the best fit: + +- Markdown stays easy to edit in Git +- static HTML output is simple to host +- it fits a Rust project better than a Node-heavy doc stack +- navigation and search come out of the box diff --git a/backend/src/main.rs b/backend/src/main.rs index ae0901cb8..51eada6dc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -22,7 +22,9 @@ use crate::{ model::{AppConfig, Config, Healthcheck, HealthcheckConfig, ProcessTargets, SourcesConfig}, processing::processor::exec_processing, repository::run_startup_migrations, - utils::{config_file_reader, db_viewer, init_logger, request::create_client, resolve_env_var}, + utils::{ + config_file_reader, db_viewer, init_bootstrap_logger, init_logger, request::create_client, resolve_env_var, + }, }; use arc_swap::{access::Access, ArcSwap}; use chrono::{DateTime, Utc}; @@ -143,6 +145,12 @@ const BUILD_TIMESTAMP: &str = env!("VERGEN_BUILD_TIMESTAMP"); async fn main() { let args = Args::parse(); + // Initialize a minimal stdout logger immediately so that any error that + // occurs before `init_logger` (e.g. during path resolution) is visible. + // `init_logger` below will attempt a second `try_init` which silently + // fails; the format and module filters set here remain in effect. + init_bootstrap_logger(args.log_level.as_deref()); + db_viewer(&args.db_viewer_args()); if args.genpwd { @@ -310,7 +318,9 @@ fn get_file_paths(args: &Args) -> ConfigPaths { let storage_path = if Path::new(&config_file).exists() { match utils::read_config_file(&config_file, true, false) { Ok(cfg) => resolve_storage_path(&home_path, cfg.storage_dir.as_deref()), - Err(err) => exit!("Can't read config file {} while resolving storage path: {err}", config_file), + Err(err) => { + exit!("Can't read config file {config_file} while resolving storage path: {err}") + } } } else { resolve_storage_path(&home_path, None) diff --git a/backend/src/utils/db_viewer.rs b/backend/src/utils/db_viewer.rs index a1ef11998..3afa99f6e 100644 --- a/backend/src/utils/db_viewer.rs +++ b/backend/src/utils/db_viewer.rs @@ -1,7 +1,6 @@ use crate::api::model::{MetadataRetryDbKey, MetadataRetryDbValue}; use crate::repository::{BPlusTreeDiskIterator, BPlusTreeQuery, VirtualIdRecord}; -use env_logger::{Builder, Target}; -use log::{error, LevelFilter}; +use log::error; use serde::{Deserialize, Serialize}; use shared::model::{EpgChannel, M3uPlaylistItem, XtreamPlaylistItem}; use std::io::Write; @@ -76,8 +75,6 @@ pub fn db_viewer(args: &DbViewerArgs<'_>) { return; } - init_db_viewer_logger(); - let mut any_processed = false; for request in requests { if let Some(filename) = request.filename { @@ -93,12 +90,6 @@ pub fn db_viewer(args: &DbViewerArgs<'_>) { } } -fn init_db_viewer_logger() { - let mut log_builder = Builder::from_default_env(); - log_builder.target(Target::Stderr); - log_builder.filter_level(LevelFilter::Info); - let _ = log_builder.try_init(); -} fn try_dump_typed_db(path: &Path) -> bool where diff --git a/backend/src/utils/logging.rs b/backend/src/utils/logging.rs index 40e424cde..b4dc05044 100644 --- a/backend/src/utils/logging.rs +++ b/backend/src/utils/logging.rs @@ -27,6 +27,37 @@ fn get_log_level(log_level: &str) -> LevelFilter { } } +fn apply_log_format(builder: &mut Builder) { + builder.format(|buf, record| { + let now = Local::now(); + let timestamp = now.to_rfc3339_opts(SecondsFormat::Secs, now.offset().fix().local_minus_utc() == 0); + writeln!(buf, "[{timestamp} {} {}] {}", record.level(), record.target(), record.args()) + }); +} + +/// Initializes a minimal stdout logger early in startup so that errors before +/// `init_logger` is called (e.g. during path resolution) are visible. +/// Reads log level from the CLI argument and the `TULIPROX_LOG` env var only — +/// config-file log level is not available yet at this point. +/// `init_logger` will attempt a second `try_init` which silently fails; +/// the format and module filters set here remain active. +pub fn init_bootstrap_logger(user_log_level: Option<&str>) { + let env_log_level = std::env::var("TULIPROX_LOG").ok(); + let log_level = user_log_level + .map(std::string::ToString::to_string) + .or(env_log_level) + .unwrap_or_else(|| "info".to_string()); + + let mut log_builder = Builder::from_default_env(); + log_builder.target(Target::Stdout); + apply_log_format(&mut log_builder); + log_builder.filter_level(get_log_level(&log_level)); + for module in LOG_ERROR_LEVEL_MOD { + log_builder.filter_module(module, LevelFilter::Error); + } + let _ = log_builder.try_init(); +} + pub fn init_logger(user_log_level: Option<&str>, config_file: &str) { @@ -40,17 +71,7 @@ pub fn init_logger(user_log_level: Option<&str>, config_file: &str) { let mut log_builder = Builder::from_default_env(); log_builder.target(Target::Stdout); - log_builder.format(move |buf, record| { - let now = Local::now(); - let timestamp = now.to_rfc3339_opts(SecondsFormat::Secs, now.offset().fix().local_minus_utc() == 0); - writeln!( - buf, - "[{timestamp} {} {}] {}", - record.level(), - record.target(), - record.args() - ) - }); + apply_log_format(&mut log_builder); // priority CLI-Argument, Env-Var, Config, Default let log_level = user_log_level diff --git a/config/README.md b/config/README.md index e76b50795..4a59aa550 100644 --- a/config/README.md +++ b/config/README.md @@ -12,4 +12,4 @@ Generate password hashes with `tuliprox --genpwd`. ## RBAC groups The optional `groups.txt` file defines permission groups in `group_name:permission1,permission2,...` format. -See the [Config Reference](docs/src/configuration/main-config.md) for the full permission list and file format. +See the [Config Reference](docs/src/configuration/config.md) for the full permission list and file format. diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a091d137d..5442564ba 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -1,13 +1,18 @@ # Summary -- [Introduction](index.md) -- [Getting Started](getting-started.md) -- [Core Features](features.md) -- [Configuration](configuration/overview.md) - - [Config Reference](configuration/main-config.md) - - [Sources And Targets](configuration/sources-and-targets.md) - - [API Proxy](configuration/api-proxy.md) -- [Streaming And Proxy](streaming-and-proxy.md) -- [Mapping And Templates](mapping-and-templates.md) -- [Deployment](deployment.md) -- [Examples And Recipes](examples-and-recipes.md) +- [Introduction & Architecture](./index.md) +- [Quick-Start](./getting-started.md) +- [Build & Deploy](./build-and-deploy.md) +- [Installation](./installation.md) +- [Configuration & Setup (Overview)](./configuration/overview.md) + - [config.yml (Core System)](./configuration/config.md) + - [Streaming & Proxy Behavior](./configuration/reverse-proxy.md) + - [Metadata Updater & FFprobe](./configuration/metadata-update.md) + - [Local Media Library](./configuration/local-library.md) + - [source.yml (Inputs, Panel API & Targets)](./configuration/source.md) + - [api-proxy.yml (Server, Users & RBAC)](./configuration/api-proxy.md) + - [template.yml (Macros & Regex)](./configuration/template.md) + - [mapping.yml (Mapper DSL & Logic)](./configuration/mapping-dsl.md) +- [Operations & Debugging (CLI & DB Dumps)](./operations-debugging.md) +- [Examples,Recipes & Ecosystem Stacks](./examples-recipes.md) +- [Troubleshooting & Resilience](./troubleshooting.md) diff --git a/docs/src/build-and-deploy.md b/docs/src/build-and-deploy.md new file mode 100644 index 000000000..36d25219e --- /dev/null +++ b/docs/src/build-and-deploy.md @@ -0,0 +1,235 @@ +# 🛠️ Build & Deploy (For Professionals) + +This guide covers advanced topics for developers and professionals who want to compile Tuliprox from source code, generate the documentation locally, +build custom Docker images, or deploy full ecosystem stacks. + +## System Architecture + +Tuliprox is built for extreme performance and modularity, consisting of three core components: + +* **Rust Backend:** A high-performance, asynchronous engine handling stream brokering, metadata enrichment, and playlist processing. +* **Yew/WebAssembly Frontend:** A reactive management interface compiled to highly optimized WASM. +* **Static Assets:** Documentation and UI assets served directly from the configured web root. + +The repository ships with various helper scripts located under `bin/` to simplify cross-platform toolchain setups: + +* `bin/build_docs.sh`: Script to handle mdBook generation. +* `bin/build_fe.sh`: The shared entry point for building docs plus frontend assets. +* `bin/build_local.sh`: Compiles the backend locally. +* `bin/build_docker.sh`: Executes the multi-stage Docker build pipeline. +* `bin/release.sh`: Orchestrates a full production release build. + +This project uses a `Makefile` to automate development tasks, tool installations, and build processes. This ensures a consistent +environment across different machines. + +**Tool installations:** + +* `make install-tools`: Full Setup: Installs Rustup, Cross, Trunk, wasm-bindgen, cargo-edit, mdBook, and markdownlint. +* `make rustup`: Installs the Rust toolchain and Cargo. +* `make cross`: Installs cross for multi-platform compilation. +* `make trunk`: Installs trunk for managing frontend builds. +* `make wasm-bindgen`: Installs the CLI tool for WebAssembly bindings. +* `make cargo-set-version`: Installs cargo-edit for version management. +* `make mdbook`: Installs mdBook for documentation generation. +* `make markdownlint`: Installs markdownlint-cli2 (requires Node.js/npm). + +**Testing & Linting**: + +* `make test`: Runs all workspace tests using the Stable toolchain. +* `make lint`: Runs clippy to find common mistakes and improve code quality. +* `make lint-fix`: Automatically applies clippy suggestions (where possible). +* `make markdown-lint`: Checks all .md files for formatting consistency. + +**Formatting**: + +We use specific Nightly rules to ensure the code style remains consistent across the project. + +* `make fmt`: Formats all code in the workspace. +* `make fmt-check`: Verifies if the code is correctly formatted (used in CI). + +--- + +## 1. Documentation Delivery & Generation + +Tuliprox embeds its own documentation directly into the Web UI. The documentation workflow is completely Markdown-based to fit naturally into a Rust +project, avoiding the overhead of a full Node/React documentation stack. + +**How it works:** + +1. Write docs as Markdown in `docs/src`. +2. Generate static HTML with `mdBook` into `frontend/build/docs`. +3. Copy the generated site into `frontend/dist/static/docs` during the web build. + +This gives you editable source files in Git without committing hand-written HTML. + +### Main Build Commands (`make`) + +Ensure you have `mdBook` installed (`cargo install mdbook`). + +| Command | Purpose | +| :--- | :--- | +| `make docs` | Builds only the static documentation via mdBook. | +| `make docs-serve` | Serves the documentation locally on a dev port with live-reload. | +| `make web-dist` | Builds the frontend WASM app (via Trunk) and the docs together. Copies the docs into the frontend dist folder automatically. | + +--- + +## 2. Building the Frontend + +### WASM Optimization (`wasm-opt`) + +Wasm optimization is handled by Trunk during the build (configured via `data-wasm-opt` in `index.html`). Trunk requires a compatible `wasm-opt` +binary in your system `PATH`. + +**Recommended local setup script sequence:** + +```bash +# Installs a specific version of wasm-tools locally in the repo +./bin/install_wasm_tools.sh 128 + +# Add it to the PATH +export PATH="$PWD/.tools/wasm-tools/version_128/bin:$PATH" + +# Build the optimized release frontend +./bin/build_fe.sh release +``` + +--- + +## 3. Building from Source & Cross-Compilation + +If you need to run Tuliprox on specific hardware (e.g., Raspberry Pi) or want a fully static Linux binary without `glibc` dependencies, follow these steps. + +### Static Binary Builds (Linux MUSL) + +For a portable Linux binary, the `musl` target is recommended. + +**Prerequisite install on Debian/Ubuntu:** + +```bash +rustup update +sudo apt-get install pkg-config musl-tools libssl-dev +rustup target add x86_64-unknown-linux-musl +``` + +**Build:** + +```bash +cargo build -p tuliprox --release --target x86_64-unknown-linux-musl +``` + +### Cross-Compilation (ARM / Windows) + +To compile for architectures other than your host, use the `cross` tool. + +**For Raspberry Pi (ARMv7):** + +```bash +cargo install cross +env RUSTFLAGS="--remap-path-prefix $HOME=~" cross build -p tuliprox --release --target armv7-unknown-linux-musleabihf +``` + +**For Windows (via Linux Cross-Compiler):** + +```bash +rustup target add x86_64-pc-windows-gnu +sudo apt-get install gcc-mingw-w64 +cargo build -p tuliprox --release --target x86_64-pc-windows-gnu +``` + +--- + +## 4. Custom Docker Builds (Multi-Arch) + +Tuliprox utilizes an advanced Multi-Stage Docker build to compile the Rust backend, the Yew frontend, and extract static FFmpeg resources in a +single pipeline. This setup distinguishes between the **Docker Stage** (image flavor) and the **Rust Compilation Target** (CPU architecture). + +### Full Pipeline Build + +You can strictly target specific environments using the `--target` and `--build-arg` flags. These refer to the **destination system** where the +container will run, regardless of your local build machine's architecture: + +* **`--target`**: Choose the image base. Use `scratch-final` for a hardened, minimal footprint or `alpine-final` if you need a shell for debugging. +* **`--build-arg RUST_TARGET`**: Defines the Instruction Set Architecture (ISA) for the binary. + +```bash +# Build for x86_64 Linux (Standard Cloud/Desktop) +docker build --rm -f docker/Dockerfile -t tuliprox \ + --target scratch-final \ + --build-arg RUST_TARGET=x86_64-unknown-linux-musl . + +# Build for ARM 64-bit (Apple Silicon / Raspberry Pi 4 & 5) +docker build --rm -f docker/Dockerfile -t tuliprox \ + --target scratch-final \ + --build-arg RUST_TARGET=aarch64-unknown-linux-musl . + +# Build for ARMv7 (Raspberry Pi 3 / Older IoT) +docker build --rm -f docker/Dockerfile -t tuliprox \ + --target scratch-final \ + --build-arg RUST_TARGET=armv7-unknown-linux-musleabihf . + +``` + +### Manual Docker Image + +If you want to build the binary and web folder manually on your host and only package them into an image: + +1. Compile the static binary (`bin/build_lin_static.sh`). +2. Compile the frontend (`yarn build`). +3. Change into the `docker` directory and copy the required files. +4. Run the manual Dockerfile build: + + ```bash + docker build -f Dockerfile-manual -t tuliprox . + ``` + +*(Note: When running your custom local image via docker-compose, ensure you change `image: ghcr.io/euzu/tuliprox:latest` to `image: tuliprox` in +your `docker-compose.yml`, and set your timezone appropriately (`TZ=${TZ:-Europe/Paris}`).)* + +--- + +## 5. Docker Container Templates — Deployment Guide + +The Tuliprox repository contains ready-to-use Docker Compose templates for a secure reverse proxy stack with VPN egress and CrowdSec protection. + +**Location:** `docker/container-templates/` +**Software baseline:** Traefik v3.5, a current Rust toolchain, and a current Docker/Compose setup. + +### Legend & Port Overview + +| Template | Folder | Purpose | Notable Ports (Internal) | +| :--- | :--- | :--- | :--- | +| **Traefik** | `traefik/` | Reverse proxy & TLS (ACME/DNS), dashboard, dynamic security middlewares, optional CrowdSec bouncer. | 80 `web`, 443 `websecure` | +| **Gluetun** | `gluetun/` | VPN egress via WireGuard; sidecars provide **SOCKS5**, **HTTP**, and **Shadowsocks** proxies bound to Gluetun’s network stack. | 1080/tcp (HTTP)
1388/tcp+udp (SOCKS5)
9388/tcp+udp (Shadowsocks) | +| **CrowdSec** | `crowdsec/` | LAPI + bouncers (Traefik & firewall) to protect services from brute-force and L7 attacks. | LAPI on `127.0.0.1:8080` (host) | +| **Tuliprox** | `tuliprox/` | Example application container with Traefik labels and `expose: 8901` for reverse proxying. | 8901 (internal) | + +### Wiring up the Stack + +1. **Networks:** Create external Docker networks first: + + ```bash + docker network create proxy-net + docker network create crowdsec-net + ``` + +2. **Gluetun (VPN & Proxy Sidecar):** + + Provide your Wireguard details in `gluetun-01/.env.wg-01` and set a user/pass in `.env.socks5-proxy`. Once started (`docker-compose up -d`), it + securely routes all traffic attached to its network through the VPN. + +3. **Tuliprox Integration:** + + In your Tuliprox `config.yml`, point the outgoing proxy to the SOCKS5 sidecar: + + ```yaml + proxy: + url: socks5://socks5-01:1388 + username: "" + password: "" + ``` + + Ensure Tuliprox is in the `proxy-net` network to reach the sidecar. All Provider-API, TMDB, and Stream-Proxy traffic will now strictly egress + through the WireGuard tunnel! + +--- diff --git a/docs/src/configuration/api-proxy.md b/docs/src/configuration/api-proxy.md index 2c1d3e379..23eb9c056 100644 --- a/docs/src/configuration/api-proxy.md +++ b/docs/src/configuration/api-proxy.md @@ -1,185 +1,194 @@ -# API Proxy +# 🛡️ Pillar 3: `api-proxy.yml` (Server, Users & RBAC) -`api-proxy.yml` tells Tuliprox which public server URLs to advertise and which users may access which targets. +The `api-proxy.yml` file acts as your Edge Gateway. It defines the public-facing URLs (virtual servers) that Tuliprox advertises +in its playlists, manages your end-users, and dictates which playlists (`targets`) those users can access, along with their +specific permissions, proxy modes, and priorities. ## Top-level entries -- `server` -- `user` -- `use_user_db` -- `auth_error_status` +```yaml +auth_error_status: 403 +use_user_db: false +server: +user: +``` -## `server` +| Parameter | Type | Impact | +| :--- | :--- | :--- | +| `auth_error_status` | Int (Default `403`) | The HTTP status code Tuliprox returns when a player sends invalid credentials or tokens. (Only applies to [Xtream/M3U API Endpoints](#api-endpoints-for-clients-players), stream paths, and resource paths, NOT the Web UI / REST API). | +| `use_user_db` | Bool (Default `false`) | If set to `true`, Tuliprox migrates all users from this YAML file into a highly performant SQLite database (`api_user.db`). **From then on, Tuliprox ignores the users in the YAML file!** You must subsequently manage users entirely via the Web UI Dashboard. Switching it back to `false` migrates them back to the YAML file. | +| `server` | List (Default `empty`) | See [Server Definitions](#1-server-definitions-server) for how to define servers | +| `user` | List (Default `empty`) | See [User Definitions](#2-user-definitions-user) for how to define users & permissions | -You can define multiple named servers. -Usually one is local and one is external. -One server should be named `default`. +--- + +## 1. Server Definitions (`server`) + +Here you define multiple named "virtual servers". A server object describes the host structure that Tuliprox injects into the +stream URLs when generating M3U or Xtream playlists. + +Typically, you define at least two: one for internal LAN access and one for external access via a reverse proxy (like Traefik or Nginx). +One server **must** strictly be named `default`. ```yaml server: - name: default protocol: http host: 192.168.1.9 - port: "8901" - timezone: Europe/Paris + port: '8901' + timezone: Europe/Berlin message: Welcome to tuliprox - name: external protocol: https - host: tv.example.com - port: "443" - timezone: Europe/Paris - message: Welcome to tuliprox - path: tuliprox + host: tv.my-domain.com + port: '443' + timezone: Europe/Berlin + path: iptv ``` -Fields: +### Server Parameters -- `name` -- `protocol` -- `host` -- `port` -- `timezone` -- `message` -- `path` +| Parameter | Type | Technical Impact & Background | +| :--- | :--- | :--- | +| `name` | String | Internal reference ID (e.g., `external`). | +| `protocol` | String | `http` or `https`. *(Note: Tuliprox does not perform TLS termination itself; you need a proxy like Traefik/Nginx in front of it for HTTPS).* | +| `host` | String | The domain or IP address transmitted to the client. | +| `port` | String | The port your external proxy listens on (usually `443` for HTTPS). | +| `timezone` | String | Defines the timezone sent to the client via the Xtream API. | +| `message` | String | The welcome message displayed in IPTV players supporting the Xtream API. | +| `path` | String | **Background:** If you host Tuliprox not on a subdomain (`tv.dom.com`) but in a subdirectory (`dom.com/iptv`), specify `iptv` here. Tuliprox will automatically prefix all output URLs with this path. | -If Tuliprox is behind another reverse proxy, `path` simplifies URL rewriting. +--- -## `user` +## 2. User Definitions (`user`) -Users are defined per target. -Each target can expose multiple credentials. +Users in Tuliprox are strictly bound to a specific `target` (defined in `source.yml`). A single target can have multiple user +credentials attached to it. ```yaml user: - - target: xc_m3u + - target: my_livingroom_target credentials: - - username: demo - password: secret1 - token: token1 + - username: john + password: mysecurepassword + token: auth_token_abc proxy: reverse server: default - exp_date: 1672705545 max_connections: 1 + epg_timeshift: Europe/Paris status: Active - priority: 0 + user_ui_enabled: true + priority: -10 ``` -Credential fields: +**Crucial Concept:** By default, Tuliprox acts purely as a stream mapper. If you want Tuliprox to actively evaluate the `status`, +enforce the `exp_date`, or kick users who breach their `max_connections`, you **must** set `user_access_control: true` globally +in your `config.yml`. Without it, these fields are purely cosmetic! -- `username` -- `password` -- `token` -- `proxy` -- `server` -- `epg_timeshift` -- `max_connections` -- `status` -- `exp_date` -- `priority` -- `user_ui_enabled` -- `user_access_control` +## Credential Parameters (Deep-Dive) -`username` and `password` are mandatory. -`token` is optional and must be unique if set. +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `username` / `password` | String | | **Mandatory.** The standard Xtream-Codes / M3U credentials used for authentication. | +| `token` | String | | Optional. Allows login via a URL parameter (`?token=XYZ`) instead of user/pass. Must be globally unique if set. | +| `proxy` | String | `reverse` | Defines the proxy mode for this user (see [proxy modes](#proxy-modes-proxy) below). | +| `server` | String | `default` | Which server block (host/port) is rendered into the playlist for this user. | +| `epg_timeshift` | String | | Shifts EPG times for users in different time zones. Formats supported: hour offsets (e.g., `-2:30`, `1:45`, `+0:15`, `2`) or exact timezones (e.g., `Europe/Paris`). Only applies when `epg_url` is configured in the source. | +| `max_connections` | Int | `0` | Hard limit of concurrent streams for *this* user. `0` = Unlimited. **Requires** `user_access_control: true` in `config.yml` to be enforced. | +| `status` | Enum | `Active` | Possible values: `Active`, `Trial`, `Expired`, `Banned`, `Disabled`, `Pending`. **Requires** `user_access_control: true` in `config.yml` to block non-active streaming. | +| `exp_date` | UnixTs | | Locks the user out after this Unix timestamp. **Requires** `user_access_control: true` in `config.yml` to be enforced. | +| `user_ui_enabled` | Bool | `true` | Allows this specific user to log into the Web UI to manage their own favorites/bouquets. | +| `priority` | Int (i8) | `0` | Stream preemption priority. Lower numbers equal higher priority. Negative numbers allowed. (see [user priority](#user-priorities-priority) below) | -## Proxy mode +--- -`proxy` can be: +### Proxy Modes (`proxy`) -- `redirect` -- `reverse` -- `reverse[live]` -- `reverse[live,vod]` +This is the most crucial field governing traffic flow for the user. When to use which? -Meaning: +* **`redirect`**: Tuliprox responds to the client with an HTTP 302 Redirect, pointing directly to the upstream provider's URL + (or rotating through DNS failover IPs). + * *When to use:* To save massive bandwidth on your server (Tuliprox only acts as a matchmaker). + * *Tradeoff:* **No** connection limits, buffering, bandwidth throttling, or custom fallback videos are applied! +* **`reverse`**: Tuliprox downloads the video stream from the provider onto your server and pipes it to the client. + * *When to use:* This is required for connection limits, fallback videos, caching, bandwidth throttling, and shared streams to function. +* **Partial Syntax**: You can mix and match! `reverse[live]` forces Live-TV through Tuliprox (allowing shared streams) but redirects + VODs (saving bandwidth). `reverse[live,vod]` routes everything except Series episodes through Tuliprox. -- `redirect`: Tuliprox returns provider URLs -- `reverse`: Tuliprox proxies stream traffic itself -- subset syntax: reverse only for selected content types +### User Priorities (`priority`) -## Priority +**Architecture Detail:** Tuliprox utilizes a *Unix Nice-Scale* (value range `-128` to `127`). A **lower** number means a **higher** +priority. The default is `0`. -User priority is optional and defaults to `0`. +**Practical Use Cases:** -Rules: +1. **Admin Override:** Set your personal user to `-10`. Set your friends to `0`. If provider limits are exhausted, you will + forcefully kick a friend to watch TV. +2. **Family vs Guests:** Set your TV to `0`, kids to `10`, and guests to `20`. Guests get kicked first. -- lower number = higher priority -- negative values are allowed -- higher-priority users can preempt lower-priority traffic when provider capacity is exhausted -- equal priority does not preempt a different running stream -- `max_connections` is independent of priority +**The Preemption Scenario:** +Your upstream provider allows 2 concurrent connections. User A (Priority `0`) is watching TV. User B (Priority `0`) is watching +a VOD. The provider limit is exhausted. +Now you, the Admin (User C with Priority `-10`), want to watch. -Probe tasks use the same style of priority scale via `metadata_update.probe.user_priority`. +1. Because your priority is *higher* (lower number), Tuliprox scans for active connections with the *lowest* priority on that + specific provider. +2. Since A and B are tied (both `0`), Tuliprox targets the stream that has been running the longest (tie-breaker based on stream age). +3. Tuliprox forcefully terminates User A's provider connection, serves User A a fallback video (`low_priority_preempted.ts`), and + instantly claims the freed provider slot for you. -## Access control +*(Note: Internal FFprobe metadata probe tasks run by default at the absolute lowest priority (`127`) and are immediately +preempted/killed if any real user needs the slot.)* -When `user_access_control` is enabled in `config.yml`, Tuliprox also evaluates: +--- -- `status` -- `exp_date` -- `max_connections` +  -for each user. +## Additional Information -## `use_user_db` +## API Endpoints for Clients (Players) -If `use_user_db: true` is enabled, users are stored in the user database instead of the YAML file. -The Web UI should then be used to add, edit or remove users. +After configuring the api-proxy, you can use these endpoints in players like TiviMate, IPTV Smarters, or VLC. -Tuliprox migrates users automatically when switching between YAML and DB mode. +*(Replace `:` with your Server definition).* -## `auth_error_status` - -HTTP status code returned when authentication fails (invalid or missing credentials). -Defaults to `403` (Forbidden). - -```yaml -auth_error_status: 403 -``` - -This setting applies to the streaming and playlist API endpoints -(`player_api.php`, `get.php`, `xmltv.php`, stream paths, resource paths). -It does **not** affect the Web UI / REST API (`/api/v1/…`) or HDHomeRun endpoints, -which always use their own fixed status codes. - -## Access URLs - -Common access patterns: - -Xtream: +**Xtream Codes API:** ```text -http://host:port/player_api.php?username=USER&password=PASS -http://host:port/player_api.php?token=TOKEN +http://:/player_api.php?username=&password= +http://:/player_api.php?token= ``` -M3U: +**M3U Playlist URL:** ```text -http://host:port/get.php?username=USER&password=PASS -http://host:port/get.php?token=TOKEN +http://:/get.php?username=&password= +http://:/get.php?token= ``` -XMLTV: +**XMLTV EPG URL:** ```text -http://host:port/xmltv.php?username=USER&password=PASS +http://:/xmltv.php?username=&password= ``` -REST-friendly aliases also work: +Tuliprox also offers **REST-friendly aliases** in case restrictive firewalls or ISP blocks target `.php` extensions: -- `m3u` instead of `get.php` -- `xtream` instead of `player_api.php` -- `epg` instead of `xmltv.php` +* `/xtream` instead of `player_api.php` +* `/m3u` instead of `get.php` +* `/epg` instead of `xmltv.php` -## Reverse proxy in front of Tuliprox +## Reverse Proxy in front of Tuliprox -If another proxy sits in front of Tuliprox, make sure it forwards: +If another proxy sits in front of Tuliprox (like Nginx or Traefik), you must ensure it forwards the correct headers so +Tuliprox's IP-based rate limiting and connection kicking works. -- `X-Real-IP` -- `X-Forwarded-For` +Make sure it forwards: -Example nginx block: +* `X-Real-IP` +* `X-Forwarded-For` + +Example Nginx block: ```nginx location /tuliprox { @@ -187,6 +196,8 @@ location /tuliprox { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass http://192.168.1.9:8901/; + + # ABSOLUTELY CRITICAL FOR VIDEO STREAMS: proxy_redirect off; proxy_buffering off; proxy_request_buffering off; @@ -201,6 +212,7 @@ Example Traefik labels: ```yaml labels: - "traefik.enable=true" - - "traefik.http.routers.tuliprox.rule=Host(`tv.my-domain.io`) && (PathPrefix(`/tv`) || PathPrefix(`/tuliprox`))" + - "traefik.http.routers.tuliprox.rule=Host(`tv.example.com`) && (PathPrefix(`/tv`) || PathPrefix(`/tuliprox`))" - "traefik.http.middlewares.tuliprox-strip.stripprefix.prefixes=/tv" + - "traefik.http.routers.tuliprox.middlewares=tuliprox-strip@docker,forward-real-ip@file" ``` diff --git a/docs/src/configuration/config.md b/docs/src/configuration/config.md new file mode 100644 index 000000000..21e10235f --- /dev/null +++ b/docs/src/configuration/config.md @@ -0,0 +1,384 @@ +# 🏛️ config.yml (Core System) + +The `config.yml` is the primary configuration file of Tuliprox. It dictates the engine's core runtime behavior, memory management, +caching mechanisms, background schedulers, and external integrations (like HDHomeRun, GeoIP, and the Web UI). + +## Top-level entries + +```yaml +process_parallel: false +disk_based_processing: false +storage_dir: ./data +default_user_agent: Tuliprox/... +backup_dir: ./data/backup +user_config_dir: ./data/user +mapping_path: mapping.yml +template_path: template.yml +update_on_boot: false +config_hot_reload: false +accept_insecure_ssl_certificates: false +sleep_timer_mins: null +connect_timeout_secs: 10 +user_access_control: false +custom_stream_response_path: null +custom_stream_response_timeout_secs: 0 + +api: +web_ui: +log: +schedules: +messaging: +video: +proxy: +ipcheck: +hdhomerun: +library: +reverse_proxy: +metadata_update: +``` + +### 1. Global System & Storage Settings (Flat Keys) + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :--- | :--- | :---: | :--- | :--- | +| `process_parallel` | Bool | No | `false` | Activates multi-threading during playlist processing. **Background:** If you have 5 providers, Tuliprox processes them sequentially by default. Setting this to `true` processes all 5 simultaneously using multiple CPU cores. *Warning:* This establishes parallel downloads to your providers. Verify this does not violate your provider's connection limits! | +| `disk_based_processing` | Bool | No | `false` | **Tradeoff Guidance:** Normally, Tuliprox loads playlists into RAM. With `true`, every chunk is manipulated directly on disk (using a B+Tree database). **Use this on low-end hardware (e.g., Raspberry Pi) or with massive playlists (>500k streams)** to prevent Out-Of-Memory crashes. It significantly increases Disk I/O load, so it is slower but much safer for tight memory footprints. | +| `storage_dir` | String | No | `./data` | Root directory for all runtime data (B+Tree databases, downloads, caches). Relative paths are resolved against the Tuliprox Home Directory. | +| `default_user_agent` | String | No | `Tuliprox/...` | Fallback HTTP `User-Agent` used for upstream provider requests if the input definition or client request does not explicitly provide one. | +| `backup_dir` | String | No | `{storage_dir}/backup` | Storage location for config backups (e.g., triggered via "Save Configuration" in the Web UI). | +| `user_config_dir` | String | No | `{storage_dir}/user` | Storage location for user-specific configurations (like favorites or custom bouquets created via the Web UI). | +| `mapping_path` | String | No | `mapping.yml` | Path to the mapping file. **Pro-Tip:** If you specify a folder path here (e.g., `./config/mappings/`), Tuliprox loads *all* `.yml` files in that folder in alphanumeric order! Ideal for structuring complex setups. | +| `template_path` | String | No | `template.yml` | Path to the template macro file. Specifying a folder here is also possible and highly recommended. | +| `update_on_boot` | Bool | No | `false` | Forces Tuliprox to immediately query all providers and rebuild all playlists upon startup. If `false`, the proxy serves the local DB cache from the last run until the scheduler triggers the next update. | +| `config_hot_reload` | Bool | No | `false` | **Background:** Spawns a filesystem watcher for `mapping.yml` and `api-proxy.yml`. Upon saving, mappings and user credentials become active immediately *without* requiring a server restart. | +| `accept_insecure_ssl_certificates` | Bool | No | `false` | Set to `true` if your upstream provider uses expired, self-signed, or improperly configured HTTPS certificates. Otherwise, the HTTP client drops the connection securely. | +| `sleep_timer_mins` | Int | No | `null` | Automatic kill-switch for proxied streams. Forcibly terminates active stream connections after X minutes (Protects against users falling asleep with the TV on). | +| `connect_timeout_secs` | Int | No | `10` | Maximum time (in seconds) Tuliprox waits to establish the initial TCP connection to a provider. `0` disables the timeout (Warning: risk of hanging threads!). | +| `user_access_control` | Bool | No | `false` | **Security:** If `true`, Tuliprox actively enforces `status` (Active/Banned), `exp_date`, and `max_connections` constraints for users defined in `api-proxy.yml`. If false, those fields are ignored. | +| `custom_stream_response_path` | String | No | `null` | Directory path where Tuliprox looks for custom fallback `.ts` files (e.g., `user_connections_exhausted.ts`, `channel_unavailable.ts`). See section [Custom Stream Response](#custom-stream-responses-fallback-videos) for more details. | +| `custom_stream_response_timeout_secs` | Int | No | `0` | Hard timeout (in seconds) that forces the fallback video stream to terminate to prevent infinite bandwidth usage. `0` means endless loop. | + +### 2. Subsections (Object Keys) + +| Block | Description | Link | +| :--- | :--- | :--- | +| `api` | Internal web server binding settings. | [See section](#3-api-server-api) | +| `web_ui` | Web Dashboard, RBAC, and Authentication. | [See section](#4-web-ui--administration-web_ui) | +| `log` | Console output verbosity and sanitization. | [See section](#5-logging-log) | +| `schedules` | Automated background tasks (Cronjobs). | [See section](#6-schedules-schedules) | +| `messaging` | Webhooks & Push-Notifications (Telegram, Discord, etc.). | [See section](#7-messaging-messaging) | +| `video` | Extension mapping and Web UI download behavior. | [See section](#8-video--web-search-video) | +| `proxy` | SOCKS5/HTTP proxy settings for outgoing requests. | [See section](#9-outgoing-proxy-proxy) | +| `ipcheck` | IP detection to verify in the Web UI which public IP Tuliprox is currently using. | [See section](#10-ip-check-ipcheck) | +| `hdhomerun` | Virtual DVB-C/T network tuner emulation. | [See section](#11-hdhomerun-emulation-hdhomerun) | +| `library` | Local Media Library integration. | [See Local Library](./local-library.md) | +| `reverse_proxy` | Streaming buffers, rate limits, caching. | [See Reverse Proxy](./reverse-proxy.md) | +| `metadata_update` | TMDB matching, FFprobe processing, Job Queues. | [See Metadata Update](./metadata-update.md) | + +*(Note: The advanced topics **Local Library**, **Reverse Proxy** and **Metadata Update** are extremely extensive and have their own +dedicated subchapters. Here we cover the global base settings.)* + +--- + +## 3. API Server (`api`) + +Controls the internal web server of Tuliprox. This does *not* dictate the public URLs given to clients (those belong in +`api-proxy.yml`), but rather the physical socket binding on your host machine. + +```yaml +api: + host: 0.0.0.0 + port: 8901 + web_root: ./web +``` + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :--- | :--- | :---: | :--- | :--- | +| `host` | String | No | `0.0.0.0` | Bind interface. `0.0.0.0` listens on all network cards. `127.0.0.1` restricts access to localhost (useful if you force traffic through a local Nginx/Traefik reverse proxy). | +| `port` | Int | No | `8901` | The listening port for proxy streams, the Web UI, and all REST APIs. | +| `web_root` | String | No | `./web` | Physical path to the compiled Wasm/JS/CSS frontend assets of the Web UI. | + +--- + +## 4. Web UI & Administration (`web_ui`) + +Tuliprox ships with a comprehensive Web Dashboard containing a Web Player, Playlist Editor, User Management, and Live Logs. + +```yaml +web_ui: + enabled: true + user_ui_enabled: true + path: admin + player_server: default + kick_secs: 90 + combine_views_stats_streams: false + auth: + enabled: true + issuer: tuliprox + secret: "YOUR_SECRET_JWT_KEY_HERE" + token_ttl_mins: 30 + userfile: user.txt + groupfile: groups.txt +``` + +### Web UI Parameters + +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `enabled` | Bool | `true` | Completely toggles the Web Dashboard and its REST API endpoints on or off. | +| `user_ui_enabled` | Bool | `true` | Allows standard proxy users (not just admins) to log into the Web UI to manage their own favorite bouquets. | +| `path` | String | `""` | Base path for the UI (e.g., `admin`). Critical for reverse proxy subfolder setups so assets load from `example.com/admin/assets/`. | +| `player_server` | String | `default` | Determines which virtual server block from `api-proxy.yml` is used to construct the streaming URLs when playing a channel directly within the Web UI player. | +| `kick_secs` | Int | `90` | **Background:** When you kick a user via the Dashboard, they are not only disconnected but hard-blocked at the IP/User level for X seconds. This prevents their IPTV player's auto-reconnect logic from instantly stealing the provider slot back. | +| `combine_views_stats_streams` | Bool | `false` | Combines the "Server Stats" and "Active Streams" views into a single unified window in the UI. | + +### Authentication & RBAC (`web_ui.auth`) + +Tuliprox features a robust Role-Based Access Control (RBAC) system. + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :--- | :--- | :---: | :--- | :--- | +| `secret` | String | Yes | `(Random)` | Critical for JWT cookie encryption. Generate a random 32-char hex string (e.g., `openssl rand -hex 16`). If omitted, Tuliprox generates one in-memory, but all active logins will invalidate on every server restart! | +| `token_ttl_mins` | Int | No | `30` | How long a login session remains valid. Setting this to `0` makes the token effectively valid for 100 years (Extreme Security Risk!). | +| `userfile` | String | No | `user.txt` | The file storing Admins and Web Users. | +| `groupfile` | String | No | `groups.txt` | The RBAC (Role-Based Access Control) definition file. | + +#### Structure of `user.txt` + +This file stores users, Argon2 password hashes, and RBAC groups. Generate secure passwords via CLI: `./tuliprox --genpwd`. +Format: `username:argon2_hash[:group1,group2]` + +```text +# A normal Admin (No group specified = Fallback to built-in Admin role) +admin:$argon2id$v=19$m=19456,t=2,p=1$QUp... + +# An Editor assigned to specific permission groups +editor:$argon2id$v=19$m=19456,t=2,p=1$Y2F...:playlist_manager,user_manager +``` + +#### Structure of `groups.txt` + +Define group permissions here. An editor might be allowed to update playlists (`playlist.write`) but forbidden from viewing or +changing `config.yml` (`config.read`). +Format: `group_name:permission1,permission2,...` + +```text +viewer:config.read,source.read,playlist.read,system.read,library.read +playlist_manager:playlist.read,playlist.write,source.read +``` + +Available Permissions: `config.read/write`, `source.read/write`, `user.read/write`, `playlist.read/write`, `library.read/write`, +`system.read/write`, `epg.read/write`. Note: Write does not imply Read. A group must explicitly grant both if users need to view +and edit content. + +--- + +## 5. Logging (`log`) + +Controls console output verbosity and sanitization. + +```yaml +log: + sanitize_sensitive_info: true + log_active_user: false + log_level: info +``` + +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `log_level` | String | `info` | Verbosity. Possible values: `trace`, `debug`, `info`, `warn`, `error`. Can be overridden per-module (e.g., `tuliprox=debug,hyper_util=warn`). | +| `sanitize_sensitive_info` | Bool | `true` | **Critical:** Masks passwords, provider URLs, and external client IPs in the logs with `***`. Highly recommended to keep `true` so you can safely share logs on GitHub/Discord for support without leaking credentials. | +| `log_active_user` | Bool | `false` | Periodically writes the current active client connection count as an INFO message to the log file. | + +--- + +## 6. Schedules (`schedules`) + +Automate your background updates here. +*Important:* Tuliprox uses standard cron syntax, **but strictly includes seconds as the first field** +(7 fields in total: `Sec Min Hour Day Month Day-of-Week Year`)! + +```yaml +schedules: + # Every morning at 08:00:00 (Seconds = 0, Minutes = 0, Hours = 8) + - schedule: "0 0 8 * * * *" + type: PlaylistUpdate + targets: [ "m3u_target", "xtream_target" ] # Optional: Only update specific targets + + # Every evening at 20:00:00 + - schedule: "0 0 20 * * * *" + type: LibraryScan + + # Every Monday at 04:00:00 + - schedule: "0 0 4 * * 1 *" + type: GeoIpUpdate +``` + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `schedule` | String | Cron expression with 7 fields (Seconds included at the start). | +| `type` | Enum | The task to execute. Valid values: `PlaylistUpdate`, `LibraryScan`, `GeoIpUpdate`. | +| `targets` | List | *(Optional, only for PlaylistUpdate)* List of target names to restrict the update to. If omitted, all enabled targets are updated. | + +--- + +## 7. Messaging (`messaging`) + +Tuliprox can proactively notify you via Push-Notifications when updates fail, finish, or when specific channels are added/removed +from a watched group. **Why is this useful?** Because it allows you to instantly detect upstream provider issues or simply let you +know when new movies are added to your playlist. + +*(Note: You must explicitly opt-in via `notify_on` to receive messages!)* + +```yaml +messaging: + notify_on:[ "info", "stats", "error", "watch" ] + telegram: + markdown: true + bot_token: "" + chat_ids: + - "" + - ":" # For group topics + templates: + stats: 'file:///config/messaging_templates/telegram_stats.templ' + discord: + url: "" + pushover: + token: "" + user: "" + rest: + url: "https://my-api.local/alert" + method: "POST" + headers: + - "Content-Type: application/json" +``` + +**Template Auto-File Behavior & Variables (Handlebars):** +You can reference custom formatting templates (`file://` or HTTP URLs). If the file doesn't exist, Tuliprox can automatically +populate the `messaging_templates/` folder with default templates if missing. Variables injected during runtime: + +* `{{message}}`: The raw text. +* `{{kind}}`: The event type (`info`, `error`, `stats`, `watch`). +* `{{timestamp}}`: RFC3339 timestamp. +* `{{stats}}`: Array of metrics per Source/Input (`raw.groups`, `processed.channels`, `took`). +* `{{processing.errors}}`: A combined string of all errors encountered during the update run. +* `{{watch}}`: Contains data about added/removed items triggered by the `watch` event in your targets. + +--- + +## 8. Video & Web Search (`video`) + +Optional video-related behaviors, mostly utilized by the Web UI. + +```yaml +video: + web_search: "https://www.imdb.com/search/title/?title={}" + extensions: ["mkv", "mp4", "avi", "ts", "webm"] + download: + directory: /tmp/tuliprox_downloads + organize_into_directories: true + episode_pattern: '.*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*' +``` + +* `extensions`: Defines which file endings Tuliprox categorizes as VOD/Video content when transforming M3U to Xtream. +* `web_search`: A template URL used in the Web UI to quickly search for a movie title (replaces `{}` with the title). +* `download`: Configuration for the Web UI's "Download Video" button. + * `directory`: Where the downloaded files are saved. + * `organize_into_directories`: If true, Tuliprox automatically creates neat subfolders for series. + * `episode_pattern`: Crucial for the directory organization. It uses the mandatory Named Capture Group `(?P...)` in the + Regex to identify and strip the episode identifier (e.g., `S01E01`) from the filename, ensuring all episodes of a show land in + the exact same base-show folder. + +--- + +## 9. Outgoing Proxy (`proxy`) + +If Tuliprox itself must operate behind a corporate proxy or VPN (e.g., a Gluetun WireGuard container): + +```yaml +proxy: + url: socks5://192.168.1.6:8123 + username: "opt_user" + password: "opt_password" +``` + +Setting this forces **every** outgoing request (Playlist downloads, TMDB API calls, FFprobe stream analysis, and Reverse-Proxy +Video Streaming) through this proxy. + +--- + +## 10. IP-Check (`ipcheck`) + +To verify in the Web UI which public IP Tuliprox is currently using (crucial when verifying VPN routing or diagnosing geo-blocks), +Tuliprox queries a detection API: + +```yaml +ipcheck: + url_ipv4: https://ipinfo.io/ip +``` + +You can also define `url_ipv6`, or generic `url` alongside `pattern_ipv4` or `pattern_ipv6` regexes to extract the IP from +complex JSON responses. + +--- + +## 11. HDHomeRun Emulation (`hdhomerun`) + +**Deep-Dive Feature:** Tuliprox can masquerade on the local network as a physical "SiliconDust HDHomeRun" DVB-C/T network tuner. +Media servers like **Plex**, **Jellyfin**, **Emby**, or **TVHeadend** will automatically discover Tuliprox via UPnP as a real +hardware antenna and ingest Live-TV natively into their Live-DVR systems. + +Tuliprox utilizes standardized UPnP/SSDP (UDP Port 1900) and the proprietary SiliconDust UDP discovery protocol (UDP Port 65001). + +```yaml +hdhomerun: + enabled: true + auth: false # If true, lineup.json requires Basic Auth (using the assigned user's credentials) + devices: + - name: hdhr1 # MUST match the 'device' name in the output target of source.yml! + tuner_count: 4 # Tells Plex that 4 channels can be watched simultaneously + port: 5004 # Unique TCP Port for this specific virtual antenna API + device_id: "107ABCDF" # 8-char hex code (Port 65001). If left blank, Tuliprox generates a valid one with correct checksums. + device_udn: "uuid:..." # Unique Device Name (Port 1900). Recommended to leave blank for auto-generation. + friendly_name: "Tuliprox Living Room" +``` + +Both `device_id` and `device_udn` are necessary for Tuliprox to satisfy both the official SiliconDust apps and third-party UPnP +scanners simultaneously. + +--- + +  + +## Additional Information + +## Custom Stream Responses (Fallback Videos) + +When a stream fails to load at the provider (HTTP 404/502) or a user reaches their connection limit, Tuliprox can seamlessly +substitute a fallback info-video (as a `.ts` stream) instead of brutally closing the TCP connection. Dropping connections often +causes hardware players or Smart TVs to freeze. + +```yaml +custom_stream_response_path: /home/tuliprox/resources +custom_stream_response_timeout_secs: 20 +``` + +| Name | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `custom_stream_response_path` | String | `(Empty)` | Directory path where Tuliprox looks for exactly named `.ts` files. | +| `custom_stream_response_timeout_secs` | Int | `0` | Hard timeout (in seconds) that forces the fallback video stream to terminate to prevent infinite bandwidth usage. `0` means the fallback loops endlessly until the user switches channels. | + +**Filenames searched for in the directory:** + +* `channel_unavailable.ts` (Provider returns 404/502/Timeout) +* `user_connections_exhausted.ts` (User hit their `max_connections` limit) +* `provider_connections_exhausted.ts` (Provider has no free slots left) +* `low_priority_preempted.ts` (User was kicked by an Admin with higher priority) +* `user_account_expired.ts` (User's `exp_date` reached) +* `panel_api_provisioning.ts` (Loops while a new Provider Account is generated via Panel API) + +**How it works:** +Tuliprox searches the specified folder for exactly named `.ts` files. If found, they are looped back to the client. The +`custom_stream_response_timeout_secs` parameter hard-kills the fallback stream after X seconds to prevent infinite bandwidth usage. + +--- diff --git a/docs/src/configuration/local-library.md b/docs/src/configuration/local-library.md new file mode 100644 index 000000000..9c1db84b8 --- /dev/null +++ b/docs/src/configuration/local-library.md @@ -0,0 +1,138 @@ +# 📂 Local Media Library + +Tuliprox is not limited to external IPTV providers. Through the **Library** module, it can recursively scan, catalog, and +seamlessly integrate local movie and TV show collections (much like Plex or Jellyfin do) directly into the Xtream/M3U outputs +for your IPTV clients. + +## Core Features + +* **Recursive Scanning:** Traverses directories looking for supported video formats (`.mkv`, `.mp4`, etc.). +* **Auto-Classification:** Automatically detects whether a file is a Movie or a Series episode (e.g., `Breaking.Bad.S01E01.mkv`) + using the internal PTT (Parse Torrent Title) engine. +* **Multi-Source Metadata:** Reads Kodi/Jellyfin/Emby compatible `.nfo` files. If no `.nfo` is present, it automatically queries + the TMDB API for covers, plots, cast, and trailers. +* **Incremental Scans:** Uses file modification timestamps to only scan and process new or altered files, ensuring extremely fast + updates. +* **Stable Virtual IDs:** Generates stable, deterministic UUIDs for local files, ensuring that channel/stream IDs in your IPTV + client remain constant across updates. + +--- + +## Configuration (`config.yml`) + +You enable the library globally in the `library` block of your `config.yml`. The metadata caches and resolved TMDB data are +physically stored inside `metadata_update.cache_path/library` (relative to your `storage_dir`). + +```yaml +library: + enabled: true + scan_directories: + - enabled: true + path: "/media/movies" + content_type: movie # Forces everything in this folder to be treated as a movie + recursive: true + - enabled: true + path: "/media/shows" + content_type: auto # Uses PTT to guess S01E01 structures + recursive: true + supported_extensions: + - "mp4" + - "mkv" + - "avi" + - "ts" + metadata: + fallback_to_filename: true # Uses parsed filename details if TMDB/NFO fails + read_existing: + kodi: true + plex: false + jellyfin: false + formats: + - "nfo" + playlist: + movie_category: "Local Movies" + series_category: "Local Shows" +``` + +### Library Configuration Parameters + +| Block / Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `enabled` | Bool | `false` | Master switch to turn on the local media library feature. | +| **`scan_directories`** | List | | Folders to monitor. | +| ↳ `enabled` | Bool | `true` | Allows temporarily disabling specific folders. | +| ↳ `path` | String | | The absolute or relative path to your media directory. | +| ↳ `content_type` | Enum | `auto` | Forces classification. Options: `auto` (guess via filename), `movie`, `series`. | +| ↳ `recursive` | Bool | `true` | If true, Tuliprox crawls all subdirectories within `path`. | +| `supported_extensions` | List | `[mp4, mkv, avi, ts, ...]` | File extensions that Tuliprox considers as playable video files. | +| **`metadata`** | Object | | Instructions on how to fetch or fallback for movie details. | +| ↳ `fallback_to_filename` | Bool | `true` | If NFO or TMDB fails, uses the filename to construct basic metadata (Title, Year). | +| ↳ `read_existing.kodi` | Bool | `true` | Attempts to read Kodi-compatible `.nfo` files residing next to the media. | +| ↳ `read_existing.plex` | Bool | `false` | Attempts to read Plex metadata formats. | +| ↳ `read_existing.jellyfin` | Bool | `false` | Attempts to read Jellyfin metadata formats. | +| ↳ `formats` | List | `["nfo"]` | Metadata file formats to read when `read_existing` is enabled (e.g., `nfo`). | + +*Note: For TMDB enrichments to work on local files, `metadata_update.tmdb.enabled: true` must be set in your config! The library +utilizes the exact same API limits, API keys, and caches as the IPTV streams.* + +--- + +## Integration as an Input (`source.yml`) + +To make your local movies visible in your M3U/Xtream targets, you attach the library as a standard `input` of `type: library` +in your `source.yml`: + +```yaml +inputs: + - name: my_local_library + type: library + enabled: true + +sources: + - inputs: + - my_local_library + - my_iptv_provider + targets: + - name: mixed_target + filter: 'Group ~ ".*"' + output: + - type: xtream +``` + +In this setup, the target `mixed_target` now merges the Live-TV channels from your IPTV provider and your local `.mkv` movies +into a single, clean Xtream API output for the client. The local files bypass the Reverse Proxy routing and stream directly off +your disk when requested by the IPTV player! + +--- + +## Triggering Scans (CLI & API) + +By default, the library scan can be automated using standard Cron syntax under the `schedules:` block (`type: LibraryScan`). +However, you can also force it manually. + +**Via CLI (Command Line):** + +```bash +# Incremental Delta-Scan (Only processes new/modified files) +./tuliprox --scan-library + +# Ignores the cache and forces TMDB/PTT to re-evaluate ALL local files +./tuliprox --force-library-rescan +``` + +**Via REST API (e.g., triggered from a post-download script like Radarr/Sonarr):** + +```http +POST /api/v1/library/scan +Content-Type: application/json +Authorization: Bearer + +{"force_rescan": false} +``` + +**Status Polling:** + +```http +GET /api/v1/library/status +``` + +Returns JSON information about the number of detected files, errors, and the progress of the background scan. diff --git a/docs/src/configuration/main-config.md b/docs/src/configuration/main-config.md deleted file mode 100644 index a3bdf4d92..000000000 --- a/docs/src/configuration/main-config.md +++ /dev/null @@ -1,488 +0,0 @@ -# Config Reference - -`config.yml` controls the application runtime, storage, reverse proxy behavior and optional subsystems. - -## Top-level entries - -Common top-level fields are: - -- `api` -- `storage_dir` -- `default_user_agent` -- `process_parallel` -- `messaging` -- `video` -- `metadata_update` -- `schedules` -- `backup_dir` -- `mapping_path` -- `template_path` -- `update_on_boot` -- `web_ui` -- `reverse_proxy` -- `log` -- `user_access_control` -- `connect_timeout_secs` -- `custom_stream_response_path` -- `custom_stream_response_timeout_secs` -- `hdhomerun` -- `proxy` -- `ipcheck` -- `config_hot_reload` -- `sleep_timer_mins` -- `accept_unsecure_ssl_certificates` -- `disk_based_processing` -- `library` - -Relative paths are resolved against the tuliprox home directory: -`--home` -> `TULIPROX_HOME` -> directory of the `tuliprox` binary. - -## Core runtime fields - -### `process_parallel` - -Enable this on multi-core systems if you want parallel processing. -Be aware that multiple worker threads can also consume multiple provider connections during updates. - -### `api` - -`api` contains the server-mode settings. Example: - -```yaml -api: - host: localhost - port: 8901 - web_root: ./web -``` - -`web_root` is resolved relative to the tuliprox home directory when it is not absolute. - -### `storage_dir` - -Storage location for persisted playlists, databases and metadata. -Example: - -```yaml -storage_dir: ./data -``` - -### `mapping_path` and `template_path` - -Tuliprox can load mappings and templates centrally from files or directories. - -- `mapping_path`: single file or directory -- `template_path`: single file or directory - -Defaults: - -- mappings: `mapping.yml` -- templates: `template.yml` - -If a path points to a directory, all `*.yml` files are loaded in alphanumeric order and merged. -Template names must remain globally unique. - -## Provider failover and rotation - -Tuliprox supports provider failover and DNS-aware rotation. -Providers can expose multiple URLs, and Tuliprox can rotate to the next candidate on supported failure conditions. - -### `provider://` scheme - -Configurations can reference providers via `provider:///...`. -Tuliprox resolves that to the current active provider URL or resolved IP. - -### Automatic failover triggers - -Failover is triggered for: - -- network timeouts -- HTTP `408` -- HTTP `500`, `502`, `503`, `504` -- HTTP `404`, `410`, `429` - -It does not trigger for `401` or `403`. - -### Provider DNS resolution - -Each provider can optionally enable a `dns` block with: - -- `enabled` -- `refresh_secs` -- `prefer`: `system` | `ipv4` | `ipv6` -- `max_addrs` -- `schemes` -- `keep_vhost` -- `overrides` -- `on_resolve_error` -- `on_connect_error` - -Resolved IPs are persisted in `provider_dns_resolved.json` below `storage_dir`. - -Example: - -```yaml -provider: - - name: my_provider - urls: - - http://provider-a.example - - http://provider-b.example - dns: - enabled: true - refresh_secs: 300 - prefer: ipv4 - schemes: [http, https] - keep_vhost: true - max_addrs: 2 - on_resolve_error: keep_last_good - on_connect_error: try_next_ip -``` - -## `messaging` - -Optional notification system for `telegram`, `discord`, `rest` and `pushover`. - -`notify_on` can include: - -- `info` -- `stats` -- `error` -- `watch` - -Templates can be raw strings or `file://` / `http(s)://` URIs. -Handlebars variables include `message`, `kind`, `timestamp`, `stats`, `watch` and `processing`. - -Example: - -```yaml -messaging: - notify_on: - - info - - stats - - error - telegram: - markdown: true - bot_token: "" - chat_ids: - - "" - - ":" -``` - -## `video` - -Optional video-related behavior: - -- `extensions`: file extensions treated as video content -- `download`: download integration for the Web UI -- `web_search`: URL template for media searches - -Example: - -```yaml -video: - web_search: "https://www.imdb.com/search/title/?title={}" - extensions: [mkv, mp4, avi] - download: - directory: /tmp - organize_into_directories: true - episode_pattern: '.*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*' -``` - -## `metadata_update` - -Controls metadata resolve, TMDB, probing and FFprobe behavior. - -Important groups: - -- `log` -- `resolve` -- `probe` -- `tmdb` -- `ffprobe` -- `retry_delay` -- `worker_idle_timeout` -- `max_queue_size` -- `no_change_cache_ttl_secs` -- `probe_fairness_resolve_burst` - -Key fields: - -- `resolve.max_retry_backoff` -- `resolve.min_retry_base` -- `resolve.max_attempts` -- `resolve.exhaustion_reset_gap` -- `probe.cooldown` -- `probe.max_attempts` -- `probe.retry_backoff_step_1` -- `probe.retry_backoff_step_2` -- `probe.retry_backoff_step_3` -- `probe.retry_load_retry_delay` -- `probe.backoff_jitter_percent` -- `probe.user_priority` -- `tmdb.enabled` -- `tmdb.api_key` -- `tmdb.rate_limit_ms` -- `tmdb.cache_duration_days` -- `tmdb.language` -- `tmdb.cooldown` -- `tmdb.match_threshold` -- `ffprobe.enabled` -- `ffprobe.timeout` -- `ffprobe.analyze_duration` -- `ffprobe.probe_size` -- `ffprobe.live_analyze_duration` -- `ffprobe.live_probe_size` - -Example: - -```yaml -metadata_update: - cache_path: metadata - log: - queue_interval: 30s - progress_interval: 15s - resolve: - max_retry_backoff: 1h - min_retry_base: 5s - max_attempts: 3 - exhaustion_reset_gap: 1h - probe: - cooldown: 7d - retry_load_retry_delay: 1m - retry_backoff_step_1: 10m - retry_backoff_step_2: 30m - retry_backoff_step_3: 1h - max_attempts: 3 - backoff_jitter_percent: 20 - user_priority: 127 -``` - -Duration fields support `s`, `m`, `h`, `d` or plain seconds. -Size fields support `B`, `KB`, `MB`, `GB`, `TB` or plain bytes. - -## `schedules` - -Tuliprox schedules use cron expressions with a leading seconds field. - -Example: - -```yaml -schedules: - - schedule: "0 0 8 * * * *" - type: PlaylistUpdate - targets: - - m3u - - schedule: "0 0 20 * * * *" - type: LibraryScan -``` - -Supported schedule types: - -- `PlaylistUpdate` -- `LibraryScan` -- `GeoIpUpdate` - -## `reverse_proxy` - -This block controls stream delivery, buffering, caching, headers, rate limiting and resource retries. -The detailed stream behavior lives in [Streaming And Proxy](../streaming-and-proxy.md). - -Notable `reverse_proxy.stream` flags include: - -- `retry` -- `metrics_enabled` -- `buffer` -- `throttle` -- `grace_period_millis` -- `grace_period_timeout_secs` -- `grace_period_hold_stream` -- `shared_burst_buffer_mb` -- `hls_session_ttl_secs` -- `catchup_session_ttl_secs` - -## `backup_dir` - -Location for configuration backups written from the Web UI. - -## `update_on_boot` - -If `true`, a playlist update starts automatically at application start. - -## `log` - -Supported fields: - -- `sanitize_sensitive_info` -- `log_active_user` -- `log_level` - -`log_level` can be a plain level such as `debug` or a module list such as: - -```yaml -log: - log_level: hyper_util::client::legacy::connect=error,tuliprox=debug -``` - -## `web_ui` - -Controls the Web UI itself: - -- `enabled` -- `user_ui_enabled` -- `content_security_policy` -- `path` -- `player_server` -- `kick_secs` -- `combine_views_stats_streams` -- `auth` - -`auth` supports: - -- `enabled` -- `issuer` -- `secret` -- `token_ttl_mins` -- `userfile` -- `groupfile` - -Example: - -```yaml -web_ui: - enabled: true - user_ui_enabled: true - auth: - enabled: true - issuer: tuliprox - secret: "" - userfile: user.txt - groupfile: groups.txt -``` - -Password hashes can be generated with `tuliprox --genpwd`. - -### `userfile` format - -The `userfile` uses an extended format that is backward compatible with the previous `username:hash` format: - -```text -# username:argon2_hash[:group1,group2,...] -admin:$argon2id$v=19$... # no groups field -> defaults to "admin" -alice:$argon2id$v=19$...:viewer -bob:$argon2id$v=19$...:user_manager,source_manager -``` - -- Lines starting with `#` are comments, empty lines are ignored. -- The optional third field (after the second `:`) assigns group memberships. -- If the third field is missing, the user defaults to the `admin` group (backward compatibility). -- Users belonging to the `admin` group have full access; other groups are silently ignored for admin users. -- Users can belong to multiple groups; permissions are the union of all group permissions. - -### `groupfile` format - -The `groupfile` (default: `groups.txt` in the same directory as `userfile`) defines permission groups: - -```text -# group_name:permission1,permission2,... -viewer:config.read,source.read,user.read,playlist.read,library.read,system.read,epg.read -user_manager:user.read,user.write -source_manager:source.read,source.write -config_manager:config.read,config.write -playlist_manager:playlist.read,playlist.write -full_manager:config.read,config.write,source.read,source.write,user.read,user.write,playlist.read,playlist.write,library.read,library.write,system.read,system.write,epg.read -``` - -- The `admin` group is built-in (always grants all permissions) and must not be defined in this file. -- If the file does not exist, only the built-in `admin` group is available. -- Groups and users can also be managed through the RBAC admin panel in the Web UI. - -### Permission domains - -| Domain | `.read` | `.write` | -|------------|----------------------|---------------------------------| -| `config` | view main config | edit main config | -| `source` | view sources | edit sources | -| `user` | list proxy users | create/edit/delete proxy users | -| `playlist` | view playlists | update/refresh playlists | -| `library` | view library | scan/manage library | -| `system` | view status/streams | GeoIP update, file downloads | -| `epg` | view EPG data | *(reserved for future use)* | - -Write does not imply read. A group must explicitly grant both `domain.read` and `domain.write` if users need to view and edit content. - -## Access control and stream fallback - -### `user_access_control` - -If enabled, Tuliprox checks provider or configured user status, expiration date and max-connections data. - -### `connect_timeout_secs` - -Defines the connect-phase timeout for provider requests. -`0` disables the connect timeout. - -### `custom_stream_response_path` - -Directory that contains fallback transport streams: - -- `channel_unavailable.ts` -- `user_connections_exhausted.ts` -- `provider_connections_exhausted.ts` -- `low_priority_preempted.ts` -- `user_account_expired.ts` -- `panel_api_provisioning.ts` - -`custom_stream_response_timeout_secs` can cap the playback duration of those fallback streams. - -## Other optional sections - -### `user_config_dir` - -Storage location for user-specific configuration such as bouquets. - -### `hdhomerun` - -Enables HDHomeRun emulation for Plex, Jellyfin, Emby or TVHeadend integration. -It supports multiple virtual devices, optional lineup auth and SSDP plus SiliconDust discovery. - -### `proxy` - -Global outgoing proxy for upstream requests. -Supported schemes include `http`, `https` and `socks5`. - -### `ipcheck` - -Defines endpoints and optional regexes for public IP detection: - -- `url` -- `url_ipv4` -- `url_ipv6` -- `pattern_ipv4` -- `pattern_ipv6` - -### `config_hot_reload` - -If enabled, mapping files and API-proxy configuration are hot reloaded. - -### `library` - -Local media library integration: - -- recursive directory scanning -- movie vs series classification -- NFO reading and writing -- TMDB enrichment -- incremental scans -- stable virtual IDs - -Example: - -```yaml -library: - enabled: true - scan_directories: - - enabled: true - path: "/projects/media" - content_type: auto - recursive: true - supported_extensions: ["mp4", "mkv", "avi", "mov", "ts", "m4v", "webm"] -``` diff --git a/docs/src/configuration/mapping-dsl.md b/docs/src/configuration/mapping-dsl.md new file mode 100644 index 000000000..ba7c5052d --- /dev/null +++ b/docs/src/configuration/mapping-dsl.md @@ -0,0 +1,207 @@ +# 🗺️ Pillar 4: `mapping.yml` (Mapper DSL & Logic) + +While the filter syntax in `source.yml` only determines *whether* a channel is let through, the Mapping Engine allows you to +perform deep, structural manipulations on the stream object *before* it is written to the final playlist. + +Tuliprox utilizes a blazingly fast, embedded **DSL (Domain Specific Language)** specifically built for this purpose. + +## Context: Why use the DSL? + +Without the DSL, your provider dictates how your channels are named and categorized. The DSL gives you the power to completely +restructure chaotic upstream provider lists into a perfectly clean, personalized format. For example, you can extract the year +out of a messy title, rename the category based on that year, fix resolutions, or filter out specific terms globally. + +## Top-level entries + +```yaml +mappings: + templates: + mapping: +``` + +* **`templates`**: *(Legacy)* Inline templates for filter macros. Prefer [template.yml](configuration/template.md). +* **`mapping`**: A list of mapping rule objects. + +## Mapping Rule Structure (`mapping`) + +```yaml +mappings: + mapping: + - id: map_sports_clean + match_as_ascii: true + mapper: + - filter: 'Group ~ ".*Sports.*"' + script: | + @Group = "Live Sports HD" + counter: + - filter: 'Group ~ "Live Sports HD"' + value: 100 + field: chno + modifier: assign +``` + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| **`id`** | String | The identifier of the mapping. It is referenced in `source.yml` under the respective Target (`mapping_ids:[map_sports_clean]`). | +| **`match_as_ascii`** | Bool | **Background:** If `true`, Tuliprox normalizes (de-unicodes) values on-the-fly during regex evaluation. `Cinéma` is treated as `Cinema`. The actual assignment in the DSL, however, retains the original accents! | +| **`mapper`** | List | A list of scripts (executed sequentially) containing the DSL logic. Can optionally be gated by a `filter`. | +| **`counter`** | List | Logic for assigning channel numbers sequentially (see below). | + +--- + +## Filter & Operator Basics + +Before writing advanced scripts, you must understand the logical operators available for evaluating `filter` strings (Applies to +`source.yml` & `mapping.yml`): + +* **Logical Operators:** `AND`, `OR`, `NOT` +* **Regex Match:** `~` (Tilde executes a Regex match against the specified field) +* **Type Match:** `=` (E.g., `Type = live`, `Type = vod`, `Type = series`, `Type = movie`) + +**Evaluatable Fields:** `Group`, `Title`, `Name`, `Caption`, `Url`, `Genre`, `Input`, `Type` + +*Example:* `((Group ~ "^DE.*") AND (NOT Title ~ ".*Shopping.*")) OR (Group ~ "^AU.*")` + +--- + +## The Mapper DSL (`mapper`) + +The language supports logical constructs, regex evaluations, and assignments. The fields of the currently processed playlist item +are always accessed using the `@` prefix. + +**Readable & Writable `@Fields`:** +`@name`, `@title`, `@caption`, `@group`, `@id`, `@chno`, `@logo`, `@logo_small`, `@parent_code`, `@audio_track`, `@time_shift`, +`@rec`, `@url`, `@epg_channel_id`, `@epg_id`, `@genre`. + +*(Special note on `@Caption`: Acts as an alias for Title/Name. If you write to `@Caption`, Tuliprox updates both the `Title` AND +the `Name` to the same value).* + +### 1. Built-in Functions + +| Function | Explanation | Example | +| :--- | :--- | :--- | +| `concat(a, b, ...)` | Concatenates multiple strings. | `concat("US \| ", @Title)` | +| `uppercase(a)` | Converts text to UPPERCASE. | `uppercase(@Group)` | +| `lowercase(a)` | Converts text to lowercase. | `lowercase(@Genre)` | +| `capitalize(a)` | Title Case (Capitalizes the first letter of words). | `capitalize(@Title)` | +| `split(a, delim)` | Splits a string and returns a Named list (iterable). | `split(@Genre, ",")` | +| `trim(a)` | Removes whitespace from the edges. | `trim(@Title)` | +| `replace(a, b, c)` | Simple text Search (b) & Replace (c). | `replace(@Title, "FHD", "")` | +| `pad(val, len, char, align)` | Pads strings/numbers. `>` (Pad left), `<` (Pad right), `^` (Center). | `pad(1, 3, "0", ">")` | +| `format(fmt, ...)` | Rust-style string formatting substituting `{}`. | `format("S{}E{}", season, ep)` | +| `template(name)` | Retrieves a macro value from `template.yml`. | `template("MY_MACRO")` | +| `number(val)` | Casts a string to a float/integer. | `number("2024")` | +| `first(list)` | Returns the first element of a Named list/Regex match. | `first(@Caption ~ "(\d+)")` | +| `print(a, b, ...)` | Logs the values to the console (Requires `trace` log level). | `print("Matched:", @Title)` | +| `add_favourite(grp)` | **Background:** Clone function! Takes the currently processed stream, changes its group to `grp`, generates a clean alias UUID, and adds the stream additionally (as a "Favorite") to the playlist. | `add_favourite("Top 10")` | + +### 2. RegEx Captures & Extraction + +Regular expressions are executed using the tilde `~` operator. The results are placed in a capture object. You can access them +via index (`.1`, `.2`) or via "Named Captures". + +```dsl +# Extract the year from the title using named captures +info = @Title ~ "(?P.*?)\s-\s(?P\d{4})" + +# Store the matches in variables +movie_title = info.Movie +movie_year = info.Year + +@Title = movie_title +``` + +### 3. Match Blocks (Switch-Case Logic) + +A `match` block allows conditional assignments. **Crucial:** Order matters. The first block whose condition evaluates to true is +executed, and the `match` block exits. + +```dsl +# Check if a regex found a specific station (e.g., FOX) +station = @Caption ~ "FOX" + +result = match { + (var1, var2) => "Both variables exist", + station => "Only the station exists", + _ => "Fallback (Default)", # The underscore matches anything +} +``` + +### 4. Map Blocks (Dictionaries & Ranges) + +Map blocks are ideal for translating hundreds of cryptic provider categories or resolutions into your own clean design. + +**Mapping Texts (with Multi-Keys `|`):** + +```dsl +quality = uppercase(@Caption ~ "\b(HD|FHD|4K|UHD)\b") + +quality = map quality { + "SHD" | "SD" => "SD", + "1080p" | "FHD" => "FHD", + "4K" | "3840p" => "UHD", + _ => quality, # If nothing matches, keep the original +} +``` + +**Mapping Numbers (Ranges `..`):** + +```dsl +year_text = @Caption ~ "(\d{4})\)?$" +year = number(year_text) # Cast String to Number + +year_group = map year { + ..2019 => "Classics (< 2020)", + 2020..2025 => "New Releases", + _ => year_text, +} +``` + +### 5. For-Each Loops (Iterating Lists) + +`for_each` iterates over Named-Results (like Regex captures or output from `split()`). Perfect for distributing movies into +multiple virtual folders based on their genres! + +```dsl +genres = split(@Genre, "[,/&]") + +genres.for_each((ignored_index, single_genre) => { + # For each genre in the string "Action, Drama", an alias stream is created! + add_favourite(concat("Genre - ", trim(single_genre))) +}) +``` + +--- + +## Counters (Sequential Numbering) (`counter`) + +Many IPTV players sort by channel number (`tvg-chno`). Counters allow sequential numbering of channels *after* they have passed +through the DSL. + +```yaml +mapping: + - id: add_channel_numbers + counter: + - filter: 'Group ~ "DE Channels"' + value: 100 + field: chno + modifier: assign + - filter: 'Group ~ "DE Channels"' + value: 1 + padding: 3 + field: title + modifier: prefix + concat: ". " +``` + +**Field Details:** + +* `filter`: A string filter determining which streams this counter applies to. +* `value`: The starting value (e.g., start counting from `100`). +* `field`: The target field to write to (`chno`, `title`, `name`, `caption`). +* `padding`: Zero-padding (e.g., `padding: 3` turns `1` into `001`). +* `modifier`: + * `assign`: Hard overwrites the `field` with the number. + * `prefix`: Prepends the number to the field (e.g., `001. ARD HD`). + * `suffix`: Appends the number to the field. +* `concat`: The separator between the number and the original field for Prefix/Suffix (e.g., `". "`). diff --git a/docs/src/configuration/metadata-update.md b/docs/src/configuration/metadata-update.md new file mode 100644 index 000000000..9ece22c32 --- /dev/null +++ b/docs/src/configuration/metadata-update.md @@ -0,0 +1,247 @@ +# 📖 Metadata Update & FFprobe (`metadata_update`) + +This chapter covers the `metadata_update` block inside `config.yml`, which determines how aggressively or gently Tuliprox +manages background tasks for resolving metadata and technical stream properties. + +Tuliprox utilizes three distinct mechanisms to ensure perfect library quality (especially for Plex/Jellyfin compatibility): + +1. **Resolve (Xtream API):** Fetching missing VOD details (Cast, Director, Plot) directly from the provider's API. +2. **TMDB:** Supplementing missing Release Years and high-resolution Covers/Backdrops via The Movie Database. +3. **Probe (FFprobe):** Physically opening the stream to analyze the exact A/V codecs (HEVC, H264) and resolution. + +## Top-level entries + +```yaml +metadata_update: + cache_path: metadata + retry_delay: 2s + worker_idle_timeout: 1m + max_queue_size: 100000 + no_change_cache_ttl_secs: 3600 + probe_fairness_resolve_burst: 200 + log: + resolve: + probe: + tmdb: + ffprobe: +``` + +### Global Options (Flat Keys) + +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `cache_path` | String | `"metadata"` | Directory where TMDB cache files and metadata are stored. Relative paths are resolved against `storage_dir`. | +| `retry_delay` | String | `"2s"` | General minimum wait time when the worker encounters temporary runtime errors (e.g., socket timeout). | +| `worker_idle_timeout` | String | `"1m"` | Time of inactivity (empty queue) after which the background worker kills itself to free RAM/CPU. | +| `max_queue_size` | Int | `100000` | RAM Safety Limit: Maximum number of metadata tasks kept in memory per input simultaneously. | +| `no_change_cache_ttl_secs` | Int | `3600` | How long (seconds) a "No Change" status is cached to avoid unnecessary DB checks across subsequent playlist updates. | +| `probe_fairness_resolve_burst` | Int | `200` | After 200 consecutive Resolve tasks, 1 Probe task is forcibly prioritized so probes don't starve. | + +--- + +## 1. Logging (`log`) + +Controls background worker logging verbosity. + +```yaml +metadata_update: + log: + queue_interval: 30s + progress_interval: 15s +``` + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `queue_interval` | String | `"30s"` | Interval to log the queue status (pending tasks). | +| `progress_interval` | String | `"15s"` | Interval for progress reports (successful/failed resolves). | + +--- + +## 2. API Resolve Limits (`resolve`) + +Controls API requests for pure metadata (Xtream Info / TMDB). + +```yaml +metadata_update: + resolve: + max_retry_backoff: 1h + min_retry_base: 5s + max_attempts: 3 + exhaustion_reset_gap: 1h +``` + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `max_retry_backoff` | String | `"1h"` | Maximum time limit for exponential wait between API failures. | +| `min_retry_base` | String | `"5s"` | Initial wait time on the very first failure. | +| `max_attempts` | Int (u8) | `3` | Max attempts per cycle before a resolve task is marked as "exhausted". | +| `exhaustion_reset_gap` | String | `"1h"` | Time window after a cycle completes before "exhausted" tasks are retried in the next run. | + +--- + +## 3. FFprobe Retries & Limits (`probe`) + +Controls technical stream probing retries via FFprobe. + +```yaml +metadata_update: + probe: + cooldown: 7d + retry_load_retry_delay: 1m + retry_backoff_step_1: 10m + retry_backoff_step_2: 30m + retry_backoff_step_3: 1h + max_attempts: 3 + backoff_jitter_percent: 20 + user_priority: 127 +``` + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `cooldown` | String | `"7d"` | Hard lock time (cooldown) during which a broken stream is completely ignored to protect the provider. | +| `retry_load_retry_delay` | String | `"1m"` | Wait time if loading the internal `metadata_retry_state.db` fails. | +| `retry_backoff_step_1` | String | `"10m"` | Wait time after the 1st FFprobe failure. | +| `retry_backoff_step_2` | String | `"30m"` | Wait time after the 2nd FFprobe failure. | +| `retry_backoff_step_3` | String | `"1h"` | Wait time from the 3rd FFprobe failure onwards. | +| `max_attempts` | Int (u8) | `3` | Max failures to probe a stream before it enters global long-term cooldown. | +| `backoff_jitter_percent` | Int (u8) | `20` | Random time deviation in percent (Jitter) so hundreds of parallel retries don't hit the exact same second. | +| `user_priority` | Int (i8) | `127` | Priority of the probe task on the Unix Nice-Scale. `127` is the absolute lowest. A real user will instantly evict the probe task. | + +--- + +## 4. TMDB API Integration (`tmdb`) + +```yaml +metadata_update: + tmdb: + enabled: false + api_key: "YOUR_KEY" + rate_limit_ms: 250 + cache_duration_days: 30 + language: en-US + cooldown: 7d + match_threshold: 86 +``` + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `enabled` | Bool | `false` | Global master switch for TMDB resolution. | +| `api_key` | String | *(Internal)* | Your own TMDB API Key. If omitted, Tuliprox uses a built-in default key. | +| `rate_limit_ms` | Int (u64) | `250` | Throttling of TMDB API calls (in ms) to prevent TMDB IP bans. | +| `cache_duration_days` | Int (u32) | `30` | How long successful TMDB results are kept in the internal cache. | +| `language` | String | `"en-US"` | Preferred metadata language (e.g., `"de-DE"`). | +| `cooldown` | String | `"7d"` | Lock time for a movie if the TMDB search was *successful* but returned *no match* for the title. | +| `match_threshold` | Int (u16) | `86` | Minimum percentage score (Jaro-Winkler Distance) for a TMDB result to be accepted as a "Match". | + +--- + +## 5. FFprobe Process Rules (`ffprobe`) + +```yaml +metadata_update: + ffprobe: + enabled: true + timeout: 60 + analyze_duration: 10s + probe_size: 10MB + live_analyze_duration: 5s + live_probe_size: 5MB +``` + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `enabled` | Bool | `false` | Global master switch for ALL stream probing. Must be `true` for input flags like `probe_vod` to work. | +| `timeout` | Int (u64) | `60` | Hard timeout (in seconds) for the OS FFprobe process. Prevents zombie processes. | +| `analyze_duration` | String | `"10s"` | Passes `-analyzeduration` to FFprobe for VODs/Series. *Warning: Requires an explicit suffix (`s`, `m`)!* | +| `probe_size` | String | `"10MB"` | Passes `-probesize` to FFprobe for VODs/Series (Data Limit). | +| `live_analyze_duration` | String | `"5s"` | Stricter time limit for Live-TV streams (minimizes latency). | +| `live_probe_size` | String | `"5MB"` | Stricter data limit for Live-TV streams. | + +**Important Note on FFprobe split:** The split design between `ffprobe` (VOD) and `ffprobe.live_...` is essential. VODs reside +statically on the server and can be analyzed generously. Live-TV streams, however, must respond quickly to avoid generating +unnecessary traffic and occupying the provider slot uselessly. + +--- + +  + +## Additional Information + +Tuliprox is not just a proxy; it is a highly intelligent **Playlist Processing Engine**. A core part of this is the asynchronous +update and metadata process that loads information from the provider, updates local databases, and fully automatically supplements +missing metadata. + +## 1. The Complete Processing Pipeline + +When a playlist update starts (via Scheduler, API, or Boot), Tuliprox runs this pipeline: + +1. **Download & Cache Check:** For each configured `input`, it checks if provider data needs re-downloading (controlled by + `cache_duration`). +2. **Input Storage (B+Tree):** The raw data (M3U, Xtream categories) is written to a local, extremely fast B+Tree database + (`input_name.db`). This drastically saves RAM. +3. **Target Processing:** For each defined `target` (output playlist), data is loaded from the input and routed through the + pipeline (`processing_order`, e.g., Filter ➔ Rename ➔ Map). +4. **Metadata Resolve & Probe:** Tuliprox analyzes the filtered entries. If data is missing (e.g., TMDB IDs, Video Codecs), + these are dispatched as "Jobs" to the `MetadataUpdateManager`. +5. **Target Storage & EPG:** The finished playlist is written to the Target databases. Only then is XMLTV EPG data matched and + assigned. + +--- + +## 2. The `MetadataUpdateManager` (Architecture) + +The `MetadataUpdateManager` is an asynchronous background engine (if `resolve_background: true` is set on the input) that +prevents blocking the main playlist update. + +### Architecture & Logic + +* **Per-Input Worker:** A dedicated, isolated *Tokio Task (Worker)* is started for each Provider-Input. This prevents a slow + provider from blocking another. +* **Task-Merging:** If a stream requires both TMDB info and an FFprobe, they are merged into a single Task. +* **Rate-Limiting & Connection-Locks:** The manager strictly respects the `max_connections` of your input. An FFprobe (Stream + Analysis) is *only* initiated if a provider connection is free. A probe task runs at the absolute lowest priority + (`user_priority: 127`). If a real user starts streaming, the FFprobe process is **immediately aborted/preempted** to free the + slot for the user! +* **Smart Retry & Cooldown:** If a fetch or probe fails (e.g., HTTP 502), an exponential backoff with Jitter (random deviation) + kicks in. If the max attempts (`max_attempts`) are reached, the task enters a global cooldown (e.g., 7 days) to stop + harassing the provider. +* **Persistence (Retry State):** The status of failed tasks is stored locally in `metadata_retry_state.db`. A server restart + does not cause Tuliprox to immediately bombard the provider with requests for broken streams. +* **Cascading Updates:** Once a worker collects a batch of metadata, it saves it in the Input DB and immediately *cascades* + (inherits) the updates into all Target DBs, without requiring a full playlist rebuild. + +--- + +## 3. Metadata Collection Mechanisms + +### How is a stream queued for analysis? + +Tuliprox checks every stream for completeness (`has_details()`). A task is queued if the switches in `inputs.options` are +active **and** one of these conditions is met: + +* **Info-Resolve (VOD/Series):** Missing provider info (Cast, Plot, Director) retrievable via Xtream API + (`get_vod_info` / `get_series_info`). +* **TMDB/Date-Resolve:** Missing `tmdb_id` or `release_date`. +* **Probing (FFprobe):** + * VOD/Series: Missing technical A/V parameters (`video_codec`, `audio_codec`, `resolution`). + * Live-TV: The `last_probed_timestamp` is older than `probe_live_interval_hours`. + +### Collection Engines + +* **Release Year / Date (PTT):** + Tuliprox uses a highly optimized internal parser (`PTT` - Parse Torrent Title). It locally analyzes the stream name and + extracts the year (e.g., from *"My Movie (2023)"*). If this fails, it queries the TMDB API. +* **TMDB Information:** + Via TMDB API and a Jaro-Winkler distance comparison (similarity scoring), it fetches IDs, release years, covers, backdrops, + genres, directors, and actors. +* **Video & Audio (FFprobe):** + Tuliprox briefly opens the stream via `ffprobe`. It extracts and normalizes: + * *Resolution:* SD, 720p, 1080p, 1440p, 4K, 8K + * *Video:* Codec (H264, HEVC, AV1), Bit-Depth (8bit, 10bit), Dynamic Range (HDR10, Dolby Vision, HLG) + * *Audio:* Codec (AAC, AC3, EAC3, DTS, TrueHD) and Channels (2.0, 5.1, 7.1) + * Tuliprox uses these tags later for the `add_quality_to_filename` target feature + (e.g., `My Movie [2160p 4K HEVC HDR].strm`). +* **Seasons & Episodes:** + For series, the Xtream API delivers a structure of seasons and episodes. Tuliprox "flattens" these into individually + playable streams (`PlaylistItemType::Series`). Each episode is treated **individually** during probing, as codecs and + resolutions can change from episode to episode. diff --git a/docs/src/configuration/overview.md b/docs/src/configuration/overview.md index 32191c1a8..9e9f5d16d 100644 --- a/docs/src/configuration/overview.md +++ b/docs/src/configuration/overview.md @@ -1,46 +1,48 @@ -# Configuration Overview +# ⚙️ Configuration & Setup (Overview) -Tuliprox is driven by a small set of files rather than one giant document. +Tuliprox intentionally avoids a gigantic, monolithic configuration file. Instead, it follows the principle of **Separation of +Concerns**. The setup consists of "5 Pillars" (files) that are logically separated. -## Main layers +## The Home-Directory Logic (Path Resolution) -- `config.yml`: server, runtime, reverse proxy, scheduling, metadata, Web UI -- `source.yml`: inputs, providers, aliases, targets -- `mapping.yml`: optional mapping logic -- `template.yml`: reusable expressions and templates -- `user.txt`: Web UI login credentials with optional group assignments -- `groups.txt`: RBAC permission group definitions (optional) +Before diving into the files, it is essential to understand how Tuliprox resolves file paths. Upon startup, Tuliprox determines +a central **Home Directory**. All relative paths in your config files (e.g., `storage_dir: ./data` or `web_root: ./web`) are +resolved strictly relative to this Home Directory. -## Practical split +The resolution order for the Home Directory is: -Use `config.yml` for: +1. **CLI Argument:** `--home` or `-H` (Highest Priority) +2. **Environment Variable:** `TULIPROX_HOME` +3. **Fallback:** The physical directory where the executed `tuliprox` binary is located. -- how the application runs -- where it stores data -- how it serves users and streams +### Default Directory Structure -Use `source.yml` for: +By default, if you just run Tuliprox in an empty folder, it will create the following structure: -- what data comes in -- how providers are grouped -- what outputs are exposed +```text +tuliprox_home/ + ├─ config/ # Contains config.yml, source.yml, mapping.yml, user.txt + ├─ data/ # Primary storage_dir for B+Tree databases (*.db) + ├─ data/backup/ # Backups initiated by the Web UI + ├─ data/user/ # User-specific configurations (like favorites) + ├─ downloads/ # Downloaded VODs + └─ web/ # Frontend assets for the Web UI + └─ cache/ # Cached resources +``` -Use mappings/templates for: +*Example:* If your home is resolved to `/opt/tuliprox` and you define `backup_dir: ./backup` in `config.yml`, Tuliprox will +securely store backups exactly under `/opt/tuliprox/backup`. -- content shaping -- repeated logic -- reusable naming or filtering expressions +--- -## Reading order for newcomers +## The 5 Pillars of Configuration -1. `config.yml` -2. `source.yml` -3. targets -4. reverse proxy settings -5. mapping/templates +To utilize Tuliprox fully, you must understand these 5 files and place them in your `config` directory: -## More detail - -- [Config Reference](main-config.md) -- [Sources And Targets](sources-and-targets.md) -- [API Proxy](api-proxy.md) +| File | Responsibility | Architecture Level | +| :--- | :--- | :--- | +| **`config.yml`** | The Core System. Defines *how* Tuliprox physically runs. Sets ports, reverse proxy buffers, paths, TMDB API keys, metadata worker limits, Web UI settings, and global logging. | **Infrastructure & Engine** | +| **`source.yml`** | The Data Flow. Defines *what* goes in (Provider URLs, Xtream credentials, Panel API limits) and *what* goes out (Targets, Filter assignments, Formats like STRM or M3U). | **Data Sources & Targets** | +| **`api-proxy.yml`** | The Gateway. Defines virtual server endpoints exposed to clients (VLC, TiviMate) and handles **Access Management** (Which user can access which target? Which proxy mode and user priority is used?). | **Network & Auth** | +| **`mapping.yml`** | The Transformation. Contains a powerful, embedded DSL (Domain Specific Language) to dynamically rename streams, reassign groups, or map IDs (including counters) based on regex filters. | **Data Enrichment** | +| **`template.yml`** | The DRY Principle (Don't Repeat Yourself). Contains globally reusable Regular Expressions (Regex) and logic macros that can be invoked in `source.yml` and `mapping.yml` via `!MACRO_NAME!`. | **Structuring** | diff --git a/docs/src/configuration/reverse-proxy.md b/docs/src/configuration/reverse-proxy.md new file mode 100644 index 000000000..2dc323817 --- /dev/null +++ b/docs/src/configuration/reverse-proxy.md @@ -0,0 +1,202 @@ +# 🌊 Reverse Proxy (Streaming, Caching & Rate Limits) + +This section documents the `reverse_proxy:` block inside `config.yml`. It is the most critical block for determining runtime +behavior when Tuliprox actively proxies video streams to clients (Reverse Proxy Mode), rather than just redirecting them. + +It manages how Tuliprox establishes upstream connections, buffers video frames, handles sudden client disconnects, and caches +static resources like EPG images and channel logos. + +## Top-level entries + +```yaml +reverse_proxy: + resource_rewrite_disabled: false + rewrite_secret: A1B2C3D4E5F60718293A4B5C6D7E8F90 + stream: + cache: + rate_limit: + disabled_header: + resource_retry: + geoip: +``` + +### General Parameters + +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `resource_rewrite_disabled` | Bool | `false` | Normally, Tuliprox rewrites all image URLs in playlists to point to itself (e.g., `http://tuliprox:8901/resource/...`). If set to `true`, original URLs are kept (clients load images directly from the provider). **Warning:** Local caching will stop working if this is enabled! | +| `rewrite_secret` | String | `""` | A 32-character Hex string (16 bytes). Tuliprox encrypts/signs the original image URLs during the rewrite process. To prevent image URLs from becoming invalid after a server restart, you MUST enter a static secret here (generate via `openssl rand -hex 16`). | + +--- + +## 1. Stream Management (`stream`) + +This sub-block defines how Tuliprox maintains stream stability, buffers data, and handles HLS/Catchup session affinity. + +```yaml +reverse_proxy: + stream: + retry: true + buffer: + enabled: true + size: 1024 + throttle_kbps: 12500 + grace_period_millis: 2000 + grace_period_timeout_secs: 4 + grace_period_hold_stream: true + hls_session_ttl_secs: 15 + catchup_session_ttl_secs: 45 + shared_burst_buffer_mb: 12 + metrics_enabled: false +``` + +### Stream Parameters in Detail + +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `retry` | Bool | `true` | **Background:** If the upstream provider unexpectedly drops the connection or a TCP network timeout occurs, Tuliprox immediately opens a new connection to the provider in the background and seamlessly pipes the new bytes to the end-client. The client's player (e.g., VLC) might stutter for a fraction of a second but will not abort playback. | +| `buffer.enabled` | Bool | `false` | Enables an asynchronous ring-buffer in RAM between the provider download stream and the client upload stream. Necessary if the provider stream is faster than the consumer can process. | +| `buffer.size` | Int | `0` | The size of the buffer in *Chunks* (1 Chunk = 8192 Bytes). A value of `1024` equals approximately 8 Megabytes of RAM per active stream. | +| `throttle_kbps` | Int | `0` | **Background:** Some players download VODs (Movies) at maximum line speed ("Bursting"). Providers often view this as abuse or scraping and will ban the IP. By throttling (e.g., to `12500` kbps), you force the download into a constant, inconspicuous flow. Supports units like `KB/s`, `MB/s`, `kbps`, `Mibps`. | +| `metrics_enabled` | Bool | `false` | **Monitoring:** If active, Tuliprox samples the live bandwidth (in kbps) and transferred bytes for every active reverse-proxied stream and pushes them via WebSockets to the Web UI. It adds a tiny bit of CPU overhead but is invaluable for debugging buffering issues. | +| `grace_period_millis` | Int | `2000` | The exact time window in ms where a temporary over-allocation is allowed (see notes on [The VLC Seek Problem](#the-vlc-seek-problem--grace-periods) for details). | +| `grace_period_timeout_secs` | Int | `4` | A hard timeout limit for overlapping "ghost sessions" to expire. | +| `grace_period_hold_stream` | Bool | `true` | Tuliprox artificially holds back the video data to the client, waiting for the grace check to finish, so it doesn't trigger the provider prematurely. | +| `hls_session_ttl_secs` | Int | `15` | Keeps the virtual provider slot open between HLS segment (`.ts`) requests to prevent provider bans for "Account Hopping". | +| `catchup_session_ttl_secs` | Int | `45` | The same session-holding principle applied to Archive/Catchup TV. See notes on section [Session TTLs for HLS & Catchup](#session-ttls-for-hls-m3u8--catchup) for details. | +| `shared_burst_buffer_mb` | Int | `12` | Minimum burst buffer size (in MB) used for shared live streams to immediately synchronize new clients without Keyframe dropouts. See notes on section [Shared Live Streams](#shared-live-streams) for details. | + +--- + +## 2. Resource Caching (`cache`) + +Tuliprox caches channel logos, posters, and EPG images on your disk so your clients don't stress the provider's servers on every +playlist reload. + +```yaml +reverse_proxy: + cache: + enabled: true + size: 1GB + directory: ./cache +``` + +An LRU (Least Recently Used) disk cache. If it hits the limit (e.g., `1GB`), Tuliprox automatically deletes the oldest images. +Note: Fails if `resource_rewrite_disabled` is true. + +--- + +## 3. Rate Limiting (`rate_limit`) + +```yaml +reverse_proxy: + rate_limit: + enabled: true + period_millis: 500 + burst_size: 10 +``` + +Implements an IP-based Token-Bucket rate limiter. In this example, an IP can fire 10 requests immediately (`burst_size`). After +that, it receives exactly one new token every 500ms (`period_millis`). This prevents DDOS attacks from malfunctioning scrapers. +*(Ensure your upstream Nginx/Traefik passes `X-Forwarded-For` for this to work correctly!)* + +--- + +## 4. Header Stripping (`disabled_header`) + +When Tuliprox makes requests to the upstream provider, it can strip revealing headers that might expose which player you are +actually using or the fact that you are proxying traffic. + +```yaml +reverse_proxy: + disabled_header: + referer_header: true # Removes the 'Referer' header + x_header: true # Removes all 'X-*' headers (like X-Real-IP) + cloudflare_header: true# Removes 'CF-*' headers + custom_header: + - my-custom-tracker +``` + +--- + +## 5. Resource Retries (`resource_retry`) + +Defines how aggressively Tuliprox tries to retry failed logo or EPG downloads when proxying them from the provider. + +```yaml +reverse_proxy: + resource_retry: + max_attempts: 3 + backoff_millis: 250 + backoff_multiplier: 1.5 + failover_redirect_patterns: + - "service-abuse" +``` + +* **Retries:** After the first error, Tuliprox waits 250ms. After the second error, it waits `250 * 1.5 = 375ms`, and so on. +* **`failover_redirect_patterns`:** A list of Regex patterns. If an upstream resource responds with an HTTP Redirect (302) + containing these patterns (e.g., pointing to a "service-abuse" warning image from the provider), Tuliprox treats it as a + failure instead of blindly serving the abuse image to your clients. + +--- + +## 6. GeoIP Resolution (`geoip`) + +To see the country flags of connected clients in the Web UI ("Active Streams" tab), Tuliprox can resolve IP addresses locally. + +```yaml +reverse_proxy: + geoip: + enabled: true + url: "https://raw.githubusercontent.com/sapics/ip-location-db/refs/heads/main/asn-country/asn-country-ipv4.csv" +``` + +The CSV file must have exactly 3 columns: `range_start,range_end,country_code`. (The DB is periodically updated via the +`schedules` block using the `GeoIpUpdate` task type). + +--- + +  + +## Additional Information + +## The "VLC Seek Problem" & Grace Periods + +When a user fast-forwards or rewinds a VOD, the player calculates the new byte offset, drops the old TCP connection, and +immediately fires a new HTTP GET request (with a `Range` header) to Tuliprox. + +**The Problem:** It takes milliseconds to seconds for the upstream provider to realize the old connection is dead. If you have +a `max_connections: 1` limit at the provider, they will view this new seek-request as a *second concurrent stream* and reject +it with an HTTP 509 (Bandwidth Exceeded) or HTTP 401 error. + +**The Tuliprox Solution:** + +* `grace_period_millis: 2000`: Tuliprox grants the user a temporary over-allocation (Grace) for exactly this duration. +* `grace_period_hold_stream: true`: Tuliprox artificially holds back the video data to the client, waiting for the grace check + to finish, so it doesn't trigger the provider prematurely. +* After the milliseconds expire, Tuliprox checks internally: Is the old connection truly gone now? If Yes ➔ Data flows. + If No ➔ The new connection is hard-killed (serving the `user_connections_exhausted.ts` video) because the user is actually + illegally watching twice. +* `grace_period_timeout_secs: 4`: A hard timeout limit for overlapping "ghost sessions" to expire. + +## Session TTLs for HLS (`.m3u8`) & Catchup + +HLS streams do not consist of an endless TCP pipe. Instead, the player downloads small `.ts` segments every few seconds +(e.g., `seg1.ts`, `seg2.ts`). + +If Tuliprox released and re-acquired the provider slot for every single segment, providers would block the account for "Account +Hopping" or spam. Tuliprox simulates a continuous session: + +* `hls_session_ttl_secs: 15`: After a `.ts` segment finishes downloading, the physical slot to the provider is closed, but the + "Virtual Slot" for this specific user remains reserved for 15 seconds. No other user can steal this slot during this window. + Channel switches from the same client can immediately take over the reservation. +* The same principle applies to Archive/Catchup TV (`catchup_session_ttl_secs: 45`), which shares the same fragmentation and + seeking issues. + +## Shared Live Streams + +Tuliprox can share a live stream (`share_live_streams: true` in the target options of `source.yml`). If 5 users watch the same +Live-TV channel, Tuliprox pulls the stream only 1x from the provider and multicasts the bytes locally to 5 clients. +To ensure a user who tunes in 10 seconds later doesn't get player errors due to missing I-Frames/Keyframes, Tuliprox +continuously keeps the last X Megabytes (`shared_burst_buffer_mb`, default `12`) in RAM. It fires this burst buffer at new +subscribers so their decoders can instantly synchronize. diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md new file mode 100644 index 000000000..b21bb27d7 --- /dev/null +++ b/docs/src/configuration/source.md @@ -0,0 +1,406 @@ +# 🔌 Pillar 2: `source.yml` (Inputs, Panel API & Targets) + +The `source.yml` is the central hub for data flows. Here you define your upstream providers (`inputs`), pool them using aliases, +configure automated reseller provisioning (`panel_api`), and define the output channels for your end devices (`targets`). + +## Top-level entries + +```yaml +templates: +provider: +inputs: +sources: +``` + +| Block | Description | Link | +| :--- | :--- | :--- | +| `templates` | *(Legacy)* Inline templates for filter macros. Prefer `template.yml`. | | +| `provider` | Provider Failover & DNS Rotation definitions. | [See section](#1-provider-failover--dns-rotation-provider) | +| `inputs` | Data Sources (Providers, Files, Batches, Library). | [See section](#2-inputs-data-sources-inputs) | +| `sources` | Routing logic combining inputs to output targets. | [See section](#3-routing--targets-sources) | + +--- + +## 1. Provider Failover & DNS Rotation (`provider`) + +Tuliprox includes a robust failover engine for unstable IPTV providers. You can define backup URLs and intelligent IP rotation. + +Define a `provider` block globally in `source.yml` to specify multiple backup URLs: + +```yaml +provider: + - name: my_failover_provider + urls: + - http://primary.example.com + - http://backup.example.com + dns: + enabled: true + refresh_secs: 300 + prefer: ipv4 # system, ipv4, ipv6 + schemes: [http, https] + keep_vhost: true + max_addrs: 2 + on_resolve_error: keep_last_good # or fallback_to_hostname + on_connect_error: try_next_ip # or rotate_provider_url + overrides: + "primary.example.com": + - 203.0.113.10 +``` + +### DNS Rotation Parameters (`provider.dns`) + +| Parameter | Type | Default | Technical Impact | +| :--- | :--- | :--- | :--- | +| `refresh_secs` | Int | `300` | The interval in seconds the background task resolves the hostnames. (Minimum effective value is 10). | +| `prefer` | Enum | `system` | Which IP protocol to prefer during DNS resolution. Options: `system`, `ipv4`, `ipv6`. | +| `max_addrs` | Int | `None` | Hard limit on the number of resolved IPs to retain per host. | +| `schemes` | List | `[http, https]` | The HTTP schemes that IP connection rotation applies to. | +| `keep_vhost` | Bool | `false` | If `true`, the `Host` header retains the original `hostname[:port]`. If `false`, it uses `IP[:port]`. Essential for reverse proxies upstream! | +| `on_resolve_error` | Enum | `keep_last_good` | Policy on DNS resolution failure. Options: `keep_last_good` (uses cached IPs), `fallback_to_hostname` (clears cache, forcing host lookup). | +| `on_connect_error` | Enum | `try_next_ip` | Policy on TCP connection failure. Options: `try_next_ip` (cycles to the next resolved IP for the same host), `rotate_provider_url` (instantly fails over to the next URL in the `urls` list). | + +### Failover Triggers + +Tuliprox automatically switches URLs or DNS IPs on failure. +Failover **DOES** occur on: + +* Network Timeouts +* HTTP 5xx errors (500, 502, 503, 504) +* HTTP 404 / 410 / 429 + +Failover **DOES NOT** trigger on: + +* HTTP 401 / 403 (Authentication errors, to avoid rotating due to a banned account). + +--- + +## 2. Inputs (Data Sources) (`inputs`) + +An `input` represents an upstream provider or a local media library. + +```yaml +inputs: + - name: my_provider + type: xtream + url: provider://my_failover_provider + username: my_user + password: my_password + enabled: true + cache_duration: 1d + persist: playlist_{}.m3u + method: GET + headers: {} + options: {} + epg: {} + aliases:[] + staged: {} + panel_api: {} +``` + +### Input Base Parameters + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :--- | :--- | :---: | :--- | :--- | +| `name` | String | Yes | | Internal reference ID for Tuliprox (e.g., `provider_alpha`). Must be strictly unique. Critical for persistent UUID generation! | +| `type` | Enum | No | `m3u` | Allowed: `m3u`, `xtream`, `library` (Local files), and `m3u_batch`/`xtream_batch` (CSV offloading). | +| `url` | String | Yes | | The Provider URL. Tuliprox supports magic scheme prefixes: `http://`, `https://`, `file://`, `batch://`, and **`provider://my_failover_provider`** (for the Failover System above). | +| `username` / `password` | String | Often | | Mandatory if `type` = `xtream`. | +| `enabled` | Bool | No | `true` | If `false`, this input is completely ignored in all processing. | +| `cache_duration` | String | No | `0` | **Crucial:** Determines how often Tuliprox actually downloads the raw list from the provider. At `1d` (1 day), Tuliprox serves from its local `.db` for 24 hours, even if you trigger hourly updates. This heavily protects against provider bans! Supports suffixes `s`, `m`, `h`, `d`. | +| `persist` | String | No | | Optional path template (e.g., `playlist_{}.m3u`) to permanently store the downloaded raw provider list locally on your disk. | +| `epg` | Object | No | | Allows mapping of external XMLTV files (see below). | +| `method` | Enum | No | `GET` | HTTP Request method for playlist downloads (`GET` or `POST`). | +| `headers` | Dict | No | | Custom HTTP headers for the download (e.g., `User-Agent: My-Player`). | +| `aliases` | List | No | | Connection pooling / Sub-accounts (see below). | +| `staged` | Object | No | | Hybrid architecture feature (see below). | +| `panel_api` | Object | No | | Automated reseller account generation (see below). | + +--- + +### Input Options (`options`) + +Controls the behavior during download and asynchronous metadata resolution (see the *Metadata Update* chapter) for this specific +provider. + +| Parameter | Type | Default | Technical Impact & Background | +| :--- | :--- | :--- | :--- | +| `xtream_skip_live` / `vod` / `series` | Bool | `false` | Immediately ignores entire categories during the Xtream API download. Saves massive amounts of RAM and runtime if you only want Live-TV from a specific provider, for instance. | +| `xtream_live_stream_without_extension` | Bool | `false` | Strips `.ts` from generated stream URLs. | +| `xtream_live_stream_use_prefix` | Bool | `true` | Injects the `/live/` prefix into URLs. | +| `disable_hls_streaming` | Bool | `false` | Forces Tuliprox to play Live-TV as a raw MPEG-TS (`.ts`) stream, skipping HLS (`.m3u8`) reverse-proxy handling, and forcing direct TS endpoints. | +| `resolve_tmdb` | Bool | `false` | Enables TMDB queries for this specific input based on parsed titles to fill missing posters and release years. | +| `probe_stream` | Bool | `false` | Allows Tuliprox to open a provider connection (`max_connections`) to read A/V details (Codecs, HDR, 4K) via FFprobe. | +| `resolve_background` | Bool | `true` | Metadata scans run asynchronously in the background so the general playlist update (which blocks clients) finishes instantly. | +| `resolve_series` / `resolve_vod` | Bool | `false` | Fetches missing details like Plot or Cast via the Provider's API (`get_vod_info` / `get_series_info`). | +| `probe_series` / `probe_vod` | Bool | `false` | Allows explicit FFprobe analysis of movies or entire TV show seasons. | +| `probe_live` | Bool | `false` | Allows FFprobe to periodically tap into Live-TV streams in the background. | +| `probe_live_interval_hours` | Int | `120` | Interval after which a Live stream is re-analyzed (Important as backup streams often change resolutions). | +| `resolve_delay` / `probe_delay` | Int | `2` | **Ban Protection:** Hard wait time (in seconds) between API or Probe requests to the *same* provider! Prevents API spamming. | + +--- + +### EPG Assignment & Smart Match (`epg`) + +Tuliprox can load external XMLTV files and map them extremely intelligently (Fuzzy-Matching) to streams missing a valid EPG-ID. + +```yaml +epg: + sources: + # 'auto' automatically generates the XMLTV URL from your Xtream credentials + - url: auto + priority: -2 + logo_override: true + - url: http://localhost/my_custom_epg.xml + priority: 0 + smart_match: + enabled: true + fuzzy_matching: true + match_threshold: 80 + best_match_threshold: 99 + name_prefix: { suffix: "." } + name_prefix_separator:[':', '|', '-'] + strip:["3840p", "uhd", "fhd", "hd", "sd", "4k"] + normalize_regex: '[^a-zA-Z0-9\-]' +``` + +**Smart Match Parameters:** + +| Parameter | Type | Default | Technical Impact | +| :--- | :--- | :--- | :--- | +| `fuzzy_matching` | Bool | `false` | Fallback to phonetic and Jaro-Winkler similarity matching if exact ID match fails. | +| `match_threshold` | Int | `80` | Minimum percentage score (10-100) to accept a fuzzy match. | +| `best_match_threshold` | Int | `99` | Score at which Tuliprox stops searching for better matches and immediately accepts the EPG assignment. | +| `name_prefix` | Enum | `Ignore` | How to treat extracted country prefixes (`US`, `FR`). Options: `Ignore`, `Suffix` (appends to end), `Prefix` (appends to start). Example: `{ suffix: "." }` turns `US: HBO` into `hbo.us`. | +| `name_prefix_separator` | List | `[':', '\|', '-']` | Characters used by the provider to delimit the country prefix from the channel name. | +| `strip` | List | *(HD/4K tags)* | Terms aggressively stripped from the channel name before attempting to match against the XMLTV database. | +| `normalize_regex` | String | `[^a-zA-Z0-9\-]` | Regex pattern used to clean names. Default strips all non-alphanumeric characters (except dashes). | + +**How Smart-Matching works:** +If a stream is missing the `tvg-id`, Tuliprox tries to map the channel name to the XMLTV file. +If a channel is named `US: HBO HD 4K`, Tuliprox uses the `name_prefix_separator` logic. It splits at `:`, recognizes `US` +as a country code, strips "4K" and "HD", cleans the string to "hbo", and appends the `name_prefix.suffix` (`.`) ➔ The EPG +Fuzzy-Matching (using Double Metaphone phonetic encoding) now actively searches for the ID `hbo.us` in the XMLTV file! + +--- + +### Provider Aliases (`aliases` & `batch://`) + +**Why use Aliases?** If you bought 3 subscriptions from the *same* provider, you can pool them. Tuliprox merges the lists and +tracks free connections in one logical pool. + +```yaml +inputs: + - type: xtream + name: my_provider + url: http://provider.net + username: sub_1 + password: pw1 + max_connections: 1 + aliases: + - name: alias_sub_2 + url: http://provider.net + username: sub_2 + password: pw2 + max_connections: 2 +``` + +Tuliprox merges the lists and tracks: "For `my_provider` I have 1 + 2 = 3 free connections in one logical pool." + +**Batch CSV Offloading:** +If you manage dozens of aliases, you can use `type: xtream_batch` and set the URL to `batch://./aliases.csv` to offload the +list. + +The CSV format for Xtream is: `name;username;password;url;max_connections;priority;exp_date`. +*(Note: For batch inputs, the first valid row in the CSV assumes the identity/name of the root input to keep UUIDs stable).* + +--- + +### Provider Panel API (`panel_api`) + +Automates the creation of sub-accounts on the provider's reseller panel when connections are needed, and delete/ignore them +when they expire. + +```yaml + panel_api: + url: 'https://panel.provider.com/api.php' + api_key: 'YOUR_ADMIN_KEY' + provisioning: + timeout_sec: 65 + method: GET + probe_interval_sec: 10 + cooldown_sec: 120 + offset: 12h + alias_pool: + size: { min: auto, max: auto } + remove_expired: true + query_parameter: + client_new: + - { key: action, value: new } + - { key: type, value: m3u } + - { key: username, value: auto } +``` + +* `min: auto` & `max: auto`: Tuliprox compares the number of your active/enabled users in `api-proxy.yml` mapped to targets + of this input and generates exactly that many alias accounts via Panel API. +* `provisioning.offset`: Tuliprox doesn't wait until an account expires. `12h` means Tuliprox fires the `client_renew` + API call 12 hours before expiration during the boot/update cycle to prevent downtime. +* `remove_expired: true`: Automatically cleans up the `source.yml` or CSV files and deletes dead accounts. +* `value: auto`: Instructs Tuliprox to inject the actual runtime values (like the generated username or the globally defined + `api_key`) dynamically into the HTTP query parameters. + +### Staged Inputs (`staged`) + +Merge a perfectly maintained M3U file (e.g. from GitHub) for Live-TV with your Xtream Provider for VOD into a *single provider* +in Tuliprox! + +**Background:** You buy a Premium Xtream account for VODs. However, the Live-TV section of this provider is terribly sorted. +But you have a perfectly maintained M3U file (e.g. found on a GitHub repository) for Live-TV. +With `staged`, you can logically merge these physical sources into a *single provider* in Tuliprox! + +```yaml + staged: + enabled: true + type: m3u + url: https://github.com/m3u_list... + live_source: staged + vod_source: input + series_source: skip +``` + +Here, Tuliprox pulls `live` from the m3u file on GitHub url and uses it for Live (Staged source), but continues to use your +Xtream input for VOD. + +--- + +## 3. Routing & Targets (`sources`) + +This block links your inputs to specific output targets and applies transformation filters. +The Target defines the final list your clients download. Under `sources:` you link Targets with one or multiple `inputs`. + +```yaml +sources: + - inputs: + - my_provider + targets: + - name: my_target + output: [] + filter: 'Group ~ ".*"' + rename:[] + sort: {} + mapping: [] + favourites: [] + watch:[] +``` + +### Target Parameters + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :--- | :--- | :---: | :--- | :--- | +| `name` | String | Yes | | Unique Target name, appears in the delivery URL (e.g., `http://host/get.php?username=X&password=Y` delivers the target assigned to this user). | +| `enabled` | Bool | No | `true` | Skips this target during building. | +| `filter` | String | Yes | | Your global filter DSL. Allows operators like `NOT`, `AND`, `OR`. Example: `(!TEMPLATE_TRASH!) AND Type = live`. | +| `processing_order` | Enum | No | `frm` | Execution order: **F**ilter, **R**ename, **M**ap. With `rmf`, it renames first, then maps, then filters. | +| `rename` | List | No | | Simple Regex Search & Replace on specific fields (e.g., `@Group`). | +| `mapping` | List | No | | References IDs from `mapping.yml` for deep DSL logic. | +| `sort` | Object | No | | Sorting logic with Regex Sequences and Orders (`asc`, `desc`). | +| `favourites` | List | No | | Duplicates final channels into a named Fav-group after all transformations. | +| `watch` | List | No | | Regex on group names. If channels in these groups change during an update, Tuliprox generates a Messaging-Event ("Channels added/removed"). | +| `use_memory_cache` | Bool | No | `false` | Puts the entire compiled target playlist into RAM. Extreme speed advantages during M3U download by clients, but costs system memory. | + +--- + +### Output Formats (`output`) + +A Target can be exported to multiple formats simultaneously. Filter logic applies globally, but each output formats the result +differently. + +**1. `xtream`:** + +```yaml +output: + - type: xtream + skip_live_direct_source: true + update_strategy: instant + trakt: + api: { api_key: "XXX", version: "2", url: "https://api.trakt.tv" } + lists: + - { user: "gary", list_slug: "latest-tv", category_name: "Trending TV", content_type: series, fuzzy_match_threshold: 80 } +``` + +* `skip_live_direct_source`: Forces players to use Tuliprox's Xtream logic (Reverse Proxy/Redirect) instead of calling the + provider's direct bypass URL. +* `update_strategy`: `instant` writes changes to disk immediately. `bundled` queues updates to reduce Disk I/O. +* `trakt`: **Deep-Dive:** Tuliprox queries lists from Trakt.tv and searches your playlist for matching movies using Jaro-Winkler + fuzzy logic. If it finds hits, it creates a virtual VOD category in Xtream (e.g., "Trending TV") and copies the movies there! + +**2. `m3u`:** + +```yaml +output: + - type: m3u + filename: custom_playlist.m3u + include_type_in_url: false + mask_redirect_url: false +``` + +* `include_type_in_url`: If true, adds the stream type (`live`, `movie`, `series`) to the URL. +* `mask_redirect_url`: If true, uses URLs from `api_proxy.yml` for users in `redirect` proxy mode. Necessary if you have + multiple providers and want to cycle/failover in redirect mode without exposing the direct provider IP initially. + +**3. `strm`:** + +```yaml +output: + - type: strm + directory: /media/strm + style: plex + flat: true + add_quality_to_filename: true + cleanup: true + strm_props:["#KODIPROP:seekable=true", "#KODIPROP:inputstream=inputstream.ffmpeg"] +``` + +Generates local `.strm` files for Emby, Plex, or Jellyfin. + +| Parameter | Type | Default | Technical Impact | +| :--- | :--- | :--- | :--- | +| `directory` | String | | **Mandatory.** The output folder on your local disk where `.strm` files will be written. | +| `style` | Enum | `kodi` | Naming convention styles for scrapers. Options: `kodi`, `plex`, `emby`, `jellyfin`. (E.g., Plex style outputs: `Movie Name (Year) {tmdb-ID}/Movie Name (Year).strm`). | +| `flat` | Bool | `false` | If true, creates a flat directory structure, skipping category/group subfolders. | +| `cleanup` | Bool | `false` | **Warning:** Deletes orphaned files from the directory that have been removed from the Target. Do not point this directly at your actual media files folder! | +| `underscore_whitespace` | Bool | `false` | Replaces all whitespaces in paths and filenames with `_`. | +| `add_quality_to_filename` | Bool | `false` | Appends tags like `[2160p 4K HEVC HDR]` to the filename. (Requires `ffprobe` probing enabled on the Input!). | +| `strm_props` | List | | Properties injected into `.strm` files to configure Kodi's internal player (e.g., `#KODIPROP:seekable=true`). | + +**4. `hdhomerun`:** + +```yaml +output: + - type: hdhomerun + device: hdhr1 # Must match a device name from config.yml + username: local_user # Must match a user from api-proxy.yml + use_output: xtream # m3u or xtream +``` + +Physically binds this Target to the simulated hardware tuner from `config.yml`. The `username` dictates which user's connection +limits and reverse proxy rules apply when Plex streams from the virtual antenna. + +#### Favourites (`favourites`) + +You can duplicate final, transformed channels into dedicated Favorite groups *after* all filtering and mapping is complete. + +```yaml +favourites: + - cluster: series + group: "My Favourites" + filter: 'Name ~ "Cinema"' + match_as_ascii: true +``` + +* **`match_as_ascii`**: (Bool) Normalizes accented characters during the filter match (allowing "Cinema" to match "Cinéma"). + The final output channel name retains its original accents. + +#### Watch (`watch`) + +Regex on group names. If channels in these groups change during an update, Tuliprox generates a Messaging-Event ("Channels +added/removed"). diff --git a/docs/src/configuration/sources-and-targets.md b/docs/src/configuration/sources-and-targets.md deleted file mode 100644 index b66524f04..000000000 --- a/docs/src/configuration/sources-and-targets.md +++ /dev/null @@ -1,414 +0,0 @@ -# Sources And Targets - -`source.yml` defines upstream inputs, provider aliases and the targets Tuliprox publishes to clients. - -## Top-level entries - -- `templates` -- `inputs` -- `sources` - -For new setups, prefer global `template_path` in `config.yml`, but inline `templates` remain supported. - -## `templates` - -Templates reduce repeated regular expressions and can reference other templates via `!name!`. - -Example: - -```yaml -templates: - - name: delimiter - value: '[\s_-]*' - - name: quality - value: '(?i)(?PHD|LQ|4K|UHD)?' -``` - -## `inputs` - -Each input can define: - -- `name` -- `type`: `m3u`, `xtream`, `library`, `m3u_batch`, `xtream_batch` -- `enabled` -- `persist` -- `url` -- `epg` -- `headers` -- `method` -- `username` -- `password` -- `panel_api` -- `cache_duration` -- `exp_date` -- `options` -- `aliases` -- `staged` - -### URL schemes - -Supported input URL schemes: - -- `http://` / `https://` -- `file://` -- `provider:///...` -- `batch://...` - -If an `m3u` or `xtream` input uses a `batch://` URL, Tuliprox treats it as `m3u_batch` or `xtream_batch`. - -### Common input options - -Important `options` fields include: - -- `xtream_skip_live` -- `xtream_skip_vod` -- `xtream_skip_series` -- `xtream_live_stream_without_extension` -- `xtream_live_stream_use_prefix` -- `disable_hls_streaming` -- `resolve_tmdb` -- `probe_stream` -- `resolve_background` -- `resolve_series` -- `resolve_vod` -- `probe_series` -- `probe_vod` -- `probe_live` -- `probe_live_interval_hours` -- `resolve_delay` -- `probe_delay` - -Example Xtream input: - -```yaml -inputs: - - name: my_provider - type: xtream - url: http://provider.example:8080 - username: user - password: secret - options: - resolve_tmdb: true - probe_stream: true - resolve_series: true - resolve_vod: true -``` - -### EPG configuration - -Inputs can define multiple XMLTV sources with priorities and smart matching: - -```yaml -epg: - sources: - - url: auto - priority: -2 - logo_override: true - - url: http://localhost:3001/xmltv.php?epg_id=1 - priority: -1 - smart_match: - enabled: true - fuzzy_matching: true - match_threshold: 80 - best_match_threshold: 99 -``` - -### Aliases - -Aliases let one logical provider input expose multiple credentials. - -```yaml -inputs: - - type: xtream - name: my_provider - url: http://provider.net - username: primary - password: secret1 - aliases: - - name: my_provider_2 - url: http://provider.net - username: secondary - password: secret2 - max_connections: 2 -``` - -### Batch inputs - -Batch inputs load aliases from CSV files. - -`xtream_batch` CSV fields: - -- `name` -- `username` -- `password` -- `url` -- `max_connections` -- `priority` -- `exp_date` - -`m3u_batch` CSV fields: - -- `url` -- `max_connections` -- `priority` - -### `panel_api` - -Tuliprox can provision or renew provider accounts through a panel API. -Supported operations include: - -- `account_info` -- `client_info` -- `client_new` -- `client_renew` -- `client_adult_content` - -Important controls: - -- `alias_pool.size.min` -- `alias_pool.size.max` -- `alias_pool.remove_expired` -- `provisioning.timeout_sec` -- `provisioning.method` -- `provisioning.probe_interval_sec` -- `provisioning.cooldown_sec` -- `provisioning.offset` - -Example: - -```yaml -panel_api: - url: https://panel.example/api.php - api_key: "1234567890" - provisioning: - timeout_sec: 65 - method: GET - probe_interval_sec: 10 - cooldown_sec: 120 - offset: 12h -``` - -### Staged inputs - -`staged` lets Tuliprox read playlist data from another source during updates while still using the main input for streaming and details. - -Important fields: - -- `enabled` -- `type` -- `url` -- `headers` -- `method` -- `username` -- `password` -- `live_source` -- `vod_source` -- `series_source` - -## `sources` - -Each `source` contains: - -- `inputs` -- `targets` - -The `inputs` list references input names from the top-level `inputs` section. - -## `targets` - -Each target can define: - -- `enabled` -- `name` -- `sort` -- `output` -- `processing_order` -- `options` -- `filter` -- `rename` -- `mapping` -- `favourites` -- `watch` -- `use_memory_cache` - -### `output` - -Supported output types: - -- `xtream` -- `m3u` -- `strm` -- `hdhomerun` - -Important output-specific fields: - -`xtream` - -- `skip_live_direct_source` -- `skip_video_direct_source` -- `skip_series_direct_source` -- `update_strategy` -- `trakt` -- `filter` - -`m3u` - -- `filename` -- `include_type_in_url` -- `mask_redirect_url` -- `filter` - -`strm` - -- `directory` -- `username` -- `underscore_whitespace` -- `cleanup` -- `style` -- `flat` -- `strm_props` -- `add_quality_to_filename` -- `filter` - -`hdhomerun` - -- `device` -- `username` -- `use_output` - -### Target `options` - -- `ignore_logo` -- `share_live_streams` -- `remove_duplicates` -- `force_redirect` - -If `share_live_streams` is enabled, each shared live channel consumes buffer memory even when several clients share the same upstream stream. - -### `sort` - -Sorting consists of ordered rules with: - -- `target`: `group` or `channel` -- `field` -- `filter` -- `order` -- `sequence` - -Example: - -```yaml -sort: - rules: - - target: group - order: asc - filter: Group ~ ".*" - field: group - sequence: - - '^Freetv' - - '^Shopping' - - '^Entertainment' -``` - -### `processing_order` - -Controls the order of Filter, Rename and Map. -Valid values: - -- `frm` -- `fmr` -- `rfm` -- `rmf` -- `mfr` -- `mrf` - -### `filter` - -Filters support `NOT`, `AND`, `OR`, regex matches and `Type` comparisons. -Fields include: - -- `Group` -- `Title` -- `Name` -- `Caption` -- `Url` -- `Genre` -- `Input` -- `Type` - -Example: - -```text -((Group ~ "^DE.*") AND (NOT Title ~ ".*Shopping.*")) OR (Group ~ "^AU.*") -``` - -### `rename` - -Rename rules contain: - -- `field` -- `pattern` -- `new_name` - -Example: - -```yaml -rename: - - field: group - pattern: '^DE(.*)' - new_name: '1. DE$1' -``` - -### `mapping` - -Targets can reference mapping IDs defined in `mapping.yml`. - -### `favourites` - -Allows explicit favorite groups after mapping and resolution. - -Example: - -```yaml -favourites: - - cluster: series - group: "My Favourites" - filter: 'Name ~ "Cinema"' - match_as_ascii: true -``` - -### `watch` - -Watches final group names and emits notifications when those groups change. - -```yaml -watch: - - 'FR - Movies \(202[34]\)' - - 'FR - Series' -``` - -## Example `source.yml` - -```yaml -templates: - - name: ALL_CHAN - value: 'Group ~ ".*"' -inputs: - - type: xtream - name: my_provider - url: http://provider.example:8080 - username: user - password: secret -sources: - - inputs: - - my_provider - targets: - - name: all_channels - output: - - type: xtream - - type: m3u - options: - ignore_logo: false - share_live_streams: true - filter: "!ALL_CHAN!" -``` diff --git a/docs/src/configuration/template.md b/docs/src/configuration/template.md new file mode 100644 index 000000000..46bd8bf41 --- /dev/null +++ b/docs/src/configuration/template.md @@ -0,0 +1,106 @@ +# 🧩 Pillar 5: `template.yml` (Macros & DRY) + +In a large IPTV setup, you will quickly realize that you are repeating the same regular expressions (Regex) or complex filters +(like blocking Adult content) across dozens of targets and mappings. + +This leads to unreadable and highly unmaintainable configurations. Tuliprox solves this elegantly using **Templates** +(applying the DRY principle: Don't Repeat Yourself). + +You define complex strings or regex patterns exactly once. Afterward, you can invoke them in all other configuration files +(like `source.yml` or `mapping.yml`) by wrapping the template name in exclamation marks: `!MACRO_NAME!`. + +## Global Path (Recommended Setup) + +It is highly recommended to set the `template_path` in `config.yml` to a directory rather than a single file: + +```yaml +template_path: ./config/templates.d +``` + +Upon startup, Tuliprox reads all `.yml` files in this directory in alphanumeric order (e.g., `01-regex.yml`, `02-filters.yml`) +and merges them into one massive global macro catalog. + +*Important: The names of the templates (`name`) must be globally unique across all files!* + +--- + +## Top-level entries + +```yaml +templates: + - name: DELIMITER + value: '[\s_-]*' +``` + +## Structure & Variable Resolution + +```yaml +templates: + # A simple regex snippet for delimiters (spaces, underscores) + - name: DELIMITER + value: '[\s_-]*' + + # A capture-group regex for common TV qualities + - name: QUALITY + value: '(?i)(?PHD|LQ|4K|UHD)?' + + # A nested logical filter condition + - name: FILTER_NO_TRASH + value: 'NOT (Group ~ "(?i).*Shopping.*" OR Group ~ "(?i).*Commercials.*")' + + # The Magic: Macros can call other Macros! + - name: FILTER_DE_CLEAN + value: 'Group ~ "^DE.*" AND !FILTER_NO_TRASH!' + + # Lists for Sequence-Sorting + - name: CHAN_SEQ + value: + - '(?i)\bUHD\b' + - '(?i)\bFHD\b' +``` + +Tuliprox recursively resolves the entire template tree during system startup. +*(Security Feature: The system detects cyclic dependencies—Macro A calls Macro B, which calls Macro A—and aborts the startup +with a log error to prevent infinite loops).* + +--- + +## Practical Application + +### 1. In `source.yml` (As a Target Filter) + +Instead of writing a monstrous 500-character line into your target, you build it out of logical template blocks. + +```yaml +targets: + - name: clean_german_tv + filter: "!FILTER_DE_CLEAN! AND Type = live" +``` + +### 2. In `source.yml` (As a Sequence Sort) + +For the "Sort Sequence" feature (sorting by the occurrence of tags in the name), templates defined as lists (`value:` as an array) +can be injected directly into the sequence array. + +```yaml +sort: + rules: + - target: channel + field: caption + order: asc + sequence: + - "!CHAN_SEQ!" + - '(?i)\bHD\b' +``` + +### 3. In `mapping.yml` (As a Regex Component) + +In the Mapper DSL, Tuliprox injects the resolved regex pattern exactly where the exclamation mark macro is placed. This prevents complex regex typos. + +```dsl +# Extracts "UHD" from "Sky Sport UHD" and writes it to the variable 'quality' +quality = uppercase(@Caption ~ "!QUALITY!") + +# Replaces all arbitrary spaces and underscores with a clean separator +@Title = replace(@Title, "!DELIMITER!", " - ") +``` diff --git a/docs/src/deployment.md b/docs/src/deployment.md deleted file mode 100644 index b19244e85..000000000 --- a/docs/src/deployment.md +++ /dev/null @@ -1,134 +0,0 @@ -# Deployment - -## Backend and frontend - -Tuliprox consists of: - -- a Rust backend -- a Yew/WebAssembly frontend -- static assets served from the configured web root - -## Documentation delivery - -The recommended documentation workflow is: - -1. write docs as Markdown in `docs/src` -2. generate static HTML with `mdBook` into `frontend/build/docs` -3. copy the generated site into `frontend/dist/static/docs` during the web build - -That gives you editable source files in Git without committing hand-written HTML. - -## Why `mdBook` - -For this repository, `mdBook` is the most pragmatic choice because it: - -- keeps source in Markdown -- generates static HTML -- fits naturally into a Rust-based project -- avoids the overhead of a full Node/React documentation stack - -## Main build commands - -Build docs only: - -```bash -make docs -``` - -Preview docs locally: - -```bash -make docs-serve -``` - -Build frontend plus docs together: - -```bash -make web-dist -``` - -This does: - -1. `mdbook build` -> `frontend/build/docs` -2. `trunk build` -> `frontend/dist` -3. copy docs -> `frontend/dist/static/docs` - -## Docker build - -Build the project image manually: - -```bash -docker build --rm -f docker/Dockerfile -t tuliprox . -``` - -For the local development image, adapt your compose file to point to the locally built image. - -## Static binary builds - -Recommended musl build: - -```bash -cross build -p tuliprox --release --target x86_64-unknown-linux-musl -``` - -Manual prerequisite install on Debian or Ubuntu: - -```bash -rustup update -sudo apt-get install pkg-config musl-tools libssl-dev -rustup target add x86_64-unknown-linux-musl -``` - -Then: - -```bash -cargo build -p tuliprox --release --target x86_64-unknown-linux-musl -``` - -## Multi-platform helper scripts - -The repository ships helper scripts under `bin/`: - -- `bin/build_docs.sh` -- `bin/build_fe.sh` -- `bin/build_local.sh` -- `bin/build_docker.sh` -- `bin/release.sh` - -`bin/build_fe.sh` is the shared entry point for docs plus frontend assets. - -Wasm optimization is handled by Trunk during the build (via `data-wasm-opt` in `index.html`). -Trunk requires a compatible `wasm-opt` in `PATH`. - -Recommended local setup: - -```bash -./bin/install_wasm_tools.sh 128 -export PATH="$PWD/.tools/wasm-tools/version_128/bin:$PATH" -./bin/build_fe.sh release -``` - -## Healthcheck - -Tuliprox supports: - -```bash -tuliprox --healthcheck -``` - -This can be wired into Docker healthchecks. - -## Cross-compilation notes - -Windows: - -```bash -rustup target add x86_64-pc-windows-gnu -cargo build -p tuliprox --release --target x86_64-pc-windows-gnu -``` - -Raspberry Pi / armv7: - -```bash -cross build -p tuliprox --release --target armv7-unknown-linux-musleabihf -``` diff --git a/docs/src/examples-and-recipes.md b/docs/src/examples-and-recipes.md deleted file mode 100644 index 42dbafc8e..000000000 --- a/docs/src/examples-and-recipes.md +++ /dev/null @@ -1,162 +0,0 @@ -# Examples And Recipes - -This page collects the practical examples that previously lived in the monolithic README. - -## Xtream provider quick setup - -Provider data: - -- URL: `http://fantastic.provider.xyz:8080` -- username: `tvjunkie` -- password: `junkie.secret` - -Minimal `config.yml`: - -```yaml -api: - host: 0.0.0.0 - port: 8901 - web_root: ./web -storage_dir: ./data -update_on_boot: true -``` - -Minimal `source.yml`: - -```yaml -templates: - - name: ALL_CHAN - value: 'Group ~ ".*"' -inputs: - - type: xtream - name: my_provider - url: http://fantastic.provider.xyz:8080 - username: tvjunkie - password: junkie.secret -sources: - - inputs: - - my_provider - targets: - - name: all_channels - output: - - type: xtream - filter: "!ALL_CHAN!" -``` - -Minimal `api-proxy.yml`: - -```yaml -server: - - name: default - protocol: http - host: 192.168.1.41 - port: 8901 - timezone: Europe/Berlin - message: Welcome to tuliprox -user: - - target: all_channels - credentials: - - username: xt - password: xt.secret - proxy: redirect - server: default -``` - -## Filtering examples - -Include all categories: - -```text -Group ~ ".*" -``` - -Only shopping: - -```text -Group ~ "(?i).*Shopping.*" -``` - -Exclude shopping: - -```text -NOT(Group ~ "(?i).*Shopping.*") -``` - -More complex example: - -```text -(Group ~ "^FR.*" AND NOT(Group ~ "^FR.*SERIES.*" OR Group ~ "^DE.*EINKAUFEN.*")) OR (Group ~ "^AU.*") -``` - -## Template-based filter composition - -```yaml -templates: - - name: NO_SHOPPING - value: 'NOT(Group ~ "(?i).*Shopping.*" OR Group ~ "(?i).*Einkaufen.*")' - - name: GERMAN_CHANNELS - value: 'Group ~ "^DE: .*"' - - name: FRENCH_CHANNELS - value: 'Group ~ "^FR: .*"' - - name: MY_CHANNELS - value: '!NO_SHOPPING! AND (!GERMAN_CHANNELS! OR !FRENCH_CHANNELS!)' -``` - -## Excluding a single channel - -```text -NOT(Title ~ "FR: TV5Monde") -``` - -Or only inside one group: - -```text -NOT(Group ~ "FR: TF1" AND Title ~ "FR: TV5Monde") -``` - -## VLC seek problem with `user_access_control` - -Seeking can generate very fast reconnects and byte-range requests. -If stale provider connections have not yet disappeared, the user can briefly appear above `max_connections`. - -Typical mitigation: - -```yaml -reverse_proxy: - stream: - grace_period_millis: 2000 - grace_period_timeout_secs: 5 -``` - -## Enable per-stream metrics in the Web UI - -To show live bandwidth and transferred bytes in the streams table, enable stream metrics: - -```yaml -reverse_proxy: - stream: - metrics_enabled: true -``` - -This is useful for operator troubleshooting and live monitoring of active reverse-proxied streams. - -## Local library CLI examples - -```bash -./tuliprox --scan-library -./tuliprox --force-library-rescan -./tuliprox --dbx /opt/tuliprox/data/all_channels/xtream/video.db -./tuliprox --dbm /opt/tuliprox/data/all_channels/m3u.db -./tuliprox --dbe /opt/tuliprox/data/all_channels/xtream/epg.db -``` - -## Custom fallback video generation - -You can turn a still image into a `.ts` fallback video: - -```bash -ffmpeg -y -nostdin -loop 1 -framerate 30 -i blank_screen.jpg -f lavfi \ - -i anullsrc=channel_layout=stereo:sample_rate=48000 -t 10 -shortest -c:v libx264 \ - -pix_fmt yuv420p -preset veryfast -crf 23 -x264-params "keyint=30:min-keyint=30:scenecut=0:bframes=0:open_gop=0" \ - -c:a aac -b:a 128k -ac 2 -ar 48000 -mpegts_flags +resend_headers -muxdelay 0 -muxpreload 0 -f mpegts blank_screen.ts -``` diff --git a/docs/src/examples-recipes.md b/docs/src/examples-recipes.md new file mode 100644 index 000000000..60990520b --- /dev/null +++ b/docs/src/examples-recipes.md @@ -0,0 +1,175 @@ +# 🧪 Examples, Recipes & Ecosystem Stacks + +This chapter provides concrete copy & paste solutions for common scenarios spanning the different pillars of the Tuliprox architecture, concluding +with the ultimate Docker deployment stack. + +## 1. Quickstart: Minimal Xtream Setup + +You want to simply "pass through" your Xtream provider while utilizing the Web UI. + +**Minimal `config.yml`:** + +```yaml +api: + host: 0.0.0.0 + port: 8901 + web_root: ./web +storage_dir: ./data +update_on_boot: true +``` + +**Minimal `source.yml`:** + +```yaml +templates: + - name: ALL_CHAN + value: 'Group ~ ".*"' +inputs: + - type: xtream + name: my_provider + url: http://fantastic.provider.xyz:8080 + username: tvjunkie + password: junkie.secret +sources: + - inputs: + - my_provider + targets: + - name: clean_list + output: + - type: xtream + filter: "!ALL_CHAN!" # Lets everything through +``` + +**Minimal `api-proxy.yml`:** + +```yaml +server: + - name: default + protocol: http + host: 192.168.1.41 + port: 8901 + timezone: Europe/Berlin + message: Welcome to tuliprox +user: + - target: clean_list + credentials: + - username: me + password: mypass + proxy: redirect # Tuliprox does not proxy the stream, it just redirects the player + server: default +``` + +--- + +## 2. Advanced Filtering (Exclusion Logic) + +DSL filters are applied in `source.yml` at the `target` level. Regular expressions require the tilde `~` operator. + +**Allow if the word "Shopping" is included (Case-Insensitive):** + +```text +Group ~ "(?i).*Shopping.*" +``` + +**Reverse the logic (No Shopping!):** + +```text +NOT (Group ~ "(?i).*Shopping.*") +``` + +**Complex Matrix (Bracket Logic):** +Allow everything from DE and FR, except Series and Commercials. Always allow Australia. + +```text +(Group ~ "^(DE|FR).*" AND NOT (Group ~ "(?i).*SERIES.*" OR Group ~ "(?i).*COMMERCIAL.*")) OR (Group ~ "^AU.*") +``` + +**Filtering a specific channel, even if it appears in multiple groups:** + +```text +NOT (Title ~ "FR: TV5Monde" AND Group ~ "FR: TF1") +``` + +--- + +## 3. Generating Custom Fallback Videos (FFmpeg) + +If a provider stream returns a 404 or the user hits their limit, it is much more elegant to play an info video ("Channel Offline") than letting +the player hang endlessly on a dropped HTTP connection. Tuliprox searches the `custom_stream_response_path` folder for exactly named `.ts` files +(MPEG-TS). + +You can turn a simple image (`blank_screen.jpg`) into a clean, 10-second fallback stream with a silent audio track (crucial for player A/V +sync!) using FFmpeg: + +```bash +ffmpeg -y -nostdin -loop 1 -framerate 30 -i blank_screen.jpg -f lavfi \ + -i anullsrc=channel_layout=stereo:sample_rate=48000 -t 10 -shortest -c:v libx264 \ + -pix_fmt yuv420p -preset veryfast -crf 23 -x264-params "keyint=30:min-keyint=30:scenecut=0:bframes=0:open_gop=0" \ + -c:a aac -b:a 128k -ac 2 -ar 48000 -mpegts_flags +resend_headers -muxdelay 0 -muxpreload 0 -f mpegts channel_unavailable.ts +``` + +Move `channel_unavailable.ts` into your resources folder. Tuliprox will loop this file from RAM until the user switches channels or the +`custom_stream_response_timeout_secs` limit is reached. + +--- + +## 4. The Ultimate Docker Ecosystem Stack + +Running Tuliprox bare-metal is fine, but running it behind a modern DevSecOps stack provides SSL termination, VPN routing to bypass ISP +blocks, and firewall protection against brute-force attacks. + +In the repository under `docker/container-templates/`, you will find ready-to-use Docker Compose files. + +### Components of the Stack + +1. **Traefik (Reverse Proxy):** Handles incoming port 443 traffic, auto-renews Let's Encrypt certificates (via Cloudflare DNS-01 challenge), and + applies strict Content-Security-Policy headers. +2. **Gluetun (VPN Egress):** A WireGuard/OpenVPN client container. It connects to Mullvad/ProtonVPN. We attach a **Socks5 Sidecar** to its network + namespace. Tuliprox can then be configured to route its upstream provider requests through this Socks5 proxy, completely hiding your server's IP + from the IPTV provider. +3. **CrowdSec (WAF & Bouncer):** Analyzes Traefik access logs in real-time. If someone tries to brute-force your Tuliprox Web UI or run + path-traversal attacks, CrowdSec instructs the Traefik Bouncer plugin to drop their IP at the edge. + +### How to wire it up + +**1. Create the Docker Networks:** + +```bash +docker network create proxy-net +docker network create crowdsec-net +``` + +**2. Configure Gluetun & Socks5:** +In `container-templates/gluetun/gluetun-01/.env.wg-01`, add your Wireguard details. +In `container-templates/gluetun/.env.socks5-proxy`, set user/pass for the proxy. +Start it: `docker-compose up -d`. It exposes port 1388 internally. + +**3. Configure Tuliprox to use the VPN:** +In your `config.yml`, point the global proxy setting to the Socks5 container: + +```yaml +proxy: + url: socks5://socks5-01:1388 + username: "" + password: "" +``` + +*Note: Ensure Tuliprox and the Socks5 container share the `proxy-net` network.* + +**4. Protect Tuliprox with Traefik Labels:** +In your Tuliprox `docker-compose.yml`, add the Traefik labels to route traffic securely: + +```yaml + labels: + - "traefik.enable=true" + - "traefik.http.routers.tuliprox-secure.entrypoints=websecure" + - "traefik.http.routers.tuliprox-secure.rule=Host(`tv.yourdomain.com`)" + - "traefik.http.routers.tuliprox-secure.tls=true" + - "traefik.http.routers.tuliprox-secure.tls.certresolver=cloudflare" + # Attach CrowdSec Bouncer and Security Headers + - "traefik.http.routers.tuliprox-secure.middlewares=cs-bouncer-traefik-plugin@file,default-security-headers@file" + - "traefik.http.services.tuliprox.loadbalancer.server.port=8901" +``` + +This architecture ensures your IPTV provider only sees the VPN IP, your clients only see your secure `tv.yourdomain.com` domain, and malicious +bots are blocked instantly by CrowdSec before they even reach the Rust backend. diff --git a/docs/src/features.md b/docs/src/features.md index 3c0bf8285..915cdc2fd 100644 --- a/docs/src/features.md +++ b/docs/src/features.md @@ -15,14 +15,12 @@ Tuliprox can publish: - M3U - Xtream-style outputs - HDHomeRun -- STRM +- STRM Files That makes it usable both for IPTV players and for media-server-oriented workflows. ## Playlist processing -Tuliprox can: - - filter channels and groups - rename or normalize entries - apply mappings and templates @@ -31,8 +29,6 @@ Tuliprox can: ## Runtime streaming -Tuliprox can: - - reverse-proxy streams instead of redirecting them - keep provider account affinity where clients need it - share live streams across users @@ -42,13 +38,21 @@ Tuliprox can: ## Metadata and library -Tuliprox can: - - resolve VOD and series metadata - probe stream capabilities - scan local media - combine local library content with IPTV-oriented outputs +## Operational features + +Tuliprox also includes: + +- scheduled playlist refreshes +- hot config reload support +- provider failover and DNS-aware connection rotation +- notifications and monitoring hooks +- **Web UI** with monitoring and web-based configuration ability + ## Access control Tuliprox supports role-based access control (RBAC) for the Web UI: @@ -59,13 +63,3 @@ Tuliprox supports role-based access control (RBAC) for the Web UI: - compact bitmask encoding in JWT claims for low-overhead permission checks - backward-compatible user file format - Web UI admin panel for user and group management - -## Operational features - -Tuliprox also includes: - -- scheduled playlist refreshes -- hot config reload support -- provider failover and DNS-aware connection rotation -- Web UI -- notifications and monitoring hooks diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index a82efaa02..3689d51cd 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -1,5 +1,56 @@ # Getting Started +## Recommended reading order + +1. [Installation](installation.md) +2. [Configuration (Core System)](configuration/config.md) +3. [Add Sources & Targets](configuration/source.md) +4. [API Proxy](configuration/api-proxy.md) +5. [Streaming & Proxy](configuration/reverse-proxy.md) +6. [Templates](configuration/template.md) +7. [Mappings](configuration/mapping-dsl.md) + +## Run Tuliprox via docker compose + +```yaml +services: + tuliprox: + container_name: tuliprox + image: ghcr.io/euzu/tuliprox-alpine:latest + working_dir: /app + volumes: + - /opt/tuliprox/config:/app/config + - /opt/tuliprox/data:/app/data + - /opt/tuliprox/backup:/app/backup + - /opt/tuliprox/downloads:/app/downloads + - /opt/tuliprox/cache:/app/cache + environment: + - TZ=Europe/Berlin + ports: + - "8901:8901" + restart: unless-stopped + healthcheck: + test: ["CMD", "/app/tuliprox", "-p", "/app/config", "--healthcheck"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s +``` + +Open the Web UI afterward and continue with the configuration. + +### First configuration steps + +For a new setup, the usual first goal is: + +1. add one working input +2. create one target +3. confirm playlist output +4. confirm one stream works +5. only then add mapping, filtering, reverse proxy and metadata features + +That keeps failures local and makes provider-specific issues much easier to diagnose. + ## Run modes Tuliprox has two main modes: @@ -81,47 +132,6 @@ Typical directories below that home: - `data/backup/` - `downloads/` - `web/` +- `cache/` All relative paths in the configuration are resolved against that home directory. - -## Quick Docker start - -```yaml -services: - tuliprox: - container_name: tuliprox - image: ghcr.io/euzu/tuliprox-alpine:latest - working_dir: /app - volumes: - - /home/tuliprox/tuliprox:/app/tuliprox - - /home/tuliprox/config:/app/config - - /home/tuliprox/data:/app/data - - /home/tuliprox/cache:/app/cache - environment: - - TZ=Europe/Paris - ports: - - "8901:8901" - restart: unless-stopped -``` - -Open the Web UI afterwards and continue with the configuration. - -## Good first milestone - -For a new setup, the usual first goal is: - -1. add one working input -2. create one target -3. confirm playlist output -4. confirm one stream works -5. only then add mapping, filtering, reverse proxy and metadata features - -That keeps failures local and makes provider-specific issues much easier to diagnose. - -## Recommended reading order - -1. [Config Reference](configuration/main-config.md) -2. [Sources And Targets](configuration/sources-and-targets.md) -3. [API Proxy](configuration/api-proxy.md) -4. [Streaming And Proxy](streaming-and-proxy.md) -5. [Mapping And Templates](mapping-and-templates.md) diff --git a/docs/src/index.md b/docs/src/index.md index 5b3c9318c..4bc2f390a 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,42 +1,97 @@ -# tuliprox +# 📖 Tuliprox Operator Manual: Introduction & Architecture -`tuliprox` is an IPTV proxy and playlist processor for people who need more control than a raw provider playlist can offer. +Welcome to the **Tuliprox Operator Manual**. -It sits between provider inputs and client applications and gives you a place to: +**Tuliprox** is a highly optimized, resource-efficient IPTV proxy and playlist processor written in Rust 🦀. It acts as an intelligent middleware +layer between your upstream IPTV providers (or local hard drives) and your local media clients (Plex, Jellyfin, Emby, Kodi, TiviMate, etc.). -- clean up channels -- normalize outputs -- protect provider accounts -- manage user access -- handle HLS, catchup and shared streaming behavior -- mix IPTV with a local media library +## System Architecture -## What makes it different +To ensure maximum performance and a modern user experience, Tuliprox consists of three core components: -Tuliprox is not only a playlist rewriter. -It is also a runtime stream broker with provider-aware logic. +* **Rust Backend:** A high-performance, asynchronous engine handling stream brokering, authentication, and playlist processing. +* **Yew/WebAssembly Frontend:** A reactive, browser-based management interface compiled to WASM for near-native performance. +* **Static Assets:** All necessary resources are served directly from the configured web root, making the deployment self-contained and easy to proxy. -That matters when you need things like: +## Why Tuliprox? -- connection limits per user -- provider account reuse for HLS or catchup -- priority-based stream preemption -- shared live streams -- custom fallback videos when a stream cannot be delivered +Unlike simple scripts that merely perform text replacement on M3U files (such as m3u4u or xTeVe), Tuliprox is a full-fledged **Runtime Stream Broker**. +It actively intercepts video traffic, negotiates connections with providers, protects your accounts from bans, and enhances metadata locally. + +Most IPTV providers restrict access via a strict `max_connections` limit. When users fast-forward, rewind, or rapidly switch channels in modern +IPTV players (like VLC or TiviMate), "ghost connections" are often created. The provider sees multiple active streams from the same IP, leading to +immediate HTTP 509 (Bandwidth Exceeded) errors or account bans. + +Tuliprox solves this through a highly complex **Reverse-Proxy Streaming Engine**: + +1. **Session Holding (HLS/Catchup):** Tuliprox artificially holds provider slots open while the client negotiates or switches between `.m3u8` + segments, preventing "account hopping" bans. +2. **Grace Periods:** It mitigates the "VLC seek problem" by granting temporary grace periods, holding the video stream back until old connections + to the provider are safely terminated. +3. **Shared Streams:** If multiple users watch the same live event, Tuliprox opens only *one* connection to the upstream provider and multicasts the + traffic locally to all clients, saving valuable provider slots and bandwidth. +4. **Preemption (Priority Routing):** You can assign priorities to users. If all provider slots are occupied, Tuliprox automatically kicks the least + important user (e.g., a background metadata scan or a user with priority 100) to free up the stream for an admin (priority 0). + +Tuliprox downloads raw lists from the provider, stores them in lightning-fast, local **B+Tree databases** (to save RAM). + +To perfectly prepare playlists for tools like Plex or Jellyfin, metadata (covers, release years, TMDB IDs, video codecs) is often missing from +provider sources. +Tuliprox dispatches asynchronous worker tasks for **Metadata Enrichment**: + +* Parse titles locally (via PTT - Parse Torrent Title) to extract release years. +* Query the **TMDB API** to fetch high-resolution covers, backdrops, and cast information. +* Utilize local **FFprobe** to look directly into the provider's stream to extract precise technical data like Codecs (HEVC, H264), HDR formats + (Dolby Vision), and Audio Channels (5.1). + +## Network Flow Architecture + +Tuliprox integrates seamlessly into modern DevSecOps environments. Here is a high-level view of a recommended production stack (which we cover +extensively in the *Examples, Recipes & Ecosystem Stacks* chapter): + +```mermaid +sequenceDiagram + participant Client as Media Client (Plex/TiviMate) + participant WAF as CrowdSec (AppSec/Firewall) + participant RP as Traefik (Reverse Proxy) + participant Tuli as Tuliprox (Stream Broker) + participant VPN as Gluetun (WireGuard/Socks5) + participant Prov as Upstream IPTV Provider + + Client->>RP: Request Stream (HTTPS) + RP->>WAF: Validate Request (Bouncer) + WAF-->>RP: Allow + RP->>Tuli: Forward Request (HTTP) + + alt Connection Limit Reached? + Tuli->>Tuli: Check Preemption & Grace Period + Tuli-->>Client: Serve "Exhausted" Custom Video + else Slot Available + Tuli->>VPN: Fetch Stream via Socks5/HTTP Proxy + VPN->>Prov: Encrypted WireGuard Tunnel + Prov-->>VPN: MPEG-TS / HLS Stream + VPN-->>Tuli: + Tuli->>Tuli: Buffer & Throttle Stream + Tuli-->>RP: Piped Video Stream + RP-->>Client: Seamless Playback + end +``` ## Documentation map -- [Getting Started](getting-started.md): first run, main commands, file layout -- [Core Features](features.md): what tuliprox can do at a high level -- [Configuration](configuration/overview.md): config file layout and field reference -- [Sources And Targets](configuration/sources-and-targets.md): inputs, aliases, outputs and processing -- [API Proxy](configuration/api-proxy.md): users, servers, reverse/redirect and access URLs -- [Streaming And Proxy](streaming-and-proxy.md): runtime behavior, HLS/catchup affinity and reverse-proxy options -- [Mapping And Templates](mapping-and-templates.md): DSL, mapping files, counters and grouping examples -- [Deployment](deployment.md): build and ship backend, frontend and docs -- [Examples And Recipes](examples-and-recipes.md): practical setup and filtering examples +This Operator Manual will guide you through every single configuration variable to help you extract the maximum potential from this engine. -## Source format - -The documentation source is plain Markdown in `docs/src`. -Static HTML is generated from it with `mdBook` and can be shipped together with the Web UI under `/static/docs/`. +* [Build & Deploy](./build-and-deploy.md) +* **[Installation](./installation.md)** +* **[Configuration Overview](./configuration/overview.md)** + * [Main Config](./configuration/config.md) + * [Streaming & Proxy Behavior](./configuration/reverse-proxy.md) + * [Metadata Update & Probe](./configuration/metadata-update.md) + * [Local Library](./configuration/local-library.md) + * [Sources & Targets](./configuration/source.md) + * [API Proxy](./configuration/api-proxy.md) + * [Templates](./configuration/template.md) + * [Mapping](./configuration/mapping-dsl.md) +* **[Examples & Recipes](./examples-recipes.md)** +* [Operations & Debugging](./operations-debugging.md) +* [Troubleshooting & Resilience](./troubleshooting.md) diff --git a/docs/src/installation.md b/docs/src/installation.md new file mode 100644 index 000000000..e6ab9c456 --- /dev/null +++ b/docs/src/installation.md @@ -0,0 +1,151 @@ +# 🚀 Installation (Docker & Binaries) + +Tuliprox is designed to run flawlessly in modern DevOps infrastructures (Docker/Kubernetes) but can also be executed as a standalone binary on Linux, +Windows, or macOS for lightweight setups. + +This guide covers the installation using precompiled artifacts. If you want to compile Tuliprox from source, create custom Docker images, or build +the documentation yourself, please refer to the [Build & Deploy (For Professionals)](build-and-deploy.md) chapter. + +--- + +## 1. Running via Docker (Recommended) + +The recommended way for production use is Docker. Tuliprox does not require external database containers (everything uses internal embedded `.db` +B+Tree files), making the setup extremely lightweight. You can pull the ready-to-use images directly from the GitHub Container Registry +(`ghcr.io`). + +### The Ideal `docker-compose.yml` + +```yaml +services: + tuliprox: + container_name: tuliprox + image: ghcr.io/euzu/tuliprox-alpine:latest + user: "133:144" # (Recommended) Run as non-root User (UID:GID of your host system) + working_dir: /app + volumes: + - /opt/tuliprox/config:/app/config + - /opt/tuliprox/data:/app/data + - /opt/tuliprox/backup:/app/backup + - /opt/tuliprox/downloads:/app/downloads + - /opt/tuliprox/cache:/app/cache + environment: + - TZ=Europe/Berlin + ports: + - "8901:8901" + restart: unless-stopped + healthcheck: + test: ["CMD", "/app/tuliprox", "-p", "/app/config", "--healthcheck"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s +``` + +### Volume Mapping (Technical Background) + +The separation of volumes is critical for security and performance: + +| Mount Point | Explanation & Technical Background | +|:-----------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `/app/config` | Must contain your YAML files (`config.yml`, `source.yml`, etc.). Monitored by Tuliprox for file system events (`config_hot_reload`). | +| `/app/data` | The `storage_dir`. Contains **Runtime Data**. Tuliprox recursively creates B+Tree databases (`*.db`), M3U caches, and TMDB metadata here. **THIS MUST BE PERSISTENT!** If lost during a restart, Tuliprox will attempt to redownload and FFprobe tens of thousands of streams, inevitably leading to provider bans. | +| `/app/backup` | Destination folder for configuration backups triggered via the Web UI. | +| `/app/downloads` | Destination folder for local video downloads initiated via the Web UI. | +| `/app/cache` | Destination folder for local image downloads initiated via the xtream codes or m3u api use. | + +### Docker Image Variants (`scratch` vs. `alpine`) + +Two distinct flavors are available in the container registry: + +1. **`scratch-final` (`ghcr.io/euzu/tuliprox:latest`)**: + This is an extremely hardened, minimalist image (`FROM scratch`). It contains *only* the statically linked Tuliprox binary, FFmpeg/FFprobe + binaries, and CA certificates (for HTTPS/TMDB requests). There is no shell (`/bin/sh`), no package manager, and no system libraries. + **Advantage:** Maximum security and minimal attack surface (DevSecOps Best Practice). + **Disadvantage:** You cannot `docker exec` into the container for manual debugging. +2. **`alpine-final` (`ghcr.io/euzu/tuliprox-alpine:latest`)**: + Based on Alpine Linux. Contains a shell, Tini (init system), and basic tools. + **Advantage:** Ideal for debugging. You can `docker exec -it tuliprox sh` to inspect logs or run manual `curl`/`ffprobe` tests from within the + container's network namespace to verify provider blocks. + +--- + +## 2. Precompiled Standalone Binaries + +If you prefer running Tuliprox bare-metal without Docker, you can download the precompiled binaries for your operating system (Linux, Windows, +macOS, ARM). + +1. Go to the [Releases page on GitHub](https://github.com/euzu/tuliprox/releases). +2. Download the archive matching your OS and Architecture. +3. Extract the binary and place it in your desired home directory. +4. Run it via the CLI commands detailed below. + +--- + +## 3. CLI Usage & Runtime Modes + +Tuliprox can operate in two primary modes. Running the compiled binary directly is the best way to test configurations locally before deploying +them to production. + +### Server Mode (Persistent Operation) + +Run Tuliprox as a persistent IPTV proxy server: + +```bash +./tuliprox -s -c config/config.yml -i config/source.yml +``` + +This mode enables: + +* The Web UI Dashboard +* The active Reverse-Proxy Streaming Engine +* API endpoints (Xtream/M3U for players) +* Background Workers (Metadata Scanner, DNS Resolver, Scheduler) + +### CLI Mode (One-Shot Processing) + +Without the `-s` flag, Tuliprox runs as a one-time processor: + +```bash +./tuliprox -c config/config.yml -i config/source.yml +``` + +It loads the configuration, downloads provider lists, applies all filters and mappings, writes the final database/M3U files to the `storage_dir`, +and gracefully exits. +*Useful for: Generating static playlists, debugging mappings, or running via external cronjobs.* + +### CLI Arguments Reference + +You can view all arguments using `./tuliprox --help`: + +| Flag | Purpose | +| :--- | :--- | +| `-H, --home ` | Sets the home directory (base for config, storage, backup, downloads). Overrides all defaults. | +| `-p, --config-path ` | Path to the config directory. | +| `-c, --config ` | Specific path to the `config.yml`. | +| `-i, --source ` | Specific path to the `source.yml`. | +| `-m, --mapping ` | Specific path to the mapping file/directory. | +| `-T, --template ` | Specific path to the template file/directory. | +| `-t, --target ` | **Target Override:** Forces processing of the specified target *only*. Extremely useful to quickly re-render a broken list via shell without blocking the entire system, even if `enabled: false` in config. | +| `-a, --api-proxy ` | Specific path to the `api-proxy.yml`. | +| `-s, --server` | Run in continuous Server Mode. | +| `-l, --log-level ` | Override log level (e.g., `info`, `debug`, `trace`). | +| `--genpwd` | Interactively generate a secure `Argon2id` password hash for the `user.txt`. | +| `--healthcheck` | Checks the API over localhost. Returns Exit Code `0` if `{status: "ok"}`. Used by Docker. | +| `--scan-library` | Triggers an incremental scan of local media directories. | +| `--force-library-rescan` | Ignores modification timestamps and forces a full TMDB/PTT re-evaluation of local media. | +| `--dbx`, `--dbm`, `--dbe`, `--dbv`, `--dbms` | Opens internal database viewers (see *Operations & Debugging*). | + +--- + +## 4. Advanced Ecosystem Integration + +Tuliprox is built to be a team player. In the `docker/container-templates/` directory of the repository, you will find complete, +production-ready stack templates: + +* **Traefik Integration:** Automated Let's Encrypt TLS, strict Content-Security-Policy headers, and ACME DNS-01 challenges. +* **Gluetun (VPN) Integration:** Route your upstream provider requests through WireGuard tunnels to hide your server IP, utilizing SOCKS5 proxy sidecars. +* **CrowdSec Integration:** Protect your Tuliprox instance against L7 AppSec attacks, path traversal, and brute-force attempts using Traefik Bouncers. + +*(For an in-depth implementation guide on these templates, see the [Build & Deploy (For Professionals)](build-and-deploy.md) and +[Examples & Recipes](examples-recipes.md) chapters). diff --git a/docs/src/mapping-and-templates.md b/docs/src/mapping-and-templates.md deleted file mode 100644 index 83f4a68a1..000000000 --- a/docs/src/mapping-and-templates.md +++ /dev/null @@ -1,257 +0,0 @@ -# Mapping And Templates - -Tuliprox supports two layers of editorial logic: - -- reusable templates -- mapping rules written in a small DSL - -## Template loading - -Templates can live: - -- inline in `source.yml` -- inline in `mapping.yml` -- centrally in files or directories configured via `template_path` - -If a directory is used, all `*.yml` files are loaded in alphanumeric order. - -Example: - -```yaml -templates: - - name: delimiter - value: '[\s_-]*' - - name: quality - value: '(?i)(?PHD|LQ|4K|UHD)?' -``` - -Referenced as: - -```text -^.*TF1!delimiter!Series?!delimiter!Films?(!delimiter!!quality!)\s*$ -``` - -## `mapping.yml` - -Top-level structure: - -- `templates` -- `mapping` - -Each mapping entry defines: - -- `id` -- `match_as_ascii` -- `mapper` -- `counter` - -## `match_as_ascii` - -If enabled, Tuliprox deunicodes values during matching. -That is useful when filters should match `é`, `ö` or `ß` with simpler ASCII patterns. - -## Mapper DSL basics - -The DSL supports: - -- variable assignment -- string manipulation -- regex matching -- match blocks -- map blocks -- access to playlist fields with `@Field` -- a few builtin functions - -Common builtins: - -- `concat` -- `uppercase` -- `lowercase` -- `capitalize` -- `trim` -- `print` -- `number` -- `first` -- `template` -- `replace` -- `pad` -- `format` -- `add_favourite` - -## Simple example - -```yaml -mappings: - mapping: - - id: favourites_news - match_as_ascii: true - mapper: - - filter: 'Group ~ "(?i)news"' - script: | - add_favourite("Favourites") -``` - -## Match and map blocks - -Example `match` block: - -```dsl -result = match { - (var1, var2) => result1, - var2 => result2, - _ => default -} -``` - -Example `map` block: - -```dsl -quality = map quality { - "720p" => "HD", - "1080p" => "FHD", - "4K" => "UHD", - _ => quality, -} -``` - -## Regex captures - -Regex matches can expose numbered or named captures: - -```dsl -title_match = @Caption ~ "(.*?)\\:\\s*(.*)" -title_prefix = title_match.1 -title_name = title_match.2 -``` - -## `for_each` - -`for_each` iterates over named results such as split values or regex captures: - -```dsl -genres = split(@Genre, "[,/&]") -genres.for_each((_, genre) => { - add_favourite(concat("Genre - ", genre)) -}) -``` - -## Counters - -Mappings can also define counters: - -```yaml -mapping: - - id: simple - counter: - - filter: 'Group ~ ".*FR.*"' - value: 9000 - field: title - padding: 2 - modifier: suffix - concat: " - " -``` - -Counter fields: - -- `filter` -- `value` -- `field` -- `modifier` -- `concat` -- `padding` - -## Grouping example - -This groups channels into quality-oriented or category-oriented buckets: - -```dsl -group = @Group ~ "(EU|SATELLITE|NATIONAL|NEWS|MUSIC|SPORT|RELIGION|FILM|KIDS|DOCU)" -quality = @Caption ~ "\\b([F]?HD[i]?)\\b" -title_match = @Caption ~ "(.*?)\\:\\s*(.*)" -title_name = title_match.2 - -quality = map group { - "NEWS" | "NATIONAL" | "SATELLITE" => quality, - _ => null, -} - -prefix = map quality { - "HD" => "01.", - "FHD" => "02.", - "HDi" => "03.", - _ => map group { - "NEWS" => "04.", - "DOCU" => "05.", - "SPORT" => "06.", - _ => group - }, -} - -name = match { - quality => concat(prefix, " FR [", quality, "]"), - group => concat(prefix, " FR [", group, "]"), - _ => prefix -} - -@Group = name -@Caption = title_name -``` - -## Grouping by release year - -```dsl -year_text = @Caption ~ "(\\d{4})\\)?$" -year = number(year_text) -year_group = map year { - ..2019 => "< 2020", - _ => year_text, -} -@Group = concat("FR | MOVIES ", year_group) -``` - -## Example `mapping.yml` - -```yaml -mappings: - templates: - - name: QUALITY - value: '(?i)\b([FUSL]?HD|SD|4K|1080p|720p|3840p)\b' - mapping: - - id: all_channels - match_as_ascii: true - mapper: - - filter: 'Caption ~ "(?i)^(US|USA|United States).*?TNT"' - script: | - quality = uppercase(@Caption ~ "!QUALITY!") - quality = map quality { - "720p" => "HD", - "1080p" => "FHD", - "4K" => "UHD", - "3840p" => "UHD", - _ => quality, - } - @Group = "United States - Entertainment" -``` - -## Filter hints - -Filters support: - -- `NOT` -- `AND` -- `OR` -- regex matches with `~` -- `Type = live|vod|series` - -Fields: - -- `Group` -- `Title` -- `Name` -- `Caption` -- `Url` -- `Genre` -- `Input` -- `Type` - -For Rust regex testing, `regex101.com` works well if the Rust flavor is selected. diff --git a/docs/src/operations-debugging.md b/docs/src/operations-debugging.md new file mode 100644 index 000000000..071621b8b --- /dev/null +++ b/docs/src/operations-debugging.md @@ -0,0 +1,102 @@ +# 🛠️ Operations & Debugging (CLI & DB Dumps) + +Tuliprox is designed as a "Fire & Forget" stream broker. However, when streams stutter, provider connection limits block your users, or EPG data and +TMDB covers do not match, the engine provides deep, low-level insights under the hood. + +This chapter covers the Command Line Interface (CLI), Logging architecture, and the internal Database Viewers. + +## 1. Command Line Arguments (CLI Flags) + +While Tuliprox is usually run via Docker, understanding the CLI flags is crucial for debugging and manual interventions. + +| Flag | Purpose & Technical Background | +| :--- | :--- | +| `-s, --server` | Starts continuous Server Mode (API, Web UI, Background Workers). Without this flag, Tuliprox acts as a "One-Shot" playlist generator that downloads, processes, and immediately exits. | +| `-H, --home ` | Sets the Home Directory. All relative paths in the configuration are resolved against this directory. If not set, resolves via `TULIPROX_HOME` env variable, or finally the binary's directory. | +| `-c, -i, -a, -m, -T` | Overrides specific config paths (e.g., `-c /etc/tuliprox/config.yml`). Useful for testing experimental configurations without altering the production setup. | +| `-t, --target ` | **Targeted Processing:** Forces processing of the specified target *only*. **Crucial:** This bypasses the `enabled: false` state in the config! Extremely useful to quickly re-render a broken list via cron/shell without blocking the entire system with other heavy targets. | +| `--genpwd` | Interactively generates a secure `Argon2id` password hash for the `user.txt` file. Never store plaintext passwords! | +| `--healthcheck` | Docker Support: Pings the API over localhost. Returns Exit Code `0` if the server responds with `{"status": "ok"}`. | +| `--scan-library` | Triggers an incremental scan of the local media directory (if configured). | +| `--force-library-rescan` | Ignores modification timestamps and forces a full TMDB/PTT re-evaluation of all local media files. | + +--- + +## 2. Logging Levels and Module Filtering + +Tuliprox utilizes the powerful Rust `env_logger` crate. The log verbosity can be controlled at an extremely granular level via `config.yml` +(`log.log_level`), the environment variable `TULIPROX_LOG`, or the CLI flag `-l`. + +The evaluation hierarchy is: **CLI Argument > Env-Var > config.yml > Default (`info`)**. + +Available levels: `trace`, `debug`, `info`, `warn`, `error`. + +**The Magic of Module Filtering:** +Often, you do not want to set the entire system to `trace` (which would flood your console and disk), but rather investigate a specific algorithm. +You can pass comma-separated module paths: + +```bash +# Everything on Info, but the internal Mapper on Trace +# (Useful to make print() commands from the DSL visible!): +./tuliprox -s -l "info,tuliprox::foundation::mapper=trace" + +# Show me all low-level HTTP-Connection errors from the Hyper crate: +./tuliprox -s -l "info,hyper_util::client::legacy::connect=error" +``` + +*Note: If `log.sanitize_sensitive_info` is set to `true` in the config (default), Tuliprox masks passwords, provider URLs, and external client IPs in +the logs with `***`. This is strongly recommended so you can safely share logs on GitHub or Discord!* + +--- + +## 3. Database Dumps (B+Tree Analysis) + +Tuliprox is built to be extremely resource-efficient. It does not keep massive playlists (often > 200,000 entries) permanently in RAM. Instead, it +stores all parsed metadata, enriched by FFprobe and TMDB, in highly optimized local **B+Tree Database files** (`.db`). + +Sometimes you need to know *exactly* what Tuliprox has discovered in the background about a specific stream. Using the built-in dump flags, you can +output these binary files in clean JSON format to your console (or pipe them into a file). + +You must point the flag directly at the corresponding `.db` file inside your `storage_dir` (e.g., `/app/data/`): + +| Flag & Example | Usage & Purpose | +| :--- | :--- | +| **`--dbx `**
`./tuliprox --dbx ./data/input_name/xtream/video.db` | **Xtream DB:** Reads the metadata derived from the Xtream API. Shows you the final JSON payloads with resolved TMDB IDs, extracted video codecs (e.g., H264), and bitrates. | +| **`--dbm `**
`./tuliprox --dbm ./data/input_name/m3u.db` | **M3U Playlist DB:** Reads the raw M3U entries. Ideal for seeing how the fallback logic for `Tvg-ID` or `Virtual_ID` reacted to messy provider tags. | +| **`--dbe `**
`./tuliprox --dbe ./data/input_name/xtream/epg.db` | **EPG DB:** Prints the fully matched XMLTV grid. You see a list of all programmes with their correct Unix timestamps. | +| **`--dbms `**
`./tuliprox --dbms ./data/input_name/metadata_retry_state.db` | **Metadata Retry Status (Cooldowns):** Extremely important! Shows you the asynchronous backoff state. If TMDB finds no info for a stream, it lands in a cooldown here. Shows `attempts: 3`, `last_error: "404 Not Found"`, `cooldown_until_ts: 1740000000`. This explains *why* a movie isn't being updated. | +| **`--dbv `**
`./tuliprox --dbv ./data/target_name/id_mapping.db` | **Target-ID Mapping:** Tracks the stability of stream UUIDs across updates. Shows which original Provider-ID points to which internal Virtual-ID. | + +### Example output via `--dbms` + +```json +{ + "Stream_ID_4242": { + "resolve": { + "attempts": 3, + "next_allowed_at_ts": 1718000000, + "cooldown_until_ts": 1718604800, + "last_error": "TMDB lookup completed without matching result", + "tmdb":null, + "updated_at_ts":1718604910 + } + } +} +``` + +**Diagnosis:** This dump immediately tells you: Tuliprox tried three times to find the movie on TMDB, failed every time, and has now paused this +movie until `cooldown_until_ts` (e.g., 7 days in the future) to save API traffic and prevent rate-limiting. + +--- + +## 4. Hot Reloading Caveats + +Tuliprox supports hot-reloading for specific files (`mapping.yml`, `api-proxy.yml`) if `config_hot_reload: true` is set in `config.yml`. + +**Important Note for Docker Bind Mounts:** +If you edit a file on your host system that is bind-mounted into the container (e.g., `nano /home/user/tuliprox/config/mapping.yml`), the file +watcher might report the inotify event using the *original host path* instead of the container's mount point `/app/config/mapping.yml`. +Tuliprox attempts to resolve this, but depending on your host OS (Windows/WSL vs Linux), filesystem events can be flaky. If hot-reload fails to +trigger, a container restart (`docker restart tuliprox`) is the safest fallback. + +--- diff --git a/docs/src/streaming-and-proxy.md b/docs/src/streaming-and-proxy.md deleted file mode 100644 index ebac8ed32..000000000 --- a/docs/src/streaming-and-proxy.md +++ /dev/null @@ -1,216 +0,0 @@ -# Streaming And Proxy - -Tuliprox is not only a playlist transformer. -Its runtime streaming behavior is a major part of the project. - -## Reverse proxy vs redirect - -Tuliprox can either redirect to provider URLs or proxy traffic itself. -Proxy mode gives Tuliprox control over: - -- user limits -- provider limits -- custom fallback responses -- HLS and catchup session handling -- stream sharing - -## `reverse_proxy.stream` - -Important fields: - -- `retry` -- `metrics_enabled` -- `buffer` -- `throttle` -- `grace_period_millis` -- `grace_period_timeout_secs` -- `grace_period_hold_stream` -- `shared_burst_buffer_mb` -- `hls_session_ttl_secs` -- `catchup_session_ttl_secs` - -### `retry` - -If `true`, Tuliprox retries provider streams when the upstream disconnects unexpectedly. - -### `metrics_enabled` - -If `true`, Tuliprox samples active stream throughput and transferred bytes and exposes them to the Web UI streams table. - -Use this when you want live per-stream bandwidth visibility while debugging or operating the proxy. - -Notes: - -- default is `false` -- metrics are only collected for reverse-proxied streams -- enabling it adds lightweight runtime accounting for active streams -- it does not change playlist output or stream selection behavior - -### `buffer` - -`buffer` has: - -- `enabled` -- `size` - -`size` is the number of 8192-byte chunks. -`1024` is roughly `8 MB`. - -When `share_live_streams` is enabled, each shared live channel keeps at least the shared burst buffer in memory. - -### `throttle` - -Bandwidth throttling supports units such as: - -- `KB/s` -- `MB/s` -- `KiB/s` -- `MiB/s` -- `kbps` -- `mbps` -- `Mibps` - -### Grace period - -`grace_period_millis` and `grace_period_timeout_secs` protect users during rapid reconnects, seeks and channel switches. - -`grace_period_hold_stream` decides whether Tuliprox waits for the grace decision before it starts sending media data. - -## HLS and catchup session reservation - -HLS and catchup behave differently from plain TS streaming because clients repeatedly connect, fetch data, disconnect and reconnect. -Tuliprox therefore keeps a short-lived provider-account reservation rather than holding a real provider slot open the entire time. - -### `hls_session_ttl_secs` - -For HLS: - -- the real provider slot is held only during the active playlist or segment request -- between requests, Tuliprox keeps only an account reservation -- the same client/session tries to reuse the same provider account -- channel switches from the same client can take over the reservation immediately - -### `catchup_session_ttl_secs` - -Catchup has the same account-affinity problem, especially during seeks and reconnects. -Tuliprox therefore applies the same reservation model to catchup. - -Important distinction: - -- normal TS streaming does not use this family reservation model -- HLS and catchup do - -## Shared live streams - -When enabled, multiple users can attach to the same upstream live stream instead of opening separate provider connections. -That reduces provider pressure but also means stream priority has to be managed at the shared-stream level. - -Current behavior: - -- the first viewer starts a shared stream immediately -- additional viewers on the same channel join the existing shared stream -- the effective priority of the shared stream is always the highest priority of its remaining viewers -- when a viewer leaves, the shared stream priority is recalculated -- if provider capacity is full and a higher-priority user starts another stream, the lower-priority shared stream can be preempted -- equal priority does not preempt a different running stream - -Example: - -```yaml -reverse_proxy: - stream: - retry: true - metrics_enabled: true -``` - -## Priority and preemption - -Tuliprox can prioritize users and internal tasks differently. -Higher-priority user streams can displace lower-priority traffic when provider capacity is exhausted. - -The important design rule is: - -- user playback wins over low-priority internal work - -Priority uses a nice-style scale: - -- lower number = higher priority -- negative values are allowed -- equal priority does not preempt a different stream - -Probe tasks use `metadata_update.probe.user_priority`. -If preempted by a higher-priority user, they are cancelled immediately and release provider capacity right away. - -## Custom stream responses - -When Tuliprox cannot serve the real stream, it can return custom fallback videos for cases such as: - -- user connection exhausted -- provider connection exhausted -- channel unavailable -- low-priority stream preempted - -That makes failure modes easier to understand for end users and downstream clients. - -The fallback files are discovered by filename in `custom_stream_response_path`. - -## Other reverse-proxy sections - -### `cache` - -LRU cache for proxied resources such as logos. -If `resource_rewrite_disabled` is set to `true`, the cache is effectively disabled because Tuliprox can no longer rewrite and track resource URLs safely. - -### `resource_rewrite_disabled` - -Disable rewritten resource URLs when Tuliprox runs behind another proxy and you do not want Tuliprox-generated asset URLs. - -### `rate_limit` - -Per-IP rate limiting with: - -- `enabled` -- `period_millis` -- `burst_size` - -### `disabled_header` - -Controls which request headers are stripped before upstream requests: - -- `referer_header` -- `x_header` -- `cloudflare_header` -- `custom_header` - -### `resource_retry` - -Controls retries for proxied upstream resources: - -- `max_attempts` -- `backoff_millis` -- `backoff_multiplier` - -### `geoip` - -Optional country lookup from CSV IP ranges. - -### `rewrite_secret` - -Persistent secret used for generating and validating rewritten resource URLs. -Set it explicitly if rewritten URLs should survive restarts unchanged. - -## VLC seek problem and grace tuning - -Seeking often produces very fast reconnects and partial range requests. -If the previous upstream connection is not fully closed yet, the provider can still count it against max-connections. - -The usual mitigation is: - -```yaml -reverse_proxy: - stream: - grace_period_millis: 2000 - grace_period_timeout_secs: 5 -``` - -That allows a short overlap, then re-checks whether stale connections have disappeared before enforcing the limit. diff --git a/docs/src/troubleshooting.md b/docs/src/troubleshooting.md new file mode 100644 index 000000000..5b836823b --- /dev/null +++ b/docs/src/troubleshooting.md @@ -0,0 +1,40 @@ +# 🛠️ Troubleshooting & Resilience + +Running a reverse proxy for IPTV involves navigating the quirks of various video players and the strict limitations of upstream providers. This page +documents common "real-world" behaviors that can lead to stream interruptions or provider bans, and how to mitigate them using Tuliprox's resilience +features. + +The solutions below will help you fine-tune your configuration for a seamless experience. + +--- + +## 1. The VLC "Seek" Problem (Grace Periods) + +**The Problem:** A user watches a VOD movie via reverse proxy. They press "Fast forward 10 seconds" in VLC. VLC calculates the new byte offset, kills +the TCP connection, and instantly fires a new HTTP GET request (with a `Range` header) to your Tuliprox server. + +Tuliprox opens a new connection to the upstream provider. However, since the old connection takes milliseconds to officially close at the provider +side, the provider sees **two** active streams. If you only paid for 1 connection, the provider throws a 509 Bandwidth Exceeded error or bans your IP! + +**The Solution in `config.yml`:** + +```yaml +reverse_proxy: + stream: + grace_period_hold_stream: true + grace_period_millis: 2000 + grace_period_timeout_secs: 5 +``` + +**What happens now?** +Tuliprox detects the bottleneck and grants a temporary "grace" state: + +* **Hold State:** Because `grace_period_hold_stream: true` is set, Tuliprox keeps the client connection "warm" but + waits before requesting new bytes from the provider. +* **The Handover:** It waits for `grace_period_millis` (2000ms) to give the provider's server time to register the old connection as closed. +* **Resolution:** + * **Success:** If the old "ghost" connection dies within the window ➔ The new stream flows instantly. + * **Timeout:** If the old connection persists beyond `grace_period_timeout_secs` (5s) ➔ Grace is revoked, and the client receives the + `user_connections_exhausted.ts` video. + +---