diff --git a/CHANGELOG.md b/CHANGELOG.md index 082ba67f5..80147a3bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,12 +110,11 @@ ## 🌟 New Features -- **Runtime liveness watchdog and a tokio-console diagnostic build.** An optional heartbeat/watchdog detects a wedged +- **Runtime liveness watchdog.** An optional heartbeat/watchdog detects a wedged async runtime (process alive, scheduler no longer making progress, logs stop) and logs a diagnostic snapshot with runtime metrics and a per-thread `/proc/self/task` inventory. It is opt-in and off by default (`TULIPROX_WATCHDOG=1` to observe, `=2` to also restart the process on a confirmed stall), and exposes its state through the `/healthcheck` - `runtime` object. For deeper task and lock inspection, `make build-diagnostic` produces a separate `tokio-console` - binary; normal release images contain neither the subscriber nor `tokio_unstable`. + `runtime` object. - **`.env` file support for secrets and environment variables:** Tuliprox now automatically loads environment variables from a `.env` file at startup. diff --git a/Cargo.lock b/Cargo.lock index 0fe3b1564..c47091847 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,12 +472,6 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" @@ -682,46 +676,6 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" -[[package]] -name = "console-api" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8599749b6667e2f0c910c1d0dff6901163ff698a52d5a39720f61b5be4b20d3" -dependencies = [ - "futures-core", - "prost", - "prost-types", - "tonic", - "tonic-prost", - "tracing-core", -] - -[[package]] -name = "console-subscriber" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4915b7d8dd960457a1b6c380114c2944f728e7c65294ab247ae6b6f1f37592" -dependencies = [ - "console-api", - "crossbeam-channel", - "crossbeam-utils", - "futures-task", - "hdrhistogram", - "humantime", - "hyper-util", - "prost", - "prost-types", - "serde", - "serde_json", - "thread_local", - "tokio", - "tokio-stream", - "tonic", - "tracing", - "tracing-core", - "tracing-subscriber", -] - [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -809,15 +763,6 @@ dependencies = [ "winnow 0.7.15", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1850,19 +1795,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "hdrhistogram" -version = "7.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" -dependencies = [ - "base64 0.22.1", - "byteorder", - "flate2", - "nom 8.0.0", - "num-traits", -] - [[package]] name = "heck" version = "0.5.0" @@ -1952,12 +1884,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humantime" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" - [[package]] name = "hybrid-array" version = "0.4.10" @@ -2257,15 +2183,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -2481,15 +2398,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - [[package]] name = "matchit" version = "0.8.4" @@ -3143,38 +3051,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "prost-types" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" -dependencies = [ - "prost", -] - [[package]] name = "quanta" version = "0.12.6" @@ -3904,15 +3780,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shared" version = "3.3.113" @@ -4202,15 +4069,6 @@ dependencies = [ "syn 3.0.4", ] -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" version = "0.3.55" @@ -4282,7 +4140,6 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "tracing", "windows-sys 0.61.2", ] @@ -4408,17 +4265,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tonic-prost" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" -dependencies = [ - "bytes", - "prost", - "tonic", -] - [[package]] name = "tower" version = "0.5.3" @@ -4527,22 +4373,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "once_cell", - "regex-automata", - "sharded-slab", - "thread_local", - "tracing", - "tracing-core", ] [[package]] @@ -4566,7 +4396,6 @@ dependencies = [ "chrono", "chrono-tz", "clap", - "console-subscriber", "crc32fast", "cron", "dashmap", @@ -5206,12 +5035,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - [[package]] name = "vergen" version = "10.0.3" diff --git a/Makefile b/Makefile index ea362a24e..b173a43aa 100644 --- a/Makefile +++ b/Makefile @@ -215,12 +215,6 @@ build: ## Build the entire workspace in parallel using detected CPU count @echo "==> Building workspace with $(CARGO_BUILD_JOBS) jobs" @TMPDIR="$${TMPDIR:-/tmp}" $(CARGO_STABLE) build -j$(CARGO_BUILD_JOBS) --workspace -.PHONY: build-diagnostic -build-diagnostic: ## Build tuliprox with tokio-console (diagnostic only; requires tokio_unstable) - @echo "==> Building diagnostic binary (tokio-console, RUSTFLAGS=--cfg tokio_unstable)" - @RUSTFLAGS="--cfg tokio_unstable" TMPDIR="$${TMPDIR:-/tmp}" $(CARGO_STABLE) build -j$(CARGO_BUILD_JOBS) --package tuliprox --features tokio-console - @echo "==> Run with TULIPROX_TOKIO_CONSOLE=1, then connect with: tokio-console" - .PHONY: serve serve: ## Run tuliprox server with settings folder: make serve @if [ -z "$(SETTINGS_FOLDER)" ]; then \ diff --git a/backend/app/Cargo.toml b/backend/app/Cargo.toml index 40bb9d895..a1349f3dd 100644 --- a/backend/app/Cargo.toml +++ b/backend/app/Cargo.toml @@ -63,8 +63,6 @@ blake3.workspace = true bytes.workspace = true tokio-stream = { version = "0.1.19", features = ["sync"] } tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs", "process", "signal", "time", "macros"] } -# Diagnostic-only: built solely under the `tokio-console` feature. -console-subscriber = { version = "0.5", optional = true } tokio-util = { version = "0.7.19", features = ["io", "io-util"] } tempfile = "3.27.0" base64.workspace = true @@ -118,9 +116,6 @@ tokio-rustls = "0.26.4" [features] proxy-auth-regression = [] -# Diagnostic build. Requires RUSTFLAGS="--cfg tokio_unstable" at compile time; -# see `make build-diagnostic`. Inactive unless TULIPROX_TOKIO_CONSOLE=1. -tokio-console = ["dep:console-subscriber"] [[bin]] diff --git a/backend/app/src/main.rs b/backend/app/src/main.rs index f3b33fd8e..6a718751f 100644 --- a/backend/app/src/main.rs +++ b/backend/app/src/main.rs @@ -277,26 +277,8 @@ fn handle_migrate_db(db_path: &Path, db_type: Option<&str>, no_backup: bool) { } } -/// Installs the tokio-console subscriber when the diagnostic build is used and -/// explicitly requested at runtime. -/// -/// The subscriber is expensive: it records every task poll. It is therefore -/// compiled in only under the `tokio-console` feature and stays inactive unless -/// `TULIPROX_TOKIO_CONSOLE=1`. A production binary does not contain the code. -#[cfg(feature = "tokio-console")] -fn init_tokio_console() { - if std::env::var("TULIPROX_TOKIO_CONSOLE").is_ok_and(|value| value == "1") { - console_subscriber::init(); - info!("tokio-console subscriber enabled (connect with `tokio-console`)"); - } -} - -#[cfg(not(feature = "tokio-console"))] -const fn init_tokio_console() {} - #[tokio::main] async fn main() { - init_tokio_console(); api::api_utils::init_uptime_clock(); let args = Args::parse(); diff --git a/backend/core/src/utils/logging.rs b/backend/core/src/utils/logging.rs index 83e487503..4a25e0681 100644 --- a/backend/core/src/utils/logging.rs +++ b/backend/core/src/utils/logging.rs @@ -198,12 +198,6 @@ fn apply_logger_with_context(context: &LoggerContext, config_log_level: Option<& } pub fn init_logger(user_log_level: Option<&str>, config_file: &str) { - // tracing_subscriber::registry() - // .with(console_subscriber::spawn()) // Console layer - // .with(EnvFilter::from_default_env()) - // .with(fmt::layer()) // stdout logging - // .init(); - let context = LoggerContext { cli_log_level: user_log_level.map(std::string::ToString::to_string), env_log_level: std::env::var("TULIPROX_LOG").ok(), diff --git a/bin/build_docker.sh b/bin/build_docker.sh index efab248cb..1fcd8c9e0 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -30,20 +30,6 @@ esac echo "🚀 Building for branch: $BRANCH (tag: $TAG_SUFFIX)" -# The experimental image is the diagnostic build: tokio-console is compiled in -# and switched on inside the image. develop and master must stay uninstrumented, -# so both the cargo feature and tokio_unstable are confined to this branch. -CARGO_FEATURES=() -DOCKER_BUILD_ARGS=() -if [ "$BRANCH" = "experimental" ]; then - export RUSTFLAGS="${RUSTFLAGS} --cfg tokio_unstable" - CARGO_FEATURES=(--features tokio-console) - # 0.0.0.0 is required so a host-published port can reach the in-container - # gRPC server; publish it as 127.0.0.1:6669:6669 to keep it host-local. - DOCKER_BUILD_ARGS=(--build-arg TULIPROX_TOKIO_CONSOLE=1 --build-arg TOKIO_CONSOLE_BIND=0.0.0.0:6669) - echo "🔬 Experimental diagnostic build: tokio_unstable + --features tokio-console" -fi - # Directories WORKING_DIR=$(pwd) DOCKER_DIR="${WORKING_DIR}/docker" @@ -118,7 +104,7 @@ for PLATFORM in "${!ARCHITECTURES[@]}"; do echo "🔨 Building for $ARCHITECTURE" # Using cross for compilation - cross build -p tuliprox --release --target "$ARCHITECTURE" --locked ${CARGO_FEATURES[@]+"${CARGO_FEATURES[@]}"} + cross build -p tuliprox --release --target "$ARCHITECTURE" --locked SOURCE_BIN_PATH="target/${ARCHITECTURE}/release/tuliprox" cp "${SOURCE_BIN_PATH}" "${DOCKER_DIR}/binaries/tuliprox-${ARCHITECTURE}" @@ -164,7 +150,6 @@ for IMAGE_NAME in "${!MULTI_PLATFORM_IMAGES[@]}"; do --platform "linux/amd64,linux/arm64" \ --cache-from "type=gha,scope=${IMAGE_NAME}-${TAG_SUFFIX}" \ --cache-to "type=gha,mode=max,scope=${IMAGE_NAME}-${TAG_SUFFIX}" \ - ${DOCKER_BUILD_ARGS[@]+"${DOCKER_BUILD_ARGS[@]}"} \ --push \ . done diff --git a/docker/Dockerfile.manual b/docker/Dockerfile.manual index 32b0fede4..73be5990e 100644 --- a/docker/Dockerfile.manual +++ b/docker/Dockerfile.manual @@ -37,15 +37,6 @@ FROM scratch AS scratch-final ARG TZ=UTC ENV TZ=${TZ} -# Non-zero only in the experimental diagnostic image; the binary ignores any -# other value, and develop/master images do not contain the subscriber at all. -ARG TULIPROX_TOKIO_CONSOLE=0 -ENV TULIPROX_TOKIO_CONSOLE=${TULIPROX_TOKIO_CONSOLE} -# console-subscriber binds this address. Loopback by default; the experimental -# image widens it to 0.0.0.0 so a published host port can reach the gRPC server. -ARG TOKIO_CONSOLE_BIND=127.0.0.1:6669 -ENV TOKIO_CONSOLE_BIND=${TOKIO_CONSOLE_BIND} - # Copy timezone data and localtime from tz-prep COPY --from=tz-prep /output/usr/share/zoneinfo /usr/share/zoneinfo COPY --from=tz-prep /output/etc/localtime /etc/localtime @@ -72,15 +63,6 @@ FROM alpine:latest AS alpine-final ARG TZ=UTC ENV TZ=${TZ} -# Non-zero only in the experimental diagnostic image; the binary ignores any -# other value, and develop/master images do not contain the subscriber at all. -ARG TULIPROX_TOKIO_CONSOLE=0 -ENV TULIPROX_TOKIO_CONSOLE=${TULIPROX_TOKIO_CONSOLE} -# console-subscriber binds this address. Loopback by default; the experimental -# image widens it to 0.0.0.0 so a published host port can reach the gRPC server. -ARG TOKIO_CONSOLE_BIND=127.0.0.1:6669 -ENV TOKIO_CONSOLE_BIND=${TOKIO_CONSOLE_BIND} - # Install dependencies including ffmpeg RUN apk add --no-cache bash curl strace tcpdump bind-tools nano ca-certificates tini ffmpeg diff --git a/docs/src/build-and-deploy.md b/docs/src/build-and-deploy.md index 36438f374..ddc6b352e 100644 --- a/docs/src/build-and-deploy.md +++ b/docs/src/build-and-deploy.md @@ -41,7 +41,6 @@ environment across different machines. * `make test`: Runs all workspace tests using the Stable toolchain. * `make lint`: Runs clippy to find common mistakes and improve code quality. * `make lint-fix`: Automatically applies clippy suggestions (where possible). -* `make build-diagnostic`: Builds the backend with `tokio-console` support (diagnostic only, not part of normal releases). * `make markdown-lint`: Checks all .md files for formatting consistency. **Formatting**: @@ -129,22 +128,6 @@ rustup target add x86_64-unknown-linux-musl cargo build -p tuliprox --target x86_64-unknown-linux-musl --release ``` -### Diagnostic Build (tokio-console) - -To diagnose async hangs (tasks that never complete, locks held across `.await`), build the diagnostic binary: - -```bash -make build-diagnostic -TULIPROX_TOKIO_CONSOLE=1 ./target/release/tuliprox -s -p ./config -tokio-console # defaults to http://127.0.0.1:6669 -``` - -This build compiles `tokio` with `--cfg tokio_unstable` and enables the optional `console-subscriber`. It is a separate -binary and normal release builds are unaffected. The experimental Docker image -(`ghcr.io/euzu/tuliprox:experimental`) is built this way and enables the subscriber by default. See -[Runtime Liveness Watchdog](./operations-debugging.md#8-runtime-liveness-watchdog) for the always-available (opt-in) -watchdog that does not require a special build. - ### Cross-Compilation (ARM / Windows) To compile for architectures other than your host, use the `cross` tool or native mingw packages. diff --git a/docs/src/operations-debugging.md b/docs/src/operations-debugging.md index 065760ae4..9cee01763 100644 --- a/docs/src/operations-debugging.md +++ b/docs/src/operations-debugging.md @@ -502,30 +502,3 @@ While the watchdog runs, `GET /healthcheck` gains a `runtime` object: `runtime.status` is `stalled` once the heartbeat stops. The top-level `status` intentionally stays `ok` for as long as the HTTP server answers, so no orchestrator restarts the process. If the whole runtime is wedged, the endpoint cannot answer at all — in that case the watchdog log is the signal. - ---- - -## 9. tokio-console Diagnostic Build - -`tokio-console` shows live which tasks exist, where each one is waiting, and which lock it holds. It is the tool to -identify an async deadlock. It needs a special build and is never part of the standard production image. - -```bash -# Build the diagnostic binary (requires RUSTFLAGS="--cfg tokio_unstable" internally) -make build-diagnostic - -# Run it with the subscriber enabled and attach the console -TULIPROX_TOKIO_CONSOLE=1 ./target/release/tuliprox -s -p ./config -tokio-console # defaults to http://127.0.0.1:6669 -``` - -The experimental Docker image (`ghcr.io/euzu/tuliprox:experimental`) already carries this build and enables it by default. - -Notes: - -* The default release build compiles none of this: no `tokio_unstable`, no `console-subscriber`. There is no runtime cost - in normal images. -* The console gRPC server has no authentication. Keep it on loopback (the default) or, if you must publish a port, - publish it only on the host loopback (`-p 127.0.0.1:6669:6669`). - ---- diff --git a/docs/src/troubleshooting.md b/docs/src/troubleshooting.md index 1192b4c0c..f9488e948 100644 --- a/docs/src/troubleshooting.md +++ b/docs/src/troubleshooting.md @@ -235,9 +235,6 @@ the grace period (`TULIPROX_WATCHDOG_RESTART_GRACE_MS`, default 30 s), so a conf brings the container back automatically. Use mode `1` while you are still investigating a cause, and mode `2` once a restart is the acceptable recovery. -**How to find the exact task and lock:** use the `tokio-console` diagnostic build (see -[Runtime Liveness Watchdog](./operations-debugging.md#8-runtime-liveness-watchdog)) or run the experimental image. - **Immediate mitigation:** restart the container. The watchdog log is what tells you the stall happened on its own and captures what every thread was doing at that moment.