Merge remote-tracking branch 'origin/main' into t3code/pr565-review

2026-08-11 13:14:44 -04:00
parent dffa6ae34e 6dd3f1a4ae
commit 977104eac5
6 changed files with 109 additions and 46 deletions
@@ -2,9 +2,11 @@
 # For the default docker compose setup, MEDIA_ROOT is the main value you need to set.

 # Optional: layer compose files through env. Leave unset for CPU-only defaults.
-# Linux/macOS example for enabling NVIDIA override:
+# Linux example for Intel/AMD VA-API or Intel Quick Sync:
+# COMPOSE_FILE=docker-compose.yml:docker-compose.vaapi.yml
+# Linux example for NVIDIA NVENC:
 # COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
-# Windows example:
+# Windows NVIDIA example:
 # COMPOSE_FILE=docker-compose.yml;docker-compose.nvidia.yml

 # Required for docker compose: host path to your media library.
@@ -27,6 +29,15 @@ POSTGRES_USER=silo
 POSTGRES_PASSWORD=silo
 POSTGRES_DB=silo
 POSTGRES_SHM_SIZE=8gb
+# POSTGRES_PORT=5432
+# REDIS_PORT=6379
+
+# Optional Meilisearch service. Generate a key, uncomment it, then start the
+# search profile with: docker compose --profile search up -d
+# MEILI_MASTER_KEY=replace-with-output-of-openssl-rand-hex-32
+# MEILISEARCH_IMAGE=getmeili/meilisearch:latest
+# MEILISEARCH_HOST=127.0.0.1
+# MEILISEARCH_PORT=7700

 # PostgreSQL auto-tuning is enabled by default in docker-compose.yml. Silo
 # applies pgtune-style OLTP recommendations through ALTER SYSTEM at startup.
@@ -40,7 +51,8 @@ POSTGRES_SHM_SIZE=8gb
 # POSTGRES_TUNE_DB_SIZE=auto
 # POSTGRES_TUNE_CONNECTIONS=100

-# Optional host port overrides for docker-compose.yml.
+# Optional host port overrides for docker-compose.yml. The container listeners
+# remain fixed, so these values only change the host side of each port mapping.
 # PORT=8090
 # Used only after Jellyfin-compatible app support is enabled by an admin.
 # JF_PORT=8096
@@ -77,7 +89,7 @@ POSTGRES_SHM_SIZE=8gb
 # Optional Redis override when running from source or pointing workers at an external Redis.
 # REDIS_URL=redis://localhost:6379

-# Optional standalone worker identity settings for docker-compose.remote-transcode.yml.
+# Optional standalone worker identity settings for custom worker deployments.
 # NODE_NAME=transcode-01
 # NODE_URL=http://transcode-01.example.com:8082

@@ -6,9 +6,10 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution expectations, merge requ

 ## Prerequisites

- **Go** 1.24+
- **Bun** 1.0+
- **PostgreSQL** 18+
+- **Go** 1.26.4+
+- **Node.js** 22+ with **pnpm** 10.32.1
+- **PostgreSQL** 18 with pgvector
+- **Redis**
 - **FFmpeg** (for transcoding support)

 ## Local Development
@@ -16,6 +17,13 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution expectations, merge requ
 Local development remains intentionally separate from the deploy-oriented compose setup. Use [docker-compose.yml](docker-compose.yml) for local services and the source-build workflow below.

 ```sh
+# Create the local bootstrap configuration
+cp .env.example .env
+printf '\nSECRET_KEY=%s\nDATABASE_URL=%s\nREDIS_URL=%s\n' \
+  "$(openssl rand -base64 48)" \
+  'postgres://silo:silo@localhost:5432/silo?sslmode=disable' \
+  'redis://localhost:6379' >> .env
+
 # Start local PostgreSQL and Redis
 docker compose up -d postgres redis

@@ -26,7 +34,8 @@ make dev-frontend
 make dev-backend
 ```

-The main compose file expects `MEDIA_ROOT` to be set even if you only want the bundled PostgreSQL and Redis services, so set that in `.env` first (`cp .env.example .env`).
+The template supplies a non-empty `MEDIA_ROOT` because Compose validates the whole file even when
+you start only PostgreSQL and Redis. Change it before testing libraries against real media.

 If you are developing `Silo` and `silo-plugin-sdk` together, keep using the local [`go.work`](go.work) workspace. That workspace is a developer convenience only. CI and release builds run with `GOWORK=off`, so any new SDK helper used here must be pushed and tagged in `silo-plugin-sdk` before this repo can merge or release the change.

@@ -79,7 +88,7 @@ database URL should be read from a non-default env file.
 go test ./...

 # Frontend tests
-cd web && bun test
+cd web && pnpm test
 ```

 ## Linting
@@ -89,8 +98,8 @@ cd web && bun test
 golangci-lint run

 # Frontend
-cd web && bun run lint
-cd web && bun run format:check
+cd web && pnpm run lint
+cd web && pnpm run format:check
 ```

 ## Project Structure
@@ -15,14 +15,22 @@ Join the community on [Discord](https://discord.gg/4RxuUQAEnW). If Silo is usefu

 ## Deploy with Docker (recommended)

-The easiest way to run Silo is with Docker Compose. The default stack assumes you do not already have PostgreSQL and Redis available, so it bundles PostgreSQL, Redis, FFmpeg, and the application for a one-command start.
+The easiest way to run Silo is with Docker Compose 2.24 or newer. The default stack assumes you do
+not already have PostgreSQL and Redis available, so it bundles PostgreSQL, Redis, FFmpeg, and the
+application for a one-command start.

 1. **Create a `.env` file**

   ```sh
   cp .env.example .env
+   printf '\nPOSTGRES_PASSWORD=%s\nSECRET_KEY=%s\n' \
+     "$(openssl rand -hex 24)" "$(openssl rand -base64 48)" >> .env
   ```

+   This replaces the development database password from `.env.example` and creates the key Silo
+   uses to encrypt stored credentials. Back up `.env` separately from PostgreSQL; losing
+   `SECRET_KEY` makes those credentials unrecoverable.
+
 2. **Set your media path**

   Edit `.env` and set:
@@ -43,6 +51,21 @@ The easiest way to run Silo is with Docker Compose. The default stack assumes yo

   If you already have PostgreSQL and Redis available, omit those bundled service examples from compose and point Silo at your existing `DATABASE_URL` and `REDIS_URL` instead.

+   ### Optional Intel/AMD VA-API or Intel Quick Sync
+
+   The default stack is CPU-only so it starts on hosts without `/dev/dri`. On a Linux host with
+   `/dev/dri`, enable the device overlay:
+
+   ```sh
+   docker compose -f docker-compose.yml -f docker-compose.vaapi.yml up -d
+   ```
+
+   To make that the default for this installation, set:
+
+   ```dotenv
+   COMPOSE_FILE=docker-compose.yml:docker-compose.vaapi.yml
+   ```
+
   ### Optional NVIDIA/NVENC

   GPU support is kept out of the default compose file so hosts without NVIDIA drivers work unchanged.
@@ -78,32 +101,39 @@ By default, data is stored under `/opt/silo`:

 - `/opt/silo/postgres`
 - `/opt/silo/redis`
+- `/opt/silo/plugins`
+- `/opt/silo/compat`
 - `/opt/silo/transcode`
 - `/opt/silo/catalog-seeds`

+The optional `search` profile also stores its index under `/opt/silo/meilisearch`.
+
 Media is mounted into the container at `/mnt/media` from the host path you set in `MEDIA_ROOT`.

-### Optional Profiles
+### Optional Search Profile

-The main compose file is integrated-first. These profiles exist for operators testing distributed mode or mirroring a split deployment shape. Most single-host installs should stay on the default integrated service, because it already includes proxying and transcoding.
+PostgreSQL full-text search works without any optional services. Meilisearch is available when you
+want its search provider:

 | Profile | Command | Description |
 |---|---|---|
 | default | `docker compose up -d` | Integrated server plus bundled PostgreSQL and Redis |
-| `proxy` | `docker compose --profile proxy up -d` | Start a standalone proxy service for distributed-mode testing |
-| `transcode` | `docker compose --profile transcode up -d` | Start a standalone transcode service for distributed-mode testing |
+| `search` | `docker compose --profile search up -d` | Add the optional Meilisearch service |

-You can enable both optional examples together:
+Before starting the `search` profile, set `MEILI_MASTER_KEY` in `.env` to the output of
+`openssl rand -hex 32`. After Silo starts, choose Meilisearch under **Admin > Settings > Search**,
+set the URL to `http://meilisearch:7700`, enter the same key as the API key, test the connection,
+and save. Restart Silo, then rebuild the catalog search index from the same page. Silo continues
+to use PostgreSQL full-text search until you select Meilisearch.

-```sh
-docker compose --profile proxy --profile transcode up -d
-```
+### Distributed Examples

-If you are splitting workers across multiple hosts, use the separate remote worker example instead of trying to stretch the main compose file across machines.
+The main Compose file includes commented proxy and transcode service examples. Most single-host
+installs should leave them commented because the integrated service already includes proxying and
+transcoding.

-### Advanced Remote Node Example
-
-For a dedicated remote transcode worker, use [docker-compose.remote-transcode.yml](docker-compose.remote-transcode.yml). That file is intended for a separate worker host that connects back to an existing Silo deployment using shared PostgreSQL and Redis.
+Multi-host operators can use those examples as a starting point for a dedicated worker Compose
+file connected to the deployment's shared PostgreSQL and Redis services.

 ### Deployment Notes

@@ -116,7 +146,10 @@ helper and cutover guide in [docs/continuum-to-silo-docker-migration.md](docs/co

 ## Configuration

-Silo requires only a `DATABASE_URL` when running from source or against external infrastructure. In the default Docker Compose path, the stack wires the database and Redis URLs for you. All other settings — libraries, metadata providers, transcoding, users — are managed through the admin UI after first launch.
+Silo requires `DATABASE_URL` and `SECRET_KEY` when running from source or against external
+infrastructure. In the default Docker Compose path, the stack wires the database and Redis URLs
+for you. All other settings — libraries, metadata providers, transcoding, users — are managed
+through the admin UI after first launch.

 ### Server Modes

@@ -183,24 +216,26 @@ move fully to a custom `postgresql.conf`.

 If you prefer running Silo without Docker:

-1. **Install prerequisites**: Go 1.24+, Bun 1.0+, PostgreSQL 18+, and FFmpeg.
+1. **Install prerequisites**: Go 1.26.4+, Node.js 22+, pnpm 10.32.1, PostgreSQL 18 with pgvector, Redis, and FFmpeg.

-2. **Start PostgreSQL and Redis** (skip if you already have them running)
+2. **Configure the source process**
+
+   ```sh
+   cp .env.example .env
+   printf '\nSECRET_KEY=%s\nDATABASE_URL=%s\nREDIS_URL=%s\n' \
+     "$(openssl rand -base64 48)" \
+     'postgres://silo:silo@localhost:5432/silo?sslmode=disable' \
+     'redis://localhost:6379' >> .env
+   ```
+
+   Change the URLs when you use existing services instead of the bundled development defaults.
+
+3. **Start PostgreSQL and Redis** (skip if you already have them running)

   ```sh
   docker compose up -d postgres redis
   ```

-   The main compose file still expects `MEDIA_ROOT` to be set even if you only want the bundled PostgreSQL and Redis services, so set that in `.env` first.
-
-3. **Configure the database connection**
-
-   ```sh
-   cp .env.example .env
-   ```
-
-   Edit `.env` and set `DATABASE_URL` to point to your PostgreSQL instance.
-
 4. **Build and run**

   ```sh
@@ -123,7 +123,6 @@ services:
      - silo-dev-plugins:/var/lib/silo/plugins
      - silo-dev-compat:/var/lib/silo/compat
      - silo-dev-transcode:/tmp/silo-transcode
-      - silo-dev-audiobook-covers:/var/lib/silo/audiobook-covers
      - ${MEDIA_ROOT:-/tmp/silo-dev-media}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro
    depends_on:
      postgres:
@@ -169,7 +168,6 @@ services:

 volumes:
  silo-dev-air-tmp:
-  silo-dev-audiobook-covers:
  silo-dev-compat:
  silo-dev-go-build:
  silo-dev-go-mod:
@@ -0,0 +1,10 @@
+services:
+  silo:
+    devices:
+      - /dev/dri:/dev/dri
+
+  # Optional distributed-mode example. Most single-host installs should
+  # just use the integrated service above.
+  # silo-transcode:
+  #   devices:
+  #     - /dev/dri:/dev/dri
@@ -8,7 +8,7 @@ services:
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-silo}
      POSTGRES_DB: ${POSTGRES_DB:-silo}
    ports:
-      - "5432:5432"
+      - "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
    volumes:
      - ${SILO_DATA_ROOT:-/opt/silo}/postgres:/var/lib/postgresql
    command: ["postgres", "-c", "listen_addresses=*"]
@@ -22,7 +22,7 @@ services:
    image: redis:alpine
    restart: unless-stopped
    ports:
-      - "6379:6379"
+      - "127.0.0.1:${REDIS_PORT:-6379}:6379"
    volumes:
      - ${SILO_DATA_ROOT:-/opt/silo}/redis:/data
    healthcheck:
@@ -56,6 +56,10 @@ services:
    restart: unless-stopped
    environment:
      MODE: integrated
+      # Keep the container listeners fixed. PORT and JF_PORT in .env are
+      # host-side published-port overrides.
+      PORT: "8080"
+      JF_PORT: "8096"
      # Master key for at-rest credential encryption. Required — the server
      # refuses to start without it. Generate with: openssl rand -base64 48
      # Back it up SEPARATELY from your database dumps; losing it makes encrypted
@@ -71,12 +75,10 @@ services:
      - "${ABS_PORT:-13378}:13378"
    volumes:
      - ${MEDIA_ROOT:?Set MEDIA_ROOT in .env to the host media path}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro
-      - ${MEDIA_BOOKS_ROOT:-${MEDIA_ROOT}}:${MEDIA_BOOKS_CONTAINER_ROOT:-${MEDIA_CONTAINER_ROOT:-/mnt/media}/books}:ro
      - ${SILO_DATA_ROOT:-/opt/silo}/plugins:/var/lib/silo/plugins
      - ${SILO_DATA_ROOT:-/opt/silo}/compat:/var/lib/silo/compat
      - ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode
      - ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro
-      - ${SILO_DATA_ROOT:-/opt/silo}/audiobook-covers:/var/lib/silo/audiobook-covers
      - /proc/meminfo:/host/proc/meminfo:ro
    depends_on:
      postgres:
@@ -86,9 +88,6 @@ services:
    env_file:
      - path: .env
        required: false
-    devices:
-      - /dev/dri:/dev/dri
-
  # Optional distributed-mode example. Most single-host installs should
  # just use the integrated service above.
  # silo-proxy: