From dc0392dee024848c45325050c9ee42f9d9f6ea8a Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:23:21 -0400 Subject: [PATCH] Add PostgreSQL auto-tuning via pgtune --- .env.example | 13 + README.md | 122 ++-- cmd/silo/main.go | 65 ++ docker-compose.yml | 7 +- internal/database/postgres_tune.go | 806 ++++++++++++++++++++++++ internal/database/postgres_tune_test.go | 173 +++++ postgres/postgresql.conf | 38 -- 7 files changed, 1104 insertions(+), 120 deletions(-) create mode 100644 internal/database/postgres_tune.go create mode 100644 internal/database/postgres_tune_test.go delete mode 100644 postgres/postgresql.conf diff --git a/.env.example b/.env.example index e62ea990..85894ae2 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,19 @@ SILO_IMAGE=ghcr.io/silo-server/silo-server:latest POSTGRES_USER=silo POSTGRES_PASSWORD=silo POSTGRES_DB=silo +POSTGRES_SHM_SIZE=8gb + +# PostgreSQL auto-tuning is enabled by default in docker-compose.yml. Silo +# applies pgtune-style OLTP recommendations through ALTER SYSTEM at startup. +# Set POSTGRES_TUNE=off if you manage PostgreSQL tuning yourself. +# POSTGRES_TUNE=off +# POSTGRES_TUNE_PROFILE=oltp +# POSTGRES_TUNE_MEMORY=auto +# POSTGRES_TUNE_MEMORY_BUDGET_PERCENT=75 +# POSTGRES_TUNE_CPUS=auto +# POSTGRES_TUNE_STORAGE=ssd +# POSTGRES_TUNE_DB_SIZE=auto +# POSTGRES_TUNE_CONNECTIONS=100 # Optional host port overrides for docker-compose.yml. # PORT=8090 diff --git a/README.md b/README.md index 892acfd4..930e532b 100644 --- a/README.md +++ b/README.md @@ -218,93 +218,57 @@ Silo requires only a `DATABASE_URL` when running from source or against external | `proxy` | Stream proxy node that connects to the shared deployment database and Redis | | `transcode` | HLS transcode worker node that connects to the shared deployment database and Redis | -## PostgreSQL Performance Tuning +## PostgreSQL Auto-Tuning -The default PostgreSQL configuration is conservative and not optimized for any particular workload. For better performance, use [PGTune](https://pgtune.leopard.in.ua/) to generate settings tailored to your hardware. - -**Recommended PGTune settings:** - -| Setting | Value | -|---|---| -| DB Type | **Web application** | -| Total Memory | Your server's RAM | -| Number of CPUs | Your server's CPU count | -| Number of Connections | 20 (Silo's default pool size) | -| Data Storage | SSD or HDD depending on your setup | - -Silo's workload is predominantly read-heavy API queries (browse, search, metadata lookups) with the database small enough to fit in RAM for most libraries. The **Web application** profile optimizes for this: CPU-bound simple queries served from cached pages, high concurrent reads, and low write volume. - -If you have a very large library (50,000+ items) or heavy activity logging, consider switching to the **OLTP** profile instead. - -### Recommended Defaults for Modern Hardware - -Assuming you're using modern hardware with fast NVMe storage this is what I'm currently running. - -```ini -# Connections -max_connections = 100 - -# Memory (tuned to keep PG under ~8GB total) -shared_buffers = 4GB # holds the entire DB in cache (typical DB is 1-4GB) -effective_cache_size = 6GB # planner hint only, not an allocation -work_mem = 32MB # per-operation sort/hash memory -maintenance_work_mem = 512MB # VACUUM, CREATE INDEX, catalog imports - -# WAL / Write Performance -wal_buffers = 64MB -min_wal_size = 512MB -max_wal_size = 2GB -checkpoint_completion_target = 0.9 - -# Storage (NVMe/Optane) -random_page_cost = 1.1 # default 4.0; NVMe random ≈ sequential -effective_io_concurrency = 200 # NVMe handles many concurrent requests - -# Parallelism -max_worker_processes = 8 -max_parallel_workers_per_gather = 4 -max_parallel_workers = 8 -max_parallel_maintenance_workers = 4 - -# Huge Pages -huge_pages = try -``` - -These settings keep PostgreSQL under ~8GB of RAM while still caching the entire database. For HDD storage, change `random_page_cost` to `4.0` and `effective_io_concurrency` to `2`. - -### Applying Settings - -**Docker (recommended):** Save the settings above to a `postgresql.conf` file and mount it into your container: +The default Docker Compose stack does not require a checked-in `postgresql.conf`. +It enables Silo's [pgtune](https://github.com/le0pard/pgtune)-style OLTP tuning +by default: ```yaml -# docker-compose.yml -postgres: - image: pgvector/pgvector:pg18 - volumes: - - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro - command: postgres -c config_file=/etc/postgresql/postgresql.conf +POSTGRES_TUNE: auto ``` -Then restart: `docker compose restart postgres` +When enabled, Silo connects with `DATABASE_URL` and applies recommendations with +`ALTER SYSTEM`, which writes to PostgreSQL's `postgresql.auto.conf` inside the +database data directory. Reloadable settings are applied immediately with +`pg_reload_conf()`. Settings that PostgreSQL marks as restart-only are written +too, and Silo logs the setting names so you can restart PostgreSQL once: -**Existing PostgreSQL:** You can apply settings without editing config files using `ALTER SYSTEM`, which writes to `postgresql.auto.conf` inside the data directory: - -```sql -ALTER SYSTEM SET work_mem = '32MB'; -ALTER SYSTEM SET effective_cache_size = '6GB'; -ALTER SYSTEM SET random_page_cost = 1.1; -ALTER SYSTEM SET effective_io_concurrency = 200; --- Apply without restart: -SELECT pg_reload_conf(); - --- These require a full restart: -ALTER SYSTEM SET shared_buffers = '4GB'; -ALTER SYSTEM SET max_connections = 100; -ALTER SYSTEM SET max_worker_processes = 8; -ALTER SYSTEM SET huge_pages = 'try'; +```sh +docker compose restart postgres ``` -Settings like `shared_buffers`, `max_connections`, `max_worker_processes`, and `huge_pages` require a PostgreSQL restart. Memory/planner settings (`work_mem`, `effective_cache_size`, `random_page_cost`) can be reloaded live. +The default Compose database user has the required PostgreSQL permissions. If +you use an external PostgreSQL server, make sure the configured `DATABASE_URL` +user can run `ALTER SYSTEM`, or set `POSTGRES_TUNE=off` and manage +PostgreSQL yourself. + +For `POSTGRES_TUNE_MEMORY=auto`, Silo uses the first trustworthy memory source: +a finite Docker cgroup limit, the read-only `/host/proc/meminfo` mount supplied +by the bundled Compose file, then `/proc/meminfo` with container safety guards. +Auto-detected memory is treated as a PostgreSQL budget, defaulting to 75% of +detected RAM so Silo, Redis, plugins, transcodes, and the OS retain headroom. +`POSTGRES_TUNE_DB_SIZE=auto` queries `pg_database_size(current_database())` and +classifies the workload by comparing the database size to that memory budget. + +Optional tuning overrides: + +| Variable | Default | Description | +|---|---:|---| +| `POSTGRES_TUNE_PROFILE` | `oltp` | Tuning profile. Only `oltp` is currently supported. | +| `POSTGRES_TUNE_MEMORY` | `auto` | Server/container RAM, such as `8GB` or `32GB`; explicit values are used as-is. | +| `POSTGRES_TUNE_MEMORY_BUDGET_PERCENT` | `75` | Percent of auto-detected RAM used for PostgreSQL recommendations. | +| `POSTGRES_TUNE_CPUS` | `auto` | CPU count used for worker recommendations. | +| `POSTGRES_TUNE_STORAGE` | `ssd` | One of `hdd`, `ssd`, `san`, or `nvme`. | +| `POSTGRES_TUNE_DB_SIZE` | `auto` | Use `less_ram` when the database comfortably fits in RAM, `mid_ram`, or `greater_ram` for very large databases. | +| `POSTGRES_TUNE_CONNECTIONS` | `100` | PostgreSQL `max_connections`; automatically raised if Silo's app pool is configured higher. | +| `POSTGRES_SHM_SIZE` | `8gb` | Docker `/dev/shm` size for the bundled PostgreSQL container. | + +Advanced operators can still supply their own PostgreSQL configuration or +override these env vars. Set `POSTGRES_TUNE=off` when you do not want Silo to +change PostgreSQL server settings. Settings already written with `ALTER SYSTEM` +remain in `postgresql.auto.conf`; reset those PostgreSQL parameters if you later +move fully to a custom `postgresql.conf`. ## Project Structure diff --git a/cmd/silo/main.go b/cmd/silo/main.go index cde16fc4..c99ff364 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -184,6 +184,70 @@ func configureOperationalLogging( return operationalWriter, opslog.NewRepo(pool), opsPM } +func maybeApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, appMaxConnections int, mode string) { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "", "integrated", "api": + default: + return + } + + opts, err := database.LoadPostgresTuneOptionsFromEnv(appMaxConnections) + if err != nil { + slog.Warn("postgres auto-tuning disabled", "error", err) + return + } + if !opts.Enabled { + return + } + + tuneCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + result, err := database.ApplyPostgresTuning(tuneCtx, pool, opts) + for _, failure := range result.Failures { + slog.Warn("postgres auto-tuning setting failed", + "name", failure.Name, + "value", failure.Value, + "error", failure.Err, + ) + } + if err != nil { + slog.Warn("postgres auto-tuning failed", + "error", err, + "applied", result.Applied, + "failures", len(result.Failures), + ) + return + } + + slog.Info("postgres auto-tuning applied", + "profile", opts.Profile, + "postgres_major", result.PostgresMajorVersion, + "settings", result.Applied, + "resets", len(result.Reset), + "failures", len(result.Failures), + "memory_budget_bytes", opts.MemoryBudgetBytes, + "detected_memory_bytes", opts.DetectedMemoryBytes, + "memory_source", opts.MemorySource, + "memory_budget_percent", opts.MemoryBudgetPercent, + "cpus", opts.CPUs, + "connections", opts.Connections, + "storage", opts.Storage, + "db_size", result.DBSize, + "database_size_bytes", result.DatabaseSizeBytes, + ) + if len(result.RestartRequired) > 0 { + slog.Warn("postgres restart required to finish applying auto-tuned settings", + "settings", strings.Join(result.RestartRequired, ","), + ) + } + if len(result.Reset) > 0 { + slog.Info("postgres auto-tuning reset stale settings", + "settings", strings.Join(result.Reset, ","), + ) + } +} + func main() { envFile := flag.String("env", ".env", "path to .env bootstrap file") flag.Parse() @@ -314,6 +378,7 @@ func main() { slog.SetDefault(slog.New(logfilter.New(baseHandler, cfg.Server.LogQuiet))) mode := cfg.Server.Mode + maybeApplyPostgresTuning(ctx, pool, cfg.Database.MaxConnections, mode) slog.Info("silo starting", "mode", mode, "listen", cfg.Server.Listen, "log_level", cfg.Server.LogLevel, "node_id", nodeID) appCtx, appCancel := context.WithCancel(ctx) diff --git a/docker-compose.yml b/docker-compose.yml index f995e672..8eab93af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: postgres: image: pgvector/pgvector:pg18 restart: unless-stopped - shm_size: 1gb + shm_size: ${POSTGRES_SHM_SIZE:-8gb} environment: POSTGRES_USER: ${POSTGRES_USER:-silo} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-silo} @@ -11,8 +11,7 @@ services: - "5432:5432" volumes: - ${SILO_DATA_ROOT:-/opt/silo}/postgres:/var/lib/postgresql - - ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro - command: postgres -c config_file=/etc/postgresql/postgresql.conf + command: ["postgres", "-c", "listen_addresses=*"] healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-silo}"] interval: 5s @@ -40,6 +39,7 @@ services: DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable REDIS_URL: redis://redis:6379 SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins + POSTGRES_TUNE: ${POSTGRES_TUNE:-auto} ports: - "${PORT:-8090}:8080" - "${JF_PORT:-8096}:8096" @@ -48,6 +48,7 @@ services: - ${SILO_DATA_ROOT:-/opt/silo}/plugins:/var/lib/silo/plugins - ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode - ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro + - /proc/meminfo:/host/proc/meminfo:ro depends_on: postgres: condition: service_healthy diff --git a/internal/database/postgres_tune.go b/internal/database/postgres_tune.go new file mode 100644 index 00000000..fa695165 --- /dev/null +++ b/internal/database/postgres_tune.go @@ -0,0 +1,806 @@ +package database + +import ( + "bufio" + "context" + "fmt" + "math" + "os" + "runtime" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + postgresTuneProfileOLTP = "oltp" + + postgresTuneStorageHDD = "hdd" + postgresTuneStorageSSD = "ssd" + postgresTuneStorageSAN = "san" + postgresTuneStorageNVMe = "nvme" + + postgresTuneDBSizeLessRAM = "less_ram" + postgresTuneDBSizeMidRAM = "mid_ram" + postgresTuneDBSizeGreaterRAM = "greater_ram" + postgresTuneDBSizeAuto = "auto" + + defaultPostgresTuneMemoryBudgetPercent = 75 + + sizeKB = int64(1024) + sizeMB = 1024 * sizeKB + sizeGB = 1024 * sizeMB + sizeTB = 1024 * sizeGB +) + +var managedPostgresTuneSettings = []string{ + "max_connections", + "shared_buffers", + "effective_cache_size", + "maintenance_work_mem", + "checkpoint_completion_target", + "wal_buffers", + "default_statistics_target", + "random_page_cost", + "effective_io_concurrency", + "work_mem", + "huge_pages", + "jit", + "wal_compression", + "autovacuum_max_workers", + "autovacuum_work_mem", + "min_wal_size", + "max_wal_size", + "max_worker_processes", + "max_parallel_workers_per_gather", + "max_parallel_workers", + "max_parallel_maintenance_workers", + "io_method", + "io_workers", +} + +// PostgresTuneOptions controls automatic PostgreSQL tuning through ALTER SYSTEM. +type PostgresTuneOptions struct { + Enabled bool + Profile string + MemoryBudgetBytes int64 + DetectedMemoryBytes int64 + MemorySource string + MemoryBudgetPercent int + CPUs int + Connections int + Storage string + DBSize string + OSType string + IOMethod string +} + +// PostgresTuneSetting is one recommended PostgreSQL parameter. +type PostgresTuneSetting struct { + Name string + Value string + RequiresRestart bool +} + +// PostgresTuneFailure records a setting that PostgreSQL rejected. +type PostgresTuneFailure struct { + Name string + Value string + Err error +} + +// PostgresTuneResult summarizes one tuning run. +type PostgresTuneResult struct { + PostgresMajorVersion int + DatabaseSizeBytes int64 + DBSize string + Settings []PostgresTuneSetting + Reset []string + Applied int + Failures []PostgresTuneFailure + Reloaded bool + RestartRequired []string +} + +type postgresTuneMemoryBudget struct { + BudgetBytes int64 + DetectedBytes int64 + Source string + Percent int +} + +// LoadPostgresTuneOptionsFromEnv reads the opt-in PostgreSQL tuning environment. +func LoadPostgresTuneOptionsFromEnv(appMaxConnections int) (PostgresTuneOptions, error) { + rawEnabled := firstNonEmptyEnv("POSTGRES_TUNE", "SILO_POSTGRES_TUNE") + if rawEnabled == "" { + return PostgresTuneOptions{}, nil + } + + enabled, profileFromMode, err := parsePostgresTuneMode(rawEnabled) + if err != nil { + return PostgresTuneOptions{}, err + } + if !enabled { + return PostgresTuneOptions{}, nil + } + + profile := firstNonEmptyEnv("POSTGRES_TUNE_PROFILE", "SILO_POSTGRES_TUNE_PROFILE") + if profile == "" { + profile = profileFromMode + } + if profile == "" { + profile = postgresTuneProfileOLTP + } + profile = strings.ToLower(profile) + if profile != postgresTuneProfileOLTP { + return PostgresTuneOptions{}, fmt.Errorf("unsupported POSTGRES_TUNE_PROFILE %q; only %q is supported", profile, postgresTuneProfileOLTP) + } + + memory, err := loadPostgresTuneMemory() + if err != nil { + return PostgresTuneOptions{}, err + } + + cpus, err := loadPostgresTuneCPUs() + if err != nil { + return PostgresTuneOptions{}, err + } + + connections, err := loadPostgresTuneConnections(appMaxConnections) + if err != nil { + return PostgresTuneOptions{}, err + } + + storage, err := normalizePostgresTuneStorage(firstNonEmptyEnv("POSTGRES_TUNE_STORAGE", "SILO_POSTGRES_TUNE_STORAGE")) + if err != nil { + return PostgresTuneOptions{}, err + } + + dbSize, err := normalizePostgresTuneDBSize(firstNonEmptyEnv("POSTGRES_TUNE_DB_SIZE", "SILO_POSTGRES_TUNE_DB_SIZE")) + if err != nil { + return PostgresTuneOptions{}, err + } + + osType := strings.ToLower(firstNonEmptyEnv("POSTGRES_TUNE_OS", "SILO_POSTGRES_TUNE_OS")) + if osType == "" { + osType = "linux" + } + + ioMethod := strings.ToLower(firstNonEmptyEnv("POSTGRES_TUNE_IO_METHOD", "SILO_POSTGRES_TUNE_IO_METHOD")) + + return PostgresTuneOptions{ + Enabled: true, + Profile: profile, + MemoryBudgetBytes: memory.BudgetBytes, + DetectedMemoryBytes: memory.DetectedBytes, + MemorySource: memory.Source, + MemoryBudgetPercent: memory.Percent, + CPUs: cpus, + Connections: connections, + Storage: storage, + DBSize: dbSize, + OSType: osType, + IOMethod: ioMethod, + }, nil +} + +// RecommendPostgresOLTPSettings returns pgtune-style OLTP recommendations. +func RecommendPostgresOLTPSettings(opts PostgresTuneOptions, postgresMajorVersion int) []PostgresTuneSetting { + memoryBudgetKB := opts.MemoryBudgetBytes / sizeKB + if memoryBudgetKB <= 0 { + return nil + } + + connections := opts.Connections + if connections <= 0 { + connections = 100 + } + + settings := make([]PostgresTuneSetting, 0, 24) + sharedBuffers := memoryBudgetKB / 4 + effectiveCacheSize := (memoryBudgetKB * 3) / 4 + maintenanceWorkMem := memoryBudgetKB / 16 + if maintenanceWorkMem > 8*sizeGB/sizeKB { + maintenanceWorkMem = 8 * sizeGB / sizeKB + } + + settings = append(settings, + PostgresTuneSetting{Name: "max_connections", Value: strconv.Itoa(connections)}, + PostgresTuneSetting{Name: "shared_buffers", Value: formatPostgresTuneKB(sharedBuffers)}, + PostgresTuneSetting{Name: "effective_cache_size", Value: formatPostgresTuneKB(effectiveCacheSize)}, + PostgresTuneSetting{Name: "maintenance_work_mem", Value: formatPostgresTuneKB(maintenanceWorkMem)}, + PostgresTuneSetting{Name: "checkpoint_completion_target", Value: "0.9"}, + PostgresTuneSetting{Name: "wal_buffers", Value: formatPostgresTuneKB(walBuffersForSharedBuffers(sharedBuffers))}, + PostgresTuneSetting{Name: "default_statistics_target", Value: "100"}, + PostgresTuneSetting{Name: "random_page_cost", Value: randomPageCost(opts.Storage, opts.DBSize)}, + ) + + if effectiveIO := effectiveIOConcurrency(opts.OSType, opts.Storage); effectiveIO != "" { + settings = append(settings, PostgresTuneSetting{Name: "effective_io_concurrency", Value: effectiveIO}) + } + + settings = append(settings, + PostgresTuneSetting{Name: "work_mem", Value: formatPostgresTuneKB(workMem(memoryBudgetKB, sharedBuffers, connections, opts.CPUs, opts.DBSize))}, + PostgresTuneSetting{Name: "huge_pages", Value: hugePages(sharedBuffers, opts.OSType)}, + ) + + if postgresMajorVersion >= 12 { + settings = append(settings, PostgresTuneSetting{Name: "jit", Value: "off"}) + } + if postgresMajorVersion >= 15 { + settings = append(settings, PostgresTuneSetting{Name: "wal_compression", Value: "lz4"}) + } else if postgresMajorVersion >= 10 { + settings = append(settings, PostgresTuneSetting{Name: "wal_compression", Value: "on"}) + } + + if autovacuumWorkers := autovacuumMaxWorkers(opts.CPUs); autovacuumWorkers != "" { + settings = append(settings, PostgresTuneSetting{Name: "autovacuum_max_workers", Value: autovacuumWorkers}) + } + if maintenanceWorkMem >= 2*sizeGB/sizeKB { + settings = append(settings, PostgresTuneSetting{Name: "autovacuum_work_mem", Value: "2GB"}) + } + + settings = append(settings, + PostgresTuneSetting{Name: "min_wal_size", Value: "2GB"}, + PostgresTuneSetting{Name: "max_wal_size", Value: "8GB"}, + ) + settings = append(settings, parallelWorkerSettings(opts.CPUs, postgresMajorVersion)...) + + if postgresMajorVersion >= 18 { + if ioWorkers := ioWorkers(opts.CPUs); ioWorkers != "" { + settings = append(settings, PostgresTuneSetting{Name: "io_workers", Value: ioWorkers}) + } + if opts.IOMethod != "" && opts.IOMethod != "auto" { + settings = append(settings, PostgresTuneSetting{Name: "io_method", Value: opts.IOMethod}) + } + } + + return settings +} + +// ApplyPostgresTuning applies pgtune-style OLTP recommendations with ALTER SYSTEM. +func ApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, opts PostgresTuneOptions) (*PostgresTuneResult, error) { + result := &PostgresTuneResult{} + if !opts.Enabled { + return result, nil + } + + if opts.DBSize == postgresTuneDBSizeAuto { + databaseSize, err := currentDatabaseSize(ctx, pool) + if err != nil { + return result, err + } + opts.DBSize = classifyPostgresTuneDBSize(databaseSize, opts.MemoryBudgetBytes) + result.DatabaseSizeBytes = databaseSize + } + result.DBSize = opts.DBSize + + majorVersion, err := postgresMajorVersion(ctx, pool) + if err != nil { + return result, err + } + result.PostgresMajorVersion = majorVersion + + settings := RecommendPostgresOLTPSettings(opts, majorVersion) + recommended := make(map[string]struct{}, len(settings)) + for _, setting := range settings { + recommended[setting.Name] = struct{}{} + } + + for _, name := range managedPostgresTuneSettings { + if _, ok := recommended[name]; ok { + continue + } + exists, err := postgresSettingExists(ctx, pool, name) + if err != nil || !exists { + continue + } + requiresRestart, _ := postgresSettingRequiresRestart(ctx, pool, name) + if _, err := pool.Exec(ctx, fmt.Sprintf("ALTER SYSTEM RESET %s", name)); err != nil { + result.Failures = append(result.Failures, PostgresTuneFailure{Name: name, Err: err}) + continue + } + result.Reset = append(result.Reset, name) + result.Applied++ + if requiresRestart { + result.RestartRequired = append(result.RestartRequired, name) + } + } + + for _, setting := range settings { + requiresRestart, _ := postgresSettingRequiresRestart(ctx, pool, setting.Name) + setting.RequiresRestart = requiresRestart + result.Settings = append(result.Settings, setting) + + if _, err := pool.Exec(ctx, fmt.Sprintf("ALTER SYSTEM SET %s = %s", setting.Name, quotePostgresLiteral(setting.Value))); err != nil { + result.Failures = append(result.Failures, PostgresTuneFailure{Name: setting.Name, Value: setting.Value, Err: err}) + continue + } + + result.Applied++ + if requiresRestart { + result.RestartRequired = append(result.RestartRequired, setting.Name) + } + } + + if result.Applied == 0 { + return result, nil + } + + if err := pool.QueryRow(ctx, "SELECT pg_reload_conf()").Scan(&result.Reloaded); err != nil { + return result, err + } + if !result.Reloaded { + return result, fmt.Errorf("pg_reload_conf returned false") + } + + return result, nil +} + +func parsePostgresTuneMode(raw string) (bool, string, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "1", "true", "yes", "on", "auto": + return true, postgresTuneProfileOLTP, nil + case postgresTuneProfileOLTP: + return true, postgresTuneProfileOLTP, nil + case "0", "false", "no", "off", "none", "disabled": + return false, "", nil + default: + return false, "", fmt.Errorf("invalid POSTGRES_TUNE value %q", raw) + } +} + +func loadPostgresTuneMemory() (postgresTuneMemoryBudget, error) { + raw := firstNonEmptyEnv("POSTGRES_TUNE_MEMORY", "SILO_POSTGRES_TUNE_MEMORY") + if raw == "" || strings.EqualFold(raw, "auto") { + mem, source, err := detectTotalMemoryBytes() + if err != nil { + return postgresTuneMemoryBudget{}, fmt.Errorf("POSTGRES_TUNE_MEMORY=auto: %w", err) + } + budgetPercent, err := loadPostgresTuneMemoryBudgetPercent() + if err != nil { + return postgresTuneMemoryBudget{}, err + } + budget := (mem * int64(budgetPercent)) / 100 + return postgresTuneMemoryBudget{ + BudgetBytes: budget, + DetectedBytes: mem, + Source: source, + Percent: budgetPercent, + }, nil + } + + mem, err := parsePostgresTuneByteSize(raw) + if err != nil { + return postgresTuneMemoryBudget{}, fmt.Errorf("invalid POSTGRES_TUNE_MEMORY: %w", err) + } + return postgresTuneMemoryBudget{ + BudgetBytes: mem, + DetectedBytes: mem, + Source: "env", + Percent: 100, + }, nil +} + +func loadPostgresTuneMemoryBudgetPercent() (int, error) { + raw := firstNonEmptyEnv("POSTGRES_TUNE_MEMORY_BUDGET_PERCENT", "SILO_POSTGRES_TUNE_MEMORY_BUDGET_PERCENT") + if raw == "" { + return defaultPostgresTuneMemoryBudgetPercent, nil + } + percent, err := strconv.Atoi(raw) + if err != nil || percent < 1 || percent > 100 { + return 0, fmt.Errorf("invalid POSTGRES_TUNE_MEMORY_BUDGET_PERCENT %q", raw) + } + return percent, nil +} + +func loadPostgresTuneCPUs() (int, error) { + raw := firstNonEmptyEnv("POSTGRES_TUNE_CPUS", "SILO_POSTGRES_TUNE_CPUS") + if raw == "" || strings.EqualFold(raw, "auto") { + cpus := runtime.NumCPU() + if cpus < 1 { + cpus = 1 + } + return cpus, nil + } + + cpus, err := strconv.Atoi(raw) + if err != nil || cpus < 1 { + return 0, fmt.Errorf("invalid POSTGRES_TUNE_CPUS %q", raw) + } + return cpus, nil +} + +func loadPostgresTuneConnections(appMaxConnections int) (int, error) { + minConnections := 1 + if appMaxConnections > 0 { + minConnections = appMaxConnections + 20 + } + + raw := firstNonEmptyEnv("POSTGRES_TUNE_CONNECTIONS", "SILO_POSTGRES_TUNE_CONNECTIONS") + if raw != "" { + connections, err := strconv.Atoi(raw) + if err != nil || connections < 1 { + return 0, fmt.Errorf("invalid POSTGRES_TUNE_CONNECTIONS %q", raw) + } + if connections < minConnections { + connections = minConnections + } + return connections, nil + } + + connections := 100 + if minConnections > connections { + connections = minConnections + } + return connections, nil +} + +func normalizePostgresTuneStorage(raw string) (string, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "", "ssd": + return postgresTuneStorageSSD, nil + case "hdd", "hard_drive_hdd": + return postgresTuneStorageHDD, nil + case "san": + return postgresTuneStorageSAN, nil + case "nvme", "optane": + return postgresTuneStorageNVMe, nil + default: + return "", fmt.Errorf("invalid POSTGRES_TUNE_STORAGE %q", raw) + } +} + +func normalizePostgresTuneDBSize(raw string) (string, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "", "auto": + return postgresTuneDBSizeAuto, nil + case "mid", "medium", "mid_ram": + return postgresTuneDBSizeMidRAM, nil + case "less", "less_ram", "fits_ram", "fits-in-ram": + return postgresTuneDBSizeLessRAM, nil + case "greater", "greater_ram", "larger_than_ram": + return postgresTuneDBSizeGreaterRAM, nil + default: + return "", fmt.Errorf("invalid POSTGRES_TUNE_DB_SIZE %q", raw) + } +} + +func detectTotalMemoryBytes() (int64, string, error) { + for _, path := range []string{ + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + } { + if mem, err := readCgroupMemoryLimit(path); err == nil && mem > 0 { + return mem, path, nil + } + } + + if mem, err := readMeminfoTotalBytes("/host/proc/meminfo"); err == nil && mem > 0 { + return mem, "/host/proc/meminfo", nil + } + + mem, err := readMeminfoTotalBytes("/proc/meminfo") + if err != nil { + return 0, "", err + } + if runningInContainer() && mem > 128*sizeGB { + return 0, "", fmt.Errorf("/proc/meminfo reports %s inside a container without a finite cgroup memory limit or /host/proc/meminfo mount; set POSTGRES_TUNE_MEMORY explicitly or mount /proc/meminfo:/host/proc/meminfo:ro", formatByteSize(mem)) + } + return mem, "/proc/meminfo", nil +} + +func readMeminfoTotalBytes(path string) (int64, error) { + file, err := os.Open(path) + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "MemTotal:") { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + return 0, fmt.Errorf("malformed MemTotal line") + } + kb, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, err + } + return kb * sizeKB, nil + } + if err := scanner.Err(); err != nil { + return 0, err + } + return 0, fmt.Errorf("MemTotal not found") +} + +func runningInContainer() bool { + if _, err := os.Stat("/.dockerenv"); err == nil { + return true + } + raw, err := os.ReadFile("/proc/1/cgroup") + return err == nil && (strings.Contains(string(raw), "docker") || strings.Contains(string(raw), "kubepods")) +} + +func readCgroupMemoryLimit(path string) (int64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value := strings.TrimSpace(string(raw)) + if value == "" || value == "max" { + return 0, fmt.Errorf("no cgroup memory limit") + } + mem, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, err + } + // Docker may expose a huge sentinel when no concrete memory limit is set. + if mem <= 0 || mem > 1<<60 { + return 0, fmt.Errorf("no concrete cgroup memory limit") + } + return mem, nil +} + +func parsePostgresTuneByteSize(raw string) (int64, error) { + normalized := strings.ToUpper(strings.TrimSpace(raw)) + normalized = strings.ReplaceAll(normalized, " ", "") + if normalized == "" { + return 0, fmt.Errorf("empty size") + } + + units := []struct { + suffix string + value int64 + }{ + {"TIB", sizeTB}, + {"TB", sizeTB}, + {"GIB", sizeGB}, + {"GB", sizeGB}, + {"MIB", sizeMB}, + {"MB", sizeMB}, + {"KIB", sizeKB}, + {"KB", sizeKB}, + {"T", sizeTB}, + {"G", sizeGB}, + {"M", sizeMB}, + {"K", sizeKB}, + {"B", 1}, + } + + for _, unit := range units { + if !strings.HasSuffix(normalized, unit.suffix) { + continue + } + number := strings.TrimSuffix(normalized, unit.suffix) + if number == "" { + return 0, fmt.Errorf("missing numeric size in %q", raw) + } + parsed, err := strconv.ParseFloat(number, 64) + if err != nil { + return 0, err + } + if parsed <= 0 { + return 0, fmt.Errorf("size must be positive") + } + return int64(parsed * float64(unit.value)), nil + } + + return 0, fmt.Errorf("size %q must include a unit such as GB or MB", raw) +} + +func formatPostgresTuneKB(kb int64) string { + if kb%(sizeGB/sizeKB) == 0 { + return fmt.Sprintf("%dGB", kb/(sizeGB/sizeKB)) + } + if kb%(sizeMB/sizeKB) == 0 { + return fmt.Sprintf("%dMB", kb/(sizeMB/sizeKB)) + } + return fmt.Sprintf("%dkB", kb) +} + +func formatByteSize(bytes int64) string { + switch { + case bytes%sizeTB == 0: + return fmt.Sprintf("%dTB", bytes/sizeTB) + case bytes%sizeGB == 0: + return fmt.Sprintf("%dGB", bytes/sizeGB) + case bytes%sizeMB == 0: + return fmt.Sprintf("%dMB", bytes/sizeMB) + default: + return fmt.Sprintf("%d bytes", bytes) + } +} + +func walBuffersForSharedBuffers(sharedBuffersKB int64) int64 { + walBuffers := (sharedBuffersKB * 3) / 100 + maxWalBuffers := 16 * sizeMB / sizeKB + if walBuffers > maxWalBuffers { + walBuffers = maxWalBuffers + } + if walBuffers > 14*sizeMB/sizeKB && walBuffers < maxWalBuffers { + walBuffers = maxWalBuffers + } + if walBuffers < 32 { + walBuffers = 32 + } + return walBuffers +} + +func randomPageCost(storage, dbSize string) string { + if dbSize == postgresTuneDBSizeLessRAM { + return "1.1" + } + if storage == postgresTuneStorageHDD { + return "4" + } + return "1.1" +} + +func effectiveIOConcurrency(osType, storage string) string { + if osType != "linux" { + return "" + } + switch storage { + case postgresTuneStorageHDD: + return "2" + case postgresTuneStorageSSD: + return "200" + case postgresTuneStorageSAN: + return "300" + case postgresTuneStorageNVMe: + return "1000" + default: + return "" + } +} + +func workMem(memoryBudgetKB, sharedBuffersKB int64, connections, cpus int, dbSize string) int64 { + parallelForWorkMem := cpus + if parallelForWorkMem < 1 { + parallelForWorkMem = 8 + } + + workMem := (memoryBudgetKB - sharedBuffersKB) / int64((connections+parallelForWorkMem)*3) + switch dbSize { + case postgresTuneDBSizeLessRAM: + workMem = int64(math.Floor(float64(workMem) * 1.3)) + case postgresTuneDBSizeGreaterRAM: + workMem = int64(math.Floor(float64(workMem) * 0.9)) + } + + minWorkMem := 4 * sizeMB / sizeKB + if workMem < minWorkMem { + return minWorkMem + } + return workMem +} + +func currentDatabaseSize(ctx context.Context, pool *pgxpool.Pool) (int64, error) { + var size int64 + if err := pool.QueryRow(ctx, "SELECT pg_database_size(current_database())").Scan(&size); err != nil { + return 0, err + } + return size, nil +} + +func classifyPostgresTuneDBSize(databaseSizeBytes, memoryBudgetBytes int64) string { + if databaseSizeBytes <= 0 || memoryBudgetBytes <= 0 { + return postgresTuneDBSizeMidRAM + } + if databaseSizeBytes <= memoryBudgetBytes/2 { + return postgresTuneDBSizeLessRAM + } + if databaseSizeBytes > memoryBudgetBytes { + return postgresTuneDBSizeGreaterRAM + } + return postgresTuneDBSizeMidRAM +} + +func hugePages(sharedBuffersKB int64, osType string) string { + if osType == "mac" || osType == "darwin" { + return "off" + } + if sharedBuffersKB >= 2*sizeGB/sizeKB { + return "try" + } + return "off" +} + +func autovacuumMaxWorkers(cpus int) string { + if cpus >= 32 { + return "5" + } + if cpus >= 16 { + return "4" + } + return "" +} + +func parallelWorkerSettings(cpus, postgresMajorVersion int) []PostgresTuneSetting { + if cpus < 4 { + return nil + } + + workersPerGather := int(math.Ceil(float64(cpus) / 2)) + if workersPerGather > 4 { + workersPerGather = 4 + } + + settings := []PostgresTuneSetting{ + {Name: "max_worker_processes", Value: strconv.Itoa(cpus)}, + {Name: "max_parallel_workers_per_gather", Value: strconv.Itoa(workersPerGather)}, + } + if postgresMajorVersion >= 10 { + settings = append(settings, PostgresTuneSetting{Name: "max_parallel_workers", Value: strconv.Itoa(cpus)}) + } + if postgresMajorVersion >= 11 { + parallelMaintenanceWorkers := int(math.Ceil(float64(cpus) / 2)) + if parallelMaintenanceWorkers > 4 { + parallelMaintenanceWorkers = 4 + } + settings = append(settings, PostgresTuneSetting{Name: "max_parallel_maintenance_workers", Value: strconv.Itoa(parallelMaintenanceWorkers)}) + } + return settings +} + +func ioWorkers(cpus int) string { + if cpus < 1 { + return "" + } + workers := cpus / 4 + if workers < 3 { + workers = 3 + } + if workers > 32 { + workers = 32 + } + if workers <= 3 { + return "" + } + return strconv.Itoa(workers) +} + +func postgresMajorVersion(ctx context.Context, pool *pgxpool.Pool) (int, error) { + var raw string + if err := pool.QueryRow(ctx, "SHOW server_version_num").Scan(&raw); err != nil { + return 0, err + } + versionNum, err := strconv.Atoi(raw) + if err != nil { + return 0, err + } + return versionNum / 10000, nil +} + +func postgresSettingRequiresRestart(ctx context.Context, pool *pgxpool.Pool, name string) (bool, error) { + var requiresRestart bool + err := pool.QueryRow(ctx, "SELECT context = 'postmaster' FROM pg_settings WHERE name = $1", name).Scan(&requiresRestart) + return requiresRestart, err +} + +func postgresSettingExists(ctx context.Context, pool *pgxpool.Pool, name string) (bool, error) { + var exists bool + err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_settings WHERE name = $1)", name).Scan(&exists) + return exists, err +} + +func quotePostgresLiteral(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +func firstNonEmptyEnv(keys ...string) string { + for _, key := range keys { + value := strings.TrimSpace(os.Getenv(key)) + if value != "" { + return value + } + } + return "" +} diff --git a/internal/database/postgres_tune_test.go b/internal/database/postgres_tune_test.go new file mode 100644 index 00000000..4e1cfd4a --- /dev/null +++ b/internal/database/postgres_tune_test.go @@ -0,0 +1,173 @@ +package database + +import "testing" + +func TestRecommendPostgresOLTPSettings(t *testing.T) { + opts := PostgresTuneOptions{ + Enabled: true, + Profile: postgresTuneProfileOLTP, + MemoryBudgetBytes: 16 * sizeGB, + CPUs: 8, + Connections: 100, + Storage: postgresTuneStorageNVMe, + DBSize: postgresTuneDBSizeMidRAM, + OSType: "linux", + } + + settings := RecommendPostgresOLTPSettings(opts, 18) + byName := map[string]string{} + for _, setting := range settings { + byName[setting.Name] = setting.Value + } + + expected := map[string]string{ + "max_connections": "100", + "shared_buffers": "4GB", + "effective_cache_size": "12GB", + "maintenance_work_mem": "1GB", + "wal_buffers": "16MB", + "min_wal_size": "2GB", + "max_wal_size": "8GB", + "random_page_cost": "1.1", + "effective_io_concurrency": "1000", + "work_mem": "38836kB", + "huge_pages": "try", + "jit": "off", + "wal_compression": "lz4", + "max_worker_processes": "8", + "max_parallel_workers_per_gather": "4", + "max_parallel_workers": "8", + "max_parallel_maintenance_workers": "4", + } + + for key, value := range expected { + if byName[key] != value { + t.Fatalf("%s = %q, want %q", key, byName[key], value) + } + } + + if _, ok := byName["io_method"]; ok { + t.Fatal("io_method should only be emitted when explicitly configured") + } +} + +func TestLoadPostgresTuneOptionsFromEnv(t *testing.T) { + t.Setenv("POSTGRES_TUNE", "auto") + t.Setenv("POSTGRES_TUNE_MEMORY", "8GB") + t.Setenv("POSTGRES_TUNE_CPUS", "4") + t.Setenv("POSTGRES_TUNE_STORAGE", "hdd") + t.Setenv("POSTGRES_TUNE_DB_SIZE", "greater_ram") + + opts, err := LoadPostgresTuneOptionsFromEnv(20) + if err != nil { + t.Fatal(err) + } + + if !opts.Enabled { + t.Fatal("expected tuning to be enabled") + } + if opts.MemoryBudgetBytes != 8*sizeGB { + t.Fatalf("MemoryBudgetBytes = %d, want %d", opts.MemoryBudgetBytes, 8*sizeGB) + } + if opts.DetectedMemoryBytes != 8*sizeGB { + t.Fatalf("DetectedMemoryBytes = %d, want %d", opts.DetectedMemoryBytes, 8*sizeGB) + } + if opts.MemorySource != "env" { + t.Fatalf("MemorySource = %q, want env", opts.MemorySource) + } + if opts.MemoryBudgetPercent != 100 { + t.Fatalf("MemoryBudgetPercent = %d, want 100", opts.MemoryBudgetPercent) + } + if opts.CPUs != 4 { + t.Fatalf("CPUs = %d, want 4", opts.CPUs) + } + if opts.Connections != 100 { + t.Fatalf("Connections = %d, want 100", opts.Connections) + } + if opts.Storage != postgresTuneStorageHDD { + t.Fatalf("Storage = %q, want %q", opts.Storage, postgresTuneStorageHDD) + } + if opts.DBSize != postgresTuneDBSizeGreaterRAM { + t.Fatalf("DBSize = %q, want %q", opts.DBSize, postgresTuneDBSizeGreaterRAM) + } +} + +func TestLoadPostgresTuneOptionsUsesApplicationPoolForHighConnectionCounts(t *testing.T) { + t.Setenv("POSTGRES_TUNE", "oltp") + t.Setenv("POSTGRES_TUNE_MEMORY", "8GB") + t.Setenv("POSTGRES_TUNE_CPUS", "4") + + opts, err := LoadPostgresTuneOptionsFromEnv(120) + if err != nil { + t.Fatal(err) + } + if opts.Connections != 140 { + t.Fatalf("Connections = %d, want 140", opts.Connections) + } +} + +func TestLoadPostgresTuneOptionsFloorsExplicitConnectionsAtApplicationPoolHeadroom(t *testing.T) { + t.Setenv("POSTGRES_TUNE", "oltp") + t.Setenv("POSTGRES_TUNE_MEMORY", "8GB") + t.Setenv("POSTGRES_TUNE_CPUS", "4") + t.Setenv("POSTGRES_TUNE_CONNECTIONS", "80") + + opts, err := LoadPostgresTuneOptionsFromEnv(90) + if err != nil { + t.Fatal(err) + } + if opts.Connections != 110 { + t.Fatalf("Connections = %d, want 110", opts.Connections) + } +} + +func TestPostgresTuneDisabledByDefault(t *testing.T) { + opts, err := LoadPostgresTuneOptionsFromEnv(20) + if err != nil { + t.Fatal(err) + } + if opts.Enabled { + t.Fatal("expected tuning to be disabled by default") + } +} + +func TestPostgresTuneDBSizeDefaultsToAuto(t *testing.T) { + t.Setenv("POSTGRES_TUNE", "oltp") + t.Setenv("POSTGRES_TUNE_MEMORY", "8GB") + t.Setenv("POSTGRES_TUNE_CPUS", "4") + + opts, err := LoadPostgresTuneOptionsFromEnv(20) + if err != nil { + t.Fatal(err) + } + if opts.DBSize != postgresTuneDBSizeAuto { + t.Fatalf("DBSize = %q, want %q", opts.DBSize, postgresTuneDBSizeAuto) + } +} + +func TestClassifyPostgresTuneDBSize(t *testing.T) { + tests := []struct { + name string + dbSize int64 + memBudget int64 + wantDBSize string + }{ + {name: "comfortably fits", dbSize: 8 * sizeGB, memBudget: 24 * sizeGB, wantDBSize: postgresTuneDBSizeLessRAM}, + {name: "fits without headroom", dbSize: 18 * sizeGB, memBudget: 24 * sizeGB, wantDBSize: postgresTuneDBSizeMidRAM}, + {name: "larger than budget", dbSize: 35 * sizeGB, memBudget: 24 * sizeGB, wantDBSize: postgresTuneDBSizeGreaterRAM}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := classifyPostgresTuneDBSize(tt.dbSize, tt.memBudget); got != tt.wantDBSize { + t.Fatalf("classifyPostgresTuneDBSize() = %q, want %q", got, tt.wantDBSize) + } + }) + } +} + +func TestParsePostgresTuneByteSizeRequiresUnit(t *testing.T) { + if _, err := parsePostgresTuneByteSize("16"); err == nil { + t.Fatal("expected bare memory value to be rejected") + } +} diff --git a/postgres/postgresql.conf b/postgres/postgresql.conf deleted file mode 100644 index 0dddf1d9..00000000 --- a/postgres/postgresql.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Silo PostgreSQL Performance Configuration -# Tuned for: modern multi-core CPU, NVMe/Optane storage, ~8GB max RAM usage -# DB Type: Web application (read-heavy, DB fits in RAM) -# -# Adjust values for your hardware, or use https://pgtune.leopard.in.ua/ -# with DB Type "Web application" to generate settings. - -# Connections -listen_addresses = '*' -max_connections = 100 - -# Memory (tuned to keep PG under ~8GB total) -shared_buffers = 4GB # holds the entire DB in cache (typical DB is 1-4GB) -effective_cache_size = 6GB # planner hint only, not an allocation -work_mem = 32MB # per-operation sort/hash memory -maintenance_work_mem = 512MB # VACUUM, CREATE INDEX, catalog imports - -# WAL / Write Performance -wal_buffers = 64MB -min_wal_size = 512MB -max_wal_size = 2GB -checkpoint_completion_target = 0.9 - -# Storage (NVMe/Optane) -random_page_cost = 1.1 # default 4.0; NVMe random ≈ sequential -effective_io_concurrency = 200 # NVMe handles many concurrent requests - -# Parallelism -max_worker_processes = 8 -max_parallel_workers_per_gather = 4 -max_parallel_workers = 8 -max_parallel_maintenance_workers = 4 - -# Huge Pages (Linux only, no effect on macOS) -huge_pages = try - -# Logging (uncomment to log slow queries) -# log_min_duration_statement = 250