807 lines
22 KiB
Go
807 lines
22 KiB
Go
package database
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
const (
|
|
postgresTuneProfileOLTP = "oltp"
|
|
|
|
postgresTuneStorageHDD = "hdd"
|
|
postgresTuneStorageSSD = "ssd"
|
|
postgresTuneStorageSAN = "san"
|
|
postgresTuneStorageNVMe = "nvme"
|
|
|
|
postgresTuneDBSizeLessRAM = "less_ram"
|
|
postgresTuneDBSizeMidRAM = "mid_ram"
|
|
postgresTuneDBSizeGreaterRAM = "greater_ram"
|
|
postgresTuneDBSizeAuto = "auto"
|
|
|
|
defaultPostgresTuneMemoryBudgetPercent = 75
|
|
|
|
sizeKB = int64(1024)
|
|
sizeMB = 1024 * sizeKB
|
|
sizeGB = 1024 * sizeMB
|
|
sizeTB = 1024 * sizeGB
|
|
)
|
|
|
|
var managedPostgresTuneSettings = []string{
|
|
"max_connections",
|
|
"shared_buffers",
|
|
"effective_cache_size",
|
|
"maintenance_work_mem",
|
|
"checkpoint_completion_target",
|
|
"wal_buffers",
|
|
"default_statistics_target",
|
|
"random_page_cost",
|
|
"effective_io_concurrency",
|
|
"work_mem",
|
|
"huge_pages",
|
|
"jit",
|
|
"wal_compression",
|
|
"autovacuum_max_workers",
|
|
"autovacuum_work_mem",
|
|
"min_wal_size",
|
|
"max_wal_size",
|
|
"max_worker_processes",
|
|
"max_parallel_workers_per_gather",
|
|
"max_parallel_workers",
|
|
"max_parallel_maintenance_workers",
|
|
"io_method",
|
|
"io_workers",
|
|
}
|
|
|
|
// PostgresTuneOptions controls automatic PostgreSQL tuning through ALTER SYSTEM.
|
|
type PostgresTuneOptions struct {
|
|
Enabled bool
|
|
Profile string
|
|
MemoryBudgetBytes int64
|
|
DetectedMemoryBytes int64
|
|
MemorySource string
|
|
MemoryBudgetPercent int
|
|
CPUs int
|
|
Connections int
|
|
Storage string
|
|
DBSize string
|
|
OSType string
|
|
IOMethod string
|
|
}
|
|
|
|
// PostgresTuneSetting is one recommended PostgreSQL parameter.
|
|
type PostgresTuneSetting struct {
|
|
Name string
|
|
Value string
|
|
RequiresRestart bool
|
|
}
|
|
|
|
// PostgresTuneFailure records a setting that PostgreSQL rejected.
|
|
type PostgresTuneFailure struct {
|
|
Name string
|
|
Value string
|
|
Err error
|
|
}
|
|
|
|
// PostgresTuneResult summarizes one tuning run.
|
|
type PostgresTuneResult struct {
|
|
PostgresMajorVersion int
|
|
DatabaseSizeBytes int64
|
|
DBSize string
|
|
Settings []PostgresTuneSetting
|
|
Reset []string
|
|
Applied int
|
|
Failures []PostgresTuneFailure
|
|
Reloaded bool
|
|
RestartRequired []string
|
|
}
|
|
|
|
type postgresTuneMemoryBudget struct {
|
|
BudgetBytes int64
|
|
DetectedBytes int64
|
|
Source string
|
|
Percent int
|
|
}
|
|
|
|
// LoadPostgresTuneOptionsFromEnv reads the opt-in PostgreSQL tuning environment.
|
|
func LoadPostgresTuneOptionsFromEnv(appMaxConnections int) (PostgresTuneOptions, error) {
|
|
rawEnabled := firstNonEmptyEnv("POSTGRES_TUNE", "SILO_POSTGRES_TUNE")
|
|
if rawEnabled == "" {
|
|
return PostgresTuneOptions{}, nil
|
|
}
|
|
|
|
enabled, profileFromMode, err := parsePostgresTuneMode(rawEnabled)
|
|
if err != nil {
|
|
return PostgresTuneOptions{}, err
|
|
}
|
|
if !enabled {
|
|
return PostgresTuneOptions{}, nil
|
|
}
|
|
|
|
profile := firstNonEmptyEnv("POSTGRES_TUNE_PROFILE", "SILO_POSTGRES_TUNE_PROFILE")
|
|
if profile == "" {
|
|
profile = profileFromMode
|
|
}
|
|
if profile == "" {
|
|
profile = postgresTuneProfileOLTP
|
|
}
|
|
profile = strings.ToLower(profile)
|
|
if profile != postgresTuneProfileOLTP {
|
|
return PostgresTuneOptions{}, fmt.Errorf("unsupported POSTGRES_TUNE_PROFILE %q; only %q is supported", profile, postgresTuneProfileOLTP)
|
|
}
|
|
|
|
memory, err := loadPostgresTuneMemory()
|
|
if err != nil {
|
|
return PostgresTuneOptions{}, err
|
|
}
|
|
|
|
cpus, err := loadPostgresTuneCPUs()
|
|
if err != nil {
|
|
return PostgresTuneOptions{}, err
|
|
}
|
|
|
|
connections, err := loadPostgresTuneConnections(appMaxConnections)
|
|
if err != nil {
|
|
return PostgresTuneOptions{}, err
|
|
}
|
|
|
|
storage, err := normalizePostgresTuneStorage(firstNonEmptyEnv("POSTGRES_TUNE_STORAGE", "SILO_POSTGRES_TUNE_STORAGE"))
|
|
if err != nil {
|
|
return PostgresTuneOptions{}, err
|
|
}
|
|
|
|
dbSize, err := normalizePostgresTuneDBSize(firstNonEmptyEnv("POSTGRES_TUNE_DB_SIZE", "SILO_POSTGRES_TUNE_DB_SIZE"))
|
|
if err != nil {
|
|
return PostgresTuneOptions{}, err
|
|
}
|
|
|
|
osType := strings.ToLower(firstNonEmptyEnv("POSTGRES_TUNE_OS", "SILO_POSTGRES_TUNE_OS"))
|
|
if osType == "" {
|
|
osType = "linux"
|
|
}
|
|
|
|
ioMethod := strings.ToLower(firstNonEmptyEnv("POSTGRES_TUNE_IO_METHOD", "SILO_POSTGRES_TUNE_IO_METHOD"))
|
|
|
|
return PostgresTuneOptions{
|
|
Enabled: true,
|
|
Profile: profile,
|
|
MemoryBudgetBytes: memory.BudgetBytes,
|
|
DetectedMemoryBytes: memory.DetectedBytes,
|
|
MemorySource: memory.Source,
|
|
MemoryBudgetPercent: memory.Percent,
|
|
CPUs: cpus,
|
|
Connections: connections,
|
|
Storage: storage,
|
|
DBSize: dbSize,
|
|
OSType: osType,
|
|
IOMethod: ioMethod,
|
|
}, nil
|
|
}
|
|
|
|
// RecommendPostgresOLTPSettings returns pgtune-style OLTP recommendations.
|
|
func RecommendPostgresOLTPSettings(opts PostgresTuneOptions, postgresMajorVersion int) []PostgresTuneSetting {
|
|
memoryBudgetKB := opts.MemoryBudgetBytes / sizeKB
|
|
if memoryBudgetKB <= 0 {
|
|
return nil
|
|
}
|
|
|
|
connections := opts.Connections
|
|
if connections <= 0 {
|
|
connections = 100
|
|
}
|
|
|
|
settings := make([]PostgresTuneSetting, 0, 24)
|
|
sharedBuffers := memoryBudgetKB / 4
|
|
effectiveCacheSize := (memoryBudgetKB * 3) / 4
|
|
maintenanceWorkMem := memoryBudgetKB / 16
|
|
if maintenanceWorkMem > 8*sizeGB/sizeKB {
|
|
maintenanceWorkMem = 8 * sizeGB / sizeKB
|
|
}
|
|
|
|
settings = append(settings,
|
|
PostgresTuneSetting{Name: "max_connections", Value: strconv.Itoa(connections)},
|
|
PostgresTuneSetting{Name: "shared_buffers", Value: formatPostgresTuneKB(sharedBuffers)},
|
|
PostgresTuneSetting{Name: "effective_cache_size", Value: formatPostgresTuneKB(effectiveCacheSize)},
|
|
PostgresTuneSetting{Name: "maintenance_work_mem", Value: formatPostgresTuneKB(maintenanceWorkMem)},
|
|
PostgresTuneSetting{Name: "checkpoint_completion_target", Value: "0.9"},
|
|
PostgresTuneSetting{Name: "wal_buffers", Value: formatPostgresTuneKB(walBuffersForSharedBuffers(sharedBuffers))},
|
|
PostgresTuneSetting{Name: "default_statistics_target", Value: "100"},
|
|
PostgresTuneSetting{Name: "random_page_cost", Value: randomPageCost(opts.Storage, opts.DBSize)},
|
|
)
|
|
|
|
if effectiveIO := effectiveIOConcurrency(opts.OSType, opts.Storage); effectiveIO != "" {
|
|
settings = append(settings, PostgresTuneSetting{Name: "effective_io_concurrency", Value: effectiveIO})
|
|
}
|
|
|
|
settings = append(settings,
|
|
PostgresTuneSetting{Name: "work_mem", Value: formatPostgresTuneKB(workMem(memoryBudgetKB, sharedBuffers, connections, opts.CPUs, opts.DBSize))},
|
|
PostgresTuneSetting{Name: "huge_pages", Value: hugePages(sharedBuffers, opts.OSType)},
|
|
)
|
|
|
|
if postgresMajorVersion >= 12 {
|
|
settings = append(settings, PostgresTuneSetting{Name: "jit", Value: "off"})
|
|
}
|
|
if postgresMajorVersion >= 15 {
|
|
settings = append(settings, PostgresTuneSetting{Name: "wal_compression", Value: "lz4"})
|
|
} else if postgresMajorVersion >= 10 {
|
|
settings = append(settings, PostgresTuneSetting{Name: "wal_compression", Value: "on"})
|
|
}
|
|
|
|
if autovacuumWorkers := autovacuumMaxWorkers(opts.CPUs); autovacuumWorkers != "" {
|
|
settings = append(settings, PostgresTuneSetting{Name: "autovacuum_max_workers", Value: autovacuumWorkers})
|
|
}
|
|
if maintenanceWorkMem >= 2*sizeGB/sizeKB {
|
|
settings = append(settings, PostgresTuneSetting{Name: "autovacuum_work_mem", Value: "2GB"})
|
|
}
|
|
|
|
settings = append(settings,
|
|
PostgresTuneSetting{Name: "min_wal_size", Value: "2GB"},
|
|
PostgresTuneSetting{Name: "max_wal_size", Value: "8GB"},
|
|
)
|
|
settings = append(settings, parallelWorkerSettings(opts.CPUs, postgresMajorVersion)...)
|
|
|
|
if postgresMajorVersion >= 18 {
|
|
if ioWorkers := ioWorkers(opts.CPUs); ioWorkers != "" {
|
|
settings = append(settings, PostgresTuneSetting{Name: "io_workers", Value: ioWorkers})
|
|
}
|
|
if opts.IOMethod != "" && opts.IOMethod != "auto" {
|
|
settings = append(settings, PostgresTuneSetting{Name: "io_method", Value: opts.IOMethod})
|
|
}
|
|
}
|
|
|
|
return settings
|
|
}
|
|
|
|
// ApplyPostgresTuning applies pgtune-style OLTP recommendations with ALTER SYSTEM.
|
|
func ApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, opts PostgresTuneOptions) (*PostgresTuneResult, error) {
|
|
result := &PostgresTuneResult{}
|
|
if !opts.Enabled {
|
|
return result, nil
|
|
}
|
|
|
|
if opts.DBSize == postgresTuneDBSizeAuto {
|
|
databaseSize, err := currentDatabaseSize(ctx, pool)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
opts.DBSize = classifyPostgresTuneDBSize(databaseSize, opts.MemoryBudgetBytes)
|
|
result.DatabaseSizeBytes = databaseSize
|
|
}
|
|
result.DBSize = opts.DBSize
|
|
|
|
majorVersion, err := postgresMajorVersion(ctx, pool)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
result.PostgresMajorVersion = majorVersion
|
|
|
|
settings := RecommendPostgresOLTPSettings(opts, majorVersion)
|
|
recommended := make(map[string]struct{}, len(settings))
|
|
for _, setting := range settings {
|
|
recommended[setting.Name] = struct{}{}
|
|
}
|
|
|
|
for _, name := range managedPostgresTuneSettings {
|
|
if _, ok := recommended[name]; ok {
|
|
continue
|
|
}
|
|
exists, err := postgresSettingExists(ctx, pool, name)
|
|
if err != nil || !exists {
|
|
continue
|
|
}
|
|
requiresRestart, _ := postgresSettingRequiresRestart(ctx, pool, name)
|
|
if _, err := pool.Exec(ctx, fmt.Sprintf("ALTER SYSTEM RESET %s", name)); err != nil {
|
|
result.Failures = append(result.Failures, PostgresTuneFailure{Name: name, Err: err})
|
|
continue
|
|
}
|
|
result.Reset = append(result.Reset, name)
|
|
result.Applied++
|
|
if requiresRestart {
|
|
result.RestartRequired = append(result.RestartRequired, name)
|
|
}
|
|
}
|
|
|
|
for _, setting := range settings {
|
|
requiresRestart, _ := postgresSettingRequiresRestart(ctx, pool, setting.Name)
|
|
setting.RequiresRestart = requiresRestart
|
|
result.Settings = append(result.Settings, setting)
|
|
|
|
if _, err := pool.Exec(ctx, fmt.Sprintf("ALTER SYSTEM SET %s = %s", setting.Name, quotePostgresLiteral(setting.Value))); err != nil {
|
|
result.Failures = append(result.Failures, PostgresTuneFailure{Name: setting.Name, Value: setting.Value, Err: err})
|
|
continue
|
|
}
|
|
|
|
result.Applied++
|
|
if requiresRestart {
|
|
result.RestartRequired = append(result.RestartRequired, setting.Name)
|
|
}
|
|
}
|
|
|
|
if result.Applied == 0 {
|
|
return result, nil
|
|
}
|
|
|
|
if err := pool.QueryRow(ctx, "SELECT pg_reload_conf()").Scan(&result.Reloaded); err != nil {
|
|
return result, err
|
|
}
|
|
if !result.Reloaded {
|
|
return result, fmt.Errorf("pg_reload_conf returned false")
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func parsePostgresTuneMode(raw string) (bool, string, error) {
|
|
switch strings.ToLower(strings.TrimSpace(raw)) {
|
|
case "1", "true", "yes", "on", "auto":
|
|
return true, postgresTuneProfileOLTP, nil
|
|
case postgresTuneProfileOLTP:
|
|
return true, postgresTuneProfileOLTP, nil
|
|
case "0", "false", "no", "off", "none", "disabled":
|
|
return false, "", nil
|
|
default:
|
|
return false, "", fmt.Errorf("invalid POSTGRES_TUNE value %q", raw)
|
|
}
|
|
}
|
|
|
|
func loadPostgresTuneMemory() (postgresTuneMemoryBudget, error) {
|
|
raw := firstNonEmptyEnv("POSTGRES_TUNE_MEMORY", "SILO_POSTGRES_TUNE_MEMORY")
|
|
if raw == "" || strings.EqualFold(raw, "auto") {
|
|
mem, source, err := detectTotalMemoryBytes()
|
|
if err != nil {
|
|
return postgresTuneMemoryBudget{}, fmt.Errorf("POSTGRES_TUNE_MEMORY=auto: %w", err)
|
|
}
|
|
budgetPercent, err := loadPostgresTuneMemoryBudgetPercent()
|
|
if err != nil {
|
|
return postgresTuneMemoryBudget{}, err
|
|
}
|
|
budget := (mem * int64(budgetPercent)) / 100
|
|
return postgresTuneMemoryBudget{
|
|
BudgetBytes: budget,
|
|
DetectedBytes: mem,
|
|
Source: source,
|
|
Percent: budgetPercent,
|
|
}, nil
|
|
}
|
|
|
|
mem, err := parsePostgresTuneByteSize(raw)
|
|
if err != nil {
|
|
return postgresTuneMemoryBudget{}, fmt.Errorf("invalid POSTGRES_TUNE_MEMORY: %w", err)
|
|
}
|
|
return postgresTuneMemoryBudget{
|
|
BudgetBytes: mem,
|
|
DetectedBytes: mem,
|
|
Source: "env",
|
|
Percent: 100,
|
|
}, nil
|
|
}
|
|
|
|
func loadPostgresTuneMemoryBudgetPercent() (int, error) {
|
|
raw := firstNonEmptyEnv("POSTGRES_TUNE_MEMORY_BUDGET_PERCENT", "SILO_POSTGRES_TUNE_MEMORY_BUDGET_PERCENT")
|
|
if raw == "" {
|
|
return defaultPostgresTuneMemoryBudgetPercent, nil
|
|
}
|
|
percent, err := strconv.Atoi(raw)
|
|
if err != nil || percent < 1 || percent > 100 {
|
|
return 0, fmt.Errorf("invalid POSTGRES_TUNE_MEMORY_BUDGET_PERCENT %q", raw)
|
|
}
|
|
return percent, nil
|
|
}
|
|
|
|
func loadPostgresTuneCPUs() (int, error) {
|
|
raw := firstNonEmptyEnv("POSTGRES_TUNE_CPUS", "SILO_POSTGRES_TUNE_CPUS")
|
|
if raw == "" || strings.EqualFold(raw, "auto") {
|
|
cpus := runtime.NumCPU()
|
|
if cpus < 1 {
|
|
cpus = 1
|
|
}
|
|
return cpus, nil
|
|
}
|
|
|
|
cpus, err := strconv.Atoi(raw)
|
|
if err != nil || cpus < 1 {
|
|
return 0, fmt.Errorf("invalid POSTGRES_TUNE_CPUS %q", raw)
|
|
}
|
|
return cpus, nil
|
|
}
|
|
|
|
func loadPostgresTuneConnections(appMaxConnections int) (int, error) {
|
|
minConnections := 1
|
|
if appMaxConnections > 0 {
|
|
minConnections = appMaxConnections + 20
|
|
}
|
|
|
|
raw := firstNonEmptyEnv("POSTGRES_TUNE_CONNECTIONS", "SILO_POSTGRES_TUNE_CONNECTIONS")
|
|
if raw != "" {
|
|
connections, err := strconv.Atoi(raw)
|
|
if err != nil || connections < 1 {
|
|
return 0, fmt.Errorf("invalid POSTGRES_TUNE_CONNECTIONS %q", raw)
|
|
}
|
|
if connections < minConnections {
|
|
connections = minConnections
|
|
}
|
|
return connections, nil
|
|
}
|
|
|
|
connections := 100
|
|
if minConnections > connections {
|
|
connections = minConnections
|
|
}
|
|
return connections, nil
|
|
}
|
|
|
|
func normalizePostgresTuneStorage(raw string) (string, error) {
|
|
switch strings.ToLower(strings.TrimSpace(raw)) {
|
|
case "", "ssd":
|
|
return postgresTuneStorageSSD, nil
|
|
case "hdd", "hard_drive_hdd":
|
|
return postgresTuneStorageHDD, nil
|
|
case "san":
|
|
return postgresTuneStorageSAN, nil
|
|
case "nvme", "optane":
|
|
return postgresTuneStorageNVMe, nil
|
|
default:
|
|
return "", fmt.Errorf("invalid POSTGRES_TUNE_STORAGE %q", raw)
|
|
}
|
|
}
|
|
|
|
func normalizePostgresTuneDBSize(raw string) (string, error) {
|
|
switch strings.ToLower(strings.TrimSpace(raw)) {
|
|
case "", "auto":
|
|
return postgresTuneDBSizeAuto, nil
|
|
case "mid", "medium", "mid_ram":
|
|
return postgresTuneDBSizeMidRAM, nil
|
|
case "less", "less_ram", "fits_ram", "fits-in-ram":
|
|
return postgresTuneDBSizeLessRAM, nil
|
|
case "greater", "greater_ram", "larger_than_ram":
|
|
return postgresTuneDBSizeGreaterRAM, nil
|
|
default:
|
|
return "", fmt.Errorf("invalid POSTGRES_TUNE_DB_SIZE %q", raw)
|
|
}
|
|
}
|
|
|
|
func detectTotalMemoryBytes() (int64, string, error) {
|
|
for _, path := range []string{
|
|
"/sys/fs/cgroup/memory.max",
|
|
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
|
|
} {
|
|
if mem, err := readCgroupMemoryLimit(path); err == nil && mem > 0 {
|
|
return mem, path, nil
|
|
}
|
|
}
|
|
|
|
if mem, err := readMeminfoTotalBytes("/host/proc/meminfo"); err == nil && mem > 0 {
|
|
return mem, "/host/proc/meminfo", nil
|
|
}
|
|
|
|
mem, err := readMeminfoTotalBytes("/proc/meminfo")
|
|
if err != nil {
|
|
return 0, "", err
|
|
}
|
|
if runningInContainer() && mem > 128*sizeGB {
|
|
return 0, "", fmt.Errorf("/proc/meminfo reports %s inside a container without a finite cgroup memory limit or /host/proc/meminfo mount; set POSTGRES_TUNE_MEMORY explicitly or mount /proc/meminfo:/host/proc/meminfo:ro", formatByteSize(mem))
|
|
}
|
|
return mem, "/proc/meminfo", nil
|
|
}
|
|
|
|
func readMeminfoTotalBytes(path string) (int64, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer file.Close()
|
|
|
|
scanner := bufio.NewScanner(file)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if !strings.HasPrefix(line, "MemTotal:") {
|
|
continue
|
|
}
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
return 0, fmt.Errorf("malformed MemTotal line")
|
|
}
|
|
kb, err := strconv.ParseInt(fields[1], 10, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return kb * sizeKB, nil
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
return 0, fmt.Errorf("MemTotal not found")
|
|
}
|
|
|
|
func runningInContainer() bool {
|
|
if _, err := os.Stat("/.dockerenv"); err == nil {
|
|
return true
|
|
}
|
|
raw, err := os.ReadFile("/proc/1/cgroup")
|
|
return err == nil && (strings.Contains(string(raw), "docker") || strings.Contains(string(raw), "kubepods"))
|
|
}
|
|
|
|
func readCgroupMemoryLimit(path string) (int64, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
value := strings.TrimSpace(string(raw))
|
|
if value == "" || value == "max" {
|
|
return 0, fmt.Errorf("no cgroup memory limit")
|
|
}
|
|
mem, err := strconv.ParseInt(value, 10, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
// Docker may expose a huge sentinel when no concrete memory limit is set.
|
|
if mem <= 0 || mem > 1<<60 {
|
|
return 0, fmt.Errorf("no concrete cgroup memory limit")
|
|
}
|
|
return mem, nil
|
|
}
|
|
|
|
func parsePostgresTuneByteSize(raw string) (int64, error) {
|
|
normalized := strings.ToUpper(strings.TrimSpace(raw))
|
|
normalized = strings.ReplaceAll(normalized, " ", "")
|
|
if normalized == "" {
|
|
return 0, fmt.Errorf("empty size")
|
|
}
|
|
|
|
units := []struct {
|
|
suffix string
|
|
value int64
|
|
}{
|
|
{"TIB", sizeTB},
|
|
{"TB", sizeTB},
|
|
{"GIB", sizeGB},
|
|
{"GB", sizeGB},
|
|
{"MIB", sizeMB},
|
|
{"MB", sizeMB},
|
|
{"KIB", sizeKB},
|
|
{"KB", sizeKB},
|
|
{"T", sizeTB},
|
|
{"G", sizeGB},
|
|
{"M", sizeMB},
|
|
{"K", sizeKB},
|
|
{"B", 1},
|
|
}
|
|
|
|
for _, unit := range units {
|
|
if !strings.HasSuffix(normalized, unit.suffix) {
|
|
continue
|
|
}
|
|
number := strings.TrimSuffix(normalized, unit.suffix)
|
|
if number == "" {
|
|
return 0, fmt.Errorf("missing numeric size in %q", raw)
|
|
}
|
|
parsed, err := strconv.ParseFloat(number, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if parsed <= 0 {
|
|
return 0, fmt.Errorf("size must be positive")
|
|
}
|
|
return int64(parsed * float64(unit.value)), nil
|
|
}
|
|
|
|
return 0, fmt.Errorf("size %q must include a unit such as GB or MB", raw)
|
|
}
|
|
|
|
func formatPostgresTuneKB(kb int64) string {
|
|
if kb%(sizeGB/sizeKB) == 0 {
|
|
return fmt.Sprintf("%dGB", kb/(sizeGB/sizeKB))
|
|
}
|
|
if kb%(sizeMB/sizeKB) == 0 {
|
|
return fmt.Sprintf("%dMB", kb/(sizeMB/sizeKB))
|
|
}
|
|
return fmt.Sprintf("%dkB", kb)
|
|
}
|
|
|
|
func formatByteSize(bytes int64) string {
|
|
switch {
|
|
case bytes%sizeTB == 0:
|
|
return fmt.Sprintf("%dTB", bytes/sizeTB)
|
|
case bytes%sizeGB == 0:
|
|
return fmt.Sprintf("%dGB", bytes/sizeGB)
|
|
case bytes%sizeMB == 0:
|
|
return fmt.Sprintf("%dMB", bytes/sizeMB)
|
|
default:
|
|
return fmt.Sprintf("%d bytes", bytes)
|
|
}
|
|
}
|
|
|
|
func walBuffersForSharedBuffers(sharedBuffersKB int64) int64 {
|
|
walBuffers := (sharedBuffersKB * 3) / 100
|
|
maxWalBuffers := 16 * sizeMB / sizeKB
|
|
if walBuffers > maxWalBuffers {
|
|
walBuffers = maxWalBuffers
|
|
}
|
|
if walBuffers > 14*sizeMB/sizeKB && walBuffers < maxWalBuffers {
|
|
walBuffers = maxWalBuffers
|
|
}
|
|
if walBuffers < 32 {
|
|
walBuffers = 32
|
|
}
|
|
return walBuffers
|
|
}
|
|
|
|
func randomPageCost(storage, dbSize string) string {
|
|
if dbSize == postgresTuneDBSizeLessRAM {
|
|
return "1.1"
|
|
}
|
|
if storage == postgresTuneStorageHDD {
|
|
return "4"
|
|
}
|
|
return "1.1"
|
|
}
|
|
|
|
func effectiveIOConcurrency(osType, storage string) string {
|
|
if osType != "linux" {
|
|
return ""
|
|
}
|
|
switch storage {
|
|
case postgresTuneStorageHDD:
|
|
return "2"
|
|
case postgresTuneStorageSSD:
|
|
return "200"
|
|
case postgresTuneStorageSAN:
|
|
return "300"
|
|
case postgresTuneStorageNVMe:
|
|
return "1000"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func workMem(memoryBudgetKB, sharedBuffersKB int64, connections, cpus int, dbSize string) int64 {
|
|
parallelForWorkMem := cpus
|
|
if parallelForWorkMem < 1 {
|
|
parallelForWorkMem = 8
|
|
}
|
|
|
|
workMem := (memoryBudgetKB - sharedBuffersKB) / int64((connections+parallelForWorkMem)*3)
|
|
switch dbSize {
|
|
case postgresTuneDBSizeLessRAM:
|
|
workMem = int64(math.Floor(float64(workMem) * 1.3))
|
|
case postgresTuneDBSizeGreaterRAM:
|
|
workMem = int64(math.Floor(float64(workMem) * 0.9))
|
|
}
|
|
|
|
minWorkMem := 4 * sizeMB / sizeKB
|
|
if workMem < minWorkMem {
|
|
return minWorkMem
|
|
}
|
|
return workMem
|
|
}
|
|
|
|
func currentDatabaseSize(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
|
|
var size int64
|
|
if err := pool.QueryRow(ctx, "SELECT pg_database_size(current_database())").Scan(&size); err != nil {
|
|
return 0, err
|
|
}
|
|
return size, nil
|
|
}
|
|
|
|
func classifyPostgresTuneDBSize(databaseSizeBytes, memoryBudgetBytes int64) string {
|
|
if databaseSizeBytes <= 0 || memoryBudgetBytes <= 0 {
|
|
return postgresTuneDBSizeMidRAM
|
|
}
|
|
if databaseSizeBytes <= memoryBudgetBytes/2 {
|
|
return postgresTuneDBSizeLessRAM
|
|
}
|
|
if databaseSizeBytes > memoryBudgetBytes {
|
|
return postgresTuneDBSizeGreaterRAM
|
|
}
|
|
return postgresTuneDBSizeMidRAM
|
|
}
|
|
|
|
func hugePages(sharedBuffersKB int64, osType string) string {
|
|
if osType == "mac" || osType == "darwin" {
|
|
return "off"
|
|
}
|
|
if sharedBuffersKB >= 2*sizeGB/sizeKB {
|
|
return "try"
|
|
}
|
|
return "off"
|
|
}
|
|
|
|
func autovacuumMaxWorkers(cpus int) string {
|
|
if cpus >= 32 {
|
|
return "5"
|
|
}
|
|
if cpus >= 16 {
|
|
return "4"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func parallelWorkerSettings(cpus, postgresMajorVersion int) []PostgresTuneSetting {
|
|
if cpus < 4 {
|
|
return nil
|
|
}
|
|
|
|
workersPerGather := int(math.Ceil(float64(cpus) / 2))
|
|
if workersPerGather > 4 {
|
|
workersPerGather = 4
|
|
}
|
|
|
|
settings := []PostgresTuneSetting{
|
|
{Name: "max_worker_processes", Value: strconv.Itoa(cpus)},
|
|
{Name: "max_parallel_workers_per_gather", Value: strconv.Itoa(workersPerGather)},
|
|
}
|
|
if postgresMajorVersion >= 10 {
|
|
settings = append(settings, PostgresTuneSetting{Name: "max_parallel_workers", Value: strconv.Itoa(cpus)})
|
|
}
|
|
if postgresMajorVersion >= 11 {
|
|
parallelMaintenanceWorkers := int(math.Ceil(float64(cpus) / 2))
|
|
if parallelMaintenanceWorkers > 4 {
|
|
parallelMaintenanceWorkers = 4
|
|
}
|
|
settings = append(settings, PostgresTuneSetting{Name: "max_parallel_maintenance_workers", Value: strconv.Itoa(parallelMaintenanceWorkers)})
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func ioWorkers(cpus int) string {
|
|
if cpus < 1 {
|
|
return ""
|
|
}
|
|
workers := cpus / 4
|
|
if workers < 3 {
|
|
workers = 3
|
|
}
|
|
if workers > 32 {
|
|
workers = 32
|
|
}
|
|
if workers <= 3 {
|
|
return ""
|
|
}
|
|
return strconv.Itoa(workers)
|
|
}
|
|
|
|
func postgresMajorVersion(ctx context.Context, pool *pgxpool.Pool) (int, error) {
|
|
var raw string
|
|
if err := pool.QueryRow(ctx, "SHOW server_version_num").Scan(&raw); err != nil {
|
|
return 0, err
|
|
}
|
|
versionNum, err := strconv.Atoi(raw)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return versionNum / 10000, nil
|
|
}
|
|
|
|
func postgresSettingRequiresRestart(ctx context.Context, pool *pgxpool.Pool, name string) (bool, error) {
|
|
var requiresRestart bool
|
|
err := pool.QueryRow(ctx, "SELECT context = 'postmaster' FROM pg_settings WHERE name = $1", name).Scan(&requiresRestart)
|
|
return requiresRestart, err
|
|
}
|
|
|
|
func postgresSettingExists(ctx context.Context, pool *pgxpool.Pool, name string) (bool, error) {
|
|
var exists bool
|
|
err := pool.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_settings WHERE name = $1)", name).Scan(&exists)
|
|
return exists, err
|
|
}
|
|
|
|
func quotePostgresLiteral(value string) string {
|
|
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
|
|
}
|
|
|
|
func firstNonEmptyEnv(keys ...string) string {
|
|
for _, key := range keys {
|
|
value := strings.TrimSpace(os.Getenv(key))
|
|
if value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|