Merge remote-tracking branch 'origin/main' into pr-397-devmerge
# Conflicts: # internal/catalog/item_repo.go
This commit is contained in:
Quick104
@@ -19,7 +19,15 @@ env:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: [self-hosted, linux]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
runs-on: ${{ matrix.runner }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
@@ -28,6 +36,13 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Prepare platform slug and lowercase image name
|
||||
run: |
|
||||
echo "PLATFORM_PAIR=${PLATFORM//\//-}" >> "$GITHUB_ENV"
|
||||
echo "IMAGE_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV"
|
||||
env:
|
||||
PLATFORM: ${{ matrix.platform }}
|
||||
|
||||
- name: Set up pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
@@ -76,9 +91,6 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
name: silo-builder
|
||||
keep-state: true
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
@@ -92,19 +104,81 @@ jobs:
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=,format=short
|
||||
|
||||
- name: Build and push
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-contexts: |
|
||||
frontend_dist=./web/dist
|
||||
build-args: |
|
||||
BUILD_REVISION=${{ github.sha }}
|
||||
BUILD_DIRTY=false
|
||||
cache-from: type=gha,scope=docker-${{ env.PLATFORM_PAIR }}
|
||||
cache-to: type=gha,scope=docker-${{ env.PLATFORM_PAIR }},mode=max
|
||||
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_LC }},push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p "${RUNNER_TEMP}/digests"
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "${RUNNER_TEMP}/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM_PAIR }}
|
||||
path: ${{ runner.temp }}/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Prepare lowercase image name
|
||||
run: echo "IMAGE_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/digests
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_LC }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=,format=short
|
||||
|
||||
- name: Create and push manifest list
|
||||
working-directory: ${{ runner.temp }}/digests
|
||||
run: |
|
||||
# shellcheck disable=SC2046
|
||||
docker buildx imagetools create \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf "${REGISTRY}/${IMAGE_LC}@sha256:%s " *)
|
||||
|
||||
- name: Inspect manifest list
|
||||
run: |
|
||||
docker buildx imagetools inspect "${REGISTRY}/${IMAGE_LC}:${GITHUB_SHA::7}"
|
||||
|
||||
+73
-1
@@ -50,6 +50,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/config"
|
||||
"github.com/Silo-Server/silo-server/internal/database"
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics"
|
||||
"github.com/Silo-Server/silo-server/internal/downloads"
|
||||
"github.com/Silo-Server/silo-server/internal/ebooks"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
@@ -67,6 +68,11 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/mdblist"
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
|
||||
// Built-in metadata providers self-register into the metadata package's
|
||||
// builtin registry on import; buildProviders resolves their seeded chain
|
||||
// entries in-process (no gRPC).
|
||||
_ "github.com/Silo-Server/silo-server/internal/metadata/nfo"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/nodeconfig"
|
||||
"github.com/Silo-Server/silo-server/internal/nodepool"
|
||||
@@ -189,6 +195,9 @@ func configureOperationalLogging(
|
||||
if err := opslog.SeedDefaults(ctx, settingsRepo); err != nil {
|
||||
log.Fatalf("seed opslog defaults: %v", err)
|
||||
}
|
||||
if err := diagnostics.SeedDefaults(ctx, settingsRepo); err != nil {
|
||||
log.Fatalf("seed diagnostics defaults: %v", err)
|
||||
}
|
||||
opsPM := partman.NewManager(pool, "operational_logs", partman.Daily, 3)
|
||||
if err := opsPM.EnsureFuturePartitions(ctx); err != nil {
|
||||
// Non-fatal: a partition hiccup must not crash-loop the server (see the
|
||||
@@ -665,6 +674,9 @@ func main() {
|
||||
// start, so this node can rebuild a Jellyfin transcode after its own
|
||||
// restart (the node hop token is recipe-less). Shares the offload Redis.
|
||||
srv.SetRecipeStore(noderecipe.NewStore(redisClient, 0))
|
||||
// Reclaim orphaned transcode dirs at boot and hourly thereafter, bound
|
||||
// to appCtx so it stops on shutdown.
|
||||
srv.StartOrphanSweeper(appCtx)
|
||||
handler = srv.Handler()
|
||||
}
|
||||
|
||||
@@ -996,6 +1008,11 @@ func main() {
|
||||
}
|
||||
out := make([]pluginhost.InstalledPluginRecord, 0, len(installations))
|
||||
for _, installation := range installations {
|
||||
// The reserved builtin row is not a plugin; keep it out
|
||||
// of the host's installed-plugin listing.
|
||||
if installation.IsBuiltin() {
|
||||
continue
|
||||
}
|
||||
capabilities, err := installationStore.ListCapabilities(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1145,6 +1162,17 @@ func main() {
|
||||
var mangaEnricher *manga.Enricher
|
||||
if needsWorkers && deps.DB != nil && deps.FileRepo != nil {
|
||||
chainRepo := metadata.NewChainRepository(deps.DB)
|
||||
// Make every existing library chain aware of the built-in providers
|
||||
// before serving: materialize legacy content_level='' chains per level,
|
||||
// then append registered builtins disabled (idempotent; also the repair
|
||||
// path after a stale chain-editor save drops a builtin row). Runs before
|
||||
// the metadata service exists, so no chain cache to invalidate here.
|
||||
syncCtx, syncCancel := context.WithTimeout(appCtx, 30*time.Second)
|
||||
syncErr := metadata.SyncBuiltinProviderChains(syncCtx, chainRepo)
|
||||
syncCancel()
|
||||
if syncErr != nil {
|
||||
log.Fatalf("sync builtin provider chains: %v", syncErr)
|
||||
}
|
||||
skippedRootRepo = metadata.NewSkippedRootRepository(deps.DB)
|
||||
itemRepo = catalog.NewItemRepository(deps.DB).WithActiveSearchProvider(activeCatalogSearchProvider)
|
||||
episodeRepo = catalog.NewEpisodeRepository(deps.DB)
|
||||
@@ -1252,6 +1280,7 @@ func main() {
|
||||
// admin image applies can succeed even if automatic metadata caching is off.
|
||||
if deps.S3Public != nil {
|
||||
imageCacher := imagecache.New(deps.S3Public)
|
||||
imageCacher.SetArtworkRevisionTracker(catalog.NewArtworkRevisionTracker(deps.DB))
|
||||
metadataService.SetImageCacher(imageCacher)
|
||||
imageCacheJobs := metadata.NewImageCacheJobRepository(deps.DB)
|
||||
metadataService.SetImageCacheJobEnqueuer(imageCacheJobs)
|
||||
@@ -1268,6 +1297,11 @@ func main() {
|
||||
People: personRepo,
|
||||
},
|
||||
)
|
||||
// Local file:// artwork (NFO sidecars): confine reads to the owning
|
||||
// library's roots and sweep stale hashed local/ prefixes on re-cache.
|
||||
// The processor host must mount the libraries, like the metadata worker.
|
||||
metadataImageCacheProcessor.SetLibraryRootResolver(deps.FolderRepo)
|
||||
metadataImageCacheProcessor.SetImagePrefixDeleter(deps.S3Public)
|
||||
metadataService.SetAutoCacheImages(cfg.Metadata.CacheImages)
|
||||
metadataImageCacheProcessor.SetEnabled(cfg.Metadata.CacheImages)
|
||||
configWatcher.OnChange(func(_, updated *config.Config) {
|
||||
@@ -1550,6 +1584,7 @@ func main() {
|
||||
|
||||
// Step 6: Create playback session manager and wire into dependencies.
|
||||
sessionMgr := playback.NewSessionManager(6, 2) // defaults from plan: max_streams=6, max_transcodes=2
|
||||
var compatTerminalRecoveryReady <-chan struct{}
|
||||
if userStoreProvider != nil {
|
||||
deps.UserStoreProvider = userStoreProvider
|
||||
}
|
||||
@@ -1561,6 +1596,13 @@ func main() {
|
||||
WithWatchState(watchstate.NewService(userStoreProvider).WithStableIdentityResolver(historyIdentity)).
|
||||
WithUserStoreProvider(userStoreProvider)
|
||||
backgroundInit = append(backgroundInit, func(ctx context.Context) {
|
||||
if compatTerminalRecoveryReady != nil {
|
||||
select {
|
||||
case <-compatTerminalRecoveryReady:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := watchProviderService.SweepOpenScrobbles(ctx); err != nil {
|
||||
slog.WarnContext(ctx, "failed to sweep open watch provider scrobbles", "component", "app", "error", err)
|
||||
}
|
||||
@@ -1879,6 +1921,11 @@ func main() {
|
||||
taskMgr.Register(tasks.NewScanLibrariesTask(deps.FolderRepo, deps.LibraryScanQueue, deps.EventBus))
|
||||
}
|
||||
taskMgr.Register(tasks.NewCleanupOrphanedMediaItemsTask(catalog.NewOrphanedProvisionalCleaner(deps.DB)))
|
||||
if deps.S3Public != nil {
|
||||
taskMgr.Register(tasks.NewCleanupArtworkRevisionsTask(
|
||||
metadata.NewArtworkRevisionGarbageCollector(deps.DB, deps.S3Public),
|
||||
))
|
||||
}
|
||||
catalogSearchIndexer := catalog.NewCatalogSearchIndexer(deps.DB, settingsRepo)
|
||||
taskMgr.Register(tasks.NewSyncCatalogSearchIndexTask(catalogSearchIndexer))
|
||||
taskMgr.Register(tasks.NewRebuildCatalogSearchIndexTask(catalogSearchIndexer))
|
||||
@@ -1895,6 +1942,15 @@ func main() {
|
||||
}
|
||||
taskMgr.Register(tasks.NewActivityLogCleanupTask(deps.DB, settingsRepo, activityPM))
|
||||
taskMgr.Register(tasks.NewOperationalLogCleanupTask(deps.DB, settingsRepo, opsPM))
|
||||
var diagnosticsStore diagnostics.ObjectStore
|
||||
if deps.S3Private != nil {
|
||||
diagnosticsStore = diagnostics.NewS3ObjectStore(deps.S3Private)
|
||||
}
|
||||
taskMgr.Register(tasks.NewClientDiagnosticsCleanupTask(
|
||||
diagnostics.NewPostgresRepository(deps.DB),
|
||||
settingsRepo,
|
||||
diagnosticsStore,
|
||||
))
|
||||
taskMgr.Register(tasks.NewPolicyDecisionLogCleanupTask(deps.DB, settingsRepo, policyPM))
|
||||
if deps.FileRepo != nil {
|
||||
// Download prepare-to-file pipeline (Phase 3): a durable, leased encode
|
||||
@@ -2359,6 +2415,7 @@ func main() {
|
||||
if (mode == "integrated" || mode == "api") && cfg.JellyfinCompat.Enabled && cfg.JellyfinCompat.Listen != "" {
|
||||
compatDeps := jellycompat.Dependencies{
|
||||
Config: cfg,
|
||||
AppContext: appCtx,
|
||||
LiveConfig: configWatcher.Config,
|
||||
DB: deps.DB,
|
||||
SecretCipher: dataCipher,
|
||||
@@ -2402,6 +2459,7 @@ func main() {
|
||||
compatDeps.SeasonRepo = seasonRepo
|
||||
compatDeps.EpisodeRepo = episodeRepo
|
||||
compatDeps.ProviderIDRepo = providerIDRepo
|
||||
compatDeps.StableIdentityResolver = watchstate.NewStableIdentityResolver(itemRepo, episodeRepo, providerIDRepo)
|
||||
compatDeps.DetailSvc = detailSvc
|
||||
compatDeps.FolderRepo = folderRepo
|
||||
compatDeps.SessionMgr = sessionMgr
|
||||
@@ -2409,6 +2467,9 @@ func main() {
|
||||
compatDeps.WatchCompletionObserver = deps.WatchCompletionObserver
|
||||
compatDeps.SettingsRepo = settingsRepo
|
||||
compatDeps.PersonRepo = personRepo
|
||||
if watchProviderService != nil {
|
||||
compatDeps.WatchScrobbler = watchProviderService
|
||||
}
|
||||
compatSearchService := catalog.NewCatalogSearchService(
|
||||
appCtx,
|
||||
settingsRepo,
|
||||
@@ -2488,7 +2549,7 @@ func main() {
|
||||
|
||||
compat := jellycompat.NewServerWithDependencies(compatDeps)
|
||||
compatServer = compat
|
||||
compat.StartBackgroundTasks(context.Background())
|
||||
compatTerminalRecoveryReady = compat.StartBackgroundTasks(context.Background())
|
||||
compatSrv = compat.HTTPServer()
|
||||
compatSrv.ReadTimeout = 30 * time.Second
|
||||
compatSrv.WriteTimeout = 0
|
||||
@@ -2784,6 +2845,12 @@ func reloadPluginImageResolvers(
|
||||
if installation == nil {
|
||||
continue
|
||||
}
|
||||
// Builtin installations resolve in-process metadata providers only;
|
||||
// registering them here would claim their capability id as a gRPC
|
||||
// image-resolver scheme with no binary behind it.
|
||||
if installation.IsBuiltin() {
|
||||
continue
|
||||
}
|
||||
capabilities, err := store.ListCapabilities(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list image resolver capabilities for installation %d: %w", installation.ID, err)
|
||||
@@ -2970,6 +3037,11 @@ func reloadMarkerPluginProviders(
|
||||
if installation == nil {
|
||||
continue
|
||||
}
|
||||
// Builtin installations expose no marker providers; defense in depth
|
||||
// alongside the capability-type filter below.
|
||||
if installation.IsBuiltin() {
|
||||
continue
|
||||
}
|
||||
capabilities, err := store.ListCapabilities(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list marker provider capabilities for installation %d: %w", installation.ID, err)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# Local NFO metadata and sidecar artwork (as built)
|
||||
|
||||
Implements #216: Kodi/Jellyfin-style `.nfo` files and sidecar images
|
||||
(`poster.jpg`, `fanart.jpg`, …) next to movies and shows are read by a builtin
|
||||
metadata provider, so curated local libraries work with partial or zero
|
||||
TMDB/TVDB coverage. Admin-facing usage lives in
|
||||
`docs/wiki/admin/nfo-local-metadata.md`; this page records the architecture for
|
||||
maintainers.
|
||||
|
||||
## Registration: a builtin provider in the plugin chain
|
||||
|
||||
Provider chains are keyed on `plugin_installations` rows, so builtins are
|
||||
registered **as data**: a reserved installation (`plugin_id='silo.builtin'`,
|
||||
`kind='builtin'`, sentinel `install_path`, `update_policy='manual'`) plus a
|
||||
`metadata_provider.v1` capability `nfo` with `default_priority` 1 for
|
||||
movie/series/season/episode and `default_enabled=false`
|
||||
(`migrations/sql/20260712100125_builtin_host_providers.sql`,
|
||||
`...124741_nfo_capability_season_episode_levels.sql`).
|
||||
|
||||
- `internal/metadata/builtin.go` is the in-process registry; providers
|
||||
self-register (`internal/metadata/nfo/register.go`) and `buildProviders`
|
||||
(`internal/metadata/chain.go`) returns the in-process provider for
|
||||
`kind='builtin'` rows instead of constructing a gRPC plugin.
|
||||
- Guard rails keep the reserved row out of every plugin surface (user plugin
|
||||
settings, installations list, image resolvers, preload, auto-update, store
|
||||
delete, mutation handlers); `silo.builtin` is a reserved manifest id. Generic
|
||||
installation reads must **not** filter builtins — the chain enabled-check
|
||||
depends on them.
|
||||
- On startup, `SyncBuiltinProviderChains` (`internal/metadata/builtin_sync.go`)
|
||||
first materializes legacy `content_level=''` chains per level, then appends
|
||||
builtin capabilities to all existing chains, disabled. New libraries pick the
|
||||
capability up through normal chain seeding. The chain-less fallback respects
|
||||
`default_enabled=false`.
|
||||
- **Everything is inert until an admin enables "NFO Files" on a library.**
|
||||
|
||||
## Identity: hint-first, never candidate-first
|
||||
|
||||
NFO `<uniqueid>` values (tmdb/imdb/tvdb) are **trusted identity hints** seeded
|
||||
before remote search (`IdentityHintProvider`, applied in
|
||||
`internal/metadata/service.go`), unlocking full remote enrichment even when
|
||||
search-by-title fails. Conflict policy by mode:
|
||||
|
||||
- initial match / scheduled refresh: NFO hints beat folder-name hints; stored
|
||||
durable IDs beat NFO hints (no background identity flips);
|
||||
- manual refresh: NFO hints beat stored IDs — the recovery path for a corrected
|
||||
NFO. The item's `content_id` re-anchors too (not just its stored provider
|
||||
IDs): a provider-anchored item whose fixed `<uniqueid>` now derives a
|
||||
different anchor is renamed to it, or merged onto the existing item already
|
||||
holding that id;
|
||||
- manual Identify: the user's choice wins; the NFO provider is skipped for the
|
||||
whole operation.
|
||||
|
||||
The NFO search candidate exists only for the title-only case and is defanged:
|
||||
ID-less candidates lose provider-priority tie-breaks to ID-bearing group
|
||||
members, `nfo` never counts as corroboration, and Phase-2 NFO results cannot
|
||||
inject provider-id keys. A title-only NFO yields a `matched` item under a
|
||||
path-deterministic `local:` content id; adding a `<uniqueid>` later promotes it
|
||||
to that provider anchor on the next refresh (this promotion from an unanchored
|
||||
id runs on scheduled refresh too — only an *already-anchored* identity is held
|
||||
stable in the background). NFO field edits propagate on **manual refresh only**
|
||||
(scheduled refresh is fill-empty by design).
|
||||
|
||||
## Sidecar artwork: copy into the S3 image cache
|
||||
|
||||
Sidecar images (`poster`/`folder`/`cover`, `fanart`/`backdrop`/`background`,
|
||||
`logo`/`clearlogo`, and `<basename>-poster/-fanart/-logo` variants; extensions
|
||||
`.jpg/.jpeg/.png/.webp`, 8 MiB cap, symlinked leaves rejected) are discovered
|
||||
by the provider's `GetImages` (`internal/metadata/nfo/images.go`). Clients —
|
||||
including jellycompat — always receive the normal presigned
|
||||
`poster_url`/`backdrop_url`/`logo_url`; library files are never served
|
||||
directly, and API nodes never need filesystem access to libraries.
|
||||
|
||||
- Sources are recorded as `file://<absolute-logical-path>` in `*_source_path`
|
||||
columns and cached by the metadata image-cache processor under
|
||||
`local/{contentType}/{contentID}/{hash8}/{imageType}/...` (`hash8` = content
|
||||
hash; the discriminator sits before the imageType segment because variant
|
||||
clamping reads the parent directory). Editing the file and refreshing rotates
|
||||
the key; the stale hashed prefix is deleted after a successful re-cache, and
|
||||
item deletion trims local paths to the content prefix.
|
||||
- The processor confines each read to the owning library's `media_folders`
|
||||
roots: a lexical check on the logical path, then a symlink-resolving re-check
|
||||
(both path and roots are resolved, so a legitimately symlinked root stays
|
||||
valid while an intermediate directory symlink escaping a root is rejected).
|
||||
Missing/unreadable/out-of-root and structurally-unusable (non-regular,
|
||||
over-cap) paths are stable failures with a long retry deferral; recovery is
|
||||
refresh-driven — the provider-artwork backfill sweep deliberately skips
|
||||
`file://` sources.
|
||||
- Generic filenames attach only when the directory holds a single content
|
||||
group; a shared `folder.jpg` in a flat multi-movie folder applies to none.
|
||||
`<basename>-poster.jpg` style names always attach to their file.
|
||||
- `applyBestImages`' final write has a local exemption so rating-0 local art
|
||||
can fill already-matched items and cannot be stickily displaced by remote
|
||||
art; the localization pass skips language-neutral local candidates.
|
||||
|
||||
**Deployment constraint:** the host running the metadata image-cache processor
|
||||
must mount the media libraries at the same paths as the scanner/metadata
|
||||
worker, otherwise local artwork jobs fail until the mount is present.
|
||||
|
||||
## Series depth and mixed libraries
|
||||
|
||||
`SeasonsRequest`/`EpisodesRequest` carry local path context; `season.nfo`
|
||||
supplies season name/plot, `<basename>.nfo` + `<basename>-thumb.ext` supply
|
||||
episode metadata/thumbs (`internal/metadata/nfo/series_depth.go`). **Naming
|
||||
supplies structure; NFOs supply metadata**: NFO season/episode numbers are
|
||||
advisory — directory/filename-derived numbers win with a warning. Episode NFOs
|
||||
work without a `season.nfo`; `SynthesizeFallbackEpisodes` still covers
|
||||
NFO-less episodes.
|
||||
|
||||
In `mixed` libraries, movie-vs-series is decided per file by naming before any
|
||||
provider runs (`internal/naming/filename.go`); the NFO provider is seeded into
|
||||
all levels and behaves as in dedicated libraries. The provider's root-element
|
||||
type guard means a `tvshow.nfo` next to a movie-classified file is ignored,
|
||||
never applied; the per-root Type override is the correction path. This is the
|
||||
contract for sports/mixed libraries (events as movies, weekly shows as series).
|
||||
|
||||
## Known limitations
|
||||
|
||||
- The admin image picker does not surface local art (automatic chain path
|
||||
only).
|
||||
- Multi-part movies: a basename-mismatched NFO in a folder holding multiple
|
||||
content groups is not found (directory candidates are suppressed there).
|
||||
- No NFO writing, no music/audiobook/ebook NFO (those ecosystems use
|
||||
OPF/metadata.json — future builtin providers via the same registry).
|
||||
- Feature detection: the `nfo` capability's presence in
|
||||
`GET /api/v1/libraries/provider-defaults` (see
|
||||
`docs/architecture/v1-scope.md`).
|
||||
@@ -17,3 +17,8 @@ When the scope locks, this file becomes the source of truth and will contain:
|
||||
|
||||
Until lock: treat any capability not tracked as `Proposed`/`Locked` on the project as out of scope
|
||||
for feature PRs (see the scope gate in `CLAUDE.md`).
|
||||
|
||||
Feature-detection precedent: clients discover which metadata providers (including the
|
||||
built-in NFO provider, #216) apply to a library type via
|
||||
`GET /api/v1/libraries/provider-defaults` rather than version sniffing. New capabilities
|
||||
should follow the same capability-endpoint pattern.
|
||||
|
||||
@@ -0,0 +1,771 @@
|
||||
# Client Diagnostics: crash reports and debug log upload
|
||||
|
||||
Status: Draft (spec + cross-repo plan)
|
||||
Date: 2026-07-19
|
||||
Repos affected: `silo-server`, `silo-android`, `silo-apple`
|
||||
|
||||
## Summary
|
||||
|
||||
Add an opt-in, self-hosted diagnostics pipeline: Silo clients detect crashes
|
||||
and abnormal exits, collect structured debug logs (playback, TV focus,
|
||||
network, lifecycle) plus a hardware snapshot (device model, display/HDMI,
|
||||
audio path, codec capabilities), and — only with explicit user consent —
|
||||
upload a compressed diagnostics bundle to the user's own Silo server. Admins
|
||||
review bundles through new acting-admin API endpoints alongside the existing
|
||||
server logs tooling.
|
||||
|
||||
Everything ships **disabled by default**:
|
||||
|
||||
- Server: a `diagnostics.uploads_enabled` server setting gates the whole
|
||||
feature (default off), and it cannot be enabled unless the server's private
|
||||
object storage is configured and validated.
|
||||
- Client: verbose "debug logging" is a per-device setting (default off).
|
||||
Crash reporting defaults to **Ask** — after an abnormal exit the app shows
|
||||
an incident-specific prompt (e.g. *"Silo closed unexpectedly during
|
||||
playback. Send a diagnostic report to your server?"*) with
|
||||
**Send / Don't send / Always send**. Nothing uploads without one of: an
|
||||
explicit prompt confirmation, a prior "Always send" election bound to this
|
||||
server and account, or the user explicitly sending a manual capture.
|
||||
|
||||
## Motivation
|
||||
|
||||
Today we are blind to client-side failures:
|
||||
|
||||
- **No crash reporting anywhere.** Neither the Android repo nor the Apple
|
||||
repo installs an uncaught-exception handler, uses MetricKit, or bundles a
|
||||
crash SDK (verified: no handler/SDK references in either repo; the only
|
||||
Firebase dependency is messaging). A user saying "the app crashed" is a
|
||||
dead end.
|
||||
- **Diagnostics don't survive to the report.** Android logs go to logcat (raw
|
||||
`android.util.Log` in ~40 files, no buffer or file sink); the Apple
|
||||
player's rich `[CMP]`/`[CMP-DIAG]` diagnostics go through `print()`
|
||||
(`PlayerLog.swift` / `PlayerCore.swift`) with no app-owned persistent sink.
|
||||
By the time a user reports a problem, the evidence is gone.
|
||||
- **The hardest bugs are device-specific.** Shield HLG quirks, Fire TV
|
||||
passthrough suppression, HDMI mode switches, TV focus traps — reproducing
|
||||
them requires knowing the exact device, panel, audio chain, and codec path.
|
||||
Users can't reliably report any of that.
|
||||
- **The server already has half the story.** `operational_logs` records
|
||||
`playback_session_id` (migration `028_log_partitioning.sql`), the admin
|
||||
logs API filters by it, and the web admin has a logs viewer. What's missing
|
||||
is the client half, and a way to join the two.
|
||||
|
||||
## Goals
|
||||
|
||||
- Capture crashes and abnormal exits (JVM/native/ANR on Android; crash and
|
||||
hang diagnostics on iOS; abnormal-exit detection on tvOS) with enough
|
||||
context to act on: stack or exit trace where the platform provides one,
|
||||
recent structured logs, and a device snapshot.
|
||||
- Capture deep opt-in debug logs for playback, TV focus behavior, networking,
|
||||
and navigation/lifecycle — everything needed to troubleshoot user issues.
|
||||
- Always include a hardware snapshot: device (Shield / Fire TV / onn /
|
||||
phone), OS, app build, display and HDMI mode with HDR capabilities, audio
|
||||
output path and passthrough capabilities, decoder inventory — with honest
|
||||
`unknown` / `not_collected` states where a platform can't provide a field.
|
||||
- One payload contract across Android and Apple; one server endpoint; one
|
||||
admin surface. The contract is a checked-in versioned JSON Schema with
|
||||
golden fixtures consumed by all three repos' CI.
|
||||
- Explicit, server-bound consent for every byte uploaded; feature fully off
|
||||
by default on both server and client.
|
||||
- Bounded cost: size caps, per-account quotas, concurrency limits, and
|
||||
server-side retention.
|
||||
|
||||
## Non-goals (v1)
|
||||
|
||||
- Live/continuous log streaming from clients to the server.
|
||||
- Screenshots or screen recordings in bundles.
|
||||
- Native/tvOS crash *stack* capture via an in-process crash reporter
|
||||
(PLCrashReporter/KSCrash-style). tvOS v1 detects abnormal exits and
|
||||
attaches breadcrumbs, not stacks (see Apple design). Adopting an in-process
|
||||
crash reporter is a separate decision with its own signal-safety and review
|
||||
implications.
|
||||
- Symbolication. Bundles store what the OS gives us; a server-side
|
||||
symbolication job can come later.
|
||||
- Native-app admin viewer screens (per current product policy, admin surfaces
|
||||
beyond STATS on the clients are a separate product decision; v1 admin
|
||||
consumption is via API + web admin).
|
||||
- macOS. The Apple repo has a macOS target, but v1 scopes to
|
||||
`android`, `android-tv`, `ios`, `tvos`; the platform enum reserves `macos`
|
||||
without accepting it.
|
||||
- Third-party telemetry, metrics aggregation, or fleet analytics. This is a
|
||||
support/debugging tool, not analytics.
|
||||
|
||||
## Why this route
|
||||
|
||||
**Upload to the user's own Silo server, not a third-party crash service.**
|
||||
Crashlytics/Sentry SaaS would send user data to a third party, require vendor
|
||||
accounts/keys we'd have to ship, and put the reports in front of *us* rather
|
||||
than the person who can actually act — the self-hosting admin. Requiring
|
||||
every admin to also run self-hosted Sentry is far too heavy. The Silo server
|
||||
is the natural, privacy-aligned destination, and it already has auth, admin
|
||||
tooling, object storage, and retention patterns to build on.
|
||||
|
||||
**Bundle-on-event, not streaming.** A bundle captured at the moment of
|
||||
failure (crash, or a user-triggered capture) contains exactly the context
|
||||
needed: recent structured logs, the playback stats timeline, and the hardware
|
||||
snapshot. Continuous streaming would cost bandwidth and privacy while mostly
|
||||
capturing noise, and would still miss pre-crash buffers.
|
||||
|
||||
**Hybrid storage: Postgres row + object-store blob.** Report *metadata* (a
|
||||
small manifest: device summary, truncated crash summary, type, timestamps)
|
||||
goes in a queryable table; the payload itself (potentially megabytes of logs
|
||||
and crash artifacts) goes to the S3-compatible private bucket as a single
|
||||
archive. Ingesting client log lines into the partitioned `operational_logs`
|
||||
table was considered and rejected: bundles are consumed as whole documents by
|
||||
a human, not queried row-wise; client lines are untrusted input with
|
||||
different retention needs; and multi-MB uploads would bloat a table tuned for
|
||||
the server's own high-churn logs. The server *does* emit normalized
|
||||
operational events about diagnostics activity (report accepted / rejected /
|
||||
downloaded / deleted, with report id, account, sizes, result) so ingest and
|
||||
admin actions are auditable and correlate with existing logs — raw client
|
||||
lines never enter `operational_logs`.
|
||||
|
||||
**A versioned archive, not a bare gzipped log stream.** Crash evidence is
|
||||
heterogeneous: JSONL logs, ANR trace text, Android native tombstones
|
||||
(protobuf on API 31+), MetricKit diagnostic JSON. A tar.gz archive with named
|
||||
entries carries each artifact losslessly with a declared media type, keeps
|
||||
large traces out of Postgres, and can evolve by adding entries.
|
||||
|
||||
**Build small in-house rather than adopt ACRA/other SDKs.** The surface we
|
||||
need on Android (one exception handler writing a bounded marker, a ring
|
||||
buffer, one upload call) is a few hundred lines; ACRA would add a dependency
|
||||
and its own report format without covering Apple. On Apple, the OS provides
|
||||
MetricKit natively (iOS; not tvOS — see Apple design).
|
||||
|
||||
## Architecture overview
|
||||
|
||||
```text
|
||||
Android / Apple client silo-server
|
||||
┌────────────────────────────────┐ ┌──────────────────────────────────────┐
|
||||
│ Safe-logging facade → ring │ │ GET /api/v1/diagnostics/status │
|
||||
│ └ debug mode: rotating │ │ └ available|disabled| │
|
||||
│ segment files │ │ storage_unavailable + limits │
|
||||
│ Crash/abnormal-exit capture │ │ POST /api/v1/diagnostics/reports │
|
||||
│ ├ UEH marker (Android) │──▶│ ├ gate + storage check │
|
||||
│ ├ ApplicationExitInfo (30+) │ │ ├ streaming, bounded multipart │
|
||||
│ ├ MetricKit (iOS) │ │ ├ account quotas + concurrency │
|
||||
│ └ exit sentinel (tvOS) │ │ ├ manifest → client_diagnostic_ │
|
||||
│ Pending reports (server-bound) │ │ │ reports (state machine) │
|
||||
│ DeviceSnapshot (probes+new │ │ └ archive → S3 private bucket │
|
||||
│ accessors) │ │ Admin (acting admin): list/detail/ │
|
||||
│ Consent UI (ask/always/never, │ │ download/delete + audit events │
|
||||
│ per server+account) │ │ Retention + orphan reconciler task │
|
||||
└────────────────────────────────┘ └──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Consent model
|
||||
|
||||
Consent is **scoped, versioned, and destination-bound** — not a global
|
||||
boolean. Both clients support multiple servers and accounts, so a device-wide
|
||||
"Always send" would let a bundle full of server A's titles and URLs upload to
|
||||
server B after a switch. Rules:
|
||||
|
||||
- Consent (including "Always send") is **account-scoped**: every consent
|
||||
record and every pending report is bound to
|
||||
`(server_instance_id, account_user_id, consent_notice_version)`, where
|
||||
`server_instance_id` is a stable opaque ID returned by the status
|
||||
endpoint. The capturing `profile_id` is recorded on reports for
|
||||
*attribution*, not as a consent scope — quotas and elections follow the
|
||||
account.
|
||||
- A pending report is only ever uploaded to the server+account it was
|
||||
captured under. Reports are never retargeted. Switching server or account
|
||||
neither uploads nor migrates pending reports; signing out of an account or
|
||||
removing a server deletes its pending reports.
|
||||
- "Always send" is an election for one server+account. Bumping
|
||||
`consent_notice_version` (when we materially change what bundles contain)
|
||||
invalidates prior elections and reverts to Ask.
|
||||
- Setting crash reports to **Never** deletes pending reports for that
|
||||
server+account and stops persistent capture (breadcrumbs/debug files); the
|
||||
in-memory ring keeps running (memory-only, never uploaded without a later
|
||||
explicit action).
|
||||
- **Child/managed profiles**: the Diagnostics section is not shown to
|
||||
profiles with `is_child`, and every diagnostics action — changing
|
||||
settings, electing Always, the crash prompt, and manual capture/send —
|
||||
requires a non-child profile. A crash captured while a child profile was
|
||||
active is held pending and prompted for the next time a non-child profile
|
||||
is active. Reports are attributed to account + capturing profile.
|
||||
- The crash prompt and the manual flow both offer **View report** — a
|
||||
human-readable summary (incident, categories and line counts, device
|
||||
identity, destination server) before anything is sent.
|
||||
- Uploads carry `consent: {mode, notice_version}` in the manifest so the
|
||||
server can reject stale-consent uploads outright.
|
||||
|
||||
## Bundle format (shared contract)
|
||||
|
||||
An upload is `multipart/form-data` with exactly two parts, in order:
|
||||
|
||||
1. `manifest` — `application/json`, ≤ 64 KiB. Queryable summary; stored in
|
||||
Postgres.
|
||||
2. `bundle` — `application/gzip`: a tar.gz archive, `manifest.json` first.
|
||||
Entry names come from a **fixed allowlist**, and each name implies its
|
||||
media type — there is no per-entry type metadata to drift:
|
||||
|
||||
```jsonc
|
||||
manifest.json (manifest minus the `archive` object; see hashing note)
|
||||
device.json (full device snapshot)
|
||||
logs.jsonl (structured log lines, newest-last)
|
||||
crash/summary.json (normalized crash info incl. provenance)
|
||||
crash/stack.txt (JVM stack / ANR trace, when present)
|
||||
crash/tombstone.pb (Android native tombstone, opaque bytes, API 31+)
|
||||
crash/metrickit.json (raw MetricKit diagnostic JSON, iOS)
|
||||
breadcrumbs.jsonl (persisted pre-crash context journal, when present)
|
||||
```
|
||||
|
||||
Hashing note: `archive.sha256` and the size fields in part 1 describe the
|
||||
finalized `bundle` part bytes as transmitted. Since the archive is hashed
|
||||
after it is built, the embedded `manifest.json` is the manifest **without**
|
||||
the `archive` object; the server verifies part 1's `archive` fields against
|
||||
the received blob. `archive.bytes` is the transmitted (gzip) size;
|
||||
`archive.uncompressed_bytes` is the total decompressed tar stream size,
|
||||
including tar headers, the end-of-archive marker, and record padding —
|
||||
i.e. the byte count between the client's tar writer and gzip writer, and
|
||||
what `gzip -l` reports. Standard end-of-archive zero padding (up to 64 KiB)
|
||||
is accepted; any other trailing data is rejected.
|
||||
|
||||
The contract lives in this repo as a versioned JSON Schema
|
||||
(`docs/design/schemas/client-diagnostics/v1/`) with canonical valid and
|
||||
invalid fixtures; Go, Kotlin, and Swift CI all validate against the same
|
||||
fixtures. Servers accept all schema versions they support and ignore unknown
|
||||
additive fields.
|
||||
|
||||
### Manifest (part 1)
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "crash", // crash | anr | native_crash | hang | abnormal_exit | manual
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_c7d…", // UUID per app run, also stamped on log lines
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android-tv", // android | android-tv | ios | tvos (macos reserved)
|
||||
"os_version": "11 (API 30)",
|
||||
"profile_id": "prof_…" // capturing profile, attribution only
|
||||
},
|
||||
"destination": { "server_instance_id": "srv_…" }, // consent binding; server rejects mismatches
|
||||
"consent": { "mode": "prompt", "notice_version": 1 }, // prompt | always | manual
|
||||
"crash": { // absent for manual
|
||||
"summary": "NullPointerException in PlaybackSessionManager.start",
|
||||
"stack_excerpt": "…truncated ≤ 8 KiB; full artifact in archive…",
|
||||
"thread": "main",
|
||||
"foreground": true,
|
||||
"source": "ueh", // ueh | exit_info | metrickit | exit_sentinel
|
||||
"provenance": "pre_failure", // pre_failure | post_restart | metric_reporting_period
|
||||
"occurred_at": "2026-07-19T18:22:31Z" // best known; MetricKit reports carry period bounds
|
||||
},
|
||||
"device_summary": { "manufacturer":"NVIDIA", "model":"SHIELD Android TV", "os":"11", "form_factor":"tv" },
|
||||
"playback_session_ids": ["ps_9f2…"],
|
||||
"log_summary": { "lines": 3841, "bytes_gz": 412034, "dropped_lines": 12, "categories": ["playback","focus","network"], "debug_logging": true },
|
||||
"archive": { "entries": ["manifest.json","device.json","logs.jsonl","crash/summary.json","crash/stack.txt"], "bytes": 498112, "uncompressed_bytes": 3110400, "sha256": "…" }
|
||||
}
|
||||
```
|
||||
|
||||
### Log line schema (`logs.jsonl`)
|
||||
|
||||
```json
|
||||
{"ts":"2026-07-19T18:22:29.412Z","run":"run_c7d…","lvl":"W","cat":"playback","tag":"AudioCapabilityMgr","msg":"passthrough suppressed: HDMI sink lost TrueHD","attrs":{"sink":"HDMI","fmt":"truehd"}}
|
||||
```
|
||||
|
||||
- `lvl`: `V|D|I|W|E`. `cat`: `playback`, `focus`, `network`, `lifecycle`,
|
||||
`browse`, `cast`, `download`, `crash`, `other`.
|
||||
- `run` (capture_session_id) distinguishes lines from prior app runs when
|
||||
breadcrumbs or debug files span sessions.
|
||||
- Playback additionally emits periodic (~5s) player-stats snapshot lines
|
||||
(`"cat":"playback","tag":"StatsSnapshot"`) built from the existing stats
|
||||
models — a timeline of decoder, resolution, HDR mode, bitrate, dropped
|
||||
frames, and audio underruns around the failure.
|
||||
|
||||
### Safe logging contract (redaction happens at collection time)
|
||||
|
||||
Best-effort scrubbing at package time is not enough: today's client logs
|
||||
already emit full URLs with query strings, token suffixes, and raw exception
|
||||
strings (e.g. the Apple `HTTPClient` response logging and player URL prints;
|
||||
Android exception messages containing backend URLs). Therefore:
|
||||
|
||||
- The facade accepts **typed attributes**, not preformatted strings, for
|
||||
anything risky: URL values are normalized (userinfo, query, and fragment
|
||||
always stripped; path preserved), exceptions pass through a sanitizing
|
||||
formatter, and free-text `msg` is authored, not interpolated from network
|
||||
data.
|
||||
- Attribute keys are **normatively registered**: a per-category key registry
|
||||
(checked in next to the JSON Schema) declares each key and its value type
|
||||
(scalar types plus a few enumerated units). Unregistered keys fail debug
|
||||
builds/CI and are dropped in production builds.
|
||||
- Existing call sites are migrated **curated, not mechanically** — each
|
||||
migrated line is reviewed against the contract. Unmigrated `Log.*`/`print`
|
||||
call sites simply never enter bundles.
|
||||
- Defense in depth at bundle time: values matching the account's current
|
||||
access/profile tokens are replaced before packaging.
|
||||
- Golden leak-fixture tests (JWTs, query tokens, signed URLs, cookies, token
|
||||
fragments) run in all three repos' CI against the facade and the bundle
|
||||
builder.
|
||||
- The network category logs `method, templated path, status, duration`;
|
||||
never headers, bodies, query strings, or full URLs.
|
||||
|
||||
### Device snapshot (`device.json`)
|
||||
|
||||
Fields are tri-state where a platform can't provide them: a value,
|
||||
`"unknown"` (probe exists, couldn't determine), or `"not_collected"`
|
||||
(no probe on this platform). Every snapshot carries `captured_at` and
|
||||
`provenance` (`pre_failure` for snapshots persisted with the incident,
|
||||
`post_restart` for snapshots taken at bundle-build time).
|
||||
|
||||
```json
|
||||
{
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"provenance": "pre_failure",
|
||||
"identity": { "manufacturer":"NVIDIA", "model":"SHIELD Android TV", "device":"mdarcy", "form_factor":"tv", "device_id":"…" },
|
||||
"display": { "mode":"3840x2160@59.94", "modes_supported":[…], "hdr_types":["HDR10","HLG","DV"], "wide_gamut":true },
|
||||
"audio": { "outputs":[{"type":"HDMI","encodings":["AC3","EAC3","TRUEHD"],"channels":8}], "passthrough":["AC3","EAC3","TRUEHD"], "suppressions":[…] },
|
||||
"video_codecs": [ { "mime":"video/hevc", "hw":true, "profiles":["Main10","DV P8"], "max":"4096x2160@60" } ],
|
||||
"network": { "transport":"ethernet" }
|
||||
}
|
||||
```
|
||||
|
||||
Android sources: `AndroidDeviceMetadataProvider` (id/model/platform/app
|
||||
version), `MediaCodecCapabilitiesProbe` (codec + HDR decode caps),
|
||||
`AudioCapabilityManager` (aggregate passthrough). **New accessors required**
|
||||
(the probes exist but don't currently export everything): current + supported
|
||||
`Display.Mode` list (DisplayHdrProbe exposes HDR support only), exact
|
||||
identity fields (`Build.DEVICE`, form factor), `AudioManager.getDevices()`
|
||||
output enumeration, and a snapshot/export API on
|
||||
`PassthroughSuppressionRegistry` (its set is currently private).
|
||||
|
||||
Apple sources: `AppleDeviceIdentity.current` (persisted id, device name,
|
||||
platform) and `ApplePlaybackV3Capabilities.snapshot()` (codec/HDR/device
|
||||
context). **Honesty constraints:** the current `outputSnapshot()` helper is
|
||||
private, reports route UIDs/types rather than full output capabilities, and
|
||||
its passthrough payload is a placeholder — so Apple v1 reports audio as
|
||||
route metadata with `passthrough: "unknown"` until a real probe exists, and
|
||||
the private helper is refactored into an internal shared accessor so
|
||||
diagnostics can reuse it without duplicating route logic. Audio route UIDs
|
||||
are persistent peripheral identifiers and are one-way hashed before
|
||||
inclusion.
|
||||
|
||||
### Playback session correlation
|
||||
|
||||
Clients include recent server playback session IDs so admins can pivot into
|
||||
the existing server-side logs (`operational_logs.playback_session_id` +
|
||||
the admin logs filter). Both clients currently retain only the *active*
|
||||
session ID (Android `PlaybackSessionManager`, Apple `PlaybackSessionBridge`),
|
||||
so each client adds a small **bounded, persisted recent-session tracker**
|
||||
(last ~10 session IDs with timestamps) that survives restarts — required for
|
||||
next-launch crash reports.
|
||||
|
||||
## Server design (`silo-server`)
|
||||
|
||||
### Feature gate
|
||||
|
||||
Borrows the *behavior* of the requests gate (status endpoint + per-call
|
||||
guard that rereads settings) but stores keys in the general `server_settings`
|
||||
store — requests uses its own dedicated table, which diagnostics doesn't
|
||||
need. Key naming follows existing namespaces (`opslog.*`,
|
||||
`audiobooks.enabled`, `playback.protocol_v3_enabled`):
|
||||
|
||||
- `diagnostics.uploads_enabled` (bool, default `false`)
|
||||
- `diagnostics.max_bundle_bytes` (default 10 MiB compressed)
|
||||
- `diagnostics.max_uncompressed_bytes` (default 64 MiB)
|
||||
- `diagnostics.max_reports_per_user_per_day` (default 20)
|
||||
- `diagnostics.retention_days` (default 30)
|
||||
- `diagnostics.max_bytes_per_user` (default 200 MiB)
|
||||
|
||||
Enabling `diagnostics.uploads_enabled` **requires a storage validation**
|
||||
(private object store configured and writable); the admin settings API
|
||||
refuses to enable it otherwise. `S3Private` is a nullable dependency in the
|
||||
router and is only populated when configured, so the feature must degrade
|
||||
predictably, not 500.
|
||||
|
||||
### Status endpoint
|
||||
|
||||
`GET /api/v1/diagnostics/status` — authenticated (`RequireAuth`), **no
|
||||
profile required**: crashes during login/profile selection must still be
|
||||
reportable, so capability discovery and upload are account-scoped with
|
||||
optional profile attribution.
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "available", // available | disabled | storage_unavailable
|
||||
"server_instance_id": "srv_…", // stable; binds consent + pending reports
|
||||
"accepted_schema_versions": [1],
|
||||
"max_bundle_bytes": 10485760,
|
||||
"max_manifest_bytes": 65536,
|
||||
"retention_days": 30,
|
||||
"consent_notice_version": 1
|
||||
}
|
||||
```
|
||||
|
||||
Clients cache this alongside existing server-driven config refresh.
|
||||
|
||||
### Ingest endpoint
|
||||
|
||||
`POST /api/v1/diagnostics/reports` — `RequireAuth`; access-token
|
||||
`Authorization` header only (no query credentials; `sa_` API keys rejected
|
||||
for diagnostics). Optional validated profile attribution. Hardened parsing —
|
||||
the avatar handler is the *precedent* (multipart + `PutObject`), but its
|
||||
buffer-everything approach (`ParseMultipartForm` + `io.ReadAll`) is not
|
||||
reused:
|
||||
|
||||
1. Guard: `diagnostics.uploads_enabled` + storage present, else stable error
|
||||
codes (`disabled`, `storage_unavailable`).
|
||||
2. `http.MaxBytesReader` with `max_bundle_bytes` + manifest overhead before
|
||||
any parsing; request deadline; per-account concurrency cap (1 in-flight)
|
||||
and a server-global concurrency ceiling, on top of the existing rate-limit
|
||||
middleware.
|
||||
3. Streaming `multipart.Reader`: exactly the two named parts in order;
|
||||
manifest capped at 64 KiB and schema-validated; unknown parts rejected.
|
||||
4. Quota reservation: in one transaction holding a per-account advisory
|
||||
lock (`pg_advisory_xact_lock(user_id)` — serializing concurrent uploads
|
||||
from multiple devices on one account), check the per-day count and
|
||||
per-user byte quota and insert the report row in state `receiving`
|
||||
(server-generated UUID + `short_id`). Failures return `quota_exceeded`
|
||||
(429, with `retry_after`) or `too_large` (413) — machine-readable so
|
||||
clients back off and keep the report pending locally.
|
||||
5. Stream the `bundle` part directly to its final key
|
||||
(`diagnostics/{user_id}/{report_id}.tar.gz`) while hashing and counting
|
||||
bytes; enforce the compressed cap during streaming. Validate the
|
||||
gzip/tar structure and per-entry limits (entry count, name allowlist,
|
||||
uncompressed total ≤ `max_uncompressed_bytes`, compression ratio bound)
|
||||
in the same bounded streaming pass. Reject on violation
|
||||
(`invalid_bundle`, `unsupported_schema`). There is no staging/rename
|
||||
step — S3-style stores have no cheap rename, and the row's `state` is
|
||||
the source of truth: a blob is not a report until its row is `ready`.
|
||||
6. On success, update the row to `ready` with sizes + sha256. On any
|
||||
failure: delete the object, mark the row `failed` (compensation); the
|
||||
reconciler (below) cleans up stragglers from process death mid-request.
|
||||
7. Respond `201 {"report_id":"…","short_id":"SILO-…"}` and emit a normalized
|
||||
operational event (accepted/rejected, account, sizes, result).
|
||||
|
||||
`short_id` is 12 Crockford-Base32 characters, server-generated with
|
||||
collision retry, case-insensitively unique, shown to the user for support
|
||||
conversations, filterable in the admin API — and explicitly not an
|
||||
authorization secret.
|
||||
|
||||
New code: `internal/diagnostics/` (service + store interface), handler in
|
||||
`internal/api/handlers/diagnostics.go`, wired through `Dependencies` in
|
||||
`internal/api/router.go`. The store interface is diagnostics-owned (the
|
||||
avatar handler's is package-private and presign-only) and covers everything
|
||||
the design uses: a **streaming** put (`io.Reader` — the existing s3client
|
||||
`PutObject` takes `[]byte`, so this is a small s3client addition),
|
||||
`GetObject` (streamed admin download fallback), `DeleteObject`,
|
||||
`ListObjects(prefix)` (reconciler), and `PresignGetURL`.
|
||||
|
||||
### Storage
|
||||
|
||||
```sql
|
||||
CREATE TABLE client_diagnostic_reports (
|
||||
id UUID PRIMARY KEY,
|
||||
short_id TEXT NOT NULL, -- unique, case-insensitive (expression index)
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
profile_id TEXT, -- capturing profile, nullable
|
||||
state TEXT NOT NULL, -- receiving | ready | failed
|
||||
captured_at TIMESTAMPTZ NOT NULL, -- client claim
|
||||
received_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- server truth; retention keys off this
|
||||
report_type TEXT NOT NULL, -- crash|anr|native_crash|hang|abnormal_exit|manual
|
||||
platform TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
crash_summary TEXT, -- truncated ≤ 8 KiB
|
||||
manifest JSONB NOT NULL, -- part 1 verbatim (≤ 64 KiB by construction)
|
||||
playback_session_ids TEXT[] NOT NULL DEFAULT '{}',
|
||||
blob_bucket TEXT,
|
||||
blob_key TEXT,
|
||||
blob_bytes BIGINT,
|
||||
uncompressed_bytes BIGINT,
|
||||
blob_sha256 TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX ON client_diagnostic_reports (lower(short_id));
|
||||
CREATE INDEX ON client_diagnostic_reports (user_id, received_at DESC);
|
||||
CREATE INDEX ON client_diagnostic_reports (received_at);
|
||||
```
|
||||
|
||||
Plain table, not partitioned — volume is orders of magnitude below
|
||||
`operational_logs`; the byte-heavy payload lives in object storage. Full
|
||||
stacks/traces live only in the archive; the row holds a truncated summary.
|
||||
|
||||
### Admin API
|
||||
|
||||
Under the acting-admin subtree (same placement as the `admin_logs.go`
|
||||
routes):
|
||||
|
||||
- `GET /api/v1/admin/diagnostics/reports` — list; filters: user, platform,
|
||||
type, date range, `short_id` (exact, case-insensitive); paginated.
|
||||
- `GET /api/v1/admin/diagnostics/reports/{id}` — row incl. manifest.
|
||||
- `GET /api/v1/admin/diagnostics/reports/{id}/download` — presigned GET URL
|
||||
(primary, matching avatar behavior) with a streamed `GetObject` proxy
|
||||
fallback for deployments where presigned URLs aren't reachable — the
|
||||
fallback is new behavior, not something the avatar path provides.
|
||||
- `DELETE /api/v1/admin/diagnostics/reports/{id}` — row + blob.
|
||||
|
||||
Admin download and delete emit audit/operational events. Web admin gets a
|
||||
list/detail/download page next to the logs viewer, with a
|
||||
playback-session-id link into the existing filtered logs view. Native apps
|
||||
add no admin screens in v1 (product policy).
|
||||
|
||||
### Retention + reconciliation
|
||||
|
||||
A taskmanager task `internal/taskmanager/tasks/cleanup_client_diagnostics.go`
|
||||
mirroring `cleanup_operational_log.go`, registered in `cmd/silo/main.go`
|
||||
with the other cleanup tasks:
|
||||
|
||||
- Delete reports older than `diagnostics.retention_days` (by `received_at`),
|
||||
then enforce `diagnostics.max_bytes_per_user` oldest-first. Blob deleted
|
||||
first (tolerating already-missing objects), then the row; deletes are
|
||||
idempotent.
|
||||
- Reconcile: remove `receiving`/`failed` rows older than a grace window,
|
||||
and (via `ListObjects`) any `diagnostics/*` objects whose row is absent,
|
||||
`failed`, or stale-`receiving` — so partial failures and process deaths
|
||||
mid-request never leak storage.
|
||||
- Account deletion cascades rows (FK) ; the reconciler then removes orphaned
|
||||
blobs.
|
||||
|
||||
## Android design (`silo-android`)
|
||||
|
||||
New package
|
||||
`android-shared/src/androidMain/kotlin/org/siloserver/silo/common/diagnostics/`.
|
||||
|
||||
### Logging
|
||||
|
||||
- `SiloLog.kt` — safe-logging facade per the contract above:
|
||||
`SiloLog.d(cat, tag, msg, attrs)` (+`i/w/e`, sanitized `Throwable`).
|
||||
Always forwards to `android.util.Log` (today's behavior unchanged) and
|
||||
offers the entry to the ring buffer.
|
||||
- `LogRing.kt` — always-on in-memory ring: fixed-size array of pre-rendered
|
||||
compact entries, **non-blocking** writes (atomic index; a torn final entry
|
||||
during crash snapshot is acceptable), ~4000 entries / ~1.5 MB cap, lazy
|
||||
attribute formatting off the hot path, dropped-entry counter. Must add no
|
||||
measurable contention to playback; a playback benchmark gate accompanies
|
||||
the PR that wires player events in.
|
||||
- `DiagnosticsFileLogger.kt` — only while debug logging is on: a
|
||||
single-writer coroutine draining a bounded channel (drop + count under
|
||||
backpressure) into **append-only framed segment files** (5 × 2 MB,
|
||||
app-private `files/diagnostics/logs/`, `noBackup`), compressed only at
|
||||
bundle-build time — a process death never corrupts more than the tail of
|
||||
the active segment (unlike a single long-lived gzip stream).
|
||||
- Playback wiring: each `PlaybackAnalyticsListener` event plus a ~5s periodic
|
||||
`PlayerStatsSnapshot` line while debug logging is on (reuses the existing
|
||||
reducer). TV focus logging hooks the TV app's focus observers under
|
||||
`cat=focus`. Network: a Ktor hook logging `method, templated path, status,
|
||||
duration_ms` only.
|
||||
|
||||
### Crash and abnormal-exit capture
|
||||
|
||||
- `CrashCapture.kt` — installed as early as possible in both `Application`
|
||||
classes; idempotent; captures the previous
|
||||
`Thread.getDefaultUncaughtExceptionHandler` once. On crash it does the
|
||||
minimum on the dying thread: render the stack (bounded), copy what the
|
||||
non-blocking ring exposes within a hard time budget, write **one bounded
|
||||
marker file** (≤ ~512 KiB) via temp-write + atomic rename, and invoke the
|
||||
prior handler in a `finally`. No locks shared with the ring's write path,
|
||||
no coroutines, no allocation-heavy JSON building (preformatted line
|
||||
buffer). Bundle assembly happens on next launch, never in the handler.
|
||||
At most 3 pending reports are kept (oldest dropped) as a crash-loop guard.
|
||||
- `ExitInfoCollector.kt` — API 30+ enhancement (minSdk is 24; many Fire TV
|
||||
devices ship below API 30, where the handler is the only source): reads
|
||||
`ActivityManager.getHistoricalProcessExitReasons()` on launch.
|
||||
`REASON_CRASH`, `REASON_CRASH_NATIVE`, `REASON_ANR` become pending
|
||||
reports. Dedupe uses a persisted bounded **fingerprint set** (process
|
||||
name, pid, timestamp, reason, status, trace hash) — not a raw timestamp
|
||||
watermark, which loses same-stamp events and breaks on clock changes.
|
||||
`getTraceInputStream()` is treated as optional opaque bytes with a bounded
|
||||
read: ANR text → `crash/stack.txt`; API 31+ native tombstone protobuf →
|
||||
`crash/tombstone.pb` (never decoded into the manifest). UEH-captured
|
||||
events are correlated by fingerprint so the same death isn't reported
|
||||
twice.
|
||||
- Next foreground launch, per the consent model: Ask → aggregated,
|
||||
incident-specific prompt (one prompt covering N pending reports; default
|
||||
focus **Don't send** on TV; "Always send" requires a second confirmation;
|
||||
full-screen review flow on TV rather than a small dialog); Always →
|
||||
silent upload, at most one bundle per crash fingerprint per day;
|
||||
Never → purge. Per-fingerprint prompt suppression with backoff prevents
|
||||
crash-loop prompt fatigue. Pending reports expire after 7 days —
|
||||
visibly (the Diagnostics screen shows pending reports with expiry), not
|
||||
silently.
|
||||
|
||||
### Device snapshot
|
||||
|
||||
`DeviceSnapshotCollector.kt` assembles `device.json` from the existing
|
||||
probes plus the new accessors listed in the contract section (display mode
|
||||
list, `Build.DEVICE`/form factor, `AudioManager.getDevices()` enumeration,
|
||||
suppression-registry export). A compact snapshot is persisted with each
|
||||
crash marker (`provenance: pre_failure`) since post-restart audio/display
|
||||
state can differ from crash-time state.
|
||||
|
||||
### Settings + consent UI
|
||||
|
||||
- `DiagnosticsSettingsStore.kt` — DataStore. Deliberately **not** the
|
||||
per-profile pattern of `AndroidPlayerSettingsStore`: consent and pending
|
||||
reports are keyed by server+account (per the consent model), and debug
|
||||
logging is per-device. Holds consent records, crash-report mode,
|
||||
debug-logging flag, fingerprint set, session tracker, and status cache.
|
||||
- Phone: "Diagnostics" section in `SettingsScreen.kt`; TV: rail category in
|
||||
`TvSettingsScreen.kt`. For non-child profiles the section is **always
|
||||
visible** and state-aware rather than hidden when the server gate is off — it distinguishes
|
||||
`available` / `disabled by server` / `storage unavailable` / `offline`,
|
||||
and always shows local pending reports (count, size, expiry) with
|
||||
review/delete controls. Uploads are simply unavailable in non-available
|
||||
states; a report captured while the gate was off is not auto-uploaded
|
||||
later unless consent for that server+account exists.
|
||||
- Manual capture is the primary support flow and is stronger than a bare
|
||||
"send now": **Start diagnostic capture** (turns on verbose capture,
|
||||
shows an indicator) → user reproduces the issue → **Stop & review** →
|
||||
summary (categories, time range, sizes, destination) → send. A one-shot
|
||||
"Send diagnostics now" remains for quick cases but warns when only the
|
||||
in-memory ring is available ("this bundle contains only the last few
|
||||
minutes of basic logs").
|
||||
- After upload, the `short_id` is displayed and kept in a small sent
|
||||
history so a user can read it to their admin later.
|
||||
|
||||
### Upload
|
||||
|
||||
- `shared/commonMain/…/network/api/DiagnosticsApi.kt` — `getStatus()` and
|
||||
`uploadReport(manifestJson, bundleBytes)` via Ktor
|
||||
`MultiPartFormDataContent` (first multipart use in the client — verified
|
||||
none exists — written as a small reusable helper), using the existing
|
||||
`safeApiCall`/`ApiResult` pattern, wired through Koin like other `*Api`
|
||||
classes.
|
||||
- `DiagnosticsUploader.kt` — builds the tar.gz (marker + ring/segment files
|
||||
overlapping the incident window + breadcrumbs + snapshot), streams it to
|
||||
the endpoint, deletes the pending report only on 201, records `short_id`.
|
||||
On `quota_exceeded`/`too_large`/`disabled`/`storage_unavailable` it keeps
|
||||
the pending report and backs off per `retry_after`; on
|
||||
`unsupported_schema` it surfaces "server needs an update" rather than
|
||||
silently retrying until expiry.
|
||||
|
||||
## Apple design (`silo-apple`)
|
||||
|
||||
Mirrors Android with platform-native mechanisms, and with tvOS scoped
|
||||
honestly — **MetricKit diagnostic payloads (crash/hang) are iOS-only; the
|
||||
SDK marks them unavailable on tvOS**, and delivery is delayed (typically
|
||||
daily, while the app runs), not "next launch".
|
||||
|
||||
- **Ring + facade**: `DiagLog` implementing the same safe-logging contract.
|
||||
The player's existing `cmpLog` helper (`PlayerLog.swift`) is **extended**
|
||||
into a tee — it keeps `print()`ing (preserving `devicectl --console`
|
||||
capture on tvOS) and also feeds the ring; remaining direct `print()` call
|
||||
sites in `Screens/Player/` migrate to it curated, per the redaction
|
||||
contract. OSLog-based subsystems are harvested at bundle-build time via
|
||||
`OSLogStore(scope: .currentProcessIdentifier)` (deployment targets are
|
||||
iOS/tvOS 26, far above the OSLogStore 15+ floor), merged into
|
||||
`logs.jsonl` with provenance.
|
||||
- **Breadcrumb journal**: because MetricKit reports arrive in a *later*
|
||||
process, the in-memory ring is gone by delivery time. While crash
|
||||
reporting is enabled, a small append-only breadcrumb journal (bounded,
|
||||
rotating, crash-safe appends: lifecycle transitions, playback start/stop,
|
||||
route changes, screen changes) persists across runs and becomes
|
||||
`breadcrumbs.jsonl`, matched to a diagnostic by `capture_session_id` and
|
||||
time window. Snapshots taken at bundle-build time are labeled
|
||||
`provenance: post_restart`; the design never presents post-restart state
|
||||
as crash-time state.
|
||||
- **iOS crash/hang detection**: an `MXMetricManager` subscriber. Each
|
||||
received `MXCrashDiagnostic`/`MXHangDiagnostic` becomes its **own**
|
||||
pending report: raw diagnostic JSON preserved as `crash/metrickit.json`,
|
||||
deduped by a canonical hash of the diagnostic (not timestamps), carrying
|
||||
its own app version and reporting-period bounds. Prompt copy is
|
||||
type-specific ("crashed" vs "was not responding") and honest about timing
|
||||
("yesterday during playback"). Delivery is verified on physical devices;
|
||||
parsing is tested via injected fixtures.
|
||||
- **tvOS abnormal-exit detection**: no crash stacks in v1. A dirty-exit
|
||||
sentinel (marker written at launch, cleared on clean
|
||||
background/termination) plus the breadcrumb journal yields
|
||||
`type: abnormal_exit` reports — "Silo did not shut down cleanly last
|
||||
time" — with breadcrumbs and device snapshot but no stack. An in-process
|
||||
crash reporter for tvOS is explicitly deferred (see Non-goals).
|
||||
- **Device snapshot**: `AppleDeviceIdentity.current` +
|
||||
`ApplePlaybackV3Capabilities.snapshot()` for identity/codec/HDR context;
|
||||
audio reported as route metadata with hashed route UIDs and
|
||||
`passthrough: "unknown"` until a real output-capability probe exists (the
|
||||
current one is a private placeholder).
|
||||
- **Settings/consent**: Diagnostics section in `SettingsView` (iOS) and
|
||||
`TVSettingsView` (tvOS) with the same state-aware presentation, manual
|
||||
capture flow, and consent rules.
|
||||
- **Upload**: a small multipart body writer added to `HTTPClient` (verified:
|
||||
JSON-only today, no multipart/uploadTask), same status/error handling as
|
||||
Android.
|
||||
- Housekeeping note: product bundle IDs are already `org.siloserver.silo`;
|
||||
only legacy keychain/logging identifiers (`com.continuum.*`) remain, so
|
||||
nothing here blocks on rebranding.
|
||||
|
||||
## Privacy summary
|
||||
|
||||
- **Consent boundaries**: no network transmission of diagnostic data without
|
||||
prompt confirmation, a server+account-bound Always election, or an
|
||||
explicit manual send. The always-on ring is memory-only; persistent
|
||||
capture (segments, breadcrumbs, markers) is app-private, size-capped,
|
||||
backup-excluded, and purged by Never/sign-out/server removal.
|
||||
- **Never collected**: credentials, `Authorization`/profile-token values,
|
||||
request bodies, query strings, full URLs (normalized at collection),
|
||||
unsanitized exception dumps. Bearer-token matching at bundle time as
|
||||
defense in depth. Golden leak fixtures in CI.
|
||||
- **Deliberately collected** (disclosed in View report): device identity
|
||||
incl. device name, media titles appearing in playback logs, templated
|
||||
server paths. The recipient is the user's own server admin, who already
|
||||
has server-side access to this information; that is the trust model this
|
||||
feature is scoped to — and why third-party destinations are out.
|
||||
- **Persistent identifiers**: the existing device IDs (already sent to the
|
||||
server today) plus hashed audio-route UIDs; each is enumerated in the
|
||||
schema docs with its purpose and retention.
|
||||
- **User control**: state-aware Diagnostics screen with pending list,
|
||||
review, delete, sent history; admin-side deletion; account deletion
|
||||
cascades reports; admin access is acting-admin-only and audited.
|
||||
- **Bounded retention**: defaults 30 days / 200 MiB per user, admin-tunable,
|
||||
enforced server-side.
|
||||
|
||||
## Rollout plan (vertical slices, each independently shippable)
|
||||
|
||||
1. **Server foundation** — settings + storage-validated gate, status
|
||||
endpoint, hardened ingest with state machine, schema + fixtures, admin
|
||||
API, retention/reconciler task, operational audit events. Feature stays
|
||||
off; nothing user-visible.
|
||||
2. **Contract fixtures in clients** — JSON Schema + golden fixtures wired
|
||||
into Android/Apple CI; safe-logging facades + rings land **dark** (no
|
||||
UI, no capture persistence, no prompts).
|
||||
3. **Android end-to-end slice** — consent model + Diagnostics settings UI +
|
||||
crash marker/ExitInfo capture + device snapshot + uploader + manual
|
||||
capture flow, with the initial curated logging migration (player tags +
|
||||
TV focus + network hook + stats snapshots). First release where a user
|
||||
can be asked for, and send, a useful report.
|
||||
4. **Android coverage expansion** — remaining curated call-site migration,
|
||||
breadcrumbs-on-Android if wanted, prompt-fatigue tuning from the field.
|
||||
5. **Apple slice** — cmpLog tee + OSLogStore harvest + breadcrumbs +
|
||||
MetricKit (iOS) + exit sentinel (tvOS) + settings/consent + multipart
|
||||
upload.
|
||||
6. **Web admin viewer + correlation UX** — list/detail/download page,
|
||||
playback-session pivot into the logs viewer.
|
||||
7. **Later / separate decisions** — symbolication job, tvOS in-process
|
||||
crash reporter, native-app admin viewer, macOS.
|
||||
|
||||
Slices 3 and 5 ship complete consent→capture→upload flows; intermediate
|
||||
releases never prompt for reports they can't send.
|
||||
|
||||
## Open questions
|
||||
|
||||
1. Should users be able to delete their **uploaded** reports from the client
|
||||
(self-service deletion), or is pending-report control + admin deletion
|
||||
enough for v1?
|
||||
2. tvOS: is abnormal-exit detection (no stacks) acceptable for v1, or does
|
||||
the Shield/Fire TV debugging need justify evaluating an in-process crash
|
||||
reporter (PLCrashReporter/KSCrash) with its signal-safety and review
|
||||
trade-offs?
|
||||
3. Do we want breadcrumbs on Android in v1 (cheap once the facade exists),
|
||||
or is the UEH marker + ExitInfo enough there?
|
||||
4. Localization of prompt/consent strings at launch vs English-first.
|
||||
5. Exact quota defaults (20/day, 200 MiB/user, 10 MiB/bundle) — tune before
|
||||
enabling anywhere real.
|
||||
6. Whether the status payload should also fold into an existing server-driven
|
||||
config refresh response later (clients poll both today); the dedicated
|
||||
endpoint ships first either way.
|
||||
|
||||
## Review notes
|
||||
|
||||
This spec was adversarially reviewed before the draft PR: two independent
|
||||
review passes (a fact-check of every codebase claim against the three repos,
|
||||
and a design attack across consent, crash-capture correctness, ingest
|
||||
robustness, storage consistency, UX, and phasing). Material corrections
|
||||
adopted include: tvOS MetricKit unavailability and honest MetricKit timing;
|
||||
server+account-bound consent; storage-validated gating (private object store
|
||||
is optional in real deployments); streaming bounded ingest with a report
|
||||
state machine and orphan reconciliation; the archive container replacing a
|
||||
bare log gzip; collection-time typed redaction replacing scrub-at-package;
|
||||
fingerprint-based ExitInfo dedupe; non-blocking ring + bounded crash marker;
|
||||
recent-session trackers (only the active playback session ID exists today);
|
||||
and honest Apple snapshot capabilities (passthrough currently unknown).
|
||||
@@ -0,0 +1,5 @@
|
||||
The canonical client-diagnostics v1 contract lives in `v1/`.
|
||||
Clients vendor the schema files, attribute registry, and fixtures from this directory.
|
||||
Servers accept supported schema versions and ignore unknown additive fields.
|
||||
Within v1, changes are additive only: new optional fields, enum values after coordinated client support, and new allowlisted archive entries.
|
||||
Breaking changes require a new versioned contract directory and explicit server support.
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"$schema": "https://silo-server.dev/schemas/client-diagnostics/v1/attr-registry.schema.json",
|
||||
"schema_version": 1,
|
||||
"policy": "Attribute keys are registered per category. Debug builds and CI should fail unregistered keys; production clients drop unregistered keys before upload.",
|
||||
"categories": {
|
||||
"playback": {
|
||||
"sink": {
|
||||
"type": "string",
|
||||
"description": "Current audio output sink, such as HDMI or speaker."
|
||||
},
|
||||
"fmt": {
|
||||
"type": "string",
|
||||
"description": "Media or passthrough format identifier, such as truehd."
|
||||
},
|
||||
"decoder": {
|
||||
"type": "string",
|
||||
"description": "Selected decoder name."
|
||||
},
|
||||
"width": {
|
||||
"type": "integer",
|
||||
"description": "Video width in pixels."
|
||||
},
|
||||
"height": {
|
||||
"type": "integer",
|
||||
"description": "Video height in pixels."
|
||||
},
|
||||
"hdr_mode": {
|
||||
"type": "string",
|
||||
"description": "Active HDR mode or selected tone-mapping mode."
|
||||
},
|
||||
"bitrate_kbps": {
|
||||
"type": "integer",
|
||||
"description": "Observed or selected bitrate in kilobits per second."
|
||||
},
|
||||
"dropped_frames": {
|
||||
"type": "integer",
|
||||
"description": "Dropped video frame count."
|
||||
},
|
||||
"audio_underruns": {
|
||||
"type": "integer",
|
||||
"description": "Audio underrun count."
|
||||
}
|
||||
},
|
||||
"focus": {
|
||||
"target": {
|
||||
"type": "string",
|
||||
"description": "Focusable element or logical destination."
|
||||
},
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": "Focus action or navigation direction."
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"method": {
|
||||
"type": "string",
|
||||
"description": "HTTP method."
|
||||
},
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "Templated path without query string."
|
||||
},
|
||||
"status": {
|
||||
"type": "integer",
|
||||
"description": "HTTP status code."
|
||||
},
|
||||
"duration_ms": {
|
||||
"type": "integer",
|
||||
"description": "Request duration in milliseconds."
|
||||
}
|
||||
},
|
||||
"lifecycle": {
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": "App lifecycle state."
|
||||
}
|
||||
},
|
||||
"crash": {
|
||||
"fingerprint": {
|
||||
"type": "string",
|
||||
"description": "Client-side dedupe fingerprint."
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Crash source matching manifest crash.source."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://silo-server.dev/schemas/client-diagnostics/v1/device.schema.json",
|
||||
"title": "Silo Client Diagnostics Device Snapshot v1",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"captured_at",
|
||||
"provenance",
|
||||
"identity",
|
||||
"display",
|
||||
"audio",
|
||||
"video_codecs",
|
||||
"network"
|
||||
],
|
||||
"properties": {
|
||||
"captured_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"maxLength": 64
|
||||
},
|
||||
"provenance": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"pre_failure",
|
||||
"post_restart"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"identity": {
|
||||
"$ref": "#/$defs/tri_state_object"
|
||||
},
|
||||
"display": {
|
||||
"$ref": "#/$defs/tri_state_object"
|
||||
},
|
||||
"audio": {
|
||||
"$ref": "#/$defs/tri_state_object"
|
||||
},
|
||||
"video_codecs": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"maxItems": 128,
|
||||
"items": {
|
||||
"$ref": "#/$defs/open_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/sentinel"
|
||||
}
|
||||
]
|
||||
},
|
||||
"network": {
|
||||
"$ref": "#/$defs/tri_state_object"
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"sentinel": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"unknown",
|
||||
"not_collected"
|
||||
]
|
||||
},
|
||||
"tri_state_object": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/open_object"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/sentinel"
|
||||
}
|
||||
]
|
||||
},
|
||||
"open_object": {
|
||||
"type": "object",
|
||||
"maxProperties": 64,
|
||||
"additionalProperties": {
|
||||
"$ref": "#/$defs/tri_state_value"
|
||||
}
|
||||
},
|
||||
"tri_state_value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"maxLength": 512
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"maxItems": 128,
|
||||
"items": {
|
||||
"$ref": "#/$defs/tri_state_value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/open_object"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_bad_archive_entry",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android",
|
||||
"os_version": "15 (API 35)",
|
||||
"profile_id": "prof_tablet"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "manual",
|
||||
"notice_version": 1
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Google",
|
||||
"model": "Pixel Tablet",
|
||||
"os": "15",
|
||||
"form_factor": "tablet"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 1,
|
||||
"bytes_gz": 100,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"other"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json",
|
||||
"../secrets.txt"
|
||||
],
|
||||
"bytes": 100,
|
||||
"uncompressed_bytes": 200,
|
||||
"sha256": "3333333333333333333333333333333333333333333333333333333333333333"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_bad_schema",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android",
|
||||
"os_version": "15 (API 35)",
|
||||
"profile_id": "prof_tablet"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "manual",
|
||||
"notice_version": 1
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Google",
|
||||
"model": "Pixel Tablet",
|
||||
"os": "15",
|
||||
"form_factor": "tablet"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 1,
|
||||
"bytes_gz": 100,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"other"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json"
|
||||
],
|
||||
"bytes": 100,
|
||||
"uncompressed_bytes": 200,
|
||||
"sha256": "2222222222222222222222222222222222222222222222222222222222222222"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_missing_consent",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android",
|
||||
"os_version": "15 (API 35)",
|
||||
"profile_id": "prof_tablet"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Google",
|
||||
"model": "Pixel Tablet",
|
||||
"os": "15",
|
||||
"form_factor": "tablet"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 1,
|
||||
"bytes_gz": 100,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"other"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json"
|
||||
],
|
||||
"bytes": 100,
|
||||
"uncompressed_bytes": 200,
|
||||
"sha256": "1111111111111111111111111111111111111111111111111111111111111111"
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_macos_reserved",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "macos",
|
||||
"os_version": "26.0",
|
||||
"profile_id": "prof_mac"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "manual",
|
||||
"notice_version": 1
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Apple",
|
||||
"model": "Mac",
|
||||
"os": "26.0",
|
||||
"form_factor": "desktop"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 1,
|
||||
"bytes_gz": 100,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"other"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json"
|
||||
],
|
||||
"bytes": 100,
|
||||
"uncompressed_bytes": 200,
|
||||
"sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
||||
}
|
||||
}
|
||||
+54
File diff suppressed because one or more lines are too long
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "kernel_panic",
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_bad_type",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android-tv",
|
||||
"os_version": "11 (API 30)",
|
||||
"profile_id": "prof_living_room"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "prompt",
|
||||
"notice_version": 1
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "NVIDIA",
|
||||
"model": "SHIELD Android TV",
|
||||
"os": "11",
|
||||
"form_factor": "tv"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 1,
|
||||
"bytes_gz": 100,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"crash"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json"
|
||||
],
|
||||
"bytes": 100,
|
||||
"uncompressed_bytes": 200,
|
||||
"sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-19T23:45:00Z",
|
||||
"capture_session_id": "run_android_manual_preprofile_006",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android",
|
||||
"os_version": "15 (API 35)"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "manual",
|
||||
"notice_version": 1
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Google",
|
||||
"model": "Pixel Tablet",
|
||||
"os": "15",
|
||||
"form_factor": "tablet"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 12,
|
||||
"bytes_gz": 2048,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"lifecycle",
|
||||
"network"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json",
|
||||
"device.json",
|
||||
"logs.jsonl"
|
||||
],
|
||||
"bytes": 4096,
|
||||
"uncompressed_bytes": 16384,
|
||||
"sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-19T23:40:00Z",
|
||||
"capture_session_id": "run_android_manual_005",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android",
|
||||
"os_version": "15 (API 35)",
|
||||
"profile_id": "prof_tablet"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "manual",
|
||||
"notice_version": 1
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Google",
|
||||
"model": "Pixel Tablet",
|
||||
"os": "15",
|
||||
"form_factor": "tablet"
|
||||
},
|
||||
"playback_session_ids": [],
|
||||
"log_summary": {
|
||||
"lines": 60,
|
||||
"bytes_gz": 5400,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"browse",
|
||||
"network"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json",
|
||||
"device.json",
|
||||
"logs.jsonl"
|
||||
],
|
||||
"bytes": 8000,
|
||||
"uncompressed_bytes": 43000,
|
||||
"sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "crash",
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"capture_session_id": "run_android_tv_ueh_001",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "android-tv",
|
||||
"os_version": "11 (API 30)",
|
||||
"profile_id": "prof_living_room"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "prompt",
|
||||
"notice_version": 1
|
||||
},
|
||||
"crash": {
|
||||
"summary": "NullPointerException in PlaybackSessionManager.start",
|
||||
"stack_excerpt": "java.lang.NullPointerException: player was null\n\tat org.siloserver.silo.PlaybackSessionManager.start(PlaybackSessionManager.kt:42)",
|
||||
"thread": "main",
|
||||
"foreground": true,
|
||||
"source": "ueh",
|
||||
"provenance": "pre_failure",
|
||||
"occurred_at": "2026-07-19T18:22:31Z"
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "NVIDIA",
|
||||
"model": "SHIELD Android TV",
|
||||
"os": "11",
|
||||
"form_factor": "tv"
|
||||
},
|
||||
"playback_session_ids": [
|
||||
"ps_9f2a"
|
||||
],
|
||||
"log_summary": {
|
||||
"lines": 3841,
|
||||
"bytes_gz": 412034,
|
||||
"dropped_lines": 12,
|
||||
"categories": [
|
||||
"playback",
|
||||
"focus",
|
||||
"network"
|
||||
],
|
||||
"debug_logging": true
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json",
|
||||
"device.json",
|
||||
"logs.jsonl",
|
||||
"crash/summary.json",
|
||||
"crash/stack.txt"
|
||||
],
|
||||
"bytes": 498112,
|
||||
"uncompressed_bytes": 3110400,
|
||||
"sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"captured_at": "2026-07-19T18:22:31Z",
|
||||
"provenance": "pre_failure",
|
||||
"identity": {
|
||||
"manufacturer": "NVIDIA",
|
||||
"model": "SHIELD Android TV",
|
||||
"device": "mdarcy",
|
||||
"form_factor": "tv",
|
||||
"device_id": "dev_8f4a"
|
||||
},
|
||||
"display": {
|
||||
"mode": "3840x2160@59.94",
|
||||
"modes_supported": [
|
||||
"3840x2160@59.94",
|
||||
"1920x1080@60"
|
||||
],
|
||||
"hdr_types": [
|
||||
"HDR10",
|
||||
"HLG",
|
||||
"DV"
|
||||
],
|
||||
"wide_gamut": true
|
||||
},
|
||||
"audio": {
|
||||
"outputs": [
|
||||
{
|
||||
"type": "HDMI",
|
||||
"encodings": [
|
||||
"AC3",
|
||||
"EAC3",
|
||||
"TRUEHD"
|
||||
],
|
||||
"channels": 8
|
||||
}
|
||||
],
|
||||
"passthrough": [
|
||||
"AC3",
|
||||
"EAC3",
|
||||
"TRUEHD"
|
||||
],
|
||||
"suppressions": []
|
||||
},
|
||||
"video_codecs": [
|
||||
{
|
||||
"mime": "video/hevc",
|
||||
"hw": true,
|
||||
"profiles": [
|
||||
"Main10",
|
||||
"DV P8"
|
||||
],
|
||||
"max": "4096x2160@60"
|
||||
}
|
||||
],
|
||||
"network": {
|
||||
"transport": "ethernet"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "hang",
|
||||
"captured_at": "2026-07-19T21:15:00Z",
|
||||
"capture_session_id": "run_ios_metric_042",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "ios",
|
||||
"os_version": "26.0",
|
||||
"profile_id": "prof_phone"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "always",
|
||||
"notice_version": 1
|
||||
},
|
||||
"crash": {
|
||||
"summary": "Main thread hang reported by MetricKit",
|
||||
"stack_excerpt": "Thread 0 name: main\n0 Silo PlayerCore.waitForRouteChange",
|
||||
"thread": "main",
|
||||
"foreground": true,
|
||||
"source": "metrickit",
|
||||
"provenance": "metric_reporting_period",
|
||||
"occurred_at": "2026-07-19T21:15:00Z"
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Apple",
|
||||
"model": "iPhone17,2",
|
||||
"os": "26.0",
|
||||
"form_factor": "phone"
|
||||
},
|
||||
"playback_session_ids": [
|
||||
"ps_ios_103"
|
||||
],
|
||||
"log_summary": {
|
||||
"lines": 902,
|
||||
"bytes_gz": 84511,
|
||||
"dropped_lines": 0,
|
||||
"categories": [
|
||||
"playback",
|
||||
"lifecycle",
|
||||
"network"
|
||||
],
|
||||
"debug_logging": true
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json",
|
||||
"device.json",
|
||||
"logs.jsonl",
|
||||
"crash/summary.json",
|
||||
"crash/metrickit.json",
|
||||
"breadcrumbs.jsonl"
|
||||
],
|
||||
"bytes": 141120,
|
||||
"uncompressed_bytes": 812400,
|
||||
"sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"ts":"2026-07-19T18:22:29.412Z","run":"run_android_tv_ueh_001","lvl":"W","cat":"playback","tag":"AudioCapabilityMgr","msg":"passthrough suppressed: HDMI sink lost TrueHD","attrs":{"sink":"HDMI","fmt":"truehd"}}
|
||||
{"ts":"2026-07-19T18:22:30Z","run":"run_android_tv_ueh_001","lvl":"I","cat":"network","tag":"HTTPClient","msg":"request completed","attrs":{"method":"GET","path":"/api/v1/playback/{id}","status":200,"duration_ms":84}}
|
||||
{"ts":"2026-07-19T18:22:31Z","run":"run_android_tv_ueh_001","lvl":"D","cat":"focus","tag":"TvFocus","msg":"focus moved","attrs":{"target":"play-button","action":"right"}}
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "abnormal_exit",
|
||||
"captured_at": "2026-07-19T22:05:11Z",
|
||||
"capture_session_id": "run_tvos_exit_007",
|
||||
"app_version": "1.4.2",
|
||||
"app_build": "20841",
|
||||
"platform": "tvos",
|
||||
"os_version": "26.0",
|
||||
"profile_id": "prof_den"
|
||||
},
|
||||
"destination": {
|
||||
"server_instance_id": "srv_home_01"
|
||||
},
|
||||
"consent": {
|
||||
"mode": "prompt",
|
||||
"notice_version": 1
|
||||
},
|
||||
"crash": {
|
||||
"summary": "Silo did not shut down cleanly last time",
|
||||
"thread": "unknown",
|
||||
"foreground": true,
|
||||
"source": "exit_sentinel",
|
||||
"provenance": "post_restart",
|
||||
"occurred_at": "2026-07-19T22:05:11Z"
|
||||
},
|
||||
"device_summary": {
|
||||
"manufacturer": "Apple",
|
||||
"model": "Apple TV",
|
||||
"os": "26.0",
|
||||
"form_factor": "tv"
|
||||
},
|
||||
"playback_session_ids": [
|
||||
"ps_tvos_11",
|
||||
"ps_tvos_10"
|
||||
],
|
||||
"log_summary": {
|
||||
"lines": 120,
|
||||
"bytes_gz": 12044,
|
||||
"dropped_lines": 3,
|
||||
"categories": [
|
||||
"lifecycle",
|
||||
"playback"
|
||||
],
|
||||
"debug_logging": false
|
||||
},
|
||||
"archive": {
|
||||
"entries": [
|
||||
"manifest.json",
|
||||
"device.json",
|
||||
"logs.jsonl",
|
||||
"crash/summary.json",
|
||||
"breadcrumbs.jsonl"
|
||||
],
|
||||
"bytes": 30001,
|
||||
"uncompressed_bytes": 104000,
|
||||
"sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://silo-server.dev/schemas/client-diagnostics/v1/logline.schema.json",
|
||||
"title": "Silo Client Diagnostics Log Line v1",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"ts",
|
||||
"run",
|
||||
"lvl",
|
||||
"cat",
|
||||
"tag",
|
||||
"msg"
|
||||
],
|
||||
"properties": {
|
||||
"ts": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"maxLength": 64
|
||||
},
|
||||
"run": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"lvl": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"V",
|
||||
"D",
|
||||
"I",
|
||||
"W",
|
||||
"E"
|
||||
],
|
||||
"maxLength": 1
|
||||
},
|
||||
"cat": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"playback",
|
||||
"focus",
|
||||
"network",
|
||||
"lifecycle",
|
||||
"browse",
|
||||
"cast",
|
||||
"download",
|
||||
"crash",
|
||||
"other"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"tag": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"msg": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 2048
|
||||
},
|
||||
"attrs": {
|
||||
"type": "object",
|
||||
"description": "Attribute keys are registered in attr-registry.json. Production clients drop unregistered keys before upload.",
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"maxLength": 512
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"maxItems": 32,
|
||||
"items": {
|
||||
"type": "string",
|
||||
"maxLength": 256
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://silo-server.dev/schemas/client-diagnostics/v1/manifest.schema.json",
|
||||
"title": "Silo Client Diagnostics Manifest v1",
|
||||
"description": "Canonical manifest contract for a client diagnostics bundle. STRING LENGTHS: every `maxLength` in this schema is a character-count sanity bound. The server enforces the matching limit as a UTF-8 *byte* length (Go `len(string)`), so a value near a cap that contains multi-byte characters (CJK, emoji, accented text) can satisfy `maxLength` here yet be rejected at ingest. Treat each numeric cap as a byte budget for non-ASCII text. JSON Schema cannot express byte length, so this divergence is intentional and documented rather than encoded.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schema_version",
|
||||
"report",
|
||||
"destination",
|
||||
"consent",
|
||||
"device_summary",
|
||||
"playback_session_ids",
|
||||
"log_summary",
|
||||
"archive"
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "integer",
|
||||
"const": 1
|
||||
},
|
||||
"report": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"type",
|
||||
"captured_at",
|
||||
"capture_session_id",
|
||||
"app_version",
|
||||
"app_build",
|
||||
"platform",
|
||||
"os_version"
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"crash",
|
||||
"anr",
|
||||
"native_crash",
|
||||
"hang",
|
||||
"abnormal_exit",
|
||||
"manual"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"captured_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"maxLength": 64
|
||||
},
|
||||
"capture_session_id": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"app_version": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 64
|
||||
},
|
||||
"app_build": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 64
|
||||
},
|
||||
"platform": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"android",
|
||||
"android-tv",
|
||||
"ios",
|
||||
"tvos"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"os_version": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"profile_id": {
|
||||
"type": "string",
|
||||
"maxLength": 128
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"destination": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"server_instance_id"
|
||||
],
|
||||
"properties": {
|
||||
"server_instance_id": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"consent": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"mode",
|
||||
"notice_version"
|
||||
],
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"prompt",
|
||||
"always",
|
||||
"manual"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"notice_version": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"crash": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"summary",
|
||||
"source",
|
||||
"provenance",
|
||||
"occurred_at"
|
||||
],
|
||||
"properties": {
|
||||
"summary": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "Free-text crash summary; may contain non-ASCII. maxLength counts characters — the server enforces 8192 UTF-8 bytes (see the schema-level string-length note).",
|
||||
"maxLength": 8192
|
||||
},
|
||||
"stack_excerpt": {
|
||||
"type": "string",
|
||||
"description": "Free-text stack excerpt; may contain non-ASCII. maxLength counts characters — the server enforces 8192 UTF-8 bytes (see the schema-level string-length note).",
|
||||
"maxLength": 8192
|
||||
},
|
||||
"thread": {
|
||||
"type": "string",
|
||||
"description": "Thread name; may contain non-ASCII. maxLength counts characters — the server enforces 128 UTF-8 bytes (see the schema-level string-length note).",
|
||||
"maxLength": 128
|
||||
},
|
||||
"foreground": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"ueh",
|
||||
"exit_info",
|
||||
"metrickit",
|
||||
"exit_sentinel"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"provenance": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"pre_failure",
|
||||
"post_restart",
|
||||
"metric_reporting_period"
|
||||
],
|
||||
"maxLength": 32
|
||||
},
|
||||
"occurred_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"maxLength": 64
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"device_summary": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"manufacturer",
|
||||
"model",
|
||||
"os",
|
||||
"form_factor"
|
||||
],
|
||||
"properties": {
|
||||
"manufacturer": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "Device manufacturer; may contain non-ASCII. maxLength counts characters — the server enforces 128 UTF-8 bytes (see the schema-level string-length note).",
|
||||
"maxLength": 128
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "Device model; may contain non-ASCII (e.g. CJK). maxLength counts characters — the server enforces 128 UTF-8 bytes, so a 128-character multi-byte value can pass this schema yet be rejected at ingest (see the schema-level string-length note).",
|
||||
"maxLength": 128
|
||||
},
|
||||
"os": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "OS name/version; may contain non-ASCII. maxLength counts characters — the server enforces 128 UTF-8 bytes (see the schema-level string-length note).",
|
||||
"maxLength": 128
|
||||
},
|
||||
"form_factor": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "Device form factor; may contain non-ASCII. maxLength counts characters — the server enforces 64 UTF-8 bytes (see the schema-level string-length note).",
|
||||
"maxLength": 64
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
"type": "string",
|
||||
"maxLength": 256
|
||||
}
|
||||
},
|
||||
"playback_session_ids": {
|
||||
"type": "array",
|
||||
"maxItems": 20,
|
||||
"items": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
}
|
||||
},
|
||||
"log_summary": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"lines",
|
||||
"bytes_gz",
|
||||
"dropped_lines",
|
||||
"categories",
|
||||
"debug_logging"
|
||||
],
|
||||
"properties": {
|
||||
"lines": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"bytes_gz": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"dropped_lines": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"categories": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"maxItems": 9,
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"playback",
|
||||
"focus",
|
||||
"network",
|
||||
"lifecycle",
|
||||
"browse",
|
||||
"cast",
|
||||
"download",
|
||||
"crash",
|
||||
"other"
|
||||
],
|
||||
"maxLength": 32
|
||||
}
|
||||
},
|
||||
"debug_logging": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"archive": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entries",
|
||||
"bytes",
|
||||
"uncompressed_bytes",
|
||||
"sha256"
|
||||
],
|
||||
"properties": {
|
||||
"entries": {
|
||||
"type": "array",
|
||||
"description": "Tar member names in bundle order. The first entry MUST be `manifest.json`: the server reads and validates the leading member before any other and compares this order against the actual bundle. `prefixItems` pins index 0, while `contains` keeps `manifest.json` required for validators that do not implement `prefixItems`.",
|
||||
"uniqueItems": true,
|
||||
"minItems": 1,
|
||||
"maxItems": 8,
|
||||
"prefixItems": [
|
||||
{
|
||||
"const": "manifest.json"
|
||||
}
|
||||
],
|
||||
"contains": {
|
||||
"const": "manifest.json"
|
||||
},
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"manifest.json",
|
||||
"device.json",
|
||||
"logs.jsonl",
|
||||
"crash/summary.json",
|
||||
"crash/stack.txt",
|
||||
"crash/tombstone.pb",
|
||||
"crash/metrickit.json",
|
||||
"breadcrumbs.jsonl"
|
||||
],
|
||||
"maxLength": 64
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"uncompressed_bytes": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"sha256": {
|
||||
"type": "string",
|
||||
"pattern": "^[A-Fa-f0-9]{64}$",
|
||||
"maxLength": 64
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"allOf": [
|
||||
{
|
||||
"$comment": "Event reports (crash/anr/native_crash/hang/abnormal_exit) must carry the crash object; mirrors ValidateManifest in internal/diagnostics/contract/contract.go.",
|
||||
"if": {
|
||||
"properties": {
|
||||
"report": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"crash",
|
||||
"anr",
|
||||
"native_crash",
|
||||
"hang",
|
||||
"abnormal_exit"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"report"
|
||||
]
|
||||
},
|
||||
"then": {
|
||||
"required": [
|
||||
"crash"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"$comment": "Manual captures must not carry a crash object.",
|
||||
"if": {
|
||||
"properties": {
|
||||
"report": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "manual"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"report"
|
||||
]
|
||||
},
|
||||
"then": {
|
||||
"not": {
|
||||
"required": [
|
||||
"crash"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
# Playback Protocol v3 Server Implementation Plan
|
||||
|
||||
**Status:** Proposed
|
||||
**Scope:** `silo-server`, with coordinated contract fixtures in `silo-android`
|
||||
**Protocol owner:** Silo server
|
||||
**Client target:** Android Media3-only playback; legacy web and Apple playback remain supported
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Implement the server half of Silo playback protocol v3 so a client can report
|
||||
its current device and output capabilities and receive one complete executable
|
||||
plan for:
|
||||
|
||||
- authenticated original HTTP playback;
|
||||
- progressive or HLS remux;
|
||||
- video-copy with audio adaptation;
|
||||
- HLS video transcode;
|
||||
- subtitle render, conversion, or burn-in; or
|
||||
- a terminal `adaptation_unavailable` outcome.
|
||||
|
||||
The client must never choose FFmpeg recipes. The server must never claim that a
|
||||
route preserves Dolby Vision, HDR, lossless audio, or subtitle fidelity unless
|
||||
the source metadata, client/output capability, selected recipe, and installed
|
||||
tooling support that claim.
|
||||
|
||||
The normative behavior remains in the sibling Android repository:
|
||||
|
||||
- `silo-android/docs/playback/01-media3-only-player-architecture.md`
|
||||
- `silo-android/docs/playback/02-migration-compatibility-validation.md`
|
||||
- `silo-android/shared/src/commonMain/kotlin/org/siloserver/silo/model/playback/PlaybackProtocolV3.kt`
|
||||
|
||||
This document maps that contract onto the current server. It does not redefine
|
||||
the protocol.
|
||||
|
||||
## 2. Verified server baseline
|
||||
|
||||
The implementation should reuse the current playback stack rather than replace
|
||||
it:
|
||||
|
||||
| Existing behavior | Current owner | v3 implication |
|
||||
| --- | --- | --- |
|
||||
| Legacy start, authorization, profile checks, track preferences, session creation | `internal/api/handlers/playback.go` | Preserve the legacy branch byte-for-byte where possible; move shared operations behind helpers. |
|
||||
| Direct/remux/transcode choice | `internal/playback/resolver.go` | Replace its broad codec/container booleans with a v3 planner for v3 requests only. Do not change legacy decisions during the first rollout. |
|
||||
| Byte-range original delivery | `internal/playback/directplay.go`, `internal/api/handlers/stream.go` | Reuse as `original_http`. |
|
||||
| Progressive MP4 remux and audio-to-AAC | `internal/playback/remux.go`, `internal/api/handlers/stream.go` | Reuse as `server_remux_progressive`; make every transformation explicit in the plan. |
|
||||
| HLS copy/transcode, timelines, restart/reconstruct | `internal/playback/transcode.go`, `internal/playback/transcode_manager.go` | Reuse as `server_remux_hls` or `server_transcode_hls`. |
|
||||
| Local and remote transcode start | `HandleStartTranscode`, `internal/transcodenode/server.go`, `internal/nodepool` | Extract a server-callable starter so v3 can return the final manifest without a second client recipe request. |
|
||||
| Playback sessions, policy and limits | `internal/playback/session.go` | Extend session state with v3 attempt/plan identity; keep current admission controls authoritative. |
|
||||
| Source video/audio/subtitle metadata | `internal/models/media.go`, `internal/scanner/probe.go` | Most required fields already exist, including DV profile, BL compatibility ID, EL presence, color/range, bit depth, layout, and channels. MEL/FEL classification and transformation provenance are not yet represented. |
|
||||
| Subtitle extract/conversion/burn-in | `internal/playback/subtitles.go`, `subtitle_stream.go`, transcode arguments | Reuse the executors; add an explicit policy decision and artifact contract. |
|
||||
| Restart-safe signed stream recipes | `internal/playback/recipecard.go`, `internal/streamtoken` | Continue using signed recipe tokens; v3 plans describe the resulting URL and timeline. |
|
||||
|
||||
The largest correctness gaps in the current resolver are expected and must not
|
||||
be carried into v3:
|
||||
|
||||
- transcode-disabled incompatibility currently falls back to attempting direct
|
||||
playback instead of returning a terminal result;
|
||||
- codec support is modeled as a string union, without profile, level, bit depth,
|
||||
frame rate, range, sink layout, or subtitle-fidelity validation;
|
||||
- clients currently supply transcode codec, bitrate, segment, and burn-in
|
||||
recipe details through `/playback/transcode/start`;
|
||||
- the start response does not contain an authoritative effective recipe,
|
||||
validation claims, transformations, or timeline for every delivery;
|
||||
- no idempotent replan or attempt-scoped route-event API exists.
|
||||
|
||||
## 3. Architecture decision
|
||||
|
||||
### 3.1 Additive dispatch on the existing start endpoint
|
||||
|
||||
Keep `POST /api/v1/playback/start` for compatibility. Refactor its handler into
|
||||
a small protocol dispatcher:
|
||||
|
||||
1. Decode only an envelope containing `protocol_version`.
|
||||
2. When it equals `3`, decode strictly as `PlaybackStartRequestV3` and call the
|
||||
v3 orchestration service.
|
||||
3. When it is absent or not `3`, pass the original body to the existing legacy
|
||||
decoder and behavior.
|
||||
|
||||
Use buffered request bytes or `json.RawMessage`; never decode the body twice
|
||||
from the socket. Unknown fields remain tolerated for additive compatibility,
|
||||
but required v3 fields and closed enum values are validated explicitly.
|
||||
|
||||
Do not overwrite the existing request/response structs with a union of legacy
|
||||
and v3 fields. That would make missing nested capabilities silently look like a
|
||||
valid legacy request and recreate the current false-direct failure mode.
|
||||
|
||||
### 3.2 Introduce one v3 orchestration service
|
||||
|
||||
Add a service in the `internal/playback` domain that owns:
|
||||
|
||||
- input normalization and validation;
|
||||
- stable track resolution;
|
||||
- source and capability evaluation;
|
||||
- ordered candidate construction;
|
||||
- policy and capacity checks;
|
||||
- deterministic plan identity and loop prevention;
|
||||
- transport startup;
|
||||
- terminal outcome mapping;
|
||||
- replan idempotency; and
|
||||
- plan/session state transitions.
|
||||
|
||||
HTTP handlers authenticate, authorize, decode, call the service, and encode the
|
||||
result. They do not contain route-selection rules.
|
||||
|
||||
Recommended files:
|
||||
|
||||
```text
|
||||
internal/playback/protocol_v3.go wire/domain types and enums
|
||||
internal/playback/plan_v3.go candidate planner
|
||||
internal/playback/capabilities_v3.go source/capability validators
|
||||
internal/playback/recipe_v3.go normalized executable recipes
|
||||
internal/playback/plan_key_v3.go canonical IDs and attempt keys
|
||||
internal/playback/tracks_v3.go stable track identities/remapping
|
||||
internal/playback/subtitle_policy_v3.go subtitle decision policy
|
||||
internal/playback/transformations_v3.go installed-tool/recipe registry
|
||||
internal/playback/protocol_v3_test.go
|
||||
internal/playback/testdata/protocol_v3/ golden contract/planner fixtures
|
||||
|
||||
internal/playback/planstore/store.go persistence interfaces
|
||||
internal/playback/planstore/postgres.go attempts, replans and route events
|
||||
|
||||
internal/api/handlers/playback_v3.go start/replan/event handlers
|
||||
internal/api/handlers/playback_transport.go reusable transport starter
|
||||
internal/api/handlers/playback_v3_test.go
|
||||
```
|
||||
|
||||
If implementation reveals that the wire and domain structs need different
|
||||
types, keep JSON structs in `handlers` and convert at the boundary. Do not add
|
||||
JSON tags throughout unrelated session and FFmpeg types merely to save mapping
|
||||
code.
|
||||
|
||||
### 3.3 Reconcile additive client-contract gaps before freezing v3
|
||||
|
||||
The current Kotlin v3 types are sufficient to negotiate and execute a plan,
|
||||
but they do not yet carry every fact the normative direct-play rules require.
|
||||
The server cannot manufacture those facts. Phase 0 therefore includes a
|
||||
coordinated additive Android contract change:
|
||||
|
||||
| Current gap | Required additive contract |
|
||||
| --- | --- |
|
||||
| Video capability is a codec-name list plus a global maximum resolution. | Add per-codec decode entries with supported profiles/levels, bit depths, maximum width/height/frame rate/bitrate, and hardware/software status. |
|
||||
| Audio passthrough is a codec-name list plus one global maximum channel count. | Add per-codec supported channel counts/layouts; retain the old fields for tolerant readers. |
|
||||
| A plan does not expose requested versus effective media file when alternate-version selection occurs. | Add requested/effective media file IDs and a normalized source descriptor to the plan. |
|
||||
| Subtitle decision does not expose the effective fidelity policy. | Add `subtitle_fidelity_policy` beside mode/artifact. |
|
||||
| Route events cannot explicitly report replan identity, local PCM recovery, retry outcome, decoder timing, or requested/effective quality. | Add optional structured fields or a versioned, bounded diagnostic schema; the server enriches delivery/recipe/source facts from stored plan state rather than trusting duplicates from the client. |
|
||||
| Android's profile-quality canonicalizer passes unknown stored labels through and does not produce `original`. | Canonicalize the closed v3 values and aliases client-side; keep server normalization tolerant for rolling upgrades. |
|
||||
|
||||
Gate the detailed behavior with additive client feature tokens such as
|
||||
`detailed_decode_capabilities`, `layout_aware_passthrough`, and
|
||||
`playback_route_diagnostics`. A v3 client missing them remains parseable, but
|
||||
unknown profile/layout facts cannot satisfy a strict direct claim. The planner
|
||||
must choose a conservative adapted route or terminal result.
|
||||
|
||||
Freeze the exact JSON field names only after Go/Kotlin golden fixtures exist.
|
||||
Do not implement a server-only guess and call the contract complete.
|
||||
|
||||
The response engine mapping is fixed:
|
||||
|
||||
| Delivery | `engine` |
|
||||
| --- | --- |
|
||||
| `original_http` | `media3_direct` |
|
||||
| `server_remux_progressive` | `media3_progressive_remux` |
|
||||
| `server_remux_hls` | `media3_hls` |
|
||||
| `server_transcode_hls` | `media3_hls` |
|
||||
|
||||
Android currently accepts header refresh modes `none` and `session`; do not
|
||||
emit `refresh_endpoint` until the client implements it.
|
||||
|
||||
Never emit legacy/client-owned runtime values in a v3 plan:
|
||||
`mpv_direct`, `client_local_loopback`, `external_player`, or
|
||||
`client_local_normalization`. They remain decode-only compatibility values on
|
||||
Android.
|
||||
|
||||
### 3.4 Keep v3 dark until the complete route is executable
|
||||
|
||||
Add a dynamic server setting `playback.protocol_v3_enabled`, default `false`.
|
||||
Read it through `PlaybackSettingsReader.Get` on each capability/start request,
|
||||
as the existing `allow_4k_transcode` path does. Do not load it only through
|
||||
`internal/config/db_loader.go`; that would turn rollback into a restart-required
|
||||
change.
|
||||
Also add `GET /api/v1/playback/capability` because `/api/v1` features use
|
||||
capability detection rather than server-version sniffing.
|
||||
|
||||
When a v3 start arrives while disabled, return a successful negotiation
|
||||
response with no allocated session and without `playback_plan_v3` in
|
||||
`server_features`. This intentionally triggers Android's
|
||||
`server_upgrade_required` path without leaking a legacy session.
|
||||
|
||||
The capability response should contain:
|
||||
|
||||
```json
|
||||
{
|
||||
"enabled": false,
|
||||
"protocol_versions": [3],
|
||||
"features": [],
|
||||
"deliveries": [],
|
||||
"transformations": [],
|
||||
"reason": "disabled"
|
||||
}
|
||||
```
|
||||
|
||||
Only advertise `playback_plan_v3` after every enabled delivery and
|
||||
transformation passes Phase 0 fixtures on the deployed build.
|
||||
|
||||
## 4. HTTP contract and ownership
|
||||
|
||||
### 4.1 Endpoints
|
||||
|
||||
| Endpoint | Success | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| `GET /api/v1/playback/capability` | `200` | Feature/protocol and installed transformation discovery. |
|
||||
| `POST /api/v1/playback/start` | `201` | Authorize source, create a session, select/start a route, return playable or terminal v3 response. |
|
||||
| `POST /api/v1/playback/{session_id}/replan` | `200` | Idempotently replace the active plan at the supplied media position. |
|
||||
| `POST /api/v1/playback/route-events` | `202` | Validate and enqueue attempt-scoped diagnostics. |
|
||||
|
||||
All four routes use normal account authentication. Start, replan, and route
|
||||
events require profile auth. `X-Profile-Id` remains authoritative; a body
|
||||
`profile_id` must match it. Replan additionally verifies session ownership and
|
||||
that `playback_attempt_id` belongs to the session.
|
||||
|
||||
Playable and `adaptation_unavailable` are protocol outcomes, not HTTP errors.
|
||||
Use HTTP errors only for transport/API failures such as malformed input,
|
||||
unauthorized access, missing files, or an idempotency conflict.
|
||||
|
||||
### 4.2 Strict request validation
|
||||
|
||||
Reject before allocating a session when any of these fail:
|
||||
|
||||
- protocol is not exactly `3`;
|
||||
- `playback_attempt_id` is absent or not a bounded UUID/ULID-like identifier;
|
||||
- file/profile authorization fails;
|
||||
- track ID and fallback index disagree;
|
||||
- output-route generation is negative or disagrees with the nested output
|
||||
context;
|
||||
- codec/container/feature lists exceed bounded counts or string lengths;
|
||||
- bandwidth, bitrate, resolution, attempt count, or position is outside sane
|
||||
bounds; or
|
||||
- a replan's failed plan is not the session's active plan.
|
||||
|
||||
Normalize codecs, containers, layouts, and dynamic-range labels once at the
|
||||
boundary. Accept bounded known quality aliases such as `4k`; fold an unknown
|
||||
rolling-upgrade quality value to `auto` with a decision warning instead of
|
||||
hard-failing playback. Preserve the original values only in bounded
|
||||
diagnostics.
|
||||
|
||||
### 4.3 Complete response invariant
|
||||
|
||||
A `playable` response is valid only after its transport can be fetched or has
|
||||
successfully entered its startup state. It includes:
|
||||
|
||||
- protocol and feature negotiation;
|
||||
- stable `session_id` and deterministic `plan_id`;
|
||||
- delivery and Media3 engine;
|
||||
- final URL, stream protocol, container/MIME, scoped headers, and supported
|
||||
header-refresh mode;
|
||||
- exact player/source timeline mapping and seek window;
|
||||
- selected stable audio/subtitle identities;
|
||||
- effective codec, range, dimensions, frame rate, bitrate, channels and layout;
|
||||
- route validation claims;
|
||||
- subtitle mode/artifact;
|
||||
- named transformations and degradation warnings; and
|
||||
- a stable decision reason code.
|
||||
|
||||
Never return a placeholder transcode URL and expect v3 Android to call the
|
||||
legacy transcode-start endpoint.
|
||||
|
||||
## 5. Domain invariants
|
||||
|
||||
### 5.1 Stable tracks
|
||||
|
||||
Use the current Android-compatible identity initially:
|
||||
|
||||
```text
|
||||
file:{media_file_id}:audio:{ffmpeg_audio_ordinal}
|
||||
file:{media_file_id}:subtitle:{combined_subtitle_ordinal}
|
||||
```
|
||||
|
||||
Freeze the combined subtitle ordering used by `buildSubtitleURLs`:
|
||||
|
||||
1. external subtitles in stored order, starting at zero;
|
||||
2. embedded subtitles at `len(external) + embedded_ordinal`; skipped legacy
|
||||
bitmap entries retain their ordinal hole; and
|
||||
3. downloaded subtitles in repository `created_at` order at
|
||||
`len(external) + len(all_embedded) + downloaded_ordinal`.
|
||||
|
||||
The identity is paired with the effective media file ID. Centralize generation,
|
||||
parsing, URL mapping, and alternate-version remapping in `tracks_v3.go`, and
|
||||
freeze the exact inventory in Phase 0 fixtures. A plan that switches file
|
||||
version must return the effective file's identity after matching the requested
|
||||
track by signature. Do not echo a requested-file track ID with an
|
||||
effective-file index.
|
||||
|
||||
Longer term, scanner-persisted immutable stream IDs can replace ordinal IDs in a
|
||||
future protocol version; do not change v3 identity semantics after release.
|
||||
|
||||
### 5.2 Deterministic plan IDs and loop prevention
|
||||
|
||||
The Android attempt key currently includes `plan_id`. Therefore a random plan
|
||||
ID would let the same failed recipe reappear under a new key and defeat loop
|
||||
prevention.
|
||||
|
||||
Derive `plan_id` deterministically within a playback attempt from:
|
||||
|
||||
- `playback_attempt_id`;
|
||||
- effective file ID;
|
||||
- delivery and stream protocol/container;
|
||||
- normalized video/audio recipe;
|
||||
- effective track IDs;
|
||||
- subtitle mode/artifact type;
|
||||
- sorted transformation names; and
|
||||
- policy/recipe version.
|
||||
|
||||
Do not include expiring URLs, JWTs, timestamps, node hostnames, or the output
|
||||
route generation in `plan_id`. The same effective recipe has the same plan ID;
|
||||
an output-route change remains distinguishable because Android includes
|
||||
`output_route_generation` in its attempt key.
|
||||
|
||||
Implement the client's FNV-1a 64-bit `v3:<16-lowercase-hex>` attempt-key
|
||||
algorithm in Go. The UTF-8 canonical string is exactly:
|
||||
|
||||
```text
|
||||
plan_id
|
||||
|KOTLIN_DELIVERY_ENUM_NAME
|
||||
|KOTLIN_STREAM_PROTOCOL_ENUM_NAME
|
||||
|lowercase_container_or_empty
|
||||
|lowercase_video_codec_or_empty
|
||||
|lowercase_audio_codec_or_empty
|
||||
|(width_or_0)x(height_or_0)
|
||||
|bitrate_kbps_or_0
|
||||
|lowercase_dynamic_range_or_empty
|
||||
|KOTLIN_SUBTITLE_MODE_ENUM_NAME
|
||||
|comma_joined_transformation_names_sorted_by_name
|
||||
|output_route_generation
|
||||
|comma_joined_local_mutations_sorted_lexically
|
||||
```
|
||||
|
||||
The newlines above are explanatory only; the hashed value is one string joined
|
||||
by literal `|` characters. Enum components are Kotlin constant names such as
|
||||
`ORIGINAL_HTTP`, `HTTP_PROGRESSIVE`, `HLS`, and `BURN_IN`, not lowercase wire
|
||||
tokens. Hash with offset basis `0xcbf29ce484222325` and prime
|
||||
`0x100000001b3`, then zero-pad to 16 lowercase hex digits.
|
||||
|
||||
Before committing a candidate, compute its base key for the request's output
|
||||
route and reject it when present in `attempted_plan_keys`. Canonical fixtures
|
||||
must be generated by the checked-in Kotlin implementation and consumed as
|
||||
opaque expected values by Go tests; a Go-generated fixture would make the
|
||||
parity test circular. Include transformation-order, empty/default-field,
|
||||
output-route, and local PCM-mutation cases.
|
||||
|
||||
Server-side ordered candidate IDs are not security tokens.
|
||||
|
||||
### 5.3 Session and replacement semantics
|
||||
|
||||
Treat replan as one replacement transaction, serialized with the per-session
|
||||
lifecycle lock:
|
||||
|
||||
1. validate policy, source, candidate and tooling without touching the active
|
||||
route;
|
||||
2. reserve the active session's existing stream/transcode capacity slot so the
|
||||
replacement is neither double-counted nor denied by its own predecessor;
|
||||
3. start the successor in a plan-scoped staging directory/transport generation,
|
||||
or under an explicitly replacement-linked new session ID;
|
||||
4. wait until the successor URL is fetchable or its startup state is validated;
|
||||
5. atomically commit effective file, tracks, recipe, plan ID, timeline and
|
||||
signed route;
|
||||
6. close and reap the predecessor; and
|
||||
7. release or transfer the capacity reservation exactly once.
|
||||
|
||||
If any pre-commit step fails, leave the old route and session usable and return
|
||||
a typed terminal/retryable result. Never run two FFmpeg writers against the
|
||||
same output directory. Progressive request-scoped transports may drain until
|
||||
the old client request disconnects; they do not require a destructive
|
||||
close-first swap.
|
||||
|
||||
Prefer retaining the same logical playback session so progress, scrobble and
|
||||
admin history remain continuous. If the transport design requires a new public
|
||||
session ID, return it explicitly and keep the old ID alive until the successor
|
||||
is ready. Android will stop the old ID afterward and ignore stop failures. Keep
|
||||
the legacy stop endpoint's behavior unchanged; internal v3 replacement and
|
||||
cleanup paths must treat an already-stopped session as success even though the
|
||||
current `SessionManager.StopSession` returns `ErrSessionNotFound`.
|
||||
|
||||
### 5.4 Replan idempotency
|
||||
|
||||
Create a timestamped Goose migration, generated with:
|
||||
|
||||
```text
|
||||
make migrate-create NAME=add_playback_protocol_v3
|
||||
```
|
||||
|
||||
Add short-lived persistence for:
|
||||
|
||||
- playback attempt/session/current-plan state;
|
||||
- `(session_id, replan_request_id)` request digest, state, lease and serialized
|
||||
response; and
|
||||
- route events.
|
||||
|
||||
Current native restart reconstruction is carried by signed stream tokens, not
|
||||
a durable plan database. This plan store is the first persisted v3 control-
|
||||
plane state; it complements rather than replaces recipe tokens and therefore
|
||||
needs explicit expiry/cleanup and token-to-plan reconciliation tests.
|
||||
|
||||
For a duplicate replan ID:
|
||||
|
||||
- same digest and completed state returns the cached response;
|
||||
- same digest and active lease waits on/single-flights the owner, then returns
|
||||
the response;
|
||||
- different digest returns `409 idempotency_key_reused`; and
|
||||
- an expired lease may be reclaimed after reconciling the session's current
|
||||
recipe/plan state.
|
||||
|
||||
Store only the minimum response needed for replay and expire it with the
|
||||
playback attempt. Never put source paths or authorization headers in route
|
||||
events. Signed URLs in cached responses inherit the attempt TTL and are removed
|
||||
by cleanup.
|
||||
|
||||
## 6. Planner rules
|
||||
|
||||
### 6.1 Candidate order
|
||||
|
||||
Build candidates in least-destructive order, but validate every candidate
|
||||
independently:
|
||||
|
||||
1. `original_http`;
|
||||
2. `server_remux_progressive` when the container alone is incompatible and
|
||||
progressive seek semantics are acceptable;
|
||||
3. `server_remux_hls` when HLS packaging is required;
|
||||
4. `server_remux_hls` with copied video plus audio adaptation, with the audio
|
||||
change named in the effective recipe, transformations, claims and warnings;
|
||||
5. `server_transcode_hls`; and
|
||||
6. terminal `adaptation_unavailable`.
|
||||
|
||||
Remux is not mandatory before transcode. Skip directly to the first valid
|
||||
candidate. A selected subtitle may independently add an artifact or force
|
||||
burn-in.
|
||||
|
||||
### 6.2 Direct eligibility
|
||||
|
||||
Original delivery requires all of:
|
||||
|
||||
- validated source file and byte-range path;
|
||||
- client container extractor support;
|
||||
- selected video codec/profile/level/bit depth/resolution/frame rate/bitrate;
|
||||
- compatible source dynamic range and current display output;
|
||||
- selected audio local decode or exact sink passthrough by codec and channel
|
||||
layout;
|
||||
- renderable selected subtitle at the required fidelity, or an independent
|
||||
converted artifact; and
|
||||
- no requested quality reduction.
|
||||
|
||||
Missing metadata is `unknown`, not compatible. A probe repair may run before
|
||||
planning, but the planner must terminal or adapt when required fields remain
|
||||
unknown.
|
||||
|
||||
### 6.3 Quality policy
|
||||
|
||||
Move target selection out of clients. Add a pure `QualityPolicy` that maps:
|
||||
|
||||
- `original` to no user-requested reduction, while still permitting mandatory
|
||||
compatibility adaptation;
|
||||
- a fixed rung to a maximum height with no upscaling; and
|
||||
- `auto` to the lowest of device maximum, bandwidth estimate with safety
|
||||
margin, user cap, metered policy, and administrator constraints.
|
||||
|
||||
Start with one server-owned ladder. Reconcile it with the existing web constants
|
||||
in `web/src/player/hooks/useTranscodeQuality.ts`; do not maintain two unrelated
|
||||
bitrate tables. The v3 result returns the actual height/bitrate and decision
|
||||
reason. Multi-variant ABR is explicitly outside this work.
|
||||
|
||||
### 6.4 Audio
|
||||
|
||||
Evaluate the selected track, not `MediaFile.CodecAudio` alone:
|
||||
|
||||
- exact sink passthrough codec plus channel count/layout permits audio copy and
|
||||
a `passthrough=true` claim;
|
||||
- client decode support permits copy with `passthrough=false`;
|
||||
- otherwise use video-copy/audio adaptation when permitted;
|
||||
- otherwise terminal with `transcoding_disabled`,
|
||||
`audio_conversion_unsupported`, or policy/capacity reason.
|
||||
|
||||
The effective recipe must report the actual output channels/layout. Do not call
|
||||
AAC stereo output Atmos-preserving. DTS core extraction, E-AC-3 conversion, or
|
||||
other lossy/core-only changes are named transformations with warnings.
|
||||
|
||||
### 6.5 HDR and Dolby Vision
|
||||
|
||||
Use `VideoTrack` fields already populated by the scanner: DV profile, BL
|
||||
compatibility ID, EL presence, range type, color metadata, bit depth, profile,
|
||||
level, and dimensions.
|
||||
|
||||
Add an explicit enhancement-layer classification:
|
||||
|
||||
```text
|
||||
none | mel | fel | unknown
|
||||
```
|
||||
|
||||
An analyzer interface may use an installed `dovi_tool`/libdovi implementation;
|
||||
without it, Profile 7 with an EL remains `unknown`. Unknown must never be
|
||||
reported as MEL, FEL, or validated native DV.
|
||||
|
||||
Represent transformations in a registry with:
|
||||
|
||||
- stable name and recipe version;
|
||||
- required binary/filter/hardware capability;
|
||||
- accepted source metadata;
|
||||
- promised output range;
|
||||
- argument builder;
|
||||
- validation fixture; and
|
||||
- failure-to-terminal mapping.
|
||||
|
||||
Initial entries should cover only paths proven by fixtures, for example:
|
||||
|
||||
- `dv_metadata_strip_to_hdr10` using a compatible HDR10 base layer;
|
||||
- `dv_p7_to_p8_1` only when the selected toolchain and fixture prove the output;
|
||||
- `hdr_to_sdr_tonemap` only when integrated and remote transcode paths both
|
||||
implement the same declared result.
|
||||
|
||||
FFmpeg's `dovi_rpu` bitstream filter can strip DV metadata without decoding,
|
||||
and `dovi_split` can separate Profile 7 layers
|
||||
([FFmpeg bitstream-filter documentation](https://ffmpeg.org/ffmpeg-bitstream-filters.html#dovi_005frpu)).
|
||||
[`dovi_tool`](https://github.com/quietvoid/dovi_tool) can inspect RPU metadata
|
||||
and convert compatible streams. Probe these capabilities at startup and expose
|
||||
only registered, available transformations. Do not silently run the current
|
||||
Profile 7 strip path and still claim Dolby Vision preservation.
|
||||
|
||||
The current progressive remux path is not v3-safe unchanged: when `dovi_rpu`
|
||||
is present it strips Profile 7 metadata automatically, while the no-filter path
|
||||
leaves the copied bitstream without that named adaptation. Refactor remux input
|
||||
to require an explicit transformation choice. A Profile 7 progressive remux is
|
||||
eligible only when it either preserves a fixture-validated native P7 stream or
|
||||
runs the registered `dv_metadata_strip_to_hdr10` transformation against a
|
||||
validated HDR10-compatible base layer. When the required filter/tool is
|
||||
missing, disqualify that remux candidate rather than emitting an ambiguous
|
||||
stream.
|
||||
|
||||
### 6.6 Subtitles
|
||||
|
||||
Resolve exactly one mode:
|
||||
|
||||
- `off`: no selected track;
|
||||
- `render`: return the source artifact when Media3 can render it with required
|
||||
fidelity;
|
||||
- `convert`: return a v3 artifact URL, MIME, format and timing origin;
|
||||
- `burn_in`: force video encoding and declare any HDR/range degradation; or
|
||||
- terminal when the selected subtitle cannot meet policy.
|
||||
|
||||
Map `preserve` to `require_authored_fidelity` and `compatible` to
|
||||
`allow_simplified_rendering`, subject to administrator policy. ASS/SSA styling,
|
||||
font attachments, PGS, VobSub and DVB must each have fixtures; do not infer all
|
||||
bitmap paths from one codec test.
|
||||
|
||||
Converted artifacts need session ownership, signed or authenticated fetches,
|
||||
bounded cache lifetime, and timing relative to the returned stream origin.
|
||||
|
||||
## 7. Transport orchestration refactor
|
||||
|
||||
Extract the body of `HandleStartTranscode` into a server-callable operation that
|
||||
accepts a validated normalized recipe and returns:
|
||||
|
||||
- final manifest URL;
|
||||
- effective local/remote node and hardware encoder;
|
||||
- effective file ID;
|
||||
- final codecs/resolution/bitrate/channels/range;
|
||||
- player/source timeline mapping;
|
||||
- seek-window semantics; and
|
||||
- typed terminal/startup failure.
|
||||
|
||||
Both the legacy endpoint and v3 service call this operation. The legacy handler
|
||||
continues accepting its existing body and maps it to the normalized recipe;
|
||||
there is no `/api/v1` breaking change.
|
||||
|
||||
Jellyfin compatibility is behaviorally out of scope for protocol v3. Its
|
||||
handlers currently start local/remote transcodes through parallel paths and a
|
||||
shared `TranscodeManager`, not through `HandleStartTranscode`. Preserve those
|
||||
wire and lifecycle paths during this work and keep their regression tests
|
||||
green. The extracted lower-level starter may be adopted by jellycompat only in
|
||||
a separately reviewed convergence change; do not half-migrate one of its local
|
||||
or remote branches.
|
||||
|
||||
Apply the same extraction to progressive remux startup where necessary. The v3
|
||||
planner chooses a recipe; the transport starter is not allowed to silently
|
||||
change it. If safety logic must change copy to encode—for example seeked HEVC or
|
||||
subtitle burn-in—the starter returns the effective normalized recipe so the
|
||||
plan, claims, warnings, recipe card, and FFmpeg process remain identical.
|
||||
|
||||
## 8. Route events and observability
|
||||
|
||||
Add `playback_route_events` with bounded columns for:
|
||||
|
||||
- playback/session/plan/plan-attempt identities;
|
||||
- event and failure classification;
|
||||
- fallback reason;
|
||||
- output-route generation;
|
||||
- sanitized diagnostics JSON;
|
||||
- user/profile/client metadata derived from auth and headers; and
|
||||
- received timestamp.
|
||||
|
||||
Accept only a closed event set initially:
|
||||
|
||||
```text
|
||||
plan_selected
|
||||
plan_invalidated
|
||||
plan_failed
|
||||
first_frame
|
||||
terminal
|
||||
stopped
|
||||
```
|
||||
|
||||
Allowlist diagnostic keys and cap key count, key/value length, body size, and
|
||||
events per user/attempt. Strip URLs, headers, tokens, file paths, free-form
|
||||
stack traces, and unknown keys. Enqueue/batch writes so telemetry never blocks
|
||||
playback planning.
|
||||
|
||||
Add exact release queries for:
|
||||
|
||||
- starts by delivery, recipe, dynamic range and client model;
|
||||
- first-frame rate and latency;
|
||||
- replan/failure rate by classification;
|
||||
- terminal reasons;
|
||||
- PCM retry and passthrough outcomes;
|
||||
- repeated-key/loop prevention; and
|
||||
- DV/HDR degradation transformations.
|
||||
|
||||
Operational logs include IDs and reason codes but no stream tokens.
|
||||
|
||||
## 9. Implementation sequence
|
||||
|
||||
Each phase should be a separate reviewable PR and link the playback capability
|
||||
epic/sub-issue.
|
||||
|
||||
### Phase 0 — Freeze the contract
|
||||
|
||||
- Add Go structs/enums mirroring the Kotlin v3 wire model.
|
||||
- Split only the start-envelope dispatch: protocol `3` returns the disabled
|
||||
no-session negotiation response, while absent/non-3 requests replay the
|
||||
buffered body through the unchanged legacy decoder.
|
||||
- Resolve the additive capability, effective-file, subtitle-policy and route-
|
||||
diagnostic gaps from section 3.3 in a coordinated Android change.
|
||||
- Add request/response JSON golden fixtures and canonical attempt-key fixtures.
|
||||
- Add strict validation tests, legacy-envelope tests, and unknown-field tests.
|
||||
- Add `GET /playback/capability` and the disabled negotiation response.
|
||||
- Keep `playback.protocol_v3_enabled=false`.
|
||||
|
||||
**Exit:** Go and Android decode the same fixtures and produce identical attempt
|
||||
keys; Android populates the detailed capability fields; legacy start tests are
|
||||
unchanged; disabled v3 starts allocate no legacy playback session.
|
||||
|
||||
### Phase 1 — Source facts and transformation registry
|
||||
|
||||
- Complete normalized source descriptors from `MediaFile` tracks.
|
||||
- Add MEL/FEL/unknown metadata and conservative probe behavior.
|
||||
- Add installed FFmpeg/dovi tool capability probes.
|
||||
- Define transformation registry and terminal reasons.
|
||||
- Add scanner/model migration or JSONB backfill handling as required.
|
||||
|
||||
**Exit:** every source fact used by the planner is present or explicitly
|
||||
unknown; no unavailable transformation is advertised.
|
||||
|
||||
### Phase 2 — Pure planner
|
||||
|
||||
- Implement capability normalization, stable tracks, quality policy, direct
|
||||
eligibility, candidate ordering, recipe claims, deterministic plan IDs and
|
||||
attempt-key rejection.
|
||||
- Cover direct, progressive remux, HLS copy, audio adaptation, full transcode,
|
||||
subtitles and terminal outcomes with table-driven tests.
|
||||
- Do not start sessions or FFmpeg from planner tests.
|
||||
|
||||
**Exit:** the full Phase 0 decision matrix passes as a pure deterministic
|
||||
function.
|
||||
|
||||
### Phase 3 — Start orchestration
|
||||
|
||||
- Wire the Phase 0 dispatcher's protocol-3 branch to the completed v3
|
||||
orchestration service; keep the legacy branch unchanged.
|
||||
- Extract reusable transcode/remux transport starters.
|
||||
- Start sessions and final transports from the v3 service.
|
||||
- Return complete plans with final URLs and timelines.
|
||||
- Persist attempt/current-plan state.
|
||||
|
||||
**Exit:** every playable fixture fetches and prepares its returned URL; a
|
||||
terminal fixture allocates no leaked session/transcode.
|
||||
|
||||
### Phase 4 — Replan and idempotency
|
||||
|
||||
- Add route, handler, persistent idempotency records and per-session
|
||||
serialization.
|
||||
- Replan track/quality/output changes and classified runtime failures.
|
||||
- Reuse session IDs when safe; close superseded transports exactly once.
|
||||
- Reject attempted recipes and stale failed plan IDs.
|
||||
- Add restart, duplicate request, concurrent request and crash-window tests.
|
||||
|
||||
**Exit:** repeated identical replan requests return the same result without a
|
||||
second FFmpeg process; a different payload under the same ID conflicts.
|
||||
|
||||
### Phase 5 — Route events and rollout controls
|
||||
|
||||
- Add sanitized asynchronous route-event ingestion and retention cleanup.
|
||||
- Add release queries/dashboards and structured plan logs.
|
||||
- Add shadow planning for legacy starts on validation servers: compute and log
|
||||
v3 choice without advertising or executing it.
|
||||
- Compare v3 shadow decisions with current production routes.
|
||||
|
||||
**Exit:** telemetry is queryable, bounded, privacy-reviewed and cannot delay a
|
||||
start/replan response.
|
||||
|
||||
### Phase 6 — Hardware and multi-repository validation
|
||||
|
||||
- Run original/remux/transcode fixtures locally and through proxy/transcode
|
||||
nodes.
|
||||
- Verify restart reconstruction, session limits, transcode-disabled policy,
|
||||
alternate versions and output-route changes.
|
||||
- Run Android Shield tests first on SDR/1080p, then the named 4K DV/AVR chain.
|
||||
- Record plan, route events, decoder, TV range indicator and AVR format for the
|
||||
same attempt.
|
||||
- Publish minimum server revision and enable v3 on validation deployments.
|
||||
|
||||
**Exit:** all Phase 0 fixtures pass on the deployed revision and Android no
|
||||
longer reports `server_upgrade_required`.
|
||||
|
||||
## 10. Test matrix
|
||||
|
||||
At minimum, include:
|
||||
|
||||
### Contract and compatibility
|
||||
|
||||
- absent/2/3/unknown protocol versions;
|
||||
- disabled and enabled feature negotiation;
|
||||
- v3 nested capabilities are consumed, not silently dropped;
|
||||
- legacy start response and HTTP status remain unchanged;
|
||||
- old web/Apple transcode-start requests still work;
|
||||
- malformed plan/terminal responses cannot be emitted.
|
||||
|
||||
### Direct and adaptation
|
||||
|
||||
- H.264/AAC MP4 direct;
|
||||
- HEVC MKV original when the exact Media3 envelope supports it;
|
||||
- container-only progressive and HLS remux;
|
||||
- unsupported TrueHD/DTS layout with video copy and audio conversion;
|
||||
- resolution/user-cap quality transcode;
|
||||
- transcode disabled, user denied, capacity unavailable, missing tool and
|
||||
policy denied terminal results;
|
||||
- alternate lower-resolution file with stable track remapping.
|
||||
|
||||
### HDR/DV
|
||||
|
||||
- HDR10, HDR10+, HLG and SDR direct gates;
|
||||
- DV profiles 5, 7 MEL, 7 FEL, 7 unknown and relevant profile 8 variants;
|
||||
- BL compatibility ID and range mismatch;
|
||||
- native DV, validated HDR10 base-layer fallback and SDR tone map;
|
||||
- missing `dovi_rpu`/`dovi_split`/dovi-tool behavior;
|
||||
- no transformation claims when output was not validated.
|
||||
|
||||
### Audio and subtitles
|
||||
|
||||
- local decode versus passthrough by exact channels/layout;
|
||||
- E-AC-3 JOC, TrueHD, DTS-HD/core and PCM fallback replan;
|
||||
- text sidecar/embedded, ASS with/without authored fidelity, PGS, VobSub, DVB,
|
||||
conversion and burn-in;
|
||||
- artifact timing after seeked remux/transcode;
|
||||
- burn-in forces encode and reports HDR degradation.
|
||||
|
||||
### Recovery and load
|
||||
|
||||
- transport/decoder failures, track/quality/output invalidation;
|
||||
- deterministic same-recipe loop rejection;
|
||||
- idempotent duplicate and concurrent replans;
|
||||
- API and transcode-node restart between plan and first manifest;
|
||||
- session expiry, explicit stop and terminal cleanup;
|
||||
- no double capacity accounting or orphan FFmpeg process;
|
||||
- route-event flood/body/diagnostic limits.
|
||||
|
||||
## 11. Verification commands
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
```text
|
||||
GOWORK=off go test ./internal/playback ./internal/api/handlers ./internal/transcodenode ./internal/nodepool
|
||||
GOWORK=off go test -race ./internal/playback ./internal/api/handlers
|
||||
make verify-local-paths
|
||||
make lint
|
||||
make build
|
||||
```
|
||||
|
||||
Add a focused integration target that starts PostgreSQL/Redis, creates a media
|
||||
fixture, calls v3 start/replan, fetches the returned stream or manifest, submits
|
||||
route events, and verifies cleanup. Contract fixtures must also run in Android
|
||||
CI against its Kotlin serializers and attempt-key function. Android tests also
|
||||
prove detailed codec/layout capability emission and a local PCM-recovery route
|
||||
event.
|
||||
|
||||
## 12. Rollout and rollback
|
||||
|
||||
1. Merge schema/types with v3 disabled.
|
||||
2. Deploy shadow planner and compare decisions without changing playback.
|
||||
3. Enable v3 only on a validation server with the Android test build.
|
||||
4. Complete SDR, HDR, DV, passthrough, subtitle, restart and node fixtures.
|
||||
5. Publish the minimum server revision.
|
||||
6. Enable v3 for production servers after the named soak and telemetry gates.
|
||||
|
||||
Rollback is the dynamic `playback.protocol_v3_enabled` setting. Disabling it
|
||||
stops advertising the feature and returns the no-session negotiation response
|
||||
to new v3 starts. Existing v3 sessions remain playable until stop/expiry; do
|
||||
not terminate active streams during rollback. Legacy web and Apple endpoints
|
||||
remain available throughout.
|
||||
|
||||
## 13. Definition of done
|
||||
|
||||
- The three Android canonical endpoints and playback capability endpoint are
|
||||
implemented with profile/session ownership checks.
|
||||
- Every playable response is complete, final and fetchable without client-owned
|
||||
recipe decisions.
|
||||
- Every enabled delivery/transformation has a passing integration fixture.
|
||||
- Direct decisions validate the selected track and full source/output facts.
|
||||
- Android supplies the detailed per-codec and per-layout capability facts those
|
||||
direct decisions consume; missing facts fail conservatively.
|
||||
- Terminal reasons replace unsafe direct fallbacks.
|
||||
- Replans are idempotent, loop-safe, restart-aware and capacity-safe.
|
||||
- Route telemetry is bounded, sanitized and queryable.
|
||||
- Existing legacy server, web, Apple, Jellyfin compatibility and audiobook
|
||||
playback tests remain green.
|
||||
- Android passes direct/remux/transcode/replan tests against the deployed server
|
||||
revision, followed by the documented 4K Dolby Vision/AVR validation.
|
||||
@@ -199,6 +199,14 @@ Silo often sees sidecar files that mirror the media basename:
|
||||
These are common and expected. Episode naming guidance in this page refers to the media files
|
||||
themselves, but sidecars may legitimately reuse the same stem.
|
||||
|
||||
Some NFO sidecars are actively read: `movie.nfo`, `tvshow.nfo`, `<media basename>.nfo`,
|
||||
`season.nfo`, `<episode basename>.nfo`, and sidecar artwork (season posters, episode
|
||||
thumbnails) feed the built-in NFO metadata provider — including `<uniqueid>` provider IDs,
|
||||
which act as trusted matching anchors just like `{tmdb-...}` folder tags. NFO files never
|
||||
change how files are grouped; naming and layout still decide series/season/episode
|
||||
structure. See [Local NFO Metadata](nfo-local-metadata.md) for the supported fields and
|
||||
merge behavior.
|
||||
|
||||
Supplemental directories next to a movie (and directly under a series root)
|
||||
are scanned as **extras** attached to that item, following the Jellyfin/Plex
|
||||
folder convention:
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
---
|
||||
title: Local NFO Metadata
|
||||
description: How Silo reads Kodi/Jellyfin-style NFO sidecar files and merges them with online metadata providers.
|
||||
summary: Supported NFO fields, merge and refresh semantics, and the naming-supplies-structure contract for curated libraries.
|
||||
tags:
|
||||
- silo
|
||||
- docs
|
||||
- wiki
|
||||
- libraries
|
||||
- metadata
|
||||
- nfo
|
||||
audience:
|
||||
- operator
|
||||
- end-user
|
||||
last_reviewed: 2026-07-12
|
||||
related:
|
||||
- ../index.md
|
||||
- media-folder-and-naming.md
|
||||
---
|
||||
|
||||
# Local NFO Metadata
|
||||
|
||||
Silo ships a built-in **NFO Files** metadata provider that reads Kodi/Jellyfin-style `.nfo`
|
||||
sidecar files. It runs inside the same provider chain as online providers (TMDB, TVDB, ...),
|
||||
so one toggle and one priority per library control it. By default it sits at priority 1: the
|
||||
NFO wins every field it declares, and online providers fill in whatever it leaves out.
|
||||
|
||||
Clients can detect whether a library type includes the NFO provider via
|
||||
`GET /api/v1/libraries/provider-defaults`.
|
||||
|
||||
## Where Silo looks for NFO files
|
||||
|
||||
For each movie or series, Silo checks (in order):
|
||||
|
||||
1. `movie.nfo` / `tvshow.nfo` in the item's folder
|
||||
2. `<media basename>.nfo` next to the media file (e.g. `My Movie (2021).nfo`)
|
||||
|
||||
For series, Silo additionally reads:
|
||||
|
||||
- `season.nfo` inside each season directory (`<season>` root: title, plot)
|
||||
- `<episode basename>.nfo` next to each episode file (`<episodedetails>` root: title,
|
||||
plot, air date, runtime, ratings)
|
||||
- Season posters: `poster.jpg` (or `folder`/`cover`) inside the season directory, or
|
||||
`seasonNN-poster.jpg` in the series root
|
||||
- Episode thumbnails: `<episode basename>-thumb.jpg`
|
||||
|
||||
A sidecar only applies when its root element matches the item type: a stray `movie.nfo` in
|
||||
a series folder is skipped in favor of `tvshow.nfo`, and a `tvshow.nfo` next to a movie
|
||||
file is ignored. Files that fail to parse are skipped silently and the next candidate is
|
||||
tried — a broken NFO never blocks matching.
|
||||
|
||||
Season and episode numbers declared inside an NFO (`<seasonnumber>`, `<season>`,
|
||||
`<episode>`) are advisory: the directory name and the `SxxEyy` filename pattern decide
|
||||
where metadata lands, and a mismatch is logged and resolved in the filename's favor.
|
||||
Multi-episode NFO documents (several `<episodedetails>` blocks in one file) are not
|
||||
supported; the first block is used and a warning is logged.
|
||||
|
||||
## Supported fields
|
||||
|
||||
| NFO element | Applies to | Silo field |
|
||||
|---|---|---|
|
||||
| `<title>` | movie, series | Title |
|
||||
| `<originaltitle>` | movie, series | Original title |
|
||||
| `<tagline>` | movie, series | Tagline |
|
||||
| `<plot>` | movie, series | Overview |
|
||||
| `<year>` | movie, series | Year (derived from the premiere date when omitted) |
|
||||
| `<runtime>` | movie, series | Runtime in minutes (non-numeric values ignored) |
|
||||
| `<premiered>` / `<releasedate>` | movie | Release date (`YYYY-MM-DD`) |
|
||||
| `<premiered>` / `<aired>` | series | First air date (`YYYY-MM-DD`) |
|
||||
| `<mpaa>` | movie, series | Content rating |
|
||||
| `<genre>` (repeated) | movie, series | Genres |
|
||||
| `<studio>` (repeated) | movie, series | Studios |
|
||||
| `<country>` (repeated) | movie, series | Countries |
|
||||
| `<tag>` (repeated) | movie, series | Keywords |
|
||||
| `<ratings>` / `<rating>` | movie, series | Ratings (named sources: `imdb`, `tmdb`/`themoviedb`, `tomatometerallcritics`, `tomatometerallaudience`; a legacy bare `<rating>` fills the IMDB slot) |
|
||||
| `<actor>` (`name`/`role`/`order`) | movie, series | Cast (actor `<thumb>` URLs are ignored) |
|
||||
| `<director>`, `<credits>` | movie, series | Crew (directors and writers) |
|
||||
| `<uniqueid type="tmdb|imdb|tvdb">` | movie, series | Trusted identity anchor for matching |
|
||||
| `<title>`, `<plot>` (`season.nfo`) | season | Season name and overview |
|
||||
| `<title>`, `<plot>`, `<aired>`, `<runtime>`, `<ratings>` (`<episodedetails>`) | episode | Episode title, overview, air date, runtime, ratings |
|
||||
|
||||
Not yet supported: `<set>` (collections) — a planned follow-up. `<userrating>` is read
|
||||
but not stored (Silo has no per-user rating field). Unknown elements are ignored, so
|
||||
exports from Kodi, Jellyfin, or tinyMediaManager work as-is.
|
||||
|
||||
## How NFO data merges with online providers
|
||||
|
||||
- **NFO wins what it declares.** With the NFO provider at priority 1, every field the NFO
|
||||
populates beats the same field from online providers.
|
||||
- **Online providers backfill the rest.** Anything the NFO leaves out (cast, artwork,
|
||||
runtime, ratings, ...) is filled by the next providers in the chain, keyed by the
|
||||
`<uniqueid>` values when present.
|
||||
- **Genres are all-or-nothing.** The first provider that supplies any genres supplies the
|
||||
whole list; later providers do not append to it. An NFO with no `<genre>` entries leaves
|
||||
genres entirely to online providers.
|
||||
- **Field locks are respected.** Fields locked in the Edit Metadata dialog (including
|
||||
Year and release/air dates) are never overwritten by any provider, NFO included.
|
||||
|
||||
## Editing an NFO after the initial match
|
||||
|
||||
Scheduled background refreshes only fill empty fields — they intentionally do **not**
|
||||
re-apply NFO changes over existing values. To pick up edits:
|
||||
|
||||
**Edit the NFO, then use Refresh metadata on the item.** A manual refresh re-reads the
|
||||
sidecar and replaces all unlocked fields with the merged provider results, NFO first.
|
||||
|
||||
## Naming supplies structure, NFO supplies presentation
|
||||
|
||||
NFO files describe *what an item is called and about* — they do not change *how files are
|
||||
grouped*. Folder layout and filenames still decide what is a series, a season, and an
|
||||
episode (see [Supported Media Folder Structures and Naming](media-folder-and-naming.md)).
|
||||
This makes fully local libraries with no online match work well, for example a fitness
|
||||
library:
|
||||
|
||||
```
|
||||
Fitness/
|
||||
P90X/
|
||||
tvshow.nfo # show title/plot (no <uniqueid> needed)
|
||||
poster.jpg fanart.jpg
|
||||
Season 01/
|
||||
season.nfo # "Course A: Classic"
|
||||
poster.jpg # season poster
|
||||
P90X S01E01 - Chest and Back.mkv
|
||||
P90X S01E01 - Chest and Back.nfo # episode title/plot
|
||||
P90X S01E01 - Chest and Back-thumb.jpg # episode thumbnail
|
||||
```
|
||||
|
||||
The folder and episode naming build the series/season/episode tree; the NFO sidecars
|
||||
supply every name, description, and image — even when no online provider knows the
|
||||
content. Episodes without an `.nfo` still appear with their generic "Episode N" title,
|
||||
so partially curated libraries degrade gracefully.
|
||||
|
||||
## Mixed libraries (sports example)
|
||||
|
||||
A single library of type **Mixed** can hold movie-shaped and show-shaped content side by
|
||||
side — the common case is sports, where pay-per-view events are movies and the weekly
|
||||
show is a series, and most of it has no TMDB/TVDB presence:
|
||||
|
||||
```
|
||||
WWE/
|
||||
WrestleMania 41 (2025)/
|
||||
WrestleMania 41 (2025).mkv
|
||||
movie.nfo # event title/plot (a <uniqueid> is optional)
|
||||
poster.jpg
|
||||
WWE SmackDown/
|
||||
tvshow.nfo
|
||||
Season 27/
|
||||
season.nfo
|
||||
poster.jpg
|
||||
WWE SmackDown S27E15.mkv
|
||||
WWE SmackDown S27E15.nfo
|
||||
WWE SmackDown S27E15-thumb.jpg
|
||||
```
|
||||
|
||||
How classification works in a mixed library:
|
||||
|
||||
- **Naming decides the type, per file, before any metadata provider runs.** An `SxxEyy`
|
||||
pattern or a `Season NN` folder routes a file to the series lane; everything else is a
|
||||
movie. An NFO never flips the type: a `tvshow.nfo` sitting next to a movie-classified
|
||||
file is ignored (the root element must match the item type), not applied.
|
||||
- **NFOs supply metadata and identity only.** Once the type is fixed, the NFO provider
|
||||
behaves exactly as in dedicated movie/series libraries — including season and episode
|
||||
depth for the show-shaped content. An event whose `movie.nfo` carries a
|
||||
`<uniqueid type="tmdb">` still gets full online enrichment, while ID-less events next
|
||||
to it fall back to their local NFO and artwork.
|
||||
- **Misclassified group?** Override its **Type** on the root in the library admin
|
||||
(Auto → Movie/Series) and rescan — that is the correction path; fixing the NFO alone
|
||||
will not re-route a file.
|
||||
|
||||
Tools that generate sports libraries (sportarr-style managers) should therefore write
|
||||
standard `SxxEyy` naming for show content and standalone movie naming (`Title (Year)/`)
|
||||
for events, and place NFOs/posters exactly as in the dedicated-library layouts above.
|
||||
|
||||
## Source References
|
||||
|
||||
- `internal/metadata/nfo/` — parser and provider
|
||||
- `internal/metadata/merge.go` — merge semantics and field locks
|
||||
@@ -35,6 +35,8 @@ rewriting.
|
||||
reference for the folder layouts and filenames Silo can scan and match today.
|
||||
- [Collection Templates](admin/collection-templates.md) - Curated, one-click starting points for
|
||||
synced library collections sourced from TMDB, Trakt, and MDBList.
|
||||
- [Local NFO Metadata](admin/nfo-local-metadata.md) - Supported NFO sidecar fields, how they merge
|
||||
with online providers, and the naming-supplies-structure contract.
|
||||
|
||||
## Deployment
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@ module github.com/Silo-Server/silo-server
|
||||
go 1.26.4
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.1
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.29
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2
|
||||
github.com/go-chi/chi/v5 v5.2.5
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
@@ -22,6 +22,7 @@ require (
|
||||
require (
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0
|
||||
github.com/abadojack/whatlanggo v1.0.1
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34
|
||||
github.com/go-chi/cors v1.2.2
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/h2non/bimg v1.1.9
|
||||
@@ -109,15 +110,15 @@ require (
|
||||
|
||||
require (
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.10.0
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect
|
||||
github.com/aws/smithy-go v1.27.3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
@@ -144,7 +145,7 @@ require (
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/text v0.38.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/grpc v1.81.1
|
||||
google.golang.org/grpc v1.82.1
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
|
||||
@@ -16,30 +16,44 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
|
||||
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek=
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.30 h1:XwsEzpTJfQYJbFicz/QMLwAZdyeNVVoOEkbF7R3gPJk=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.30/go.mod h1:Ud32SuMc+/9BGxfpSVld7HrE2o05JwKmXY4M3jOQNZU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2 h1:5C00eQYpTrgQXnp6V3P6P7zPElna3AXvlukbANE6nJI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 h1:V7ZZ300WPXGjvkyore5DGe0ljVPOxCXie/thWdtSBXE=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.4.1/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 h1:gYFYh4iLLcAOJRLNPY2aD2g9DIhKn4eof8UkIrr1rTk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.32.1/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 h1:arjT9Cm3/WYbGmD5TUZHk4UQn4Lle1fUNZs5FC6CtF0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 h1:RvfHDg+xvAeZ+5741vUEjpOVtYSIm93W2zhx10Xtydw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.44.1/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q=
|
||||
github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
|
||||
github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
@@ -442,8 +456,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ=
|
||||
google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
|
||||
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
|
||||
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@@ -124,7 +124,7 @@ func (s stubStore) DeleteHomeDismissal(context.Context, string, string, string)
|
||||
panic("unused")
|
||||
}
|
||||
func (s stubStore) AddFavorite(context.Context, string, string) error { panic("unused") }
|
||||
func (s stubStore) AddFavoriteAt(context.Context, string, string, time.Time) error {
|
||||
func (s stubStore) AddFavoriteAt(context.Context, string, string, time.Time) (bool, error) {
|
||||
panic("unused")
|
||||
}
|
||||
func (s stubStore) RemoveFavorite(context.Context, string, string) error {
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/config"
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
@@ -64,6 +65,12 @@ type ServerSettingsStore interface {
|
||||
GetAll(ctx context.Context) (map[string]string, error)
|
||||
}
|
||||
|
||||
type DiagnosticsEnablementStore interface {
|
||||
PutStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error
|
||||
DeleteObject(ctx context.Context, bucket, key string) error
|
||||
Bucket() string
|
||||
}
|
||||
|
||||
type AdminJobCreator interface {
|
||||
Create(ctx context.Context, input adminjob.CreateJobInput) (*models.AdminJob, error)
|
||||
CreateLibraryRefresh(
|
||||
@@ -97,6 +104,7 @@ type AdminHandler struct {
|
||||
EventBus cache.EventBus
|
||||
EventsHub *evt.Hub
|
||||
SettingsRepo ServerSettingsStore
|
||||
DiagnosticsStore DiagnosticsEnablementStore
|
||||
JobRepo AdminJobCreator
|
||||
ItemRefreshResolver ItemRefreshScopeResolver
|
||||
ImpersonationService ImpersonationService
|
||||
@@ -2215,6 +2223,38 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
req.Value = strconv.FormatBool(enabled)
|
||||
case diagnostics.KeyUploadsEnabled:
|
||||
enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", diagnostics.KeyUploadsEnabled+" must be true or false")
|
||||
return
|
||||
}
|
||||
req.Value = strconv.FormatBool(enabled)
|
||||
if enabled {
|
||||
if err := h.validateDiagnosticsUploadsEnabled(r.Context()); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "storage_unavailable", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
case diagnostics.KeyMaxBundleBytes,
|
||||
diagnostics.KeyMaxUncompressedBytes,
|
||||
diagnostics.KeyMaxReportsPerUserDay,
|
||||
diagnostics.KeyRetentionDays,
|
||||
diagnostics.KeyMaxBytesPerUser:
|
||||
normalized, err := h.normalizeDiagnosticsNumericSetting(r.Context(), key, req.Value)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
req.Value = normalized
|
||||
case diagnostics.KeyConsentNoticeVersion:
|
||||
n, err := strconv.Atoi(strings.TrimSpace(req.Value))
|
||||
if err != nil || n < 1 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request",
|
||||
diagnostics.KeyConsentNoticeVersion+" must be an integer greater than 0")
|
||||
return
|
||||
}
|
||||
req.Value = strconv.Itoa(n)
|
||||
case policy.SettingDecisionLogScopeSampleRate:
|
||||
n, err := strconv.Atoi(strings.TrimSpace(req.Value))
|
||||
if err != nil || n <= 0 {
|
||||
@@ -2243,10 +2283,11 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
"subtitle_ai.transcribe_quota_period must be day, week, or month")
|
||||
return
|
||||
}
|
||||
case notifications.SettingApplePushDeliveryEnabled:
|
||||
case notifications.SettingApplePushDeliveryEnabled,
|
||||
notifications.SettingAndroidPushDeliveryEnabled:
|
||||
enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "notifications.apple_push_delivery_enabled must be true or false")
|
||||
writeError(w, http.StatusBadRequest, "bad_request", key+" must be true or false")
|
||||
return
|
||||
}
|
||||
req.Value = strconv.FormatBool(enabled)
|
||||
@@ -2382,3 +2423,85 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: req.Value, RestartRequired: restartRequired})
|
||||
}
|
||||
|
||||
func (h *AdminHandler) validateDiagnosticsUploadsEnabled(ctx context.Context) error {
|
||||
if h.DiagnosticsStore == nil || strings.TrimSpace(h.DiagnosticsStore.Bucket()) == "" {
|
||||
return errors.New("diagnostics uploads require configured private object storage")
|
||||
}
|
||||
const probeKey = "diagnostics/.probe"
|
||||
if err := h.DiagnosticsStore.PutStream(
|
||||
ctx,
|
||||
h.DiagnosticsStore.Bucket(),
|
||||
probeKey,
|
||||
strings.NewReader("ok"),
|
||||
"application/octet-stream",
|
||||
); err != nil {
|
||||
return fmt.Errorf("diagnostics storage probe write failed: %w", err)
|
||||
}
|
||||
if err := h.DiagnosticsStore.DeleteObject(ctx, h.DiagnosticsStore.Bucket(), probeKey); err != nil {
|
||||
return fmt.Errorf("diagnostics storage probe delete failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *AdminHandler) normalizeDiagnosticsNumericSetting(ctx context.Context, key, raw string) (string, error) {
|
||||
const (
|
||||
mib = int64(1024 * 1024)
|
||||
gib = int64(1024 * mib)
|
||||
)
|
||||
|
||||
value, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s must be an integer", key)
|
||||
}
|
||||
|
||||
settings := diagnostics.DefaultSettings()
|
||||
if h.SettingsRepo != nil {
|
||||
loaded, loadErr := diagnostics.LoadSettings(ctx, h.SettingsRepo)
|
||||
if loadErr != nil {
|
||||
return "", fmt.Errorf("load diagnostics settings: %w", loadErr)
|
||||
}
|
||||
settings = loaded
|
||||
}
|
||||
|
||||
switch key {
|
||||
case diagnostics.KeyMaxBundleBytes:
|
||||
if value < mib || value > 256*mib {
|
||||
return "", fmt.Errorf("%s must be between 1 MiB (%d bytes) and 256 MiB (%d bytes)", key, mib, 256*mib)
|
||||
}
|
||||
if value > settings.MaxUncompressedBytes {
|
||||
return "", fmt.Errorf("%s must not exceed %s (%d bytes)", key, diagnostics.KeyMaxUncompressedBytes, settings.MaxUncompressedBytes)
|
||||
}
|
||||
// A single bundle can never exceed the per-user byte cap, or every upload
|
||||
// at this size would fail quota; keep the two bounds consistent.
|
||||
if value > settings.MaxBytesPerUser {
|
||||
return "", fmt.Errorf("%s must not exceed %s (%d bytes)", key, diagnostics.KeyMaxBytesPerUser, settings.MaxBytesPerUser)
|
||||
}
|
||||
case diagnostics.KeyMaxUncompressedBytes:
|
||||
if value < settings.MaxBundleBytes || value > gib {
|
||||
return "", fmt.Errorf("%s must be between %s (%d bytes) and 1 GiB (%d bytes)", key, diagnostics.KeyMaxBundleBytes, settings.MaxBundleBytes, gib)
|
||||
}
|
||||
case diagnostics.KeyMaxReportsPerUserDay:
|
||||
if value < 1 || value > 1000 {
|
||||
return "", fmt.Errorf("%s must be between 1 and 1000", key)
|
||||
}
|
||||
case diagnostics.KeyRetentionDays:
|
||||
if value < 1 || value > 365 {
|
||||
return "", fmt.Errorf("%s must be between 1 and 365", key)
|
||||
}
|
||||
case diagnostics.KeyMaxBytesPerUser:
|
||||
if value < 10*mib || value > 10*gib {
|
||||
return "", fmt.Errorf("%s must be between 10 MiB (%d bytes) and 10 GiB (%d bytes)", key, 10*mib, 10*gib)
|
||||
}
|
||||
// The per-user cap must leave room for at least one max-size bundle, or
|
||||
// /diagnostics/status would advertise a bundle size InsertReceiving always
|
||||
// rejects as quota_exceeded.
|
||||
if value < settings.MaxBundleBytes {
|
||||
return "", fmt.Errorf("%s must be at least %s (%d bytes)", key, diagnostics.KeyMaxBundleBytes, settings.MaxBundleBytes)
|
||||
}
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported diagnostics numeric setting %s", key)
|
||||
}
|
||||
|
||||
return strconv.FormatInt(value, 10), nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
@@ -64,8 +65,21 @@ type adminPushRelayRegisterResponse struct {
|
||||
|
||||
// HandleTest handles POST /admin/notifications/push/apple/test.
|
||||
func (h *AdminApplePushHandler) HandleTest(w http.ResponseWriter, r *http.Request) {
|
||||
h.handleTest(w, r, "Apple", func(ctx context.Context, profileID, serverDeviceID string) (*notifications.ApplePushTestResult, error) {
|
||||
return h.system.SendApplePushTest(ctx, profileID, serverDeviceID)
|
||||
})
|
||||
}
|
||||
|
||||
// HandleTestAndroid handles POST /admin/notifications/push/fcm/test.
|
||||
func (h *AdminApplePushHandler) HandleTestAndroid(w http.ResponseWriter, r *http.Request) {
|
||||
h.handleTest(w, r, "Android", func(ctx context.Context, profileID, serverDeviceID string) (*notifications.ApplePushTestResult, error) {
|
||||
return h.system.SendAndroidPushTest(ctx, profileID, serverDeviceID)
|
||||
})
|
||||
}
|
||||
|
||||
func (h *AdminApplePushHandler) handleTest(w http.ResponseWriter, r *http.Request, platformLabel string, send func(context.Context, string, string) (*notifications.ApplePushTestResult, error)) {
|
||||
if h == nil || h.system == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Apple push delivery is not available")
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", platformLabel+" push delivery is not available")
|
||||
return
|
||||
}
|
||||
var req adminApplePushTestRequest
|
||||
@@ -73,17 +87,17 @@ func (h *AdminApplePushHandler) HandleTest(w http.ResponseWriter, r *http.Reques
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
result, err := h.system.SendApplePushTest(r.Context(), req.ProfileID, req.ServerDeviceID)
|
||||
result, err := send(r.Context(), req.ProfileID, req.ServerDeviceID)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, notifications.ErrPushDeliveryInvalid):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
case errors.Is(err, notifications.ErrPushDeliveryNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Apple push device not found")
|
||||
writeError(w, http.StatusNotFound, "not_found", platformLabel+" push device not found")
|
||||
case errors.Is(err, notifications.ErrPushDeliveryUnavailable):
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Apple push delivery is not available")
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", platformLabel+" push delivery is not available")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to send Apple push test")
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to send "+platformLabel+" push test")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics"
|
||||
)
|
||||
|
||||
const diagnosticsDownloadExpiry = 15 * time.Minute
|
||||
|
||||
type AdminDiagnosticsService interface {
|
||||
ListForAdmin(ctx context.Context, filters diagnostics.ListFilters) (diagnostics.ListResult, error)
|
||||
GetReport(ctx context.Context, id string) (*diagnostics.Report, error)
|
||||
PresignReportDownload(ctx context.Context, report *diagnostics.Report, expiry time.Duration) (string, error)
|
||||
OpenReportDownload(ctx context.Context, report *diagnostics.Report) (io.ReadCloser, error)
|
||||
DeleteReport(ctx context.Context, id string) (*diagnostics.Report, error)
|
||||
}
|
||||
|
||||
type adminDiagnosticsEffectiveTTL interface {
|
||||
EffectiveReportDownloadTTL(requested time.Duration) time.Duration
|
||||
}
|
||||
|
||||
type diagnosticsDownloadURLResponse struct {
|
||||
DownloadURL string `json:"download_url"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
func RegisterAdminDiagnosticsRoutes(r chi.Router, h *DiagnosticsHandler) {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
r.Route("/diagnostics/reports", func(r chi.Router) {
|
||||
r.Get("/", h.HandleAdminListReports)
|
||||
r.Get("/{id}", h.HandleAdminGetReport)
|
||||
r.Get("/{id}/download", h.HandleAdminDownloadReport)
|
||||
r.Delete("/{id}", h.HandleAdminDeleteReport)
|
||||
})
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) HandleAdminListReports(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := h.admin()
|
||||
if !ok {
|
||||
writeDiagnosticsAdminUnavailable(w)
|
||||
return
|
||||
}
|
||||
filters, err := parseDiagnosticsReportListFilters(r)
|
||||
if err != nil {
|
||||
writeDiagnosticsAdminError(w, err)
|
||||
return
|
||||
}
|
||||
result, err := admin.ListForAdmin(r.Context(), filters)
|
||||
if err != nil {
|
||||
writeDiagnosticsAdminError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) HandleAdminGetReport(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := h.admin()
|
||||
if !ok {
|
||||
writeDiagnosticsAdminUnavailable(w)
|
||||
return
|
||||
}
|
||||
report, err := admin.GetReport(r.Context(), chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
writeDiagnosticsAdminError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, report)
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) HandleAdminDownloadReport(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := h.admin()
|
||||
if !ok {
|
||||
writeDiagnosticsAdminUnavailable(w)
|
||||
return
|
||||
}
|
||||
reportID := chi.URLParam(r, "id")
|
||||
report, err := admin.GetReport(r.Context(), reportID)
|
||||
if err != nil {
|
||||
writeDiagnosticsAdminError(w, err)
|
||||
return
|
||||
}
|
||||
if report.State != diagnostics.StateReady {
|
||||
writeDiagnosticsAdminError(w, diagnostics.ErrReportNotReady)
|
||||
return
|
||||
}
|
||||
|
||||
ttl := diagnosticsEffectiveDownloadTTL(admin, diagnosticsDownloadExpiry)
|
||||
if !diagnosticsProxyDownloadRequested(r) {
|
||||
if url, err := admin.PresignReportDownload(r.Context(), report, ttl); err == nil && strings.TrimSpace(url) != "" {
|
||||
h.auditDownload(r, report.ID)
|
||||
writeJSON(w, http.StatusOK, diagnosticsDownloadURLResponse{
|
||||
DownloadURL: url,
|
||||
ExpiresAt: time.Now().UTC().Add(ttl),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
body, err := admin.OpenReportDownload(r.Context(), report)
|
||||
if err != nil {
|
||||
writeDiagnosticsAdminError(w, err)
|
||||
return
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
h.auditDownload(r, report.ID)
|
||||
w.Header().Set("Content-Type", diagnostics.ReportDownloadContentType)
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", diagnosticsReportFilename(report)))
|
||||
if report.BlobBytes != nil && *report.BlobBytes >= 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(*report.BlobBytes, 10))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, body); err != nil {
|
||||
h.diagnosticsLogger().WarnContext(r.Context(), "diagnostic report download stream failed",
|
||||
"component", "diagnostics",
|
||||
"admin_user_id", currentAdminUserID(r),
|
||||
"report_id", report.ID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) HandleAdminDeleteReport(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := h.admin()
|
||||
if !ok {
|
||||
writeDiagnosticsAdminUnavailable(w)
|
||||
return
|
||||
}
|
||||
reportID := chi.URLParam(r, "id")
|
||||
report, err := admin.DeleteReport(r.Context(), reportID)
|
||||
if err != nil {
|
||||
writeDiagnosticsAdminError(w, err)
|
||||
return
|
||||
}
|
||||
h.diagnosticsLogger().InfoContext(r.Context(), "diagnostic report deleted",
|
||||
"component", "diagnostics",
|
||||
"admin_user_id", currentAdminUserID(r),
|
||||
"report_id", report.ID,
|
||||
)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) admin() (AdminDiagnosticsService, bool) {
|
||||
if h.adminService == nil {
|
||||
return nil, false
|
||||
}
|
||||
return h.adminService, true
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) diagnosticsLogger() *slog.Logger {
|
||||
if h.logger == nil {
|
||||
return slog.Default()
|
||||
}
|
||||
return h.logger
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) auditDownload(r *http.Request, reportID string) {
|
||||
h.diagnosticsLogger().InfoContext(r.Context(), "diagnostic report downloaded",
|
||||
"component", "diagnostics",
|
||||
"admin_user_id", currentAdminUserID(r),
|
||||
"report_id", reportID,
|
||||
)
|
||||
}
|
||||
|
||||
func parseDiagnosticsReportListFilters(r *http.Request) (diagnostics.ListFilters, error) {
|
||||
filters := diagnostics.ListFilters{
|
||||
Platform: strings.TrimSpace(r.URL.Query().Get("platform")),
|
||||
ReportType: strings.TrimSpace(r.URL.Query().Get("report_type")),
|
||||
Cursor: strings.TrimSpace(r.URL.Query().Get("cursor")),
|
||||
Limit: parseLimit(r, 50),
|
||||
}
|
||||
|
||||
userID, err := parseOptionalIntQuery(r, "user_id")
|
||||
if err != nil {
|
||||
return diagnostics.ListFilters{}, err
|
||||
}
|
||||
filters.UserID = userID
|
||||
|
||||
if from, err := parseTimeQuery(r, "from"); err != nil {
|
||||
return diagnostics.ListFilters{}, err
|
||||
} else {
|
||||
filters.From = from
|
||||
}
|
||||
if to, err := parseTimeQuery(r, "to"); err != nil {
|
||||
return diagnostics.ListFilters{}, err
|
||||
} else {
|
||||
filters.To = to
|
||||
}
|
||||
|
||||
shortID := strings.TrimSpace(r.URL.Query().Get("short_id"))
|
||||
if shortID != "" {
|
||||
normalized, err := diagnostics.ParseShortID(shortID)
|
||||
if err != nil {
|
||||
return diagnostics.ListFilters{}, err
|
||||
}
|
||||
filters.ShortID = normalized
|
||||
}
|
||||
return filters, nil
|
||||
}
|
||||
|
||||
func diagnosticsReportFilename(report *diagnostics.Report) string {
|
||||
name := strings.TrimSpace(report.ShortID)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(report.ID)
|
||||
}
|
||||
if name == "" {
|
||||
name = "report"
|
||||
}
|
||||
return "silo-diagnostics-" + name + ".tar.gz"
|
||||
}
|
||||
|
||||
func diagnosticsProxyDownloadRequested(r *http.Request) bool {
|
||||
raw := strings.TrimSpace(r.URL.Query().Get("proxy"))
|
||||
return raw == "1" || strings.EqualFold(raw, "true") || strings.EqualFold(raw, "yes")
|
||||
}
|
||||
|
||||
func diagnosticsEffectiveDownloadTTL(admin AdminDiagnosticsService, requested time.Duration) time.Duration {
|
||||
ttl := requested
|
||||
if effective, ok := admin.(adminDiagnosticsEffectiveTTL); ok {
|
||||
ttl = effective.EffectiveReportDownloadTTL(requested)
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return requested
|
||||
}
|
||||
return ttl
|
||||
}
|
||||
|
||||
func writeDiagnosticsAdminUnavailable(w http.ResponseWriter) {
|
||||
writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Diagnostics reports are not configured")
|
||||
}
|
||||
|
||||
func writeDiagnosticsAdminError(w http.ResponseWriter, err error) {
|
||||
var parseErr *requestParseError
|
||||
switch {
|
||||
case errors.As(err, &parseErr):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", parseErr.Error())
|
||||
case errors.Is(err, diagnostics.ErrInvalidShortID):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid short_id")
|
||||
case errors.Is(err, diagnostics.ErrInvalidCursor):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid cursor")
|
||||
case errors.Is(err, diagnostics.ErrNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Diagnostic report not found")
|
||||
case errors.Is(err, diagnostics.ErrReportNotReady):
|
||||
writeError(w, http.StatusConflict, "not_ready", "Diagnostic report is not ready")
|
||||
case errors.Is(err, diagnostics.ErrStorageUnavailable):
|
||||
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not available")
|
||||
case diagnostics.IsObjectNotFound(err):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Diagnostic report bundle not found")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Diagnostics report operation failed")
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,12 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
@@ -52,6 +54,7 @@ type AdminImageHandler struct {
|
||||
imageSvc ImageService
|
||||
imageResolver ImageURLResolver
|
||||
detailSvc *catalog.DetailService
|
||||
EventsHub *evt.Hub
|
||||
}
|
||||
|
||||
// NewAdminImageHandler creates a handler for admin image selection endpoints.
|
||||
@@ -110,6 +113,8 @@ type applyItemImageResponse struct {
|
||||
ContentID string `json:"content_id"`
|
||||
StoredPath string `json:"stored_path"`
|
||||
Thumbhash string `json:"thumbhash"`
|
||||
ImageURL string `json:"image_url,omitempty"`
|
||||
Revision string `json:"revision,omitempty"`
|
||||
}
|
||||
|
||||
// resolvedItem holds the result of looking up a content ID across all three tables.
|
||||
@@ -304,6 +309,18 @@ func (h *AdminImageHandler) HandleApplyItemImage(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
imageType := metadata.ImageTypeFromString(req.Type)
|
||||
// Episodes only have stills. Clients historically sent "poster" here (the
|
||||
// old flow silently dropped it), so coerce rather than reject.
|
||||
if resolved.contentType == "episode" {
|
||||
imageType = metadata.ImageStill
|
||||
}
|
||||
// Reject unsupported target/image combinations before spending a download
|
||||
// and an S3 upload on an image that can never be published.
|
||||
if err := catalog.ValidateArtworkSelectionTarget(resolved.contentType, metadata.ImageTypeToString(imageType)); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "unsupported_image_type",
|
||||
fmt.Sprintf("%s items do not accept %s images", resolved.contentType, metadata.ImageTypeToString(imageType)))
|
||||
return
|
||||
}
|
||||
providerID := req.ProviderID
|
||||
if providerID == "" {
|
||||
providerID = primaryProvider(resolved.parentItem)
|
||||
@@ -344,52 +361,54 @@ func (h *AdminImageHandler) HandleApplyItemImage(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
// Build the MetadataUpdate targeting only the relevant image field.
|
||||
upd := buildImageUpdate(imageType, result.StoredPath, result.Thumbhash)
|
||||
|
||||
// Lock FieldImages on the parent media_items row to prevent
|
||||
// automatic refreshes from overwriting the manual selection.
|
||||
if resolved.contentType == "movie" || resolved.contentType == "series" {
|
||||
locked := mergeLockedField(resolved.parentItem.LockedFields, int(metadata.FieldImages))
|
||||
upd.LockedFields = &locked
|
||||
}
|
||||
|
||||
// Persist to the correct table.
|
||||
if err := h.persistImageUpdate(r.Context(), contentID, resolved, &upd); err != nil {
|
||||
err = h.detailSvc.PublishArtworkSelection(r.Context(), catalog.ArtworkSelection{
|
||||
TargetType: resolved.contentType,
|
||||
TargetContentID: contentID,
|
||||
ParentContentID: resolved.parentItem.ContentID,
|
||||
ImageType: metadata.ImageTypeToString(imageType),
|
||||
StoredPath: result.StoredPath,
|
||||
SourcePath: req.OriginalURL,
|
||||
Thumbhash: result.Thumbhash,
|
||||
LockField: int(metadata.FieldImages),
|
||||
})
|
||||
if err != nil {
|
||||
slog.ErrorContext(r.Context(), "admin images: persist failed", "component", "api", "content_id", contentID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Image cached but failed to update item")
|
||||
if cleanupErr := h.detailSvc.QueueArtworkRevisionGC(
|
||||
r.Context(), result.StoredPath, metadata.ImageTypeToString(imageType), time.Now().Add(time.Hour),
|
||||
); cleanupErr != nil {
|
||||
slog.ErrorContext(r.Context(), "admin images: failed to queue unpublished artwork cleanup", "component", "api",
|
||||
"content_id", contentID, "stored_path", result.StoredPath, "error", cleanupErr)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, catalog.ErrItemNotFound),
|
||||
errors.Is(err, catalog.ErrSeasonNotFound),
|
||||
errors.Is(err, catalog.ErrEpisodeNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Item not found")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Image cached but failed to update item")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Also lock FieldImages on the parent series for seasons/episodes.
|
||||
if resolved.contentType == "season" || resolved.contentType == "episode" {
|
||||
parentLocked := mergeLockedField(resolved.parentItem.LockedFields, int(metadata.FieldImages))
|
||||
parentUpd := catalog.MetadataUpdate{LockedFields: &parentLocked}
|
||||
if lockErr := h.detailSvc.UpdateMediaItemMetadata(r.Context(), resolved.parentItem.ContentID, &parentUpd); lockErr != nil {
|
||||
slog.ErrorContext(r.Context(), "admin images: failed to lock FieldImages on parent series", "component", "api",
|
||||
"content_id", contentID, "parent_id", resolved.parentItem.ContentID, "error", lockErr)
|
||||
}
|
||||
publishEventMetadataUpdate(r.Context(), h.EventsHub, 0, contentID)
|
||||
if resolved.parentItem.ContentID != contentID {
|
||||
publishEventMetadataUpdate(r.Context(), h.EventsHub, 0, resolved.parentItem.ContentID)
|
||||
}
|
||||
|
||||
imageURL := ""
|
||||
if h.imageResolver != nil {
|
||||
imageURL = h.imageResolver.ResolveImageURL(r.Context(), result.StoredPath, "original")
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, applyItemImageResponse{
|
||||
ContentID: contentID,
|
||||
StoredPath: result.StoredPath,
|
||||
Thumbhash: result.Thumbhash,
|
||||
ImageURL: imageURL,
|
||||
Revision: result.Revision,
|
||||
})
|
||||
}
|
||||
|
||||
// persistImageUpdate writes the image path update to the correct table.
|
||||
func (h *AdminImageHandler) persistImageUpdate(ctx context.Context, contentID string, resolved *resolvedItem, upd *catalog.MetadataUpdate) error {
|
||||
switch resolved.contentType {
|
||||
case "season":
|
||||
return h.detailSvc.UpdateSeasonMetadata(ctx, contentID, upd)
|
||||
case "episode":
|
||||
return h.detailSvc.UpdateEpisodeMetadata(ctx, contentID, upd)
|
||||
default:
|
||||
return h.detailSvc.UpdateMediaItemMetadata(ctx, contentID, upd)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveImageFolderID finds the primary library folder for a content ID.
|
||||
func (h *AdminImageHandler) resolveImageFolderID(ctx context.Context, contentID string) (int, error) {
|
||||
if h.folders == nil {
|
||||
@@ -417,38 +436,6 @@ func buildProviderIDs(item *models.MediaItem) map[string]string {
|
||||
return ids
|
||||
}
|
||||
|
||||
// buildImageUpdate constructs a MetadataUpdate targeting a single image field.
|
||||
func buildImageUpdate(imageType metadata.ImageType, storedPath, thumbhash string) catalog.MetadataUpdate {
|
||||
var upd catalog.MetadataUpdate
|
||||
switch imageType {
|
||||
case metadata.ImageBackdrop:
|
||||
upd.BackdropPath = &storedPath
|
||||
upd.BackdropThumbhash = &thumbhash
|
||||
case metadata.ImageLogo:
|
||||
upd.LogoPath = &storedPath
|
||||
case metadata.ImageStill:
|
||||
upd.StillPath = &storedPath
|
||||
upd.StillThumbhash = &thumbhash
|
||||
default: // poster
|
||||
upd.PosterPath = &storedPath
|
||||
upd.PosterThumbhash = &thumbhash
|
||||
}
|
||||
return upd
|
||||
}
|
||||
|
||||
// mergeLockedField adds a field value to the locked fields slice if not already present.
|
||||
func mergeLockedField(existing []int, field int) []int {
|
||||
for _, f := range existing {
|
||||
if f == field {
|
||||
return existing
|
||||
}
|
||||
}
|
||||
result := make([]int, len(existing)+1)
|
||||
copy(result, existing)
|
||||
result[len(existing)] = field
|
||||
return result
|
||||
}
|
||||
|
||||
// primaryProvider returns the primary provider slug for a media item.
|
||||
func primaryProvider(item *models.MediaItem) string {
|
||||
if item.TmdbID != "" {
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics"
|
||||
)
|
||||
|
||||
const (
|
||||
diagnosticsMultipartOverheadBytes = int64(128 * 1024)
|
||||
diagnosticsBusyRetryAfter = "5"
|
||||
diagnosticsQuotaRetryAfter = "60"
|
||||
// diagnosticsUploadReadTimeout replaces the shared 30s server ReadTimeout
|
||||
// for this route only. Bundles are up to 10 MiB (admin-tunable higher) and
|
||||
// mobile crash uploads over slow uplinks routinely exceed 30s; without this
|
||||
// net/http aborts the read mid-stream before Ingest can return a retryable
|
||||
// error. The bound stays generous but finite so a stalled upload still can't
|
||||
// hold the connection open indefinitely.
|
||||
diagnosticsUploadReadTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
errDiagnosticsPartTooLarge = errors.New("diagnostics multipart part too large")
|
||||
errDiagnosticsUnexpectedPart = errors.New("diagnostics multipart contains unexpected part")
|
||||
)
|
||||
|
||||
type DiagnosticsService interface {
|
||||
Status(ctx context.Context, userID int) (diagnostics.Status, error)
|
||||
Ingest(ctx context.Context, userID int, profileID *string, manifestJSON []byte, bundle io.Reader) (diagnostics.IngestResult, error)
|
||||
}
|
||||
|
||||
type DiagnosticsHandler struct {
|
||||
service DiagnosticsService
|
||||
adminService AdminDiagnosticsService
|
||||
inflight *diagnosticsInFlightLimiter
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewDiagnosticsHandler(service DiagnosticsService) *DiagnosticsHandler {
|
||||
handler := &DiagnosticsHandler{
|
||||
service: service,
|
||||
inflight: newDiagnosticsInFlightLimiter(4),
|
||||
logger: slog.Default(),
|
||||
}
|
||||
if adminService, ok := service.(AdminDiagnosticsService); ok {
|
||||
handler.adminService = adminService
|
||||
}
|
||||
return handler
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := diagnosticsUserID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
status, err := h.service.Status(r.Context(), userID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load diagnostics status")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) HandleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
// Extend the read deadline for this request only; the server-wide
|
||||
// ReadTimeout is too short for slow, large bundle uploads (see the constant).
|
||||
rc := http.NewResponseController(w)
|
||||
if err := rc.SetReadDeadline(time.Now().Add(diagnosticsUploadReadTimeout)); err != nil {
|
||||
h.diagnosticsLogger().WarnContext(r.Context(), "diagnostics upload read deadline not extended",
|
||||
"component", "diagnostics",
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
// Extend the write deadline the same way. cmd/silo's integrated server sets
|
||||
// WriteTimeout 120s, but a slow upload can take longer than that within the
|
||||
// read window above; Ingest may store and mark the report ready and then the
|
||||
// final writeJSON would miss the 120s write deadline, so the client sees a
|
||||
// timeout and retries a report that already succeeded.
|
||||
if err := rc.SetWriteDeadline(time.Now().Add(diagnosticsUploadReadTimeout)); err != nil {
|
||||
h.diagnosticsLogger().WarnContext(r.Context(), "diagnostics upload write deadline not extended",
|
||||
"component", "diagnostics",
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
|
||||
userID, ok := diagnosticsUserID(w, r)
|
||||
if !ok {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims != nil && claims.TokenType == auth.TokenTypeAPIKey {
|
||||
h.logRejected(r.Context(), claims.UserID, "api_key_not_allowed")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
status, err := h.service.Status(r.Context(), userID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load diagnostics status")
|
||||
return
|
||||
}
|
||||
maxBundleBytes := status.MaxBundleBytes
|
||||
if maxBundleBytes <= 0 {
|
||||
maxBundleBytes = diagnostics.DefaultMaxBundleBytes
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBundleBytes+diagnosticsMultipartOverheadBytes)
|
||||
|
||||
switch status.Status {
|
||||
case diagnostics.StatusDisabled:
|
||||
writeError(w, http.StatusForbidden, "disabled", "Diagnostics uploads are disabled")
|
||||
return
|
||||
case diagnostics.StatusStorageUnavailable:
|
||||
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not configured")
|
||||
return
|
||||
case diagnostics.StatusAvailable:
|
||||
default:
|
||||
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not available")
|
||||
return
|
||||
}
|
||||
|
||||
release, acquired := h.inflight.acquire(userID)
|
||||
if !acquired {
|
||||
h.logRejected(r.Context(), userID, "busy")
|
||||
w.Header().Set("Retry-After", diagnosticsBusyRetryAfter)
|
||||
writeError(w, http.StatusServiceUnavailable, "busy", "Diagnostics upload capacity is busy")
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
h.writeDiagnosticsMultipartError(r.Context(), userID, w, err)
|
||||
return
|
||||
}
|
||||
|
||||
manifestPart, err := nextDiagnosticsPart(mr, "manifest", "application/json")
|
||||
if err != nil {
|
||||
h.writeDiagnosticsMultipartError(r.Context(), userID, w, err)
|
||||
return
|
||||
}
|
||||
manifestJSON, err := readDiagnosticsPart(manifestPart, diagnostics.MaxManifestBytes)
|
||||
_ = manifestPart.Close()
|
||||
if err != nil {
|
||||
h.writeDiagnosticsMultipartError(r.Context(), userID, w, err)
|
||||
return
|
||||
}
|
||||
|
||||
bundlePart, err := nextDiagnosticsPart(mr, "bundle", diagnostics.BundleContentType)
|
||||
if err != nil {
|
||||
h.writeDiagnosticsMultipartError(r.Context(), userID, w, err)
|
||||
return
|
||||
}
|
||||
defer bundlePart.Close()
|
||||
|
||||
profileID := strings.TrimSpace(r.Header.Get("X-Profile-Id"))
|
||||
var profileIDPtr *string
|
||||
if profileID != "" {
|
||||
profileIDPtr = &profileID
|
||||
}
|
||||
|
||||
result, err := h.service.Ingest(
|
||||
r.Context(),
|
||||
userID,
|
||||
profileIDPtr,
|
||||
manifestJSON,
|
||||
&exactlyTwoPartBundleReader{part: bundlePart, mr: mr},
|
||||
)
|
||||
if err != nil {
|
||||
writeDiagnosticsServiceError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, result)
|
||||
}
|
||||
|
||||
func diagnosticsUserID(w http.ResponseWriter, r *http.Request) (int, bool) {
|
||||
if !hasBearerAuthorizationHeader(r) {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authorization bearer token required")
|
||||
return 0, false
|
||||
}
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
|
||||
return 0, false
|
||||
}
|
||||
if claims.TokenType == auth.TokenTypeAPIKey {
|
||||
writeError(w, http.StatusForbidden, "api_key_not_allowed", "API keys cannot upload diagnostics")
|
||||
return 0, false
|
||||
}
|
||||
if claims.TokenType != auth.TokenTypeAccess {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Diagnostics require a user access token")
|
||||
return 0, false
|
||||
}
|
||||
if claims.UserID <= 0 {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
|
||||
return 0, false
|
||||
}
|
||||
return claims.UserID, true
|
||||
}
|
||||
|
||||
func hasBearerAuthorizationHeader(r *http.Request) bool {
|
||||
header := r.Header.Get("Authorization")
|
||||
if header == "" {
|
||||
return false
|
||||
}
|
||||
parts := strings.SplitN(header, " ", 2)
|
||||
return len(parts) == 2 && strings.EqualFold(parts[0], "bearer") && strings.TrimSpace(parts[1]) != ""
|
||||
}
|
||||
|
||||
func nextDiagnosticsPart(mr *multipart.Reader, expectedName, expectedContentType string) (*multipart.Part, error) {
|
||||
part, err := mr.NextPart()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reject a mismatched part without calling part.Close(): Close drains the
|
||||
// unread part body first, so a max-size wrongly-named/typed first part would
|
||||
// stream up to the bundle limit — holding the per-user/global in-flight slot —
|
||||
// before we return 400. Abandoning the part returns immediately; net/http then
|
||||
// discards only a small bounded prefix of the request before closing the
|
||||
// connection, so malformed uploads fail promptly under load.
|
||||
if part.FormName() != expectedName {
|
||||
return nil, errDiagnosticsUnexpectedPart
|
||||
}
|
||||
if !diagnosticsContentTypeMatches(part.Header.Get("Content-Type"), expectedContentType) {
|
||||
return nil, errDiagnosticsUnexpectedPart
|
||||
}
|
||||
return part, nil
|
||||
}
|
||||
|
||||
func diagnosticsContentTypeMatches(raw, expected string) bool {
|
||||
mediaType, _, err := mime.ParseMediaType(raw)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(mediaType, expected)
|
||||
}
|
||||
|
||||
func readDiagnosticsPart(part *multipart.Part, limit int64) ([]byte, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(part, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, errDiagnosticsPartTooLarge
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
type exactlyTwoPartBundleReader struct {
|
||||
part *multipart.Part
|
||||
mr *multipart.Reader
|
||||
checked bool
|
||||
}
|
||||
|
||||
func (r *exactlyTwoPartBundleReader) Read(p []byte) (int, error) {
|
||||
n, err := r.part.Read(p)
|
||||
if errors.Is(err, io.EOF) && !r.checked {
|
||||
r.checked = true
|
||||
next, nextErr := r.mr.NextPart()
|
||||
if errors.Is(nextErr, io.EOF) {
|
||||
return n, io.EOF
|
||||
}
|
||||
if nextErr != nil {
|
||||
return n, nextErr
|
||||
}
|
||||
_ = next.Close()
|
||||
return n, errDiagnosticsUnexpectedPart
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) writeDiagnosticsMultipartError(ctx context.Context, userID int, w http.ResponseWriter, err error) {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
switch {
|
||||
case errors.As(err, &maxBytesErr), errors.Is(err, errDiagnosticsPartTooLarge):
|
||||
h.logRejected(ctx, userID, "too_large")
|
||||
writeError(w, http.StatusRequestEntityTooLarge, "too_large", "Diagnostics upload is too large")
|
||||
default:
|
||||
h.logRejected(ctx, userID, "invalid_bundle")
|
||||
writeError(w, http.StatusBadRequest, "invalid_bundle", "Invalid diagnostics upload")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DiagnosticsHandler) logRejected(ctx context.Context, userID int, reason string) {
|
||||
logger := h.logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
args := []any{
|
||||
"component", "diagnostics",
|
||||
"result", "rejected",
|
||||
"reason", reason,
|
||||
}
|
||||
if userID > 0 {
|
||||
args = append(args, "user_id", userID)
|
||||
}
|
||||
logger.InfoContext(ctx, "diagnostic report rejected", args...)
|
||||
}
|
||||
|
||||
func writeDiagnosticsServiceError(w http.ResponseWriter, err error) {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
switch {
|
||||
case errors.As(err, &maxBytesErr), errors.Is(err, diagnostics.ErrTooLarge):
|
||||
writeError(w, http.StatusRequestEntityTooLarge, "too_large", "Diagnostics upload is too large")
|
||||
case errors.Is(err, diagnostics.ErrDisabled):
|
||||
writeError(w, http.StatusForbidden, "disabled", "Diagnostics uploads are disabled")
|
||||
case errors.Is(err, diagnostics.ErrStorageUnavailable):
|
||||
writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not configured")
|
||||
case errors.Is(err, diagnostics.ErrQuotaExceeded):
|
||||
w.Header().Set("Retry-After", diagnosticsQuotaRetryAfter)
|
||||
writeError(w, http.StatusTooManyRequests, "quota_exceeded", "Diagnostics upload quota exceeded")
|
||||
case errors.Is(err, diagnostics.ErrUnsupportedSchema):
|
||||
writeError(w, http.StatusBadRequest, "unsupported_schema", "Diagnostics schema version is not supported")
|
||||
case errors.Is(err, diagnostics.ErrDestinationMismatch):
|
||||
writeError(w, http.StatusBadRequest, "destination_mismatch", "Diagnostics destination does not match this server")
|
||||
case errors.Is(err, diagnostics.ErrStaleConsent):
|
||||
writeError(w, http.StatusBadRequest, "stale_consent", "Diagnostics consent notice is stale")
|
||||
case errors.Is(err, diagnostics.ErrArchiveMismatch):
|
||||
writeError(w, http.StatusBadRequest, "archive_mismatch", "Diagnostics archive metadata does not match")
|
||||
case errors.Is(err, diagnostics.ErrProfileMismatch):
|
||||
writeError(w, http.StatusBadRequest, "profile_mismatch", "Diagnostics profile does not match the captured report")
|
||||
case errors.Is(err, diagnostics.ErrChildProfileForbidden):
|
||||
writeError(w, http.StatusForbidden, "child_profile_forbidden", "Diagnostics cannot be attributed to a child profile")
|
||||
case errors.Is(err, diagnostics.ErrInvalidBundle):
|
||||
writeError(w, http.StatusBadRequest, "invalid_bundle", "Invalid diagnostics bundle")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Diagnostics upload failed")
|
||||
}
|
||||
}
|
||||
|
||||
type diagnosticsInFlightLimiter struct {
|
||||
mu sync.Mutex
|
||||
active map[int]struct{}
|
||||
global chan struct{}
|
||||
}
|
||||
|
||||
func newDiagnosticsInFlightLimiter(globalLimit int) *diagnosticsInFlightLimiter {
|
||||
if globalLimit <= 0 {
|
||||
globalLimit = 1
|
||||
}
|
||||
return &diagnosticsInFlightLimiter{
|
||||
active: make(map[int]struct{}),
|
||||
global: make(chan struct{}, globalLimit),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *diagnosticsInFlightLimiter) acquire(userID int) (func(), bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if _, ok := l.active[userID]; ok {
|
||||
return nil, false
|
||||
}
|
||||
select {
|
||||
case l.global <- struct{}{}:
|
||||
l.active[userID] = struct{}{}
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
return func() {
|
||||
l.mu.Lock()
|
||||
delete(l.active, userID)
|
||||
l.mu.Unlock()
|
||||
<-l.global
|
||||
}, true
|
||||
}
|
||||
@@ -0,0 +1,876 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/textproto"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics"
|
||||
)
|
||||
|
||||
func TestDiagnosticsUploadHappyPath(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{
|
||||
{name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)},
|
||||
{name: "bundle", contentType: "application/gzip", body: []byte("bundle")},
|
||||
}, accessClaims()))
|
||||
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusCreated, rec.Body.String())
|
||||
}
|
||||
var resp diagnostics.IngestResult
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.ShortID != "SILO-ABCDEF123456" {
|
||||
t.Fatalf("short_id = %q, want SILO-ABCDEF123456", resp.ShortID)
|
||||
}
|
||||
if service.ingestCalls != 1 {
|
||||
t.Fatalf("ingest calls = %d, want 1", service.ingestCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadManifestTooBig(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
logs := captureDiagnosticsLogs(handler)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{
|
||||
{name: "manifest", contentType: "application/json", body: bytes.Repeat([]byte("x"), int(diagnostics.MaxManifestBytes)+1)},
|
||||
{name: "bundle", contentType: "application/gzip", body: []byte("bundle")},
|
||||
}, accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusRequestEntityTooLarge, "too_large")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
assertDiagnosticsRejectionLog(t, logs, "too_large", 42)
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadWrongPartOrder(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
logs := captureDiagnosticsLogs(handler)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{
|
||||
{name: "bundle", contentType: "application/gzip", body: []byte("bundle")},
|
||||
{name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)},
|
||||
}, accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "invalid_bundle")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
assertDiagnosticsRejectionLog(t, logs, "invalid_bundle", 42)
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadWrongFirstPartNotDrained(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
// A large, wrongly-named first part must be rejected without draining its
|
||||
// body: an invalid upload must not be allowed to stream its whole payload
|
||||
// (holding the in-flight slot) before it gets a 400.
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
header := make(textproto.MIMEHeader)
|
||||
header.Set("Content-Disposition", `form-data; name="bundle"`)
|
||||
header.Set("Content-Type", "application/gzip")
|
||||
w, err := writer.CreatePart(header)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart part: %v", err)
|
||||
}
|
||||
if _, err := w.Write(bytes.Repeat([]byte("x"), 1<<20)); err != nil {
|
||||
t.Fatalf("write multipart part: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
counter := &countingReader{r: &body}
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/diagnostics/reports", counter)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer token")
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), accessClaims()))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, req)
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "invalid_bundle")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
// Only the small multipart framing/headers should have been read; the 1 MiB
|
||||
// wrong-first-part body must not have been drained.
|
||||
if counter.n > 128*1024 {
|
||||
t.Fatalf("read %d bytes from request body, want the wrong first part rejected without draining (< 128 KiB)", counter.n)
|
||||
}
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadBodyTooLargeLogsRejection(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.status.MaxBundleBytes = 1
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
logs := captureDiagnosticsLogs(handler)
|
||||
|
||||
body := strings.NewReader(strings.Repeat("x\n", int(diagnosticsMultipartOverheadBytes/2)+2))
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/diagnostics/reports", body)
|
||||
req.Header.Set("Content-Type", "multipart/form-data; boundary=diagnostics-boundary")
|
||||
req.Header.Set("Authorization", "Bearer token")
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), accessClaims()))
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, req)
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusRequestEntityTooLarge, "too_large")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
assertDiagnosticsRejectionLog(t, logs, "too_large", 42)
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadMissingPart(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{
|
||||
{name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)},
|
||||
}, accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "invalid_bundle")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadDisabled(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.status.Status = diagnostics.StatusDisabled
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusForbidden, "disabled")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadStorageUnavailable(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.status.Status = diagnostics.StatusStorageUnavailable
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusServiceUnavailable, "storage_unavailable")
|
||||
if service.ingestCalls != 0 {
|
||||
t.Fatalf("ingest calls = %d, want 0", service.ingestCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadQuotaExceededSetsRetryAfter(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.ingestErr = &diagnostics.QuotaError{Kind: diagnostics.QuotaKindReportsPerDay, Limit: 20}
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusTooManyRequests, "quota_exceeded")
|
||||
if got := rec.Header().Get("Retry-After"); got != diagnosticsQuotaRetryAfter {
|
||||
t.Fatalf("Retry-After = %q, want %q", got, diagnosticsQuotaRetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadRejectsAPIKey(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
logs := captureDiagnosticsLogs(handler)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := newDiagnosticsUploadRequest(t, validDiagnosticsParts(), &auth.Claims{
|
||||
UserID: 42,
|
||||
TokenType: auth.TokenTypeAPIKey,
|
||||
})
|
||||
req.Header.Set("Authorization", "Bearer sa_test")
|
||||
handler.HandleUpload(rec, req)
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusForbidden, "api_key_not_allowed")
|
||||
if service.statusCalls != 0 {
|
||||
t.Fatalf("status calls = %d, want 0", service.statusCalls)
|
||||
}
|
||||
assertDiagnosticsRejectionLog(t, logs, "api_key_not_allowed", 42)
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadBusy(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.started = make(chan struct{})
|
||||
service.block = make(chan struct{})
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
logs := captureDiagnosticsLogs(handler)
|
||||
|
||||
firstReq := newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims())
|
||||
firstDone := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() {
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, firstReq)
|
||||
firstDone <- rec
|
||||
}()
|
||||
<-service.started
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims()))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusServiceUnavailable, "busy")
|
||||
if got := rec.Header().Get("Retry-After"); got != diagnosticsBusyRetryAfter {
|
||||
t.Fatalf("Retry-After = %q, want %q", got, diagnosticsBusyRetryAfter)
|
||||
}
|
||||
assertDiagnosticsRejectionLog(t, logs, "busy", 42)
|
||||
|
||||
close(service.block)
|
||||
first := <-firstDone
|
||||
if first.Code != http.StatusCreated {
|
||||
t.Fatalf("first status = %d, want %d; body=%s", first.Code, http.StatusCreated, first.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminListReportsParsesFilters(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
req := httptest.NewRequest(
|
||||
http.MethodGet,
|
||||
"/api/v1/admin/diagnostics/reports?user_id=42&platform=ios&report_type=crash&from=2026-07-19T10:00:00Z&to=2026-07-20T10:00:00Z&short_id=abcdef123456&limit=25&cursor=next",
|
||||
nil,
|
||||
)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleAdminListReports(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if service.listCalls != 1 {
|
||||
t.Fatalf("list calls = %d, want 1", service.listCalls)
|
||||
}
|
||||
filters := service.listFilters
|
||||
if filters.UserID == nil || *filters.UserID != 42 {
|
||||
t.Fatalf("UserID = %v, want 42", filters.UserID)
|
||||
}
|
||||
if filters.Platform != "ios" || filters.ReportType != "crash" {
|
||||
t.Fatalf("filters = %#v, want platform ios report_type crash", filters)
|
||||
}
|
||||
if filters.From == nil || !filters.From.Equal(time.Date(2026, 7, 19, 10, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("From = %v, want expected timestamp", filters.From)
|
||||
}
|
||||
if filters.To == nil || !filters.To.Equal(time.Date(2026, 7, 20, 10, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("To = %v, want expected timestamp", filters.To)
|
||||
}
|
||||
if filters.ShortID != "SILO-ABCDEF123456" || filters.Limit != 25 || filters.Cursor != "next" {
|
||||
t.Fatalf("filters = %#v, want normalized short id, limit, cursor", filters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminDownloadReturnsPresignedURLWhenAvailable(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.getReport = readyDiagnosticsReport()
|
||||
service.presignURL = "https://storage.example.test/report"
|
||||
service.effectiveTTL = 5 * time.Minute
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
before := time.Now().UTC()
|
||||
rec := httptest.NewRecorder()
|
||||
adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports/report-1/download", adminClaims()))
|
||||
after := time.Now().UTC()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
var resp diagnosticsDownloadURLResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.DownloadURL != service.presignURL {
|
||||
t.Fatalf("download_url = %q, want %q", resp.DownloadURL, service.presignURL)
|
||||
}
|
||||
if service.presignExpiry != service.effectiveTTL {
|
||||
t.Fatalf("presign expiry = %s, want %s", service.presignExpiry, service.effectiveTTL)
|
||||
}
|
||||
if resp.ExpiresAt.Before(before.Add(service.effectiveTTL)) || resp.ExpiresAt.After(after.Add(service.effectiveTTL)) {
|
||||
t.Fatalf("expires_at = %s, want within effective TTL window", resp.ExpiresAt)
|
||||
}
|
||||
if service.openCalls != 0 {
|
||||
t.Fatalf("open calls = %d, want 0", service.openCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminDownloadStreamsWhenProxyRequested(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.getReport = readyDiagnosticsReport()
|
||||
service.presignURL = "https://storage.example.test/report"
|
||||
service.openData = []byte("gzipped report")
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports/report-1/download?proxy=1", adminClaims()))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if service.presignCalls != 0 {
|
||||
t.Fatalf("presign calls = %d, want 0", service.presignCalls)
|
||||
}
|
||||
if service.openCalls != 1 {
|
||||
t.Fatalf("open calls = %d, want 1", service.openCalls)
|
||||
}
|
||||
if rec.Body.String() != "gzipped report" {
|
||||
t.Fatalf("body = %q, want streamed data", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminDownloadStreamsWhenPresignUnavailable(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.getReport = readyDiagnosticsReport()
|
||||
service.presignErr = errors.New("presign unavailable")
|
||||
service.openData = []byte("gzipped report")
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports/report-1/download", adminClaims()))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("Content-Type"); got != diagnostics.ReportDownloadContentType {
|
||||
t.Fatalf("Content-Type = %q, want %q", got, diagnostics.ReportDownloadContentType)
|
||||
}
|
||||
if !strings.Contains(rec.Header().Get("Content-Disposition"), "silo-diagnostics-SILO-ABCDEF123456.tar.gz") {
|
||||
t.Fatalf("Content-Disposition = %q, want diagnostic filename", rec.Header().Get("Content-Disposition"))
|
||||
}
|
||||
if rec.Body.String() != "gzipped report" {
|
||||
t.Fatalf("body = %q, want streamed data", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminListReportsRejectsMalformedQuery(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics/reports?user_id=abc", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleAdminListReports(rec, req)
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "bad_request")
|
||||
if service.listCalls != 0 {
|
||||
t.Fatalf("list calls = %d, want 0", service.listCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminListReportsRejectsMalformedCursor(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.listErr = diagnostics.ErrInvalidCursor
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics/reports?cursor=bad", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleAdminListReports(rec, req)
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "bad_request")
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminDeleteEmitsAuditEvent(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
service.deleteReport = readyDiagnosticsReport()
|
||||
var logs bytes.Buffer
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
handler.logger = slog.New(slog.NewJSONHandler(&logs, nil))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodDelete, "/diagnostics/reports/report-1", adminClaims()))
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusNoContent, rec.Body.String())
|
||||
}
|
||||
logLine := logs.String()
|
||||
if !strings.Contains(logLine, `"msg":"diagnostic report deleted"`) ||
|
||||
!strings.Contains(logLine, `"admin_user_id":7`) ||
|
||||
!strings.Contains(logLine, `"report_id":"report-1"`) {
|
||||
t.Fatalf("audit log = %s", logLine)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsAdminRoutesRejectNonActingAdmin(t *testing.T) {
|
||||
service := newFakeDiagnosticsService()
|
||||
handler := NewDiagnosticsHandler(service)
|
||||
router := chi.NewRouter()
|
||||
router.Use(apimw.RequireActingAdmin(nil))
|
||||
RegisterAdminDiagnosticsRoutes(router, handler)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports", &auth.Claims{
|
||||
UserID: 8,
|
||||
Role: "user",
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
}))
|
||||
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusForbidden, rec.Body.String())
|
||||
}
|
||||
if service.listCalls != 0 {
|
||||
t.Fatalf("list calls = %d, want 0", service.listCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsUploadsEnablementRequiresStorageProbe(t *testing.T) {
|
||||
settings := &fakeServerSettingsStore{values: map[string]string{}}
|
||||
handler := &AdminHandler{SettingsRepo: settings}
|
||||
router := chi.NewRouter()
|
||||
router.Put("/admin/settings/{key}", handler.HandleUpdateSetting)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/admin/settings/"+diagnostics.KeyUploadsEnabled,
|
||||
strings.NewReader(`{"value":"true"}`),
|
||||
))
|
||||
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "storage_unavailable")
|
||||
if settings.values[diagnostics.KeyUploadsEnabled] != "" {
|
||||
t.Fatalf("setting persisted = %q, want empty", settings.values[diagnostics.KeyUploadsEnabled])
|
||||
}
|
||||
|
||||
store := &fakeDiagnosticsEnablementStore{bucket: "private"}
|
||||
handler.DiagnosticsStore = store
|
||||
rec = httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/admin/settings/"+diagnostics.KeyUploadsEnabled,
|
||||
strings.NewReader(`{"value":"true"}`),
|
||||
))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if settings.values[diagnostics.KeyUploadsEnabled] != "true" {
|
||||
t.Fatalf("setting persisted = %q, want true", settings.values[diagnostics.KeyUploadsEnabled])
|
||||
}
|
||||
if !sameStringSlice(store.ops, []string{"put:diagnostics/.probe", "delete:diagnostics/.probe"}) {
|
||||
t.Fatalf("probe ops = %v, want put/delete probe", store.ops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsNumericSettingsAcceptBoundaries(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
key string
|
||||
value string
|
||||
initial map[string]string
|
||||
}{
|
||||
{name: "bundle minimum", key: diagnostics.KeyMaxBundleBytes, value: "1048576", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824"}},
|
||||
{name: "bundle maximum", key: diagnostics.KeyMaxBundleBytes, value: "268435456", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824", diagnostics.KeyMaxBytesPerUser: "10737418240"}},
|
||||
{name: "uncompressed bundle floor", key: diagnostics.KeyMaxUncompressedBytes, value: "1048576", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "1048576"}},
|
||||
{name: "uncompressed maximum", key: diagnostics.KeyMaxUncompressedBytes, value: "1073741824", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "1048576"}},
|
||||
{name: "reports minimum", key: diagnostics.KeyMaxReportsPerUserDay, value: "1"},
|
||||
{name: "reports maximum", key: diagnostics.KeyMaxReportsPerUserDay, value: "1000"},
|
||||
{name: "retention minimum", key: diagnostics.KeyRetentionDays, value: "1"},
|
||||
{name: "retention maximum", key: diagnostics.KeyRetentionDays, value: "365"},
|
||||
{name: "user bytes minimum", key: diagnostics.KeyMaxBytesPerUser, value: "10485760"},
|
||||
{name: "user bytes maximum", key: diagnostics.KeyMaxBytesPerUser, value: "10737418240"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
settings := &fakeServerSettingsStore{values: tc.initial}
|
||||
rec := updateDiagnosticsSetting(t, &AdminHandler{SettingsRepo: settings}, tc.key, tc.value)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if settings.values[tc.key] != tc.value {
|
||||
t.Fatalf("stored value = %q, want %q", settings.values[tc.key], tc.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsNumericSettingsRejectOutOfRangeValues(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
key string
|
||||
value string
|
||||
initial map[string]string
|
||||
}{
|
||||
{name: "bundle below minimum", key: diagnostics.KeyMaxBundleBytes, value: "1048575"},
|
||||
{name: "bundle above maximum", key: diagnostics.KeyMaxBundleBytes, value: "268435457", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824"}},
|
||||
{name: "bundle above uncompressed", key: diagnostics.KeyMaxBundleBytes, value: "67108865", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "67108864"}},
|
||||
{name: "uncompressed below bundle", key: diagnostics.KeyMaxUncompressedBytes, value: "10485759", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "10485760"}},
|
||||
{name: "bundle above per-user cap", key: diagnostics.KeyMaxBundleBytes, value: "104857600", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824", diagnostics.KeyMaxBytesPerUser: "10485760"}},
|
||||
{name: "per-user below bundle", key: diagnostics.KeyMaxBytesPerUser, value: "10485760", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "104857600", diagnostics.KeyMaxUncompressedBytes: "1073741824"}},
|
||||
{name: "uncompressed above maximum", key: diagnostics.KeyMaxUncompressedBytes, value: "1073741825"},
|
||||
{name: "reports below minimum", key: diagnostics.KeyMaxReportsPerUserDay, value: "0"},
|
||||
{name: "reports above maximum", key: diagnostics.KeyMaxReportsPerUserDay, value: "1001"},
|
||||
{name: "retention below minimum", key: diagnostics.KeyRetentionDays, value: "0"},
|
||||
{name: "retention above maximum", key: diagnostics.KeyRetentionDays, value: "366"},
|
||||
{name: "user bytes below minimum", key: diagnostics.KeyMaxBytesPerUser, value: "10485759"},
|
||||
{name: "user bytes above maximum", key: diagnostics.KeyMaxBytesPerUser, value: "10737418241"},
|
||||
{name: "not an integer", key: diagnostics.KeyRetentionDays, value: "thirty"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
settings := &fakeServerSettingsStore{values: tc.initial}
|
||||
rec := updateDiagnosticsSetting(t, &AdminHandler{SettingsRepo: settings}, tc.key, tc.value)
|
||||
body := rec.Body.String()
|
||||
assertDiagnosticsError(t, rec, http.StatusBadRequest, "bad_request")
|
||||
if _, stored := settings.values[tc.key]; stored {
|
||||
t.Fatalf("invalid value was stored: %#v", settings.values)
|
||||
}
|
||||
if !strings.Contains(body, tc.key) {
|
||||
t.Fatalf("error body = %s, want clear error naming %s", body, tc.key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type diagnosticsPart struct {
|
||||
name string
|
||||
contentType string
|
||||
body []byte
|
||||
}
|
||||
|
||||
func validDiagnosticsParts() []diagnosticsPart {
|
||||
return []diagnosticsPart{
|
||||
{name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)},
|
||||
{name: "bundle", contentType: "application/gzip", body: []byte("bundle")},
|
||||
}
|
||||
}
|
||||
|
||||
func newDiagnosticsUploadRequest(t *testing.T, parts []diagnosticsPart, claims *auth.Claims) *http.Request {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
for _, part := range parts {
|
||||
header := make(textproto.MIMEHeader)
|
||||
header.Set("Content-Disposition", `form-data; name="`+part.name+`"`)
|
||||
header.Set("Content-Type", part.contentType)
|
||||
w, err := writer.CreatePart(header)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart part: %v", err)
|
||||
}
|
||||
if _, err := w.Write(part.body); err != nil {
|
||||
t.Fatalf("write multipart part: %v", err)
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/diagnostics/reports", &body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer token")
|
||||
if claims != nil {
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), claims))
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func accessClaims() *auth.Claims {
|
||||
return &auth.Claims{
|
||||
UserID: 42,
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
}
|
||||
}
|
||||
|
||||
func assertDiagnosticsError(t *testing.T, rec *httptest.ResponseRecorder, status int, code string) {
|
||||
t.Helper()
|
||||
if rec.Code != status {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, status, rec.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode error response: %v", err)
|
||||
}
|
||||
if resp.Error != code {
|
||||
t.Fatalf("error = %q, want %q", resp.Error, code)
|
||||
}
|
||||
}
|
||||
|
||||
func captureDiagnosticsLogs(handler *DiagnosticsHandler) *bytes.Buffer {
|
||||
var logs bytes.Buffer
|
||||
handler.logger = slog.New(slog.NewJSONHandler(&logs, nil))
|
||||
return &logs
|
||||
}
|
||||
|
||||
func assertDiagnosticsRejectionLog(t *testing.T, logs *bytes.Buffer, reason string, userID int) {
|
||||
t.Helper()
|
||||
line := logs.String()
|
||||
for _, want := range []string{
|
||||
`"msg":"diagnostic report rejected"`,
|
||||
`"component":"diagnostics"`,
|
||||
`"result":"rejected"`,
|
||||
`"reason":"` + reason + `"`,
|
||||
`"user_id":` + strconv.Itoa(userID),
|
||||
} {
|
||||
if !strings.Contains(line, want) {
|
||||
t.Fatalf("rejection log = %s, want %s", line, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func updateDiagnosticsSetting(t *testing.T, handler *AdminHandler, key, value string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
router := chi.NewRouter()
|
||||
router.Put("/admin/settings/{key}", handler.HandleUpdateSetting)
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/admin/settings/"+key,
|
||||
strings.NewReader(`{"value":"`+value+`"}`),
|
||||
))
|
||||
return rec
|
||||
}
|
||||
|
||||
func readyDiagnosticsReport() *diagnostics.Report {
|
||||
blobBucket := "private"
|
||||
blobKey := "diagnostics/7/report-1.tar.gz"
|
||||
blobBytes := int64(13)
|
||||
return &diagnostics.Report{
|
||||
ID: "report-1",
|
||||
ShortID: "SILO-ABCDEF123456",
|
||||
UserID: 7,
|
||||
State: diagnostics.StateReady,
|
||||
ReportType: "crash",
|
||||
Platform: "android-tv",
|
||||
BlobBucket: &blobBucket,
|
||||
BlobKey: &blobKey,
|
||||
BlobBytes: &blobBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func adminDiagnosticsRouter(handler *DiagnosticsHandler) chi.Router {
|
||||
router := chi.NewRouter()
|
||||
RegisterAdminDiagnosticsRoutes(router, handler)
|
||||
return router
|
||||
}
|
||||
|
||||
func adminDiagnosticsRequest(method, target string, claims *auth.Claims) *http.Request {
|
||||
req := httptest.NewRequest(method, target, nil)
|
||||
if claims != nil {
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), claims))
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func adminClaims() *auth.Claims {
|
||||
return &auth.Claims{
|
||||
UserID: 7,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
}
|
||||
}
|
||||
|
||||
type fakeDiagnosticsService struct {
|
||||
mu sync.Mutex
|
||||
status diagnostics.Status
|
||||
statusCalls int
|
||||
ingestCalls int
|
||||
ingestErr error
|
||||
started chan struct{}
|
||||
block chan struct{}
|
||||
listCalls int
|
||||
listFilters diagnostics.ListFilters
|
||||
listResult diagnostics.ListResult
|
||||
listErr error
|
||||
getReport *diagnostics.Report
|
||||
getErr error
|
||||
presignURL string
|
||||
presignErr error
|
||||
presignCalls int
|
||||
presignExpiry time.Duration
|
||||
effectiveTTL time.Duration
|
||||
openCalls int
|
||||
openData []byte
|
||||
openErr error
|
||||
deleteReport *diagnostics.Report
|
||||
deleteErr error
|
||||
}
|
||||
|
||||
func newFakeDiagnosticsService() *fakeDiagnosticsService {
|
||||
return &fakeDiagnosticsService{
|
||||
status: diagnostics.Status{
|
||||
Status: diagnostics.StatusAvailable,
|
||||
ServerInstanceID: "server-1",
|
||||
AcceptedSchemaVersions: []int{1},
|
||||
MaxBundleBytes: diagnostics.DefaultMaxBundleBytes,
|
||||
MaxManifestBytes: diagnostics.MaxManifestBytes,
|
||||
RetentionDays: diagnostics.DefaultRetentionDays,
|
||||
ConsentNoticeVersion: diagnostics.DefaultConsentNoticeVer,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) Status(context.Context, int) (diagnostics.Status, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.statusCalls++
|
||||
return f.status, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) Ingest(_ context.Context, _ int, _ *string, _ []byte, bundle io.Reader) (diagnostics.IngestResult, error) {
|
||||
f.mu.Lock()
|
||||
f.ingestCalls++
|
||||
started := f.started
|
||||
block := f.block
|
||||
err := f.ingestErr
|
||||
f.mu.Unlock()
|
||||
|
||||
if started != nil {
|
||||
close(started)
|
||||
}
|
||||
if block != nil {
|
||||
<-block
|
||||
}
|
||||
if err != nil {
|
||||
return diagnostics.IngestResult{}, err
|
||||
}
|
||||
if _, err := io.ReadAll(bundle); err != nil {
|
||||
return diagnostics.IngestResult{}, err
|
||||
}
|
||||
return diagnostics.IngestResult{
|
||||
ReportID: "11111111-1111-1111-1111-111111111111",
|
||||
ShortID: "SILO-ABCDEF123456",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) ListForAdmin(_ context.Context, filters diagnostics.ListFilters) (diagnostics.ListResult, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.listCalls++
|
||||
f.listFilters = filters
|
||||
if f.listErr != nil {
|
||||
return diagnostics.ListResult{}, f.listErr
|
||||
}
|
||||
return f.listResult, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) GetReport(context.Context, string) (*diagnostics.Report, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.getErr != nil {
|
||||
return nil, f.getErr
|
||||
}
|
||||
if f.getReport != nil {
|
||||
return f.getReport, nil
|
||||
}
|
||||
return readyDiagnosticsReport(), nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) PresignReportDownload(_ context.Context, _ *diagnostics.Report, expiry time.Duration) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.presignCalls++
|
||||
f.presignExpiry = expiry
|
||||
if f.presignErr != nil {
|
||||
return "", f.presignErr
|
||||
}
|
||||
return f.presignURL, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) EffectiveReportDownloadTTL(requested time.Duration) time.Duration {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.effectiveTTL > 0 {
|
||||
return f.effectiveTTL
|
||||
}
|
||||
return requested
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) OpenReportDownload(context.Context, *diagnostics.Report) (io.ReadCloser, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.openCalls++
|
||||
if f.openErr != nil {
|
||||
return nil, f.openErr
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(f.openData)), nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsService) DeleteReport(context.Context, string) (*diagnostics.Report, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.deleteErr != nil {
|
||||
return nil, f.deleteErr
|
||||
}
|
||||
if f.deleteReport != nil {
|
||||
return f.deleteReport, nil
|
||||
}
|
||||
return readyDiagnosticsReport(), nil
|
||||
}
|
||||
|
||||
type fakeDiagnosticsEnablementStore struct {
|
||||
bucket string
|
||||
ops []string
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsEnablementStore) PutStream(_ context.Context, _ string, key string, r io.Reader, _ string) error {
|
||||
if _, err := io.ReadAll(r); err != nil {
|
||||
return err
|
||||
}
|
||||
f.ops = append(f.ops, "put:"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsEnablementStore) DeleteObject(_ context.Context, _ string, key string) error {
|
||||
f.ops = append(f.ops, "delete:"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticsEnablementStore) Bucket() string {
|
||||
return f.bucket
|
||||
}
|
||||
|
||||
func sameStringSlice(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -108,6 +108,21 @@ func (h *EventsHandler) HandleWebSocket(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
conn, err := wsUpgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
// The Android/KMP client has a long history of silent handshake
|
||||
// failures here (gorilla writes the 4xx itself); log the exact
|
||||
// upgrade-relevant request shape so a failing client is diagnosable
|
||||
// from the server alone.
|
||||
slog.WarnContext(r.Context(), "events websocket upgrade failed", "component", "api",
|
||||
"error", err,
|
||||
"user_id", claims.UserID,
|
||||
"proto", r.Proto,
|
||||
"method", r.Method,
|
||||
"connection_header", r.Header.Get("Connection"),
|
||||
"upgrade_header", r.Header.Get("Upgrade"),
|
||||
"ws_version", r.Header.Get("Sec-Websocket-Version"),
|
||||
"ws_key_present", r.Header.Get("Sec-Websocket-Key") != "",
|
||||
"user_agent", r.UserAgent(),
|
||||
)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
@@ -344,6 +344,7 @@ type episodeResponse struct {
|
||||
StillThumbhash string `json:"still_thumbhash,omitempty"`
|
||||
UserData *catalog.SeasonUserData `json:"user_data,omitempty"`
|
||||
Files []episodeFileResponse `json:"files,omitempty"`
|
||||
OverlaySummary *models.OverlaySummary `json:"overlay_summary,omitempty"`
|
||||
}
|
||||
|
||||
type episodeImageFallback struct {
|
||||
@@ -959,8 +960,10 @@ func (h *ItemsHandler) buildEpisodeResponses(r *http.Request, episodes []*models
|
||||
}
|
||||
for i := range resp {
|
||||
resp[i].StillURL = stillURLs[stillPaths[i]].URL
|
||||
resp[i].Files = episodeFileResponses(filesByEpisode[resp[i].ContentID], filter)
|
||||
files := catalog.FilterMediaFilesByAccess(filesByEpisode[resp[i].ContentID], filter)
|
||||
resp[i].Files = episodeFileResponses(files, filter)
|
||||
resp[i].UserData = userData[resp[i].ContentID]
|
||||
resp[i].OverlaySummary = overlays.BuildSummary(files)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -31,6 +30,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
"github.com/Silo-Server/silo-server/internal/rootcheck"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/Silo-Server/silo-server/internal/scantrigger"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
@@ -251,6 +251,11 @@ type libraryMountCheckRootResponse struct {
|
||||
Reachable bool `json:"reachable"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
ErrorMessage *string `json:"error_message"`
|
||||
// SuspectEmpty is set when the root is reachable but the library holds
|
||||
// only missing-marked files under it — the signature of a lost mount
|
||||
// exposing an empty mountpoint directory, which a reachability probe
|
||||
// alone cannot detect. Additive field; absent/false for healthy roots.
|
||||
SuspectEmpty bool `json:"suspect_empty"`
|
||||
}
|
||||
|
||||
type libraryMountCheckResponse struct {
|
||||
@@ -847,8 +852,9 @@ func (h *LibraryHandler) HandleCheckLibraryMount(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
resp := checkLibraryMount(folder)
|
||||
if resp.Healthy && folder.ScanWarningCode != nil && *folder.ScanWarningCode == "empty_root" {
|
||||
resp := h.checkLibraryMount(r.Context(), folder)
|
||||
if resp.Healthy && folder.ScanWarningCode != nil &&
|
||||
(*folder.ScanWarningCode == "empty_root" || *folder.ScanWarningCode == "dead_root") {
|
||||
if err := h.folderRepo.ClearScanWarning(r.Context(), folder.ID); err != nil {
|
||||
if errors.Is(err, catalog.ErrFolderNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Library not found")
|
||||
@@ -1256,7 +1262,7 @@ func scanBoolMetric(result *libraryingest.Result, pick func(*scanner.ScanResult)
|
||||
return pick(result.ScanResult)
|
||||
}
|
||||
|
||||
func checkLibraryMount(folder *models.MediaFolder) libraryMountCheckResponse {
|
||||
func (h *LibraryHandler) checkLibraryMount(ctx context.Context, folder *models.MediaFolder) libraryMountCheckResponse {
|
||||
resp := libraryMountCheckResponse{
|
||||
Status: "ok",
|
||||
LibraryID: folder.ID,
|
||||
@@ -1273,40 +1279,58 @@ func checkLibraryMount(folder *models.MediaFolder) libraryMountCheckResponse {
|
||||
}
|
||||
|
||||
unreachable := 0
|
||||
for _, path := range folder.Paths {
|
||||
emptyPaths := make([]string, 0, len(folder.Paths))
|
||||
probes := rootcheck.ProbeManyWithTimeout(ctx, folder.Paths, rootcheck.DefaultProbeTimeout)
|
||||
for i, path := range folder.Paths {
|
||||
probe := probes[i]
|
||||
root := libraryMountCheckRootResponse{
|
||||
Path: path,
|
||||
Reachable: true,
|
||||
Reachable: probe.Reachable,
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
code, message := classifyMountCheckError(err, false)
|
||||
root.Reachable = false
|
||||
root.ErrorCode = stringPtr(code)
|
||||
root.ErrorMessage = stringPtr(message)
|
||||
} else if !info.IsDir() {
|
||||
root.Reachable = false
|
||||
root.ErrorCode = stringPtr("not_directory")
|
||||
root.ErrorMessage = stringPtr("Path is not a directory")
|
||||
} else if _, err := os.ReadDir(path); err != nil {
|
||||
code, message := classifyMountCheckError(err, true)
|
||||
root.Reachable = false
|
||||
root.ErrorCode = stringPtr(code)
|
||||
root.ErrorMessage = stringPtr(message)
|
||||
}
|
||||
|
||||
if !root.Reachable {
|
||||
if !probe.Reachable {
|
||||
root.ErrorCode = stringPtr(probe.ErrorCode)
|
||||
root.ErrorMessage = stringPtr(probe.ErrorMessage)
|
||||
unreachable++
|
||||
resp.Healthy = false
|
||||
} else if probe.Empty {
|
||||
emptyPaths = append(emptyPaths, path)
|
||||
}
|
||||
resp.Roots = append(resp.Roots, root)
|
||||
}
|
||||
|
||||
switch unreachable {
|
||||
case 0:
|
||||
// A reachable but literally empty root can be a lost mount that left its
|
||||
// bare mountpoint behind. Cross-check against the catalog: an empty root
|
||||
// holding only missing-marked files is suspect, and reporting it healthy
|
||||
// would both mislead the operator and wrongly clear the dead_root
|
||||
// warning. Best-effort — probe results stand on a query error.
|
||||
suspect := 0
|
||||
if h.pool != nil && len(emptyPaths) > 0 {
|
||||
suspectRoots, err := scanner.NewFileRepository(h.pool).ListRootsWithOnlyMissingFiles(ctx, folder.ID, emptyPaths)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "mount check: suspect-empty root query failed", "component", "api", "library_id", folder.ID, "error", err)
|
||||
} else if len(suspectRoots) > 0 {
|
||||
suspectSet := make(map[string]bool, len(suspectRoots))
|
||||
for _, root := range suspectRoots {
|
||||
suspectSet[root] = true
|
||||
}
|
||||
for i := range resp.Roots {
|
||||
if resp.Roots[i].Reachable && suspectSet[resp.Roots[i].Path] {
|
||||
resp.Roots[i].SuspectEmpty = true
|
||||
suspect++
|
||||
resp.Healthy = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case unreachable == 0 && suspect == 0:
|
||||
resp.Summary = "All configured roots are reachable"
|
||||
case 1:
|
||||
case unreachable == 0:
|
||||
resp.Summary = fmt.Sprintf("%d of %d roots reachable but empty while the library still has cataloged files (lost mount?)", suspect, len(folder.Paths))
|
||||
case suspect > 0:
|
||||
resp.Summary = fmt.Sprintf("%d of %d roots unreachable; %d more reachable but empty while the library still has cataloged files", unreachable, len(folder.Paths), suspect)
|
||||
case unreachable == 1:
|
||||
resp.Summary = fmt.Sprintf("1 of %d roots unreachable", len(folder.Paths))
|
||||
default:
|
||||
resp.Summary = fmt.Sprintf("%d of %d roots unreachable", unreachable, len(folder.Paths))
|
||||
@@ -1315,19 +1339,6 @@ func checkLibraryMount(folder *models.MediaFolder) libraryMountCheckResponse {
|
||||
return resp
|
||||
}
|
||||
|
||||
func classifyMountCheckError(err error, isRead bool) (string, string) {
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return "not_found", "Path does not exist"
|
||||
case errors.Is(err, os.ErrPermission):
|
||||
return "permission_denied", "Permission denied"
|
||||
case isRead:
|
||||
return "read_failed", "Failed to read directory"
|
||||
default:
|
||||
return "stat_failed", "Failed to stat path"
|
||||
}
|
||||
}
|
||||
|
||||
func stringPtr(v string) *string {
|
||||
return &v
|
||||
}
|
||||
@@ -2191,22 +2202,7 @@ func buildSeededChainEntries(level string, candidates []seedCandidate) []metadat
|
||||
}
|
||||
|
||||
func metadataContentLevelsForLibraryType(libraryType string) []string {
|
||||
switch libraryType {
|
||||
case "series":
|
||||
return []string{"series", "season", "episode"}
|
||||
case "movies", "movie":
|
||||
return []string{"movie"}
|
||||
case "audiobooks", "audiobook":
|
||||
return []string{"audiobook"}
|
||||
case "ebooks", "ebook":
|
||||
return []string{"ebook"}
|
||||
case "manga":
|
||||
return []string{"manga"}
|
||||
case "mixed":
|
||||
return []string{"movie", "series", "season", "episode", "audiobook", "ebook"}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return metadata.ContentLevelsForLibraryType(libraryType)
|
||||
}
|
||||
|
||||
// HandleListStaleIDs handles GET /libraries/stale-ids.
|
||||
|
||||
@@ -388,21 +388,30 @@ func (h *NotificationsHandler) HandleCapability(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
}
|
||||
applePush := capabilityPush{Available: false, Provider: "off", SupportedModes: []string{"in_app_only"}}
|
||||
androidPush := capabilityPush{Available: false, Provider: "off", SupportedModes: []string{"in_app_only"}}
|
||||
// Like web push above, availability requires both the wiring (cipher/store)
|
||||
// and the admin delivery toggle: Available must mean "setup will actually
|
||||
// deliver", not "the server could store a token".
|
||||
if h.system.PushDevices != nil && h.system.PushDevices.Available() &&
|
||||
h.system.Settings.ApplePushDeliveryEnabled(r.Context()) {
|
||||
applePush = capabilityPush{
|
||||
Available: true,
|
||||
Provider: notifications.PushProviderSiloRelay,
|
||||
SupportedModes: []string{notifications.PushModePrivatePush, notifications.PushModeInAppOnly},
|
||||
if h.system.PushDevices != nil && h.system.PushDevices.Available() {
|
||||
if h.system.Settings.ApplePushDeliveryEnabled(r.Context()) {
|
||||
applePush = capabilityPush{
|
||||
Available: true,
|
||||
Provider: notifications.PushProviderSiloRelay,
|
||||
SupportedModes: []string{notifications.PushModePrivatePush, notifications.PushModeInAppOnly},
|
||||
}
|
||||
}
|
||||
if h.system.Settings.AndroidPushDeliveryEnabled(r.Context()) {
|
||||
androidPush = capabilityPush{
|
||||
Available: true,
|
||||
Provider: notifications.PushProviderSiloRelay,
|
||||
SupportedModes: []string{notifications.PushModePrivatePush, notifications.PushModeInAppOnly},
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, capabilityResponse{
|
||||
InApp: capabilityInApp{Enabled: h.system.Settings.UIEnabled(r.Context())},
|
||||
ApplePush: applePush,
|
||||
AndroidPush: capabilityPush{Available: false, Provider: "off", SupportedModes: []string{"in_app_only"}},
|
||||
AndroidPush: androidPush,
|
||||
WebPush: webPush,
|
||||
Webhooks: webhooks,
|
||||
Email: email,
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type applePushRegisterRequest struct {
|
||||
@@ -73,3 +74,80 @@ func (h *NotificationsHandler) HandleRegisterApplePushDevice(w http.ResponseWrit
|
||||
PushMode: device.PushMode,
|
||||
})
|
||||
}
|
||||
|
||||
type pushDeviceRegisterRequest struct {
|
||||
Platform string `json:"platform"`
|
||||
Token string `json:"token"`
|
||||
DeviceID string `json:"device_id"`
|
||||
PushMode string `json:"push_mode"`
|
||||
}
|
||||
|
||||
// HandleRegisterPushDevice handles POST /notifications/push/devices, the
|
||||
// platform-generic registration used by the Android client. Apple installs
|
||||
// keep the dedicated /devices/push/apple route because their registration
|
||||
// carries APNs-specific fields (environment, topic).
|
||||
func (h *NotificationsHandler) HandleRegisterPushDevice(w http.ResponseWriter, r *http.Request) {
|
||||
service := h.pushDevices()
|
||||
if service == nil || !service.Available() {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available")
|
||||
return
|
||||
}
|
||||
|
||||
var req pushDeviceRegisterRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.Platform != notifications.PushPlatformAndroid {
|
||||
writeError(w, http.StatusUnprocessableEntity, "unsupported_push_device", "platform is not supported on this endpoint")
|
||||
return
|
||||
}
|
||||
|
||||
device, err := service.RegisterFCM(r.Context(), apimw.GetUserID(r.Context()), apimw.GetProfileID(r.Context()), notifications.FCMPushRegistrationInput{
|
||||
DeviceID: req.DeviceID,
|
||||
FCMToken: req.Token,
|
||||
PushMode: req.PushMode,
|
||||
})
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, notifications.ErrPushDeviceInvalid):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
case errors.Is(err, notifications.ErrPushDeviceUnsupported):
|
||||
writeError(w, http.StatusUnprocessableEntity, "unsupported_push_device", err.Error())
|
||||
case errors.Is(err, notifications.ErrPushDeviceUnavailable):
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to register push device")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, applePushRegisterResponse{
|
||||
ID: device.ID,
|
||||
ServerDeviceID: device.ServerDeviceID,
|
||||
Enabled: device.Enabled,
|
||||
PushMode: device.PushMode,
|
||||
})
|
||||
}
|
||||
|
||||
// HandleUnregisterPushDevice handles DELETE /notifications/push/devices/{device_id}.
|
||||
func (h *NotificationsHandler) HandleUnregisterPushDevice(w http.ResponseWriter, r *http.Request) {
|
||||
service := h.pushDevices()
|
||||
if service == nil || !service.Available() {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available")
|
||||
return
|
||||
}
|
||||
err := service.Unregister(r.Context(), apimw.GetProfileID(r.Context()), chi.URLParam(r, "device_id"))
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, notifications.ErrPushDeviceInvalid):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
case errors.Is(err, notifications.ErrPushDeviceUnavailable):
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to unregister push device")
|
||||
}
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -15,10 +15,12 @@ import (
|
||||
)
|
||||
|
||||
type handlerPushStore struct {
|
||||
got notifications.ApplePushDeviceRegistration
|
||||
calls int
|
||||
device *notifications.PushDevice
|
||||
err error
|
||||
got notifications.ApplePushDeviceRegistration
|
||||
gotFCM notifications.FCMPushDeviceRegistration
|
||||
calls int
|
||||
device *notifications.PushDevice
|
||||
err error
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *handlerPushStore) UpsertApple(ctx context.Context, registration notifications.ApplePushDeviceRegistration, cipher *secret.Cipher) (*notifications.PushDevice, error) {
|
||||
@@ -38,6 +40,29 @@ func (f *handlerPushStore) UpsertApple(ctx context.Context, registration notific
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *handlerPushStore) UpsertFCM(ctx context.Context, registration notifications.FCMPushDeviceRegistration, cipher *secret.Cipher) (*notifications.PushDevice, error) {
|
||||
f.calls++
|
||||
f.gotFCM = registration
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if f.device != nil {
|
||||
return f.device, nil
|
||||
}
|
||||
return ¬ifications.PushDevice{
|
||||
ID: "push-row-android",
|
||||
Platform: notifications.PushPlatformAndroid,
|
||||
ServerDeviceID: "server-device-android",
|
||||
Enabled: true,
|
||||
PushMode: registration.PushMode,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *handlerPushStore) DeleteByProfileDevice(ctx context.Context, profileID, deviceID string) error {
|
||||
f.deleted = append(f.deleted, profileID+"/"+deviceID)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func handlerPushCipher(t *testing.T) *secret.Cipher {
|
||||
t.Helper()
|
||||
cipher, err := secret.New([]byte("01234567890123456789012345678901"))
|
||||
@@ -88,6 +113,68 @@ func TestHandleRegisterApplePushDevice(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func newPushDevicesRequest(method, target, body string) *http.Request {
|
||||
req := httptest.NewRequest(method, target, strings.NewReader(body))
|
||||
ctx := apimw.SetClaims(req.Context(), &auth.Claims{UserID: 42, Role: "user", TokenType: auth.TokenTypeAccess})
|
||||
ctx = apimw.SetProfileID(ctx, "profile-1")
|
||||
return req.WithContext(ctx)
|
||||
}
|
||||
|
||||
func TestHandleRegisterPushDeviceAndroid(t *testing.T) {
|
||||
store := &handlerPushStore{}
|
||||
handler := NewNotificationsHandler(¬ifications.System{
|
||||
PushDevices: notifications.NewPushDeviceService(store, handlerPushCipher(t)),
|
||||
}, nil)
|
||||
|
||||
token := strings.Repeat("F", 140)
|
||||
body := `{
|
||||
"platform":"android",
|
||||
"token":"` + token + `",
|
||||
"device_id":"local-device",
|
||||
"push_mode":"private_push"
|
||||
}`
|
||||
rr := httptest.NewRecorder()
|
||||
handler.HandleRegisterPushDevice(rr, newPushDevicesRequest(http.MethodPost, "/api/v1/notifications/push/devices", body))
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var response applePushRegisterResponse
|
||||
if err := json.NewDecoder(rr.Body).Decode(&response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if response.ID != "push-row-android" || response.PushMode != notifications.PushModePrivatePush {
|
||||
t.Fatalf("unexpected response: %+v", response)
|
||||
}
|
||||
if store.gotFCM.UserID != 42 || store.gotFCM.ProfileID != "profile-1" || store.gotFCM.FCMToken != token {
|
||||
t.Fatalf("unexpected stored registration: %+v", store.gotFCM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegisterPushDeviceRejectsUnknownPlatformAndBadToken(t *testing.T) {
|
||||
newHandler := func() *NotificationsHandler {
|
||||
return NewNotificationsHandler(¬ifications.System{
|
||||
PushDevices: notifications.NewPushDeviceService(&handlerPushStore{}, handlerPushCipher(t)),
|
||||
}, nil)
|
||||
}
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
newHandler().HandleRegisterPushDevice(rr, newPushDevicesRequest(http.MethodPost,
|
||||
"/api/v1/notifications/push/devices",
|
||||
`{"platform":"apple","token":"`+strings.Repeat("F", 140)+`","device_id":"local-device"}`))
|
||||
if rr.Code != http.StatusUnprocessableEntity {
|
||||
t.Fatalf("apple platform status = %d, want 422", rr.Code)
|
||||
}
|
||||
|
||||
rr = httptest.NewRecorder()
|
||||
newHandler().HandleRegisterPushDevice(rr, newPushDevicesRequest(http.MethodPost,
|
||||
"/api/v1/notifications/push/devices",
|
||||
`{"platform":"android","token":"short","device_id":"local-device"}`))
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("bad token status = %d, want 400", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleRegisterApplePushDeviceErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
+956
-345
File diff suppressed because it is too large
Load Diff
@@ -72,6 +72,32 @@ type playbackSessionRow struct {
|
||||
RequestedVideoResolution string `json:"requested_video_resolution,omitempty"`
|
||||
VideoDecision string `json:"video_decision,omitempty"`
|
||||
AudioDecision string `json:"audio_decision,omitempty"`
|
||||
EffectivePlayMethod string `json:"effective_play_method,omitempty"`
|
||||
IsJellyfinClient bool `json:"is_jellyfin_client,omitempty"`
|
||||
}
|
||||
|
||||
// playbackSessionsCapabilitiesResponse advertises the additive fields of the
|
||||
// live admin session payload so independently deployed clients (Android,
|
||||
// Apple) can feature-detect them. Both fields are omitempty on the wire, so
|
||||
// absence on a row is otherwise indistinguishable from an older server.
|
||||
type playbackSessionsCapabilitiesResponse struct {
|
||||
// EffectivePlayMethod reports that rows carry effective_play_method.
|
||||
EffectivePlayMethod bool `json:"effective_play_method"`
|
||||
// EffectivePlayMethodValues is the closed bucket vocabulary a supported
|
||||
// server emits (absent field = unknown).
|
||||
EffectivePlayMethodValues []string `json:"effective_play_method_values"`
|
||||
// IsJellyfinClient reports that rows carry is_jellyfin_client.
|
||||
IsJellyfinClient bool `json:"is_jellyfin_client"`
|
||||
}
|
||||
|
||||
// HandleGetSessionsCapabilities exposes additive feature support for the live
|
||||
// admin session payload (GET /admin/sessions/capabilities).
|
||||
func (h *AdminHandler) HandleGetSessionsCapabilities(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, playbackSessionsCapabilitiesResponse{
|
||||
EffectivePlayMethod: true,
|
||||
EffectivePlayMethodValues: []string{"direct", "remux", "transcode", "audio"},
|
||||
IsJellyfinClient: true,
|
||||
})
|
||||
}
|
||||
|
||||
// PlaybackSessionsQuery scopes live session listing.
|
||||
@@ -248,6 +274,8 @@ func enrichPlaybackSessionRow(row *playbackSessionRow, audioTracksJSON []byte) {
|
||||
}
|
||||
|
||||
row.VideoDecision, row.AudioDecision = sessionComponentDecision(row.PlayMethod, row.TranscodeAudio, row.TargetVideoCodec)
|
||||
row.EffectivePlayMethod = effectivePlayMethod(row.VideoDecision, row.AudioDecision)
|
||||
row.IsJellyfinClient = isJellyfinEcosystemClient(row.ClientName, row.ClientUserAgent)
|
||||
|
||||
var audioTracks []models.AudioTrack
|
||||
if len(audioTracksJSON) > 0 {
|
||||
@@ -335,6 +363,79 @@ func sessionComponentDecision(playMethod string, transcodeAudio bool, targetVide
|
||||
}
|
||||
}
|
||||
|
||||
// effectivePlayMethod reduces the per-stream decisions to the single bucket
|
||||
// the admin activity views aggregate on. Raw play_method is misleading there:
|
||||
// an HLS repackage reports play_method "transcode" with a copied video stream,
|
||||
// and an audio-only re-encode reports "remux" — the decisions carry what
|
||||
// actually costs CPU.
|
||||
// - video re-encoded -> "transcode"
|
||||
// - only audio re-encoded -> "audio"
|
||||
// - streams only repackaged -> "remux"
|
||||
// - nothing touched -> "direct"
|
||||
//
|
||||
// Returns "" when the decisions are unknown (empty or unrecognized
|
||||
// play_method, e.g. a stale row from an older node), so consumers can
|
||||
// distinguish "unknown" from a definite bucket instead of inventing one.
|
||||
func effectivePlayMethod(videoDecision, audioDecision string) string {
|
||||
switch {
|
||||
case videoDecision == "" && audioDecision == "":
|
||||
return ""
|
||||
case videoDecision == "transcode":
|
||||
return "transcode"
|
||||
case audioDecision == "transcode":
|
||||
return "audio"
|
||||
case videoDecision == "direct" && audioDecision == "direct":
|
||||
return "direct"
|
||||
default:
|
||||
return "remux"
|
||||
}
|
||||
}
|
||||
|
||||
// jellyfinClientTokens identifies Jellyfin-ecosystem clients — sessions served
|
||||
// through the Jellyfin compatibility surface — by client name (parsed from the
|
||||
// MediaBrowser auth header) or user agent. This is the single source of truth
|
||||
// behind is_jellyfin_client, which the admin UIs surface as the "JF" pill;
|
||||
// keep it in step with the display-labeling rules in
|
||||
// playbackClientDisplayName so a client that gets a label also gets the flag.
|
||||
// Kodi and mpv reach Silo only through the Jellyfin compat surface today
|
||||
// (jellyfin-kodi/JellyCon, jellyfin-mpv-shim). Generic browser tokens are
|
||||
// deliberately excluded: the native web player shares those user agents.
|
||||
var jellyfinClientTokens = []string{
|
||||
"jellyfin",
|
||||
"findroid",
|
||||
"streamyfin",
|
||||
"swiftfin",
|
||||
"jellycon",
|
||||
"wholphin",
|
||||
"fladder",
|
||||
"vidhub",
|
||||
"senplayer",
|
||||
"infuse",
|
||||
"delfin",
|
||||
"finamp",
|
||||
"kodi",
|
||||
"mpv",
|
||||
}
|
||||
|
||||
// isJellyfinEcosystemClient reports whether the session's client metadata
|
||||
// matches a known Jellyfin-ecosystem client. This is a heuristic: an
|
||||
// unrecognized fork simply gets no JF pill (cosmetic). Stamping a compat-origin
|
||||
// flag on the session at creation would be exact and is the eventual fix.
|
||||
func isJellyfinEcosystemClient(clientName, userAgent string) bool {
|
||||
for _, value := range []string{clientName, userAgent} {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
for _, token := range jellyfinClientTokens {
|
||||
if strings.Contains(value, token) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func firstNonEmptyValue(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package handlers
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T) {
|
||||
videoDecision, audioDecision := sessionComponentDecision("transcode", false, "copy")
|
||||
@@ -12,3 +17,89 @@ func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T)
|
||||
t.Fatalf("audioDecision = %q, want remux", audioDecision)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEffectivePlayMethodBuckets pins the bucket for every decision pair
|
||||
// sessionComponentDecision can produce, plus the unknown case.
|
||||
func TestEffectivePlayMethodBuckets(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
playMethod string
|
||||
transcodeAudio bool
|
||||
targetVideo string
|
||||
want string
|
||||
}{
|
||||
{"direct play", "direct", false, "", "direct"},
|
||||
{"plain remux", "remux", false, "", "remux"},
|
||||
{"audio-only re-encode via remux", "remux", true, "", "audio"},
|
||||
{"full video transcode", "transcode", true, "h264", "transcode"},
|
||||
{"video transcode with copied audio", "transcode", false, "h264", "transcode"},
|
||||
{"video-copy HLS repackage", "transcode", false, "copy", "remux"},
|
||||
{"video-copy HLS with audio re-encode", "transcode", true, "copy", "audio"},
|
||||
// Unknown play_method (stale row from an older node): the bucket must
|
||||
// stay empty rather than inventing a method from transcode_audio.
|
||||
{"unknown method with transcode_audio set", "hls", true, "", ""},
|
||||
{"empty method", "", false, "", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
video, audio := sessionComponentDecision(tc.playMethod, tc.transcodeAudio, tc.targetVideo)
|
||||
if got := effectivePlayMethod(video, audio); got != tc.want {
|
||||
t.Fatalf("effectivePlayMethod(%q, %q) = %q, want %q", video, audio, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionsCapabilitiesAdvertisesActivityFields pins the feature-detection
|
||||
// contract: the additive session fields are omitempty on the wire, so this
|
||||
// endpoint is how independently deployed clients distinguish an older server
|
||||
// from a supported one reporting an unknown method / non-Jellyfin session.
|
||||
func TestSessionsCapabilitiesAdvertisesActivityFields(t *testing.T) {
|
||||
rr := httptest.NewRecorder()
|
||||
(&AdminHandler{}).HandleGetSessionsCapabilities(rr, httptest.NewRequest(http.MethodGet, "/admin/sessions/capabilities", nil))
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rr.Code)
|
||||
}
|
||||
var resp playbackSessionsCapabilitiesResponse
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode capabilities: %v", err)
|
||||
}
|
||||
if !resp.EffectivePlayMethod || !resp.IsJellyfinClient {
|
||||
t.Fatalf("capabilities must advertise both fields: %+v", resp)
|
||||
}
|
||||
want := []string{"direct", "remux", "transcode", "audio"}
|
||||
if len(resp.EffectivePlayMethodValues) != len(want) {
|
||||
t.Fatalf("bucket vocabulary = %v, want %v", resp.EffectivePlayMethodValues, want)
|
||||
}
|
||||
for i, v := range want {
|
||||
if resp.EffectivePlayMethodValues[i] != v {
|
||||
t.Fatalf("bucket vocabulary = %v, want %v", resp.EffectivePlayMethodValues, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsJellyfinEcosystemClient(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
clientName string
|
||||
userAgent string
|
||||
want bool
|
||||
}{
|
||||
{"jellyfin web by name", "Jellyfin Web", "", true},
|
||||
{"findroid by name", "Findroid", "Findroid/0.15", true},
|
||||
{"infuse by user agent only", "", "Infuse-Direct/8.4.6", true},
|
||||
{"kodi addon by name", "Kodi", "Kodi/21.0", true},
|
||||
{"mpv shim by user agent", "", "mpv 0.38.0", true},
|
||||
{"native android client", "Silo Android", "okhttp/4.12", false},
|
||||
{"generic browser", "", "Mozilla/5.0 (X11) Chrome/120.0 Safari/537.36", false},
|
||||
{"no metadata", "", "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isJellyfinEcosystemClient(tc.clientName, tc.userAgent); got != tc.want {
|
||||
t.Fatalf("isJellyfinEcosystemClient(%q, %q) = %v, want %v", tc.clientName, tc.userAgent, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1377
-124
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,84 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/transcodenode"
|
||||
)
|
||||
|
||||
// startLocalPlaybackTransport is the shared local ffmpeg launch primitive for
|
||||
// legacy and protocol-v3 orchestration. Callers retain ownership of lifecycle
|
||||
// locking and decide whether registration is immediate or transactionally
|
||||
// staged.
|
||||
func (h *PlaybackHandler) startLocalPlaybackTransport(ctx context.Context, opts playback.TranscodeOpts) (*playback.TranscodeSession, error) {
|
||||
return playback.StartTranscode(context.WithoutCancel(ctx), opts)
|
||||
}
|
||||
|
||||
// startRemotePlaybackTransport is the shared remote-node launch primitive.
|
||||
// It returns the node's HTTP status separately so legacy and v3 can preserve
|
||||
// their existing public error envelopes while executing identical transport
|
||||
// startup and response parsing.
|
||||
func (h *PlaybackHandler) startRemotePlaybackTransport(ctx context.Context, nodeURL string, request transcodenode.TranscodeStartRequest) (transcodenode.TranscodeStartResponse, int, error) {
|
||||
body, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return transcodenode.TranscodeStartResponse{}, 0, err
|
||||
}
|
||||
requestCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
httpRequest, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodeURL+"/transcode/start", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return transcodenode.TranscodeStartResponse{}, 0, err
|
||||
}
|
||||
httpRequest.Header.Set("Content-Type", "application/json")
|
||||
httpRequest.Header.Set("Authorization", "Bearer "+h.JWTSecret)
|
||||
response, err := http.DefaultClient.Do(httpRequest)
|
||||
if err != nil {
|
||||
return transcodenode.TranscodeStartResponse{}, 0, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusAccepted {
|
||||
// Drain the (small) error body so the transport can reuse the
|
||||
// connection instead of tearing it down on every failed start.
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||||
return transcodenode.TranscodeStartResponse{}, response.StatusCode, nil
|
||||
}
|
||||
var result transcodenode.TranscodeStartResponse
|
||||
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
|
||||
slog.WarnContext(ctx, "remote transcode start response decode failed", "component", "api", "node", nodeURL, "error", err)
|
||||
}
|
||||
return result, response.StatusCode, nil
|
||||
}
|
||||
|
||||
func fetchRemoteTranscodeCapabilities(ctx context.Context, nodeURL, jwtSecret string) (playback.HWAccelInfo, error) {
|
||||
requestCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, strings.TrimRight(nodeURL, "/")+"/hw-capabilities", nil)
|
||||
if err != nil {
|
||||
return playback.HWAccelInfo{}, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+jwtSecret)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
return playback.HWAccelInfo{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return playback.HWAccelInfo{}, fmt.Errorf("node returned %d", response.StatusCode)
|
||||
}
|
||||
var info playback.HWAccelInfo
|
||||
if err := json.NewDecoder(response.Body).Decode(&info); err != nil {
|
||||
return playback.HWAccelInfo{}, err
|
||||
}
|
||||
info.Source = "transcode_node"
|
||||
info.NodeURL = nodeURL
|
||||
return info, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,121 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
const playbackV3ShadowSetting = "playback.protocol_v3_shadow_enabled"
|
||||
|
||||
func (h *PlaybackHandler) protocolV3ShadowEnabled(ctx context.Context) bool {
|
||||
return h.settingFlagCachedV3(ctx, playbackV3ShadowSetting)
|
||||
}
|
||||
|
||||
// shadowLegacyPlaybackV3 compares the production legacy route with a v3 plan
|
||||
// without advertising or executing that plan. Legacy capabilities lack the
|
||||
// detailed v3 fields, so exact source facts are used only as an explicitly
|
||||
// labelled validation inference; this is never a production compatibility
|
||||
// claim and cannot enable v3 playback.
|
||||
func (h *PlaybackHandler) shadowLegacyPlaybackV3(ctx context.Context, req startPlaybackRequest, requestedFile, effectiveFile *models.MediaFile, audioIndex int, productionMethod playback.PlayMethod, productionTranscodeAudio bool, sessionID string) {
|
||||
// The shadow planner runs on a bare goroutine off the production start
|
||||
// path; an escaped panic here would take down the whole process for a
|
||||
// telemetry-only comparison, so it must fail closed instead.
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
slog.ErrorContext(ctx, "playback v3 shadow planner panicked",
|
||||
"component", "playback", "playback_session_id", sessionID,
|
||||
"panic", recovered, "stack", string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
if !h.protocolV3ShadowEnabled(ctx) || requestedFile == nil || effectiveFile == nil {
|
||||
return
|
||||
}
|
||||
request := legacyShadowRequestV3(req, effectiveFile, audioIndex, sessionID)
|
||||
result := playback.PlanPlaybackV3(playback.PlannerInputV3{
|
||||
Request: request,
|
||||
RequestedFile: requestedFile,
|
||||
EffectiveFile: effectiveFile,
|
||||
AudioTrackIndex: audioIndex,
|
||||
Settings: h.plannerSettingsV3(ctx),
|
||||
Registry: h.transformationRegistryV3(ctx),
|
||||
HLSRegistry: h.lazyHLSPlanningRegistryV3(ctx),
|
||||
Now: time.Now(),
|
||||
})
|
||||
attrs := []any{
|
||||
"component", "playback",
|
||||
"playback_session_id", sessionID,
|
||||
"requested_file_id", requestedFile.ID,
|
||||
"effective_file_id", effectiveFile.ID,
|
||||
"production_method", productionMethod,
|
||||
"production_transcode_audio", productionTranscodeAudio,
|
||||
"capability_basis", "legacy_source_exact_inference",
|
||||
}
|
||||
if result.Plan != nil {
|
||||
attrs = append(attrs,
|
||||
"shadow_plan_id", result.Plan.PlanID,
|
||||
"shadow_delivery", result.Plan.Delivery,
|
||||
"shadow_method", result.PlayMethod,
|
||||
"shadow_transcode_audio", result.TranscodeAudio,
|
||||
"shadow_reason", result.Plan.DecisionReason,
|
||||
"route_match", result.PlayMethod == productionMethod && result.TranscodeAudio == productionTranscodeAudio,
|
||||
)
|
||||
} else if result.Terminal != nil {
|
||||
attrs = append(attrs, "shadow_terminal", result.Terminal.Reason, "route_match", false)
|
||||
}
|
||||
slog.InfoContext(ctx, "playback v3 shadow decision", attrs...)
|
||||
}
|
||||
|
||||
func legacyShadowRequestV3(req startPlaybackRequest, file *models.MediaFile, audioIndex int, sessionID string) playback.StartRequestV3 {
|
||||
source := playback.SourceDescriptorFromFileV3(file, audioIndex)
|
||||
hdr := legacyShadowHDRV3(req)
|
||||
features := []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureDetailedDecodeV3}
|
||||
capabilities := playback.ClientCodecCapabilitiesV3{
|
||||
CodecsVideo: append([]string(nil), req.CodecsVideo...),
|
||||
CodecsAudio: append([]string(nil), req.CodecsAudio...),
|
||||
Containers: append([]string(nil), req.Containers...),
|
||||
MaxResolution: req.MaxResolution,
|
||||
HDR: req.HDR,
|
||||
HDRDetails: hdr,
|
||||
}
|
||||
if containsStringFoldV3(req.CodecsVideo, source.VideoCodec) {
|
||||
capabilities.VideoDecode = []playback.VideoDecodeCapabilityV3{{
|
||||
Codec: source.VideoCodec, Profiles: []string{source.VideoProfile}, Levels: []int{source.VideoLevel}, BitDepths: []int{source.BitDepth},
|
||||
MaxWidth: source.Width, MaxHeight: source.Height, MaxFrameRate: source.FrameRate, MaxBitrateKbps: source.BitrateKbps, Hardware: true,
|
||||
}}
|
||||
}
|
||||
var passthrough *playback.AudioPassthroughV3
|
||||
if req.AudioPassthrough != nil {
|
||||
passthrough = &playback.AudioPassthroughV3{PassthroughCodecs: append([]string(nil), req.AudioPassthrough.PassthroughCodecs...), SpatializerEnabled: req.AudioPassthrough.SpatializerEnabled, MaxChannels: req.AudioPassthrough.MaxChannels}
|
||||
if containsStringFoldV3(req.AudioPassthrough.PassthroughCodecs, source.AudioCodec) && source.AudioChannels > 0 && source.AudioLayout != "" && source.AudioChannels <= req.AudioPassthrough.MaxChannels {
|
||||
passthrough.Entries = []playback.AudioPassthroughEntryV3{{Codec: source.AudioCodec, ChannelCounts: []int{source.AudioChannels}, Layouts: []string{source.AudioLayout}}}
|
||||
features = append(features, playback.FeatureLayoutPassthrough)
|
||||
}
|
||||
capabilities.AudioPassthrough = passthrough
|
||||
}
|
||||
engines := map[string]playback.EngineCapabilityV3{
|
||||
string(playback.EngineMedia3DirectV3): {Enabled: true, SupportedOnDevice: true, Subtitles: playback.EngineSubtitleCapabilitiesV3{EmbeddedText: true, SidecarText: true, EmbeddedBitmap: true, SidecarBitmap: true}},
|
||||
string(playback.EngineMedia3ProgressiveRemuxV3): {Enabled: true, SupportedOnDevice: true},
|
||||
string(playback.EngineMedia3HLSV3): {Enabled: true, SupportedOnDevice: true},
|
||||
}
|
||||
return playback.StartRequestV3{
|
||||
ProtocolVersion: playback.ProtocolV3, ClientFeatures: features, FileID: file.ID, ProfileID: req.ProfileID,
|
||||
PlaybackAttemptID: "shadow-" + sessionID, QualityPreference: "original", SubtitleFidelityPreference: playback.SubtitleFidelityCompatibleV3,
|
||||
StartPosition: req.StartPosition, AudioTrackIndex: &audioIndex, Capabilities: capabilities,
|
||||
ClientPlaybackContext: playback.ClientPlaybackContextV3{ProtocolVersion: playback.ProtocolV3, Features: features, Platform: "legacy-shadow", Output: playback.OutputContextV3{HDRDetails: hdr, AudioPassthrough: passthrough}, Engines: engines},
|
||||
}
|
||||
}
|
||||
|
||||
func legacyShadowHDRV3(req startPlaybackRequest) *playback.HDRCapabilitiesV3 {
|
||||
if req.HdrDetails != nil {
|
||||
return &playback.HDRCapabilitiesV3{HDR10: req.HdrDetails.HDR10, HDR10Plus: req.HdrDetails.HDR10Plus, HLG: req.HdrDetails.HLG, DolbyVisionProfiles: append([]int(nil), req.HdrDetails.DolbyVisionProfiles...)}
|
||||
}
|
||||
if req.HDR {
|
||||
return &playback.HDRCapabilitiesV3{HDR10: true}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,198 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodepool"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
// enumeratingNodePlannerV3 is a SessionPlanner stub that also exposes pooled
|
||||
// transcode node URLs, matching *nodepool.Planner's production shape.
|
||||
type enumeratingNodePlannerV3 struct {
|
||||
staticNodePlannerV3
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (p enumeratingNodePlannerV3) TranscodeNodeURLs() []string { return p.urls }
|
||||
|
||||
// presetLocalRegistryV3 pins the handler's local transformation registry so
|
||||
// tests never probe the machine's real ffmpeg.
|
||||
func presetLocalRegistryV3(h *PlaybackHandler, registry *playback.TransformationRegistryV3) {
|
||||
h.v3RegistryOnce.Do(func() {})
|
||||
h.v3Registry = registry
|
||||
}
|
||||
|
||||
func TestHLSPlanningRegistryV3UnionsPooledNodeCapabilities(t *testing.T) {
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/hw-capabilities" {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{
|
||||
{Name: "video_to_h264", Executor: "server", RecipeVersion: "1"},
|
||||
{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"},
|
||||
}})
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
handler := NewPlaybackHandler(playback.NewSessionManager(0, 0))
|
||||
handler.JWTSecret = "test-secret"
|
||||
presetLocalRegistryV3(handler, playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{
|
||||
{Name: "video_to_h264", RecipeVersion: "1"},
|
||||
{Name: "audio_to_aac", RecipeVersion: "1"},
|
||||
{Name: "server_dv7_to_hdr10", RecipeVersion: "1"},
|
||||
}))
|
||||
handler.NodePlanner = enumeratingNodePlannerV3{urls: []string{remote.URL}}
|
||||
|
||||
registry := handler.hlsPlanningRegistryV3(context.Background())
|
||||
if !registry.Available("video_to_h264") || !registry.Available("audio_to_aac") {
|
||||
t.Fatal("pooled node capabilities must widen the HLS planning registry")
|
||||
}
|
||||
if registry.Available("server_dv7_to_hdr10") {
|
||||
t.Fatal("transformations no node advertises must stay unavailable")
|
||||
}
|
||||
if handler.transformationRegistryV3(context.Background()).Available("video_to_h264") {
|
||||
t.Fatal("the local registry must not be widened by node capabilities")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlanningRegistryV3WithoutEnumeratorIsLocal(t *testing.T) {
|
||||
handler := NewPlaybackHandler(playback.NewSessionManager(0, 0))
|
||||
local := playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{{Name: "audio_to_aac", RecipeVersion: "1", Available: true}})
|
||||
presetLocalRegistryV3(handler, local)
|
||||
handler.NodePlanner = staticNodePlannerV3{plan: nodepool.Plan{}}
|
||||
|
||||
if registry := handler.hlsPlanningRegistryV3(context.Background()); registry != local {
|
||||
t.Fatal("a planner without node enumeration must plan from the local registry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteTransformationsV3FailureCacheSplit(t *testing.T) {
|
||||
hits := 0
|
||||
fail := true
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits++
|
||||
if fail {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}}})
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
handler := NewPlaybackHandler(playback.NewSessionManager(0, 0))
|
||||
handler.JWTSecret = "test-secret"
|
||||
if _, err := handler.remoteTransformationsPlanningV3(context.Background(), remote.URL); err == nil {
|
||||
t.Fatal("fetch against a failing node must error")
|
||||
}
|
||||
if _, err := handler.remoteTransformationsPlanningV3(context.Background(), remote.URL); err == nil {
|
||||
t.Fatal("planning lookups must surface the memoized failure")
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Fatalf("failing node was fetched %d times; planning must memoize the failure", hits)
|
||||
}
|
||||
|
||||
// The transport path must fetch through the memoized failure: it may
|
||||
// have been produced by a planning deadline far shorter than this
|
||||
// path's budget, and rejecting the already-selected node on it would
|
||||
// fail a start a fresh fetch could still validate.
|
||||
fail = false
|
||||
transformations, err := handler.remoteTransformationsV3(context.Background(), remote.URL)
|
||||
if err != nil || len(transformations) != 1 {
|
||||
t.Fatalf("transport lookup must refetch through a memoized failure: %v %#v", err, transformations)
|
||||
}
|
||||
if hits != 2 {
|
||||
t.Fatalf("transport lookup fetched %d times, want 2", hits)
|
||||
}
|
||||
// The refetched success replaces the failure for planning too.
|
||||
if _, err := handler.remoteTransformationsPlanningV3(context.Background(), remote.URL); err != nil {
|
||||
t.Fatalf("planning lookup after transport success: %v", err)
|
||||
}
|
||||
if hits != 2 {
|
||||
t.Fatalf("cached success was refetched (%d hits)", hits)
|
||||
}
|
||||
}
|
||||
|
||||
// In a heterogeneous pool, a plan that needs server transformations must be
|
||||
// placed on a node advertising them even when load balancing prefers an
|
||||
// incapable node, while transformation-free plans keep load-based selection.
|
||||
func TestPlanNodeSessionV3PrefersCapabilityMatchingNode(t *testing.T) {
|
||||
capable := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{
|
||||
{Name: "video_to_h264", Executor: "server", RecipeVersion: "1"},
|
||||
{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"},
|
||||
}})
|
||||
}))
|
||||
defer capable.Close()
|
||||
incapable := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, playback.HWAccelInfo{})
|
||||
}))
|
||||
defer incapable.Close()
|
||||
|
||||
transcodes := nodepool.NewTranscodePool()
|
||||
transcodes.SetNodes([]*nodepool.Node{
|
||||
{ID: 1, Name: "incapable", Type: nodepool.NodeTypeTranscode, URL: incapable.URL, Enabled: true, Healthy: true, ActiveJobs: 0},
|
||||
{ID: 2, Name: "capable", Type: nodepool.NodeTypeTranscode, URL: capable.URL, Enabled: true, Healthy: true, ActiveJobs: 5},
|
||||
})
|
||||
handler := NewPlaybackHandler(playback.NewSessionManager(0, 0))
|
||||
handler.JWTSecret = "test-secret"
|
||||
handler.NodePlanner = nodepool.NewPlanner(nodepool.NewProxyPool(), transcodes)
|
||||
|
||||
plan := &playback.PlanV3{
|
||||
PlanID: "plan:heterogeneous",
|
||||
Delivery: playback.DeliveryTranscodeHLSV3,
|
||||
Transformations: []playback.TransformationV3{
|
||||
{Name: "video_to_h264", Executor: "server", RecipeVersion: "1"},
|
||||
{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"},
|
||||
},
|
||||
}
|
||||
selected := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-hetero"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode})
|
||||
if selected.TranscodeNode == nil || selected.TranscodeNode.URL != capable.URL {
|
||||
t.Fatalf("capability-requiring plan selected %+v, want the capable node", selected.TranscodeNode)
|
||||
}
|
||||
|
||||
free := &playback.PlanV3{PlanID: "plan:copy", Delivery: playback.DeliveryRemuxHLSV3, Transformations: []playback.TransformationV3{}}
|
||||
loadBased := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-copy"}, playback.PlannerResultV3{Plan: free, PlayMethod: playback.PlayRemux})
|
||||
if loadBased.TranscodeNode == nil || loadBased.TranscodeNode.URL != incapable.URL {
|
||||
t.Fatalf("transformation-free plan selected %+v, want load-based selection", loadBased.TranscodeNode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareTransportV3LocalFallbackRejectsUnavailableTransformations(t *testing.T) {
|
||||
handler := NewPlaybackHandler(playback.NewSessionManager(0, 0))
|
||||
presetLocalRegistryV3(handler, playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{
|
||||
{Name: "video_to_h264", RecipeVersion: "1"},
|
||||
{Name: "audio_to_aac", RecipeVersion: "1"},
|
||||
}))
|
||||
plan := &playback.PlanV3{
|
||||
PlanID: "plan:local-capability",
|
||||
Delivery: playback.DeliveryTranscodeHLSV3,
|
||||
Transformations: []playback.TransformationV3{
|
||||
{Name: "video_to_h264", Executor: "server", RecipeVersion: "1"},
|
||||
{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"},
|
||||
},
|
||||
}
|
||||
request := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
_, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-local-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"})
|
||||
if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" || !transportErr.retryable {
|
||||
t.Fatalf("transport error = %#v", transportErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRequiresServerTransformationsV3(t *testing.T) {
|
||||
if planRequiresServerTransformationsV3(nil) {
|
||||
t.Fatal("nil plan must not require server transformations")
|
||||
}
|
||||
clientOnly := &playback.PlanV3{Transformations: []playback.TransformationV3{{Name: playback.ClientDV7ToDV81V3, Executor: "client", RecipeVersion: "1"}}}
|
||||
if planRequiresServerTransformationsV3(clientOnly) {
|
||||
t.Fatal("client-executed transformations must not require a server executor")
|
||||
}
|
||||
server := &playback.PlanV3{Transformations: []playback.TransformationV3{{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}}}
|
||||
if !planRequiresServerTransformationsV3(server) {
|
||||
t.Fatal("server-executed transformations must require executor validation")
|
||||
}
|
||||
}
|
||||
@@ -157,6 +157,7 @@ type pluginInstallationResponse struct {
|
||||
Version string `json:"version"`
|
||||
InstallPath string `json:"install_path"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Kind string `json:"kind"`
|
||||
UpdatePolicy string `json:"update_policy"`
|
||||
AvailableVersion *string `json:"available_version,omitempty"`
|
||||
SourceKind string `json:"source_kind"`
|
||||
@@ -320,6 +321,11 @@ type pluginUserSettingsSummary struct {
|
||||
UserConfigSchema []pluginConfigSchemaJSON `json:"user_config_schema"`
|
||||
Routes []pluginRouteJSON `json:"routes"`
|
||||
Assets []pluginAssetJSON `json:"assets"`
|
||||
// Category is the manifest's optional slash-delimited grouping path
|
||||
// (e.g. "Tools/Utilities") used to group plugin entries in the
|
||||
// user-facing Apps navigation. Empty (omitted) when the manifest
|
||||
// declares no category. Additive-only per v1 API rules.
|
||||
Category string `json:"category,omitempty"`
|
||||
}
|
||||
|
||||
type pluginUserSettingsListResponse struct {
|
||||
@@ -877,6 +883,10 @@ func (h *PluginHandler) HandleUpdateInstallation(w http.ResponseWriter, r *http.
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load plugin installation")
|
||||
return
|
||||
}
|
||||
if currentInstallation.IsBuiltin() {
|
||||
writeError(w, http.StatusConflict, "builtin_installation", "Built-in host providers cannot be modified")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Enabled != nil && !*req.Enabled && currentInstallation.Enabled && h.service != nil {
|
||||
if err := h.service.Stop(id); err != nil && !errors.Is(err, pluginhost.ErrClientNotFound) {
|
||||
@@ -928,6 +938,9 @@ func (h *PluginHandler) HandleApplyUpdate(w http.ResponseWriter, r *http.Request
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid installation ID")
|
||||
return
|
||||
}
|
||||
if h.rejectBuiltinInstallation(w, r, id) {
|
||||
return
|
||||
}
|
||||
|
||||
installation, err := h.service.UpdateToAvailableVersion(r.Context(), id)
|
||||
if err != nil {
|
||||
@@ -969,6 +982,9 @@ func (h *PluginHandler) HandlePutInstallationConfig(w http.ResponseWriter, r *ht
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Plugin service not configured")
|
||||
return
|
||||
}
|
||||
if h.rejectBuiltinInstallation(w, r, id) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.service.SetGlobalConfig(r.Context(), id, req.Key, req.Value); err != nil {
|
||||
var validationErr *plugins.ConfigValidationError
|
||||
@@ -1007,6 +1023,9 @@ func (h *PluginHandler) HandleTestInstallationConfig(w http.ResponseWriter, r *h
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "key is required")
|
||||
return
|
||||
}
|
||||
if h.rejectBuiltinInstallation(w, r, id) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.service.TestGlobalConfig(r.Context(), id, req.Key, req.Value); err != nil {
|
||||
if errors.Is(err, plugins.ErrInstallationNotFound) {
|
||||
@@ -1041,6 +1060,10 @@ func (h *PluginHandler) HandlePutAuthBinding(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if h.rejectBuiltinInstallation(w, r, id) {
|
||||
return
|
||||
}
|
||||
|
||||
var req pluginAuthBindingRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
@@ -1074,6 +1097,10 @@ func (h *PluginHandler) HandlePutTaskBinding(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if h.rejectBuiltinInstallation(w, r, id) {
|
||||
return
|
||||
}
|
||||
|
||||
capabilityID := chi.URLParam(r, "capability_id")
|
||||
if strings.TrimSpace(capabilityID) == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "capability_id is required")
|
||||
@@ -1101,6 +1128,21 @@ func (h *PluginHandler) HandlePutTaskBinding(w http.ResponseWriter, r *http.Requ
|
||||
writeJSON(w, http.StatusOK, pluginTaskBindingUpdateResponse{RestartRequired: true})
|
||||
}
|
||||
|
||||
// rejectBuiltinInstallation writes a 409 and returns true when the target
|
||||
// installation is the reserved builtin row, which no plugin-management
|
||||
// endpoint may mutate. Lookup errors are left to the caller's own handling.
|
||||
func (h *PluginHandler) rejectBuiltinInstallation(w http.ResponseWriter, r *http.Request, id int) bool {
|
||||
installation, err := h.installations.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if !installation.IsBuiltin() {
|
||||
return false
|
||||
}
|
||||
writeError(w, http.StatusConflict, "builtin_installation", "Built-in host providers cannot be modified")
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleDeleteInstallation(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := parseNamedIDParam(r, "id")
|
||||
if err != nil {
|
||||
@@ -1113,6 +1155,10 @@ func (h *PluginHandler) HandleDeleteInstallation(w http.ResponseWriter, r *http.
|
||||
writeError(w, http.StatusNotFound, "not_found", "Plugin installation not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, plugins.ErrBuiltinInstallationImmutable) {
|
||||
writeError(w, http.StatusConflict, "builtin_installation", "Built-in host providers cannot be uninstalled")
|
||||
return
|
||||
}
|
||||
slog.ErrorContext(r.Context(), "deleting plugin installation", "component", "api", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete plugin installation")
|
||||
return
|
||||
@@ -1151,6 +1197,11 @@ func (h *PluginHandler) HandleListUserPluginSettings(w http.ResponseWriter, r *h
|
||||
Installations: make([]pluginUserSettingsSummary, 0, len(installations)),
|
||||
}
|
||||
for _, installation := range installations {
|
||||
// The reserved builtin row has no manifest on disk; without this skip
|
||||
// the whole user-scoped settings list would 500.
|
||||
if installation.IsBuiltin() {
|
||||
continue
|
||||
}
|
||||
manifest, err := plugins.LoadManifestFile(plugins.InstalledManifestPath(installation.InstallPath))
|
||||
if err != nil {
|
||||
slog.ErrorContext(r.Context(), "loading plugin manifest", "component", "api", "installation_id", installation.ID, "error", err)
|
||||
@@ -1233,7 +1284,7 @@ func (h *PluginHandler) loadUserConfigInstallation(
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load plugin installation")
|
||||
return nil, nil, err
|
||||
}
|
||||
if !installation.Enabled {
|
||||
if !installation.Enabled || installation.IsBuiltin() {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Plugin installation not found")
|
||||
return nil, nil, plugins.ErrInstallationNotFound
|
||||
}
|
||||
@@ -1277,6 +1328,12 @@ func (h *PluginHandler) buildInstallationResponses(
|
||||
}
|
||||
response := make([]pluginInstallationResponse, 0, len(installations))
|
||||
for _, installation := range installations {
|
||||
// The reserved builtin row is not a manageable plugin: old web builds
|
||||
// would render a phantom entry with uninstall/upgrade buttons that
|
||||
// error, and the chain editor does not need it in this list.
|
||||
if installation.IsBuiltin() {
|
||||
continue
|
||||
}
|
||||
item, err := h.buildInstallationResponseWithBindings(
|
||||
ctx,
|
||||
installation,
|
||||
@@ -1390,6 +1447,7 @@ func (h *PluginHandler) buildInstallationResponseWithBindings(
|
||||
Version: installation.Version,
|
||||
InstallPath: installation.InstallPath,
|
||||
Enabled: installation.Enabled,
|
||||
Kind: installation.Kind,
|
||||
UpdatePolicy: installation.UpdatePolicy,
|
||||
AvailableVersion: installation.AvailableVersion,
|
||||
SourceKind: sourceKind,
|
||||
@@ -1465,6 +1523,7 @@ func toUserPluginSettingsSummary(
|
||||
UserConfigSchema: configSchemasToJSON(manifest.GetUserConfigSchema()),
|
||||
Routes: routesToJSON(manifest.GetHttpRoutes()),
|
||||
Assets: assetsToJSON(manifest.GetAssets()),
|
||||
Category: manifest.GetCategory(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
)
|
||||
|
||||
func pluginBuiltinTestPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("SILO_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
return pool
|
||||
}
|
||||
|
||||
func builtinTestHandler(pool *pgxpool.Pool) *PluginHandler {
|
||||
return &PluginHandler{
|
||||
repositories: plugins.NewRepositoryStore(pool),
|
||||
installations: plugins.NewInstallationStore(pool),
|
||||
configs: plugins.NewRuntimeConfigStore(pool),
|
||||
}
|
||||
}
|
||||
|
||||
func seedHandlerBuiltinInstallation(t *testing.T, pool *pgxpool.Pool) int {
|
||||
t.Helper()
|
||||
var id int
|
||||
pluginID := fmt.Sprintf("test.builtin.handler-%d", time.Now().UnixNano())
|
||||
err := pool.QueryRow(context.Background(),
|
||||
`INSERT INTO plugin_installations (plugin_id, version, install_path, enabled, update_policy, kind)
|
||||
VALUES ($1, '0', '/nonexistent/silo-builtin-test', true, 'manual', 'builtin')
|
||||
RETURNING id`, pluginID).Scan(&id)
|
||||
if err != nil {
|
||||
t.Fatalf("seed builtin installation: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM plugin_installations WHERE id = $1`, id)
|
||||
})
|
||||
return id
|
||||
}
|
||||
|
||||
func requestWithIDParam(method, target, param string, id int) *http.Request {
|
||||
return requestWithIDParamBody(method, target, param, id, nil)
|
||||
}
|
||||
|
||||
func requestWithIDParamBody(method, target, param string, id int, body io.Reader) *http.Request {
|
||||
req := httptest.NewRequest(method, target, body)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add(param, strconv.Itoa(id))
|
||||
return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
}
|
||||
|
||||
// The manifest-less builtin row must not 500 the user-scoped plugin settings
|
||||
// list the moment the migration lands (ship-blocker: the web sidebar fetches
|
||||
// this for every logged-in user).
|
||||
func TestHandleListUserPluginSettings_SkipsBuiltinRow(t *testing.T) {
|
||||
pool := pluginBuiltinTestPool(t)
|
||||
builtinID := seedHandlerBuiltinInstallation(t, pool)
|
||||
h := builtinTestHandler(pool)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListUserPluginSettings(rec, httptest.NewRequest(http.MethodGet, "/api/v1/settings/plugins", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
Installations []struct {
|
||||
ID int `json:"id"`
|
||||
} `json:"installations"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
for _, inst := range resp.Installations {
|
||||
if inst.ID == builtinID {
|
||||
t.Errorf("builtin installation %d leaked into user plugin settings", builtinID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadUserConfigInstallation must 404 builtin ids.
|
||||
func TestHandleGetUserPluginSettings_BuiltinIDIs404(t *testing.T) {
|
||||
pool := pluginBuiltinTestPool(t)
|
||||
builtinID := seedHandlerBuiltinInstallation(t, pool)
|
||||
h := builtinTestHandler(pool)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetUserPluginSettings(rec, requestWithIDParam(http.MethodGet, "/api/v1/settings/plugins/0", "installation_id", builtinID))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want 404; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// GET /plugins/installations must filter builtin rows server-side, and the
|
||||
// remaining rows carry the additive kind field.
|
||||
func TestHandleListInstallations_FiltersBuiltinAndExposesKind(t *testing.T) {
|
||||
pool := pluginBuiltinTestPool(t)
|
||||
builtinID := seedHandlerBuiltinInstallation(t, pool)
|
||||
h := builtinTestHandler(pool)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListInstallations(rec, httptest.NewRequest(http.MethodGet, "/api/v1/plugins/installations", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var resp []struct {
|
||||
ID int `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
for _, inst := range resp {
|
||||
if inst.ID == builtinID {
|
||||
t.Errorf("builtin installation %d leaked into installations list", builtinID)
|
||||
}
|
||||
if inst.Kind == "" {
|
||||
t.Errorf("installation %d missing additive kind field", inst.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mutating endpoints must reject the builtin row with a clear 4xx.
|
||||
func TestBuiltinInstallationMutationsRejected(t *testing.T) {
|
||||
pool := pluginBuiltinTestPool(t)
|
||||
builtinID := seedHandlerBuiltinInstallation(t, pool)
|
||||
h := builtinTestHandler(pool)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
call func(rec *httptest.ResponseRecorder)
|
||||
}{
|
||||
{"delete", func(rec *httptest.ResponseRecorder) {
|
||||
h.HandleDeleteInstallation(rec, requestWithIDParam(http.MethodDelete, "/api/v1/plugins/installations/0", "id", builtinID))
|
||||
}},
|
||||
{"update", func(rec *httptest.ResponseRecorder) {
|
||||
req := requestWithIDParamBody(http.MethodPatch, "/api/v1/plugins/installations/0", "id", builtinID,
|
||||
strings.NewReader(`{"enabled": false}`))
|
||||
h.HandleUpdateInstallation(rec, req)
|
||||
}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
tc.call(rec)
|
||||
if rec.Code < 400 || rec.Code >= 500 {
|
||||
t.Fatalf("status = %d, want 4xx; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
)
|
||||
|
||||
func TestToUserPluginSettingsSummaryIncludesManifestCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
installation := &plugins.Installation{
|
||||
ID: 42,
|
||||
PluginID: "example-audiobooks",
|
||||
Version: "1.2.3",
|
||||
}
|
||||
manifest := &pluginv1.PluginManifest{
|
||||
Category: "Tools/Utilities",
|
||||
}
|
||||
|
||||
got := toUserPluginSettingsSummary(installation, manifest)
|
||||
|
||||
if got.Category != "Tools/Utilities" {
|
||||
t.Fatalf("Category = %q, want %q", got.Category, "Tools/Utilities")
|
||||
}
|
||||
if got.ID != 42 || got.PluginID != "example-audiobooks" || got.Version != "1.2.3" {
|
||||
t.Fatalf("identity fields = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToUserPluginSettingsSummaryOmitsEmptyCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
installation := &plugins.Installation{
|
||||
ID: 7,
|
||||
PluginID: "example-plain",
|
||||
Version: "0.1.0",
|
||||
}
|
||||
|
||||
got := toUserPluginSettingsSummary(installation, &pluginv1.PluginManifest{})
|
||||
|
||||
if got.Category != "" {
|
||||
t.Fatalf("Category = %q, want empty", got.Category)
|
||||
}
|
||||
|
||||
// The field is additive-only; ensure it stays absent from the JSON
|
||||
// payload for manifests without a category so existing clients see an
|
||||
// unchanged response shape.
|
||||
encoded, err := json.Marshal(got)
|
||||
if err != nil {
|
||||
t.Fatalf("marshaling summary: %v", err)
|
||||
}
|
||||
if strings.Contains(string(encoded), "\"category\"") {
|
||||
t.Fatalf("JSON payload should omit empty category, got %s", encoded)
|
||||
}
|
||||
}
|
||||
@@ -159,7 +159,7 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) {
|
||||
seekSeconds = s
|
||||
}
|
||||
}
|
||||
if err := playback.ServeRemux(w, r, file.FilePath, "mp4", seekSeconds, session.TranscodeAudio, session.AudioTrackIndex, file.PrimaryDVProfile()); err != nil {
|
||||
if err := playback.ServeRemuxWithDVMode(w, r, file.FilePath, "mp4", seekSeconds, session.TranscodeAudio, session.AudioTrackIndex, file.PrimaryDVProfile(), session.RemuxDVMode, h.ffmpegPath()); err != nil {
|
||||
h.handleTransportStartFailure(r.Context(), session, file, err)
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
setPlaybackSessionLogContext(r, sessionID)
|
||||
|
||||
trackParam := chi.URLParam(r, "track")
|
||||
trackIndex, _, err := playback.ParseSubtitleTrackParam(trackParam)
|
||||
trackIndex, requestedFormat, err := playback.ParseSubtitleTrackParam(trackParam)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid subtitle track index")
|
||||
return
|
||||
@@ -224,7 +224,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
sub := file.ExternalSubtitles[trackIndex]
|
||||
|
||||
// Serve ASS/SSA external subtitles as raw data for client-side rendering.
|
||||
if playback.IsASS(sub.Format) {
|
||||
if playback.IsASS(sub.Format) && requestedFormat != "vtt" {
|
||||
data, err := playback.LoadExternalSubtitleRaw(sub.Path)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error",
|
||||
@@ -264,7 +264,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
// demuxed, so the first byte lands within ~1s even on network
|
||||
// storage. Works identically for direct-play, remux, and
|
||||
// transcode because it doesn't depend on any other ffmpeg.
|
||||
h.streamEmbeddedSubtitle(w, r, file, embeddedIndex, session)
|
||||
h.streamEmbeddedSubtitle(w, r, file, embeddedIndex, session, requestedFormat)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -295,7 +295,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Serve ASS/SSA downloaded subtitles as raw data.
|
||||
if playback.IsASS(string(dl.Format)) {
|
||||
if playback.IsASS(string(dl.Format)) && requestedFormat != "vtt" {
|
||||
playback.ServeSubtitle(w, data, "ass")
|
||||
return
|
||||
}
|
||||
@@ -307,7 +307,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Convert to VTT using the playback conversion pipeline.
|
||||
vttData, err := playback.ConvertToVTT(data, string(dl.Format))
|
||||
vttData, err := playback.ConvertToVTTWithFFmpeg(r.Context(), data, string(dl.Format), h.ffmpegPath())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "convert_error", "Failed to convert subtitle")
|
||||
return
|
||||
@@ -488,7 +488,7 @@ func (h *StreamHandler) handleTransportStartFailure(ctx context.Context, session
|
||||
// track, seeked to the best-known playback position, and pipes its
|
||||
// stdout directly to w. Because this ffmpeg is independent of the video
|
||||
// pipeline, it works the same for direct play, remux, and transcode.
|
||||
func (h *StreamHandler) streamEmbeddedSubtitle(w http.ResponseWriter, r *http.Request, file *models.MediaFile, embeddedIndex int, session *playback.Session) {
|
||||
func (h *StreamHandler) streamEmbeddedSubtitle(w http.ResponseWriter, r *http.Request, file *models.MediaFile, embeddedIndex int, session *playback.Session, requestedFormat ...string) {
|
||||
track := file.SubtitleTracks[embeddedIndex]
|
||||
outFormat := "vtt"
|
||||
switch {
|
||||
@@ -537,6 +537,20 @@ func (h *StreamHandler) streamEmbeddedSubtitle(w http.ResponseWriter, r *http.Re
|
||||
AllowWindow: allowWindow,
|
||||
FFmpegPath: h.ffmpegPath(),
|
||||
}
|
||||
if len(requestedFormat) > 0 && requestedFormat[0] == "vtt" {
|
||||
// Only text sources can be converted to WebVTT. A bitmap track (PGS
|
||||
// reaches here because it is deliverable as .sup; DVD/DVB are rejected
|
||||
// upstream) carries no text, so honoring the override would spawn an
|
||||
// ffmpeg that always fails after the 200 and headers are committed.
|
||||
// Reject before any spawn or header write.
|
||||
if playback.NeedsBurnIn(track.Codec) {
|
||||
writeError(w, http.StatusUnsupportedMediaType, "unsupported_media_type",
|
||||
"Bitmap subtitle tracks cannot be converted to WebVTT")
|
||||
return
|
||||
}
|
||||
opts.TargetFormat = "vtt"
|
||||
outFormat = "vtt"
|
||||
}
|
||||
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
|
||||
@@ -196,6 +196,51 @@ func TestHandleSubtitle_NilMediaFileReturns404(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A .vtt request for a bitmap (PGS) embedded track must be rejected up front:
|
||||
// PGS passes the burn-in guard because it is deliverable as .sup, but it has
|
||||
// no text to convert, so forcing WebVTT would spawn an ffmpeg that always
|
||||
// fails after the 200 and headers are committed.
|
||||
func TestHandleSubtitle_BitmapTrackVTTRequestReturns415(t *testing.T) {
|
||||
file := &models.MediaFile{
|
||||
ID: 42,
|
||||
ContentID: "movie-1",
|
||||
FilePath: "/tmp/movie.mkv",
|
||||
Duration: 3600,
|
||||
SubtitleTracks: []models.SubtitleTrack{
|
||||
{Index: 0, Language: "eng", Codec: "hdmv_pgs_subtitle"},
|
||||
},
|
||||
}
|
||||
baseMgr := playback.NewSessionManager(0, 0)
|
||||
session, err := baseMgr.StartSession(1, "profile-1", 42, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
|
||||
handler := NewStreamHandler(baseMgr, testPlaybackFileResolver{file: file})
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/stream/"+session.ID+"/subtitles/0.vtt", nil)
|
||||
req = req.WithContext(newAuthorizedPlaybackContext())
|
||||
routeCtx := chi.NewRouteContext()
|
||||
routeCtx.URLParams.Add("session_id", session.ID)
|
||||
routeCtx.URLParams.Add("track", "0.vtt")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
handler.HandleSubtitle(rr, req)
|
||||
|
||||
if rr.Code != http.StatusUnsupportedMediaType {
|
||||
t.Fatalf("status = %d, body = %s; want 415", rr.Code, rr.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode error body: %v (body = %s)", err, rr.Body.String())
|
||||
}
|
||||
if body.Error != "unsupported_media_type" {
|
||||
t.Fatalf("error code = %q, want %q", body.Error, "unsupported_media_type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitleSourceFileIDPinsURLAcrossEffectiveFileSwitch(t *testing.T) {
|
||||
session := &playback.Session{MediaFileID: 200, RequestedMediaFileID: 100}
|
||||
|
||||
|
||||
@@ -2,11 +2,8 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/buildinfo"
|
||||
"github.com/Silo-Server/silo-server/internal/nodepool"
|
||||
@@ -57,28 +54,5 @@ func (h *SystemHandler) HandleBuildInfo(w http.ResponseWriter, _ *http.Request)
|
||||
}
|
||||
|
||||
func (h *SystemHandler) fetchRemoteHWAccel(ctx context.Context, node *nodepool.Node) (playback.HWAccelInfo, error) {
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, node.URL+"/hw-capabilities", nil)
|
||||
if err != nil {
|
||||
return playback.HWAccelInfo{}, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+h.jwtSecret)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return playback.HWAccelInfo{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return playback.HWAccelInfo{}, fmt.Errorf("node returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var info playback.HWAccelInfo
|
||||
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
|
||||
return playback.HWAccelInfo{}, err
|
||||
}
|
||||
info.Source = "transcode_node"
|
||||
info.NodeURL = node.URL
|
||||
return info, nil
|
||||
return fetchRemoteTranscodeCapabilities(ctx, node.URL, h.jwtSecret)
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ type AuthMiddleware struct {
|
||||
sessionValidator SessionValidator
|
||||
apiKeyValidator APIKeyValidator // nil if API keys not configured
|
||||
apiKeyUserLoader APIKeyUserLoader // nil if API keys not configured
|
||||
|
||||
apiKeyLastUsed *auth.APIKeyLastUsedTracker
|
||||
}
|
||||
|
||||
// NewAuthMiddleware creates a new AuthMiddleware with the given token validator
|
||||
@@ -57,6 +59,7 @@ func NewAuthMiddleware(tv TokenValidator, sv SessionValidator, akv APIKeyValidat
|
||||
sessionValidator: sv,
|
||||
apiKeyValidator: akv,
|
||||
apiKeyUserLoader: akul,
|
||||
apiKeyLastUsed: auth.NewAPIKeyLastUsedTracker(akv, nil),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,10 +101,7 @@ func (am *AuthMiddleware) RequireAuth(next http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
|
||||
// Update last_used_at asynchronously.
|
||||
go func(id int64) {
|
||||
_ = am.apiKeyValidator.UpdateLastUsed(context.Background(), id)
|
||||
}(apiKey.ID)
|
||||
am.apiKeyLastUsed.Touch(apiKey.ID)
|
||||
|
||||
claims = &auth.Claims{
|
||||
UserID: user.ID,
|
||||
|
||||
@@ -17,7 +17,8 @@ type DemoSettingsReader interface {
|
||||
// Allowed: browsing, playback, favorites, watchlist, ratings, collections,
|
||||
// profiles, playback progress, watched state.
|
||||
//
|
||||
// Blocked: API key management, downloads, history imports, subtitle downloads.
|
||||
// Blocked: API key management, downloads, history imports, subtitle downloads,
|
||||
// diagnostics report uploads/deletes.
|
||||
type DemoGuard struct {
|
||||
settings DemoSettingsReader
|
||||
}
|
||||
@@ -41,6 +42,9 @@ var demoBlockedRoutes = []blockedRoute{
|
||||
{methods: []string{"POST"}, prefix: "/api/v1/subtitles/download"},
|
||||
{methods: []string{"POST"}, prefix: "/api/v1/subtitles/upload"},
|
||||
{methods: []string{"DELETE"}, prefix: "/api/v1/subtitles/"},
|
||||
// Diagnostics report uploads/deletes write DB rows and private-bucket blobs;
|
||||
// GET /api/v1/diagnostics/status is unaffected (GETs always pass).
|
||||
{methods: []string{"POST", "DELETE"}, prefix: "/api/v1/diagnostics/reports"},
|
||||
}
|
||||
|
||||
// Guard is an HTTP middleware that enforces demo mode restrictions.
|
||||
|
||||
+95
-8
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/catalogseed"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/config"
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics"
|
||||
"github.com/Silo-Server/silo-server/internal/downloads"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/historyimport"
|
||||
@@ -50,6 +51,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
"github.com/Silo-Server/silo-server/internal/opslog"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/playback/planstore"
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
"github.com/Silo-Server/silo-server/internal/policy"
|
||||
"github.com/Silo-Server/silo-server/internal/ratelimit"
|
||||
@@ -253,14 +255,22 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// acting-admin profile policy, degrading admin routes to the plain
|
||||
// role check.
|
||||
var checkPrimaryProfile apimw.PrimaryProfileChecker
|
||||
// lookupProfile resolves a profile for the given user, returning nil when it
|
||||
// does not exist. Shared by the acting-admin primary check and the
|
||||
// diagnostics profile-attribution validator so both read profile state
|
||||
// (IsPrimary, IsChild) the same way.
|
||||
var lookupProfile func(ctx context.Context, userID int, profileID string) (*userstore.Profile, error)
|
||||
if deps.UserStoreProvider != nil {
|
||||
userStores := deps.UserStoreProvider
|
||||
checkPrimaryProfile = func(ctx context.Context, userID int, profileID string) (bool, bool, error) {
|
||||
lookupProfile = func(ctx context.Context, userID int, profileID string) (*userstore.Profile, error) {
|
||||
store, err := userStores.ForUser(ctx, userID)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
return nil, err
|
||||
}
|
||||
profile, err := store.GetProfile(ctx, profileID)
|
||||
return store.GetProfile(ctx, profileID)
|
||||
}
|
||||
checkPrimaryProfile = func(ctx context.Context, userID int, profileID string) (bool, bool, error) {
|
||||
profile, err := lookupProfile(ctx, userID, profileID)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
@@ -307,6 +317,39 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
if deps.DB != nil {
|
||||
accessGroupStore = access.NewGroupStore(deps.DB)
|
||||
}
|
||||
var diagnosticsStore diagnostics.ObjectStore
|
||||
if deps.S3Private != nil {
|
||||
diagnosticsStore = diagnostics.NewS3ObjectStore(deps.S3Private)
|
||||
}
|
||||
var diagnosticsHandler *handlers.DiagnosticsHandler
|
||||
if deps.DB != nil {
|
||||
diagnosticsService := diagnostics.NewService(
|
||||
diagnostics.NewPostgresRepository(deps.DB),
|
||||
settingsRepo,
|
||||
diagnosticsStore,
|
||||
slog.Default(),
|
||||
)
|
||||
if lookupProfile != nil {
|
||||
// Attribute reports only to a profile that belongs to the account and
|
||||
// is not a child profile; child profiles must not perform diagnostics
|
||||
// actions per the design, so reject their attribution here.
|
||||
diagnosticsService.SetProfileAttributionValidator(diagnostics.NewProfileAttributionValidator(
|
||||
func(ctx context.Context, userID int, profileID string) (bool, bool, error) {
|
||||
profile, err := lookupProfile(ctx, userID, profileID)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if profile == nil {
|
||||
return false, false, nil
|
||||
}
|
||||
return true, profile.IsChild, nil
|
||||
},
|
||||
))
|
||||
}
|
||||
diagnosticsHandler = handlers.NewDiagnosticsHandler(
|
||||
diagnosticsService,
|
||||
)
|
||||
}
|
||||
|
||||
// Build auth handler and auth middleware if DB and config are available.
|
||||
var userRepo *auth.UserRepository
|
||||
@@ -805,6 +848,12 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
} else {
|
||||
playbackHandler = handlers.NewPlaybackHandler(deps.SessionMgr)
|
||||
}
|
||||
if deps.DB != nil {
|
||||
playbackHandler.PlanStoreV3 = planstore.NewPostgres(deps.DB)
|
||||
}
|
||||
// Maintenance also bounds the in-memory fallback store: without it a
|
||||
// DB-less deployment accumulates attempts and replans forever.
|
||||
playbackHandler.StartV3Maintenance(deps.AppContext)
|
||||
|
||||
// Wire UserStoreProvider for progress/history persistence.
|
||||
if deps.UserStoreProvider != nil {
|
||||
@@ -850,11 +899,13 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
playbackHandler.PlaybackConfig = func() config.PlaybackConfig {
|
||||
return deps.CurrentConfig().Playback
|
||||
}
|
||||
if cleaned, err := playbackHandler.CleanupOrphanedTranscodes(); err != nil {
|
||||
slog.Warn("playback transcode cleanup failed", "dir", deps.Config.Playback.TranscodeDir, "error", err)
|
||||
} else if cleaned > 0 {
|
||||
slog.Info("playback transcode cleanup removed orphaned dirs", "dir", deps.Config.Playback.TranscodeDir, "count", cleaned)
|
||||
}
|
||||
// In integrated mode this and the jellycompat sweep both scan the same
|
||||
// TranscodeDir but each snapshots only its own manager's live set, so a
|
||||
// >24h idle dir owned by the other manager can be reaped. Bounded and
|
||||
// safe: active dirs stay mtime-fresh (spared) and either side rebuilds
|
||||
// from its token/recipe, so the worst case is a wasted rebuild. A shared
|
||||
// active-set source across both managers would remove even that.
|
||||
playback.StartPeriodicOrphanCleanup(deps.AppContext, "api", deps.Config.Playback.TranscodeDir, playbackHandler.CleanupOrphanedTranscodes, playback.OrphanCleanupInterval)
|
||||
}
|
||||
playbackHandler.ProbeEnsurer = deps.ProbeEnsurer
|
||||
playbackHandler.ChapterThumbnailQueuer = deps.ChapterThumbnailQueuer
|
||||
@@ -949,6 +1000,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
adminHandler.BootstrapSensitiveValues = deps.BootstrapSensitiveValues
|
||||
adminHandler.RestartStatus = restartStatus
|
||||
adminHandler.CatalogSearchStatus = catalogSearchService
|
||||
adminHandler.DiagnosticsStore = diagnosticsStore
|
||||
if settingsRepo != nil {
|
||||
adminHandler.SettingsRepo = settingsRepo
|
||||
}
|
||||
@@ -1028,6 +1080,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
deps.PluginImageResolver,
|
||||
detailSvc,
|
||||
)
|
||||
adminImageHandler.EventsHub = deps.EventsHub
|
||||
}
|
||||
|
||||
var adminIntroHandler *handlers.AdminIntroHandler
|
||||
@@ -1757,6 +1810,28 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
})
|
||||
}
|
||||
|
||||
// Client diagnostics are account-scoped and must work before profile
|
||||
// selection, so this route intentionally uses auth only plus the
|
||||
// generic rate limiter, not viewer/profile middleware.
|
||||
if diagnosticsHandler != nil && authMiddleware != nil {
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(authMiddleware.RequireAuth)
|
||||
// Demo mode blocks non-admin report uploads (a write to the
|
||||
// private bucket and DB); the read-only status endpoint stays
|
||||
// available because DemoGuard always lets GETs through.
|
||||
if demoGuard != nil {
|
||||
r.Use(demoGuard.Guard)
|
||||
}
|
||||
if deps.RateLimitMW != nil {
|
||||
r.Use(deps.RateLimitMW.Handler)
|
||||
}
|
||||
r.Route("/diagnostics", func(r chi.Router) {
|
||||
r.Get("/status", diagnosticsHandler.HandleStatus)
|
||||
r.Post("/reports", diagnosticsHandler.HandleUpload)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// All remaining routes require auth.
|
||||
if authMiddleware != nil {
|
||||
r.Group(func(r chi.Router) {
|
||||
@@ -1817,6 +1892,10 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/preferences", notificationsHandler.HandleGetPreferences)
|
||||
r.Put("/preferences", notificationsHandler.HandleUpdatePreferences)
|
||||
r.Get("/push/apple/display/{delivery_id}", notificationsHandler.HandleApplePushDisplay)
|
||||
// Platform-generic registration used by the Android
|
||||
// client; Apple keeps its dedicated route above.
|
||||
r.Post("/push/devices", notificationsHandler.HandleRegisterPushDevice)
|
||||
r.Delete("/push/devices/{device_id}", notificationsHandler.HandleUnregisterPushDevice)
|
||||
r.Get("/email-preferences", notificationsHandler.HandleGetEmailPreferences)
|
||||
r.Put("/email-preferences", notificationsHandler.HandleUpdateEmailPreferences)
|
||||
r.Put("/email-preferences/address", notificationsHandler.HandleRequestEmailAddress)
|
||||
@@ -2279,6 +2358,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
playbackHandler.FFmpegLogSink = deps.FFmpegLogSink
|
||||
|
||||
r.Route("/playback", func(r chi.Router) {
|
||||
r.Get("/capability", playbackHandler.HandlePlaybackCapabilityV3)
|
||||
// HLS transcode delivery — no profile auth needed;
|
||||
// session ID (UUID) serves as the access token, same
|
||||
// pattern as /stream/{session_id}.
|
||||
@@ -2292,6 +2372,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(apimw.RequireProfile)
|
||||
r.Post("/start", playbackHandler.HandleStartPlayback)
|
||||
r.Post("/{session_id}/replan", playbackHandler.HandleReplanPlaybackV3)
|
||||
r.Post("/route-events", playbackHandler.HandlePlaybackRouteEventV3)
|
||||
r.Post("/{session_id}/progress", playbackHandler.HandleUpdateProgress)
|
||||
r.Patch("/{session_id}/audio", playbackHandler.HandleChangeAudioTrack)
|
||||
r.Delete("/{session_id}", playbackHandler.HandleStopPlayback)
|
||||
@@ -2475,6 +2557,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
|
||||
r.Get("/sessions", adminHandler.HandleListSessions)
|
||||
r.Get("/sessions/capabilities", adminHandler.HandleGetSessionsCapabilities)
|
||||
r.Get("/playback-history", adminHandler.HandleListPlaybackHistory)
|
||||
r.Get("/unmatched", adminHandler.HandleListUnmatched)
|
||||
r.Get("/stats", adminHandler.HandleGetStats)
|
||||
@@ -2537,6 +2620,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
applePushHandler := handlers.NewAdminApplePushHandler(deps.Notifications, settingsRepo)
|
||||
if deps.Notifications != nil {
|
||||
r.Post("/notifications/push/apple/test", applePushHandler.HandleTest)
|
||||
r.Post("/notifications/push/fcm/test", applePushHandler.HandleTestAndroid)
|
||||
}
|
||||
if settingsRepo != nil {
|
||||
r.Post("/notifications/push/relay/register", applePushHandler.HandleRegisterRelay)
|
||||
@@ -2864,6 +2948,9 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/logs/audit", adminLogsHandler.HandleListAuditLogs)
|
||||
r.Get("/logs/ws", adminLogsHandler.HandleLogStreamWebSocket)
|
||||
}
|
||||
if diagnosticsHandler != nil {
|
||||
handlers.RegisterAdminDiagnosticsRoutes(r, diagnosticsHandler)
|
||||
}
|
||||
if adminPlaybackControlHandler != nil {
|
||||
r.Post("/sessions/{session_id}/pause", adminPlaybackControlHandler.HandlePauseSession)
|
||||
r.Post("/sessions/{session_id}/resume", adminPlaybackControlHandler.HandleResumeSession)
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// Package artworkkey owns the object-key naming contract for cached artwork.
|
||||
// Legacy names such as original.webp and revisioned names such as
|
||||
// original.<revision>.webp are both supported.
|
||||
package artworkkey
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const OriginalVariant = "original"
|
||||
|
||||
// Build returns an object key for a variant under basePath.
|
||||
func Build(basePath, variant, revision, ext string) string {
|
||||
basePath = strings.TrimRight(strings.TrimSpace(basePath), "/")
|
||||
variant = strings.TrimSpace(variant)
|
||||
revision = strings.TrimSpace(revision)
|
||||
if basePath == "" || variant == "" {
|
||||
return ""
|
||||
}
|
||||
if ext == "" {
|
||||
ext = ".webp"
|
||||
} else if !strings.HasPrefix(ext, ".") {
|
||||
ext = "." + ext
|
||||
}
|
||||
if revision == "" {
|
||||
return basePath + "/" + variant + ext
|
||||
}
|
||||
return basePath + "/" + variant + "." + revision + ext
|
||||
}
|
||||
|
||||
// Original returns the original-variant key under basePath.
|
||||
func Original(basePath, revision, ext string) string {
|
||||
return Build(basePath, OriginalVariant, revision, ext)
|
||||
}
|
||||
|
||||
// Variant rewrites an original key to another variant while retaining any
|
||||
// revision and extension. Unrecognized paths pass through unchanged.
|
||||
func Variant(originalPath, variant string) string {
|
||||
if originalPath == "" || variant == "" || variant == OriginalVariant {
|
||||
return originalPath
|
||||
}
|
||||
dir := path.Dir(originalPath)
|
||||
base := path.Base(originalPath)
|
||||
if dir == "." || !strings.HasPrefix(base, OriginalVariant+".") {
|
||||
return originalPath
|
||||
}
|
||||
return strings.TrimRight(dir, "/") + "/" + variant + strings.TrimPrefix(base, OriginalVariant)
|
||||
}
|
||||
|
||||
// Directory returns the image-type prefix containing every revision and
|
||||
// variant for an artwork key, including a trailing slash.
|
||||
func Directory(objectPath string) string {
|
||||
objectPath = strings.TrimSpace(objectPath)
|
||||
if objectPath == "" || strings.Contains(objectPath, "://") {
|
||||
return ""
|
||||
}
|
||||
dir := path.Dir(objectPath)
|
||||
if dir == "." || dir == "/" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimRight(dir, "/") + "/"
|
||||
}
|
||||
|
||||
// Revision extracts the content revision from a revisioned key. Legacy keys
|
||||
// return an empty string.
|
||||
func Revision(objectPath string) string {
|
||||
name := path.Base(strings.TrimSpace(objectPath))
|
||||
ext := path.Ext(name)
|
||||
stem := strings.TrimSuffix(name, ext)
|
||||
firstDot := strings.IndexByte(stem, '.')
|
||||
if firstDot < 0 || firstDot == len(stem)-1 {
|
||||
return ""
|
||||
}
|
||||
return stem[firstDot+1:]
|
||||
}
|
||||
|
||||
// VariantWidths returns the resize widths generated for an artwork type. This
|
||||
// is the single source of truth for the variant ladder: image generation,
|
||||
// object-key expansion, and garbage collection all derive from it.
|
||||
func VariantWidths(imageType string) []int {
|
||||
switch strings.ToLower(strings.TrimSpace(imageType)) {
|
||||
case "backdrop":
|
||||
return []int{1920, 1280, 300}
|
||||
case "logo":
|
||||
return []int{500}
|
||||
default: // poster, still, profile
|
||||
return []int{500, 300}
|
||||
}
|
||||
}
|
||||
|
||||
// VariantNames returns the cached variants generated for an artwork type.
|
||||
func VariantNames(imageType string) []string {
|
||||
widths := VariantWidths(imageType)
|
||||
names := make([]string, 0, len(widths)+1)
|
||||
names = append(names, OriginalVariant)
|
||||
for _, width := range widths {
|
||||
names = append(names, "w"+strconv.Itoa(width))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// ObjectKeys expands an original key to every expected key for its image type.
|
||||
func ObjectKeys(originalPath, imageType string) []string {
|
||||
if originalPath == "" || strings.Contains(originalPath, "://") {
|
||||
return nil
|
||||
}
|
||||
names := VariantNames(imageType)
|
||||
keys := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
keys = append(keys, Variant(originalPath, name))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package artworkkey
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRevisionedArtworkKeys(t *testing.T) {
|
||||
base := "tmdb/movies/550/poster"
|
||||
original := Original(base, "abc123", ".webp")
|
||||
if original != base+"/original.abc123.webp" {
|
||||
t.Fatalf("Original() = %q", original)
|
||||
}
|
||||
if got := Variant(original, "w500"); got != base+"/w500.abc123.webp" {
|
||||
t.Fatalf("Variant() = %q", got)
|
||||
}
|
||||
if got := Revision(original); got != "abc123" {
|
||||
t.Fatalf("Revision() = %q", got)
|
||||
}
|
||||
if got := Directory(original); got != base+"/" {
|
||||
t.Fatalf("Directory() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyArtworkKeysRemainSupported(t *testing.T) {
|
||||
original := "tmdb/movies/550/poster/original.webp"
|
||||
if got := Variant(original, "w300"); got != "tmdb/movies/550/poster/w300.webp" {
|
||||
t.Fatalf("Variant() = %q", got)
|
||||
}
|
||||
if got := Revision(original); got != "" {
|
||||
t.Fatalf("Revision() = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVariantOnlyRewritesOriginalFilename(t *testing.T) {
|
||||
original := "tmdb/movies/original.segment/550/poster/original.abc123.webp"
|
||||
want := "tmdb/movies/original.segment/550/poster/w500.abc123.webp"
|
||||
if got := Variant(original, "w500"); got != want {
|
||||
t.Fatalf("Variant() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ import (
|
||||
// method set by structural typing. Anything implementing CacheAudiobookCover can
|
||||
// be passed to scanner.ExtractAndUploadAudiobookCover via this interface.
|
||||
type audiobookCoverCacher interface {
|
||||
CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (basePath string, ext string, thumbhash string, err error)
|
||||
CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (storedPath string, thumbhash string, err error)
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -570,7 +570,7 @@ func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, resu
|
||||
return
|
||||
}
|
||||
|
||||
if storedPath := cachedOriginalImagePath(cached.BasePath, cached.Ext); storedPath != "" {
|
||||
if storedPath := metadata.CachedImageOriginalPath(cached); storedPath != "" {
|
||||
result.PosterPath = storedPath
|
||||
}
|
||||
if cached.Thumbhash != "" {
|
||||
@@ -578,19 +578,6 @@ func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, resu
|
||||
}
|
||||
}
|
||||
|
||||
func cachedOriginalImagePath(basePath, ext string) string {
|
||||
if basePath == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.Contains(basePath, "/original.") {
|
||||
return basePath
|
||||
}
|
||||
if ext == "" {
|
||||
ext = ".jpg"
|
||||
}
|
||||
return strings.TrimRight(basePath, "/") + "/original" + ext
|
||||
}
|
||||
|
||||
// persist writes the enriched metadata back to the database.
|
||||
func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs map[string]string, result *metadata.MetadataResult) error {
|
||||
// Build the MetadataUpdate — only set fields that the provider returned.
|
||||
|
||||
@@ -125,7 +125,7 @@ func TestCacheRemotePosterCachesProviderURL(t *testing.T) {
|
||||
if cacher.req.ContentType != "audiobooks" || cacher.req.ContentID != "content-1" {
|
||||
t.Fatalf("cache target = %q/%q", cacher.req.ContentType, cacher.req.ContentID)
|
||||
}
|
||||
if result.PosterPath != "audiobook-metadata/audiobooks/content-1/poster/original.webp" {
|
||||
if result.PosterPath != "audiobook-metadata/audiobooks/content-1/poster/original.test-revision.webp" {
|
||||
t.Fatalf("PosterPath = %q", result.PosterPath)
|
||||
}
|
||||
if result.PosterThumbhash != "thumb" {
|
||||
@@ -155,16 +155,17 @@ type fakeAudiobookImageCacher struct {
|
||||
req metadata.CacheImageRequest
|
||||
}
|
||||
|
||||
func (f *fakeAudiobookImageCacher) CacheAudiobookCover(context.Context, []byte, string) (string, string, string, error) {
|
||||
return "", "", "", nil
|
||||
func (f *fakeAudiobookImageCacher) CacheAudiobookCover(context.Context, []byte, string) (string, string, error) {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
func (f *fakeAudiobookImageCacher) CacheImage(_ context.Context, req metadata.CacheImageRequest) (*metadata.CacheImageResult, error) {
|
||||
f.calls++
|
||||
f.req = req
|
||||
return &metadata.CacheImageResult{
|
||||
BasePath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster",
|
||||
Thumbhash: "thumb",
|
||||
Ext: ".webp",
|
||||
BasePath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster",
|
||||
OriginalPath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster/original.test-revision.webp",
|
||||
Thumbhash: "thumb",
|
||||
Ext: ".webp",
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
apiKeyLastUsedInterval = time.Minute
|
||||
apiKeyLastUsedTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// APIKeyLastUsedUpdater persists an API key's last-used timestamp.
|
||||
type APIKeyLastUsedUpdater interface {
|
||||
UpdateLastUsed(ctx context.Context, id int64) error
|
||||
}
|
||||
|
||||
// APIKeyLastUsedTracker coalesces last-used writes per key without blocking
|
||||
// authentication requests. Stale entries are pruned inline once per interval,
|
||||
// avoiding both unbounded historical-key retention and another goroutine.
|
||||
type APIKeyLastUsedTracker struct {
|
||||
updater APIKeyLastUsedUpdater
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
lastUsedAt map[int64]time.Time
|
||||
nextPrune time.Time
|
||||
}
|
||||
|
||||
// NewAPIKeyLastUsedTracker creates a tracker. A nil clock uses time.Now.
|
||||
func NewAPIKeyLastUsedTracker(updater APIKeyLastUsedUpdater, now func() time.Time) *APIKeyLastUsedTracker {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &APIKeyLastUsedTracker{
|
||||
updater: updater,
|
||||
now: now,
|
||||
lastUsedAt: make(map[int64]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
// Touch records key usage asynchronously, at most once per interval per key.
|
||||
func (t *APIKeyLastUsedTracker) Touch(id int64) {
|
||||
if t == nil || t.updater == nil || !t.shouldUpdate(id) {
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), apiKeyLastUsedTimeout)
|
||||
defer cancel()
|
||||
if err := t.updater.UpdateLastUsed(ctx, id); err != nil {
|
||||
slog.DebugContext(ctx, "api key last-used update failed", "component", "auth", "id", id, "error", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (t *APIKeyLastUsedTracker) shouldUpdate(id int64) bool {
|
||||
now := t.now()
|
||||
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
if t.nextPrune.IsZero() || !now.Before(t.nextPrune) {
|
||||
cutoff := now.Add(-apiKeyLastUsedInterval)
|
||||
for keyID, lastUsed := range t.lastUsedAt {
|
||||
if !lastUsed.After(cutoff) {
|
||||
delete(t.lastUsedAt, keyID)
|
||||
}
|
||||
}
|
||||
t.nextPrune = now.Add(apiKeyLastUsedInterval)
|
||||
}
|
||||
|
||||
if lastUsed, ok := t.lastUsedAt[id]; ok && now.Sub(lastUsed) < apiKeyLastUsedInterval {
|
||||
return false
|
||||
}
|
||||
t.lastUsedAt[id] = now
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type recordingLastUsedUpdater struct {
|
||||
calls chan lastUsedCall
|
||||
}
|
||||
|
||||
type lastUsedCall struct {
|
||||
id int64
|
||||
deadline time.Time
|
||||
}
|
||||
|
||||
func (u *recordingLastUsedUpdater) UpdateLastUsed(ctx context.Context, id int64) error {
|
||||
deadline, _ := ctx.Deadline()
|
||||
u.calls <- lastUsedCall{id: id, deadline: deadline}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAPIKeyLastUsedTrackerThrottlesPerKey(t *testing.T) {
|
||||
now := time.Date(2026, time.July, 20, 12, 0, 0, 0, time.UTC)
|
||||
tracker := NewAPIKeyLastUsedTracker(nil, func() time.Time { return now })
|
||||
|
||||
if !tracker.shouldUpdate(1) {
|
||||
t.Fatal("first use should update")
|
||||
}
|
||||
if tracker.shouldUpdate(1) {
|
||||
t.Fatal("repeat use inside the interval should be throttled")
|
||||
}
|
||||
if !tracker.shouldUpdate(2) {
|
||||
t.Fatal("a different key should update independently")
|
||||
}
|
||||
|
||||
now = now.Add(apiKeyLastUsedInterval)
|
||||
if !tracker.shouldUpdate(1) {
|
||||
t.Fatal("use at the next interval should update")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKeyLastUsedTrackerPrunesExpiredKeys(t *testing.T) {
|
||||
now := time.Date(2026, time.July, 20, 12, 0, 0, 0, time.UTC)
|
||||
tracker := NewAPIKeyLastUsedTracker(nil, func() time.Time { return now })
|
||||
|
||||
tracker.shouldUpdate(1)
|
||||
tracker.shouldUpdate(2)
|
||||
|
||||
now = now.Add(apiKeyLastUsedInterval)
|
||||
tracker.shouldUpdate(3)
|
||||
|
||||
if len(tracker.lastUsedAt) != 1 {
|
||||
t.Fatalf("retained entries = %d, want 1", len(tracker.lastUsedAt))
|
||||
}
|
||||
if _, ok := tracker.lastUsedAt[3]; !ok {
|
||||
t.Fatal("current key was not retained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKeyLastUsedTrackerAddsWriteDeadline(t *testing.T) {
|
||||
updater := &recordingLastUsedUpdater{calls: make(chan lastUsedCall, 1)}
|
||||
tracker := NewAPIKeyLastUsedTracker(updater, nil)
|
||||
tracker.Touch(42)
|
||||
|
||||
select {
|
||||
case call := <-updater.calls:
|
||||
if call.id != 42 {
|
||||
t.Fatalf("key id = %d, want 42", call.id)
|
||||
}
|
||||
if call.deadline.IsZero() {
|
||||
t.Fatal("last-used update context has no deadline")
|
||||
}
|
||||
remaining := time.Until(call.deadline)
|
||||
if remaining <= 0 || remaining > apiKeyLastUsedTimeout {
|
||||
t.Fatalf("deadline after %s, want within (0, %s]", remaining, apiKeyLastUsedTimeout)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for last-used update")
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,23 @@ func (s Snapshot) ThemeColor() string {
|
||||
return DefaultThemeColor
|
||||
}
|
||||
|
||||
// RenderKey returns a stable identity covering every snapshot field the
|
||||
// Render* functions read, so callers can cache rendered output and re-render
|
||||
// only when the branding configuration actually changes. Keep in sync with
|
||||
// RenderIndexHTML and RenderManifest.
|
||||
func (s Snapshot) RenderKey() string {
|
||||
return strings.Join([]string{
|
||||
s.ServerName,
|
||||
s.LoginSubtitle,
|
||||
s.AccentColor,
|
||||
s.DefaultTheme,
|
||||
s.assets[KindWordmark],
|
||||
s.assets[KindMark],
|
||||
s.assets[KindFavicon],
|
||||
s.assets[KindLoginBg],
|
||||
}, "\x00")
|
||||
}
|
||||
|
||||
// Favicon link literal as it appears in web/index.html. Kept in sync with that
|
||||
// file; RenderIndexHTML rewrites it to a custom favicon when one is set.
|
||||
const indexFaviconLink = `<link rel="icon" href="/favicon.ico" sizes="any" />`
|
||||
|
||||
@@ -137,10 +137,15 @@ func intInSlice(value int, values []int) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// FilterMediaFilesByAccess drops file versions that exceed the viewer's
|
||||
// effective quality ceiling.
|
||||
// FilterMediaFilesByAccess drops file versions the viewer cannot access:
|
||||
// files in libraries outside their allowed set, in libraries they disabled,
|
||||
// or above their effective quality ceiling — the same predicate as
|
||||
// FileAllowedByAccess.
|
||||
func FilterMediaFilesByAccess(files []*models.MediaFile, filter AccessFilter) []*models.MediaFile {
|
||||
if len(files) == 0 || strings.TrimSpace(filter.MaxPlaybackQuality) == "" {
|
||||
unrestricted := filter.AllowedLibraryIDs == nil &&
|
||||
len(filter.DisabledLibraryIDs) == 0 &&
|
||||
strings.TrimSpace(filter.MaxPlaybackQuality) == ""
|
||||
if len(files) == 0 || unrestricted {
|
||||
return files
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestFilterMediaFilesByAccess(t *testing.T) {
|
||||
allowed := &models.MediaFile{ID: 1, MediaFolderID: 1, Resolution: "1080p"}
|
||||
otherLibrary := &models.MediaFile{ID: 2, MediaFolderID: 2, Resolution: "2160p"}
|
||||
files := []*models.MediaFile{allowed, otherLibrary}
|
||||
|
||||
t.Run("no restrictions returns all files", func(t *testing.T) {
|
||||
got := FilterMediaFilesByAccess(files, AccessFilter{})
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 files, got %d", len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("allowed library ids filter without quality ceiling", func(t *testing.T) {
|
||||
got := FilterMediaFilesByAccess(files, AccessFilter{AllowedLibraryIDs: []int{1}})
|
||||
if len(got) != 1 || got[0].ID != allowed.ID {
|
||||
t.Fatalf("expected only file %d, got %v", allowed.ID, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("disabled library ids filter without quality ceiling", func(t *testing.T) {
|
||||
got := FilterMediaFilesByAccess(files, AccessFilter{DisabledLibraryIDs: []int{2}})
|
||||
if len(got) != 1 || got[0].ID != allowed.ID {
|
||||
t.Fatalf("expected only file %d, got %v", allowed.ID, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("quality ceiling filters", func(t *testing.T) {
|
||||
got := FilterMediaFilesByAccess(files, AccessFilter{MaxPlaybackQuality: "1080p"})
|
||||
if len(got) != 1 || got[0].ID != allowed.ID {
|
||||
t.Fatalf("expected only file %d, got %v", allowed.ID, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("matches FileAllowedByAccess predicate", func(t *testing.T) {
|
||||
filter := AccessFilter{AllowedLibraryIDs: []int{1}, MaxPlaybackQuality: "1080p"}
|
||||
got := FilterMediaFilesByAccess(files, filter)
|
||||
for _, f := range got {
|
||||
if !FileAllowedByAccess(f, filter) {
|
||||
t.Fatalf("file %d returned despite failing FileAllowedByAccess", f.ID)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultArtworkGCGracePeriod = 24 * time.Hour
|
||||
artworkTargetSeason = "season"
|
||||
artworkTargetEpisode = "episode"
|
||||
artworkImagePoster = "poster"
|
||||
artworkImageBackdrop = "backdrop"
|
||||
artworkImageLogo = "logo"
|
||||
artworkImageStill = "still"
|
||||
artworkPosterPathColumn = "poster_path"
|
||||
artworkBackdropPathColumn = "backdrop_path"
|
||||
artworkLogoPathColumn = "logo_path"
|
||||
artworkPosterSourceColumn = "poster_source_path"
|
||||
artworkBackdropSourceColumn = "backdrop_source_path"
|
||||
artworkLogoSourceColumn = "logo_source_path"
|
||||
artworkPosterThumbColumn = "poster_thumbhash"
|
||||
artworkBackdropThumbColumn = "backdrop_thumbhash"
|
||||
)
|
||||
|
||||
// ErrUnsupportedArtworkSelection reports a target/image-type combination that
|
||||
// has no artwork column. Handlers should reject these before caching anything.
|
||||
var ErrUnsupportedArtworkSelection = errors.New("catalog: unsupported artwork selection")
|
||||
|
||||
// ArtworkRevisionTracker registers every immutable artwork upload before its
|
||||
// objects are written, allowing the collector to reclaim uploads that are
|
||||
// never published. Confirmed live revisions remain dormant in the registry so
|
||||
// database triggers can later reactivate them.
|
||||
type ArtworkRevisionTracker struct {
|
||||
pool *pgxpool.Pool
|
||||
gracePeriod time.Duration
|
||||
}
|
||||
|
||||
func NewArtworkRevisionTracker(pool *pgxpool.Pool) *ArtworkRevisionTracker {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return &ArtworkRevisionTracker{pool: pool, gracePeriod: defaultArtworkGCGracePeriod}
|
||||
}
|
||||
|
||||
// TrackArtworkRevision records the exact object manifest for a revision before
|
||||
// its upload starts. Image caching calls this before uploading, so it
|
||||
// serializes with a collector that may already be deleting an older,
|
||||
// currently-unreferenced copy. Revisions parked as referenced stay dormant: a
|
||||
// re-cache of live artwork is not garbage, and displacement triggers re-arm
|
||||
// the row if the reference later moves away.
|
||||
func (t *ArtworkRevisionTracker) TrackArtworkRevision(ctx context.Context, originalPath, imageType string, objectKeys []string) error {
|
||||
if t == nil || t.pool == nil {
|
||||
return fmt.Errorf("catalog: artwork revision tracking is not configured")
|
||||
}
|
||||
originalPath = strings.TrimSpace(originalPath)
|
||||
keys := compactArtworkObjectKeys(objectKeys)
|
||||
if originalPath == "" || strings.Contains(originalPath, "://") || len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
notBefore := time.Now().Add(t.gracePeriod)
|
||||
// deleted_at is cleared because this upsert precedes a re-upload of the
|
||||
// exact manifest: the objects exist again once the cacher finishes.
|
||||
_, err := t.pool.Exec(ctx, `
|
||||
INSERT INTO artwork_revision_gc_candidates (
|
||||
original_path, image_type, object_keys, not_before, next_attempt_at
|
||||
) VALUES ($1, $2, $3, $4, $4)
|
||||
ON CONFLICT (original_path) DO UPDATE SET
|
||||
object_keys = EXCLUDED.object_keys,
|
||||
image_type = CASE
|
||||
WHEN artwork_revision_gc_candidates.image_type = '' THEN EXCLUDED.image_type
|
||||
ELSE artwork_revision_gc_candidates.image_type
|
||||
END,
|
||||
not_before = CASE
|
||||
WHEN artwork_revision_gc_candidates.next_attempt_at IS NULL THEN artwork_revision_gc_candidates.not_before
|
||||
ELSE EXCLUDED.not_before
|
||||
END,
|
||||
next_attempt_at = CASE
|
||||
WHEN artwork_revision_gc_candidates.next_attempt_at IS NULL THEN NULL
|
||||
ELSE EXCLUDED.next_attempt_at
|
||||
END,
|
||||
deleted_at = NULL,
|
||||
attempt_count = 0,
|
||||
locked_at = NULL,
|
||||
locked_by = '',
|
||||
last_error = '',
|
||||
updated_at = NOW()`, originalPath, strings.ToLower(strings.TrimSpace(imageType)), keys, notBefore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("catalog: track artwork revision: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ArtworkSelection describes a manually selected, already-cached artwork
|
||||
// revision. PublishArtworkSelection makes the database pointer and image lock
|
||||
// visible atomically, then schedules the displaced revision for delayed cleanup.
|
||||
type ArtworkSelection struct {
|
||||
TargetType string
|
||||
TargetContentID string
|
||||
ParentContentID string
|
||||
ImageType string
|
||||
StoredPath string
|
||||
SourcePath string
|
||||
Thumbhash string
|
||||
LockField int
|
||||
}
|
||||
|
||||
// ValidateArtworkSelectionTarget reports whether a target/image-type pair maps
|
||||
// to a real artwork column, so handlers can reject unsupported requests before
|
||||
// downloading or uploading anything.
|
||||
func ValidateArtworkSelectionTarget(targetType, imageType string) error {
|
||||
_, _, _, _, _, err := artworkTargetColumns(
|
||||
strings.ToLower(strings.TrimSpace(targetType)),
|
||||
strings.ToLower(strings.TrimSpace(imageType)),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// PublishArtworkSelection atomically changes the selected artwork, records its
|
||||
// source, locks automatic image refreshes, and queues the old immutable revision
|
||||
// for reference-aware garbage collection after a grace period.
|
||||
func (s *DetailService) PublishArtworkSelection(ctx context.Context, selection ArtworkSelection) error {
|
||||
if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil {
|
||||
return fmt.Errorf("catalog: artwork persistence is not configured")
|
||||
}
|
||||
selection.TargetType = strings.ToLower(strings.TrimSpace(selection.TargetType))
|
||||
selection.ImageType = strings.ToLower(strings.TrimSpace(selection.ImageType))
|
||||
selection.TargetContentID = strings.TrimSpace(selection.TargetContentID)
|
||||
selection.ParentContentID = strings.TrimSpace(selection.ParentContentID)
|
||||
selection.StoredPath = strings.TrimSpace(selection.StoredPath)
|
||||
if selection.TargetContentID == "" || selection.StoredPath == "" {
|
||||
return fmt.Errorf("catalog: artwork target and stored path are required")
|
||||
}
|
||||
|
||||
table, pathColumn, sourceColumn, thumbhashColumn, notFound, err := artworkTargetColumns(selection.TargetType, selection.ImageType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := s.itemRepo.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("catalog: begin artwork publication: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
var previousPath string
|
||||
selectSQL := fmt.Sprintf("SELECT COALESCE(%s, '') FROM %s WHERE content_id = $1 FOR UPDATE", pathColumn, table)
|
||||
if err := tx.QueryRow(ctx, selectSQL, selection.TargetContentID).Scan(&previousPath); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return notFound
|
||||
}
|
||||
return fmt.Errorf("catalog: lock artwork target: %w", err)
|
||||
}
|
||||
|
||||
setThumbhash := ""
|
||||
args := []any{selection.StoredPath, selection.SourcePath, selection.TargetContentID}
|
||||
if thumbhashColumn != "" {
|
||||
setThumbhash = fmt.Sprintf(", %s = $4", thumbhashColumn)
|
||||
args = append(args, selection.Thumbhash)
|
||||
}
|
||||
updateSQL := fmt.Sprintf(`
|
||||
UPDATE %s
|
||||
SET %s = $1, %s = $2%s, updated_at = NOW()
|
||||
WHERE content_id = $3`, table, pathColumn, sourceColumn, setThumbhash)
|
||||
if _, err := tx.Exec(ctx, updateSQL, args...); err != nil {
|
||||
return fmt.Errorf("catalog: publish artwork pointer: %w", err)
|
||||
}
|
||||
|
||||
lockContentID := selection.TargetContentID
|
||||
if selection.TargetType == artworkTargetSeason || selection.TargetType == artworkTargetEpisode {
|
||||
lockContentID = selection.ParentContentID
|
||||
}
|
||||
if lockContentID == "" {
|
||||
return fmt.Errorf("catalog: parent content ID is required for %s artwork", selection.TargetType)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE media_items
|
||||
SET locked_fields = CASE
|
||||
WHEN $2 = ANY(COALESCE(locked_fields, '{}'::integer[])) THEN COALESCE(locked_fields, '{}'::integer[])
|
||||
ELSE array_append(COALESCE(locked_fields, '{}'::integer[]), $2)
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE content_id = $1`, lockContentID, selection.LockField)
|
||||
if err != nil {
|
||||
return fmt.Errorf("catalog: lock manual artwork: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrItemNotFound
|
||||
}
|
||||
// The selected revision is referenced by construction once this transaction
|
||||
// commits, so park it dormant immediately: no first-pass verification cycle
|
||||
// is needed, and database triggers re-arm it if a later writer displaces it.
|
||||
if err := parkArtworkRevision(ctx, tx, selection.StoredPath, selection.ImageType, time.Now().Add(defaultArtworkGCGracePeriod)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if previousPath != "" && previousPath != selection.StoredPath {
|
||||
if err := queueArtworkRevisionGC(ctx, tx, previousPath, selection.ImageType, time.Now().Add(defaultArtworkGCGracePeriod)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("catalog: commit artwork publication: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// QueueArtworkRevisionGC schedules an unreferenced cached revision for cleanup.
|
||||
// It is used to reclaim an upload when publication fails after object storage
|
||||
// succeeded. Cleanup still verifies database references before deleting.
|
||||
func (s *DetailService) QueueArtworkRevisionGC(ctx context.Context, originalPath, imageType string, notBefore time.Time) error {
|
||||
if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil {
|
||||
return fmt.Errorf("catalog: artwork persistence is not configured")
|
||||
}
|
||||
return queueArtworkRevisionGC(ctx, s.itemRepo.pool, originalPath, imageType, notBefore)
|
||||
}
|
||||
|
||||
type artworkGCExecer interface {
|
||||
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||
}
|
||||
|
||||
// queueArtworkRevisionGC arms a revision for verification after notBefore. The
|
||||
// stored manifest (when one was tracked pre-upload) is kept; rows without one
|
||||
// carry the image type so the collector can expand the object keys itself.
|
||||
func queueArtworkRevisionGC(ctx context.Context, db artworkGCExecer, originalPath, imageType string, notBefore time.Time) error {
|
||||
return upsertArtworkRevision(ctx, db, originalPath, imageType, notBefore, false)
|
||||
}
|
||||
|
||||
// parkArtworkRevision registers a revision as referenced and dormant. Only a
|
||||
// displacement trigger or the collector's dormant sweep re-arms it.
|
||||
func parkArtworkRevision(ctx context.Context, db artworkGCExecer, originalPath, imageType string, notBefore time.Time) error {
|
||||
return upsertArtworkRevision(ctx, db, originalPath, imageType, notBefore, true)
|
||||
}
|
||||
|
||||
func upsertArtworkRevision(
|
||||
ctx context.Context,
|
||||
db artworkGCExecer,
|
||||
originalPath string,
|
||||
imageType string,
|
||||
notBefore time.Time,
|
||||
dormant bool,
|
||||
) error {
|
||||
originalPath = strings.TrimSpace(originalPath)
|
||||
imageType = strings.ToLower(strings.TrimSpace(imageType))
|
||||
if originalPath == "" || strings.Contains(originalPath, "://") || imageType == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := db.Exec(ctx, `
|
||||
INSERT INTO artwork_revision_gc_candidates (
|
||||
original_path, image_type, object_keys, not_before, next_attempt_at
|
||||
) VALUES ($1, $2, '{}', $3::timestamptz, CASE WHEN $4 THEN NULL ELSE $3::timestamptz END)
|
||||
ON CONFLICT (original_path) DO UPDATE SET
|
||||
image_type = CASE
|
||||
WHEN artwork_revision_gc_candidates.image_type = '' THEN EXCLUDED.image_type
|
||||
ELSE artwork_revision_gc_candidates.image_type
|
||||
END,
|
||||
not_before = EXCLUDED.not_before,
|
||||
next_attempt_at = EXCLUDED.next_attempt_at,
|
||||
attempt_count = 0,
|
||||
locked_at = NULL,
|
||||
locked_by = '',
|
||||
last_error = '',
|
||||
updated_at = NOW()`, originalPath, imageType, notBefore, dormant)
|
||||
if err != nil {
|
||||
return fmt.Errorf("catalog: queue artwork revision cleanup: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func compactArtworkObjectKeys(keys []string) []string {
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
result := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" || strings.Contains(key, "://") {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, key)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// artworkTargetColumns maps a selection target to its artwork columns. Season
|
||||
// and episode rows have dedicated tables; every other catalog item type
|
||||
// (movie, series, audiobook, ebook, manga, ...) stores artwork on its
|
||||
// media_items row.
|
||||
func artworkTargetColumns(targetType, imageType string) (table, pathColumn, sourceColumn, thumbhashColumn string, notFound error, err error) {
|
||||
switch targetType {
|
||||
case artworkTargetSeason:
|
||||
if imageType == artworkImagePoster {
|
||||
return "seasons", artworkPosterPathColumn, artworkPosterSourceColumn, artworkPosterThumbColumn, ErrSeasonNotFound, nil
|
||||
}
|
||||
case artworkTargetEpisode:
|
||||
if imageType == artworkImageStill {
|
||||
return "episodes", "still_path", "still_source_path", "still_thumbhash", ErrEpisodeNotFound, nil
|
||||
}
|
||||
default:
|
||||
table = "media_items"
|
||||
notFound = ErrItemNotFound
|
||||
switch imageType {
|
||||
case artworkImagePoster:
|
||||
return table, artworkPosterPathColumn, artworkPosterSourceColumn, artworkPosterThumbColumn, notFound, nil
|
||||
case artworkImageBackdrop:
|
||||
return table, artworkBackdropPathColumn, artworkBackdropSourceColumn, artworkBackdropThumbColumn, notFound, nil
|
||||
case artworkImageLogo:
|
||||
return table, artworkLogoPathColumn, artworkLogoSourceColumn, "", notFound, nil
|
||||
}
|
||||
}
|
||||
return "", "", "", "", nil, fmt.Errorf("%w: %s does not accept %q images", ErrUnsupportedArtworkSelection, targetType, imageType)
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func newArtworkSelectionTestPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("SILO_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
return pool
|
||||
}
|
||||
|
||||
func TestQueueAndParkArtworkRevisionUpserts(t *testing.T) {
|
||||
pool := newArtworkSelectionTestPool(t)
|
||||
ctx := context.Background()
|
||||
suffix := time.Now().UnixNano()
|
||||
path := fmt.Sprintf("tmdb/movies/%d/poster/original.rev.webp", suffix)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, path)
|
||||
})
|
||||
|
||||
// Arming registers the revision for verification after the grace period.
|
||||
if err := queueArtworkRevisionGC(ctx, pool, path, "poster", time.Now().Add(time.Hour)); err != nil {
|
||||
t.Fatalf("queueArtworkRevisionGC: %v", err)
|
||||
}
|
||||
var imageType string
|
||||
var nextAttempt *time.Time
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT image_type, next_attempt_at FROM artwork_revision_gc_candidates
|
||||
WHERE original_path = $1`, path).Scan(&imageType, &nextAttempt); err != nil {
|
||||
t.Fatalf("load armed candidate: %v", err)
|
||||
}
|
||||
if imageType != "poster" {
|
||||
t.Fatalf("image_type = %q, want poster", imageType)
|
||||
}
|
||||
if nextAttempt == nil {
|
||||
t.Fatal("armed candidate has NULL next_attempt_at")
|
||||
}
|
||||
|
||||
// Publication parks the selected revision: referenced by construction.
|
||||
if err := parkArtworkRevision(ctx, pool, path, "poster", time.Now().Add(time.Hour)); err != nil {
|
||||
t.Fatalf("parkArtworkRevision: %v", err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT next_attempt_at FROM artwork_revision_gc_candidates
|
||||
WHERE original_path = $1`, path).Scan(&nextAttempt); err != nil {
|
||||
t.Fatalf("load parked candidate: %v", err)
|
||||
}
|
||||
if nextAttempt != nil {
|
||||
t.Fatalf("parked candidate next_attempt_at = %v, want NULL", *nextAttempt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackArtworkRevisionKeepsDormantRowsDormant(t *testing.T) {
|
||||
pool := newArtworkSelectionTestPool(t)
|
||||
ctx := context.Background()
|
||||
suffix := time.Now().UnixNano()
|
||||
path := fmt.Sprintf("tmdb/movies/%d/poster/original.live.webp", suffix)
|
||||
keys := []string{path, fmt.Sprintf("tmdb/movies/%d/poster/w500.live.webp", suffix)}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, path)
|
||||
})
|
||||
|
||||
if err := parkArtworkRevision(ctx, pool, path, "poster", time.Now().Add(time.Hour)); err != nil {
|
||||
t.Fatalf("parkArtworkRevision: %v", err)
|
||||
}
|
||||
// A re-cache of live artwork must not re-arm the parked row.
|
||||
tracker := NewArtworkRevisionTracker(pool)
|
||||
if err := tracker.TrackArtworkRevision(ctx, path, "poster", keys); err != nil {
|
||||
t.Fatalf("TrackArtworkRevision: %v", err)
|
||||
}
|
||||
|
||||
var nextAttempt *time.Time
|
||||
var storedKeys []string
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT next_attempt_at, object_keys FROM artwork_revision_gc_candidates
|
||||
WHERE original_path = $1`, path).Scan(&nextAttempt, &storedKeys); err != nil {
|
||||
t.Fatalf("load candidate: %v", err)
|
||||
}
|
||||
if nextAttempt != nil {
|
||||
t.Fatalf("re-cache re-armed dormant row: next_attempt_at = %v", *nextAttempt)
|
||||
}
|
||||
if !slices.Equal(storedKeys, keys) {
|
||||
t.Fatalf("stored manifest = %v, want exact tracked manifest %v", storedKeys, keys)
|
||||
}
|
||||
}
|
||||
@@ -2253,6 +2253,16 @@ func filterCatalogSearchItems(items []*models.MediaItem, raw string) []*models.M
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
if len(filtered) == 0 && len(items) > 0 && eligibleForFuzzy(parsed) {
|
||||
// The strict per-token substring filter erases the repo layer's
|
||||
// typo-tolerant (trigram) matches: a misspelled token is never a
|
||||
// substring of the titles it fuzzy-matched, so a typo query that
|
||||
// found results via the fuzzy fallback would be filtered to zero here
|
||||
// while the plain search box shows matches. When a fuzzy-eligible
|
||||
// query would be emptied entirely, trust the repo's relevance
|
||||
// ordering instead.
|
||||
return items
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/artworkkey"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/overlays"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
@@ -3620,10 +3621,7 @@ func cachedImageVariantPath(path, imageType, size string) string {
|
||||
if variant == "" {
|
||||
return path
|
||||
}
|
||||
if strings.Contains(path, "/original.") {
|
||||
return strings.Replace(path, "/original.", "/"+variant+".", 1)
|
||||
}
|
||||
return path
|
||||
return artworkkey.Variant(path, variant)
|
||||
}
|
||||
|
||||
// imageTypeFromCachedPath returns the image type segment ("poster", "backdrop",
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/artworkkey"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
@@ -894,14 +895,26 @@ func dirSetToSlice(set map[string]struct{}) []string {
|
||||
return dirs
|
||||
}
|
||||
|
||||
// imageDeletePrefix returns the S3 prefix to sweep when deleting the content
|
||||
// that owns the cached image at path. Local artwork keys carry a content-hash
|
||||
// segment (local/{contentType}/{contentID}/{hash8}/{imageType}/...), so local
|
||||
// paths trim to local/{contentType}/{contentID}/ — deleting every stale hash
|
||||
// prefix, not just the live one. Other keys keep the per-image-type dir.
|
||||
func imageDeletePrefix(path string) string {
|
||||
if !strings.HasPrefix(path, "local/") {
|
||||
return pathDir(path)
|
||||
}
|
||||
segments := strings.SplitN(path, "/", 4)
|
||||
if len(segments) < 4 {
|
||||
return pathDir(path)
|
||||
}
|
||||
return segments[0] + "/" + segments[1] + "/" + segments[2] + "/"
|
||||
}
|
||||
|
||||
// pathDir extracts the directory portion of an S3 image path.
|
||||
// e.g. "tmdb/movies/550/poster/original.webp" → "tmdb/movies/550/poster/"
|
||||
func pathDir(path string) string {
|
||||
idx := strings.LastIndex(path, "/")
|
||||
if idx <= 0 {
|
||||
return ""
|
||||
}
|
||||
return path[:idx+1]
|
||||
return artworkkey.Directory(path)
|
||||
}
|
||||
|
||||
// rowQuerier is satisfied by both *pgxpool.Pool and pgx.Tx, letting read
|
||||
@@ -967,7 +980,7 @@ func collectRawImageDirs(ctx context.Context, q rowQuerier, contentIDs []string)
|
||||
}
|
||||
for _, p := range []string{p1, p2, p3} {
|
||||
if p != "" && !strings.Contains(p, "://") {
|
||||
if dir := pathDir(p); dir != "" {
|
||||
if dir := imageDeletePrefix(p); dir != "" {
|
||||
dirSet[dir] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -1035,6 +1048,32 @@ func filterUnreferencedImageDirs(ctx context.Context, q rowQuerier, dirs, deleti
|
||||
}
|
||||
|
||||
// UpdateLastScanned sets the last_scanned_at timestamp for the given folder.
|
||||
// LibraryRootsForContent returns the media folder root paths the given
|
||||
// content belongs to, resolved through media_item_libraries. The metadata
|
||||
// image-cache processor uses this to confine local file:// artwork sources
|
||||
// (metadata.LibraryRootResolver).
|
||||
func (r *FolderRepository) LibraryRootsForContent(ctx context.Context, contentID string) ([]string, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT DISTINCT mfp.path
|
||||
FROM media_folder_paths mfp
|
||||
JOIN media_item_libraries mil ON mil.media_folder_id = mfp.media_folder_id
|
||||
WHERE mil.content_id = $1
|
||||
`, contentID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying library roots for content %q: %w", contentID, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var paths []string
|
||||
for rows.Next() {
|
||||
var path string
|
||||
if err := rows.Scan(&path); err != nil {
|
||||
return nil, fmt.Errorf("scanning library root path: %w", err)
|
||||
}
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return paths, rows.Err()
|
||||
}
|
||||
|
||||
func (r *FolderRepository) UpdateLastScanned(ctx context.Context, id int, scannedAt time.Time) error {
|
||||
tag, err := r.pool.Exec(ctx,
|
||||
"UPDATE media_folders SET last_scanned_at = $1 WHERE id = $2",
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestLibraryRootsForContent(t *testing.T) {
|
||||
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("SILO_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
|
||||
suffix := time.Now().UnixNano()
|
||||
contentID := fmt.Sprintf("roots-test-%d", suffix)
|
||||
var folderID int
|
||||
if err := pool.QueryRow(ctx,
|
||||
`INSERT INTO media_folders (type, name, enabled) VALUES ('movies', $1, true) RETURNING id`,
|
||||
fmt.Sprintf("roots-test-%d", suffix)).Scan(&folderID); err != nil {
|
||||
t.Fatalf("seed folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_item_libraries WHERE content_id = $1`, contentID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_folder_paths WHERE media_folder_id = $1`, folderID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID)
|
||||
})
|
||||
if _, err := pool.Exec(ctx,
|
||||
`INSERT INTO media_items (content_id, type, title, status, genres) VALUES ($1, 'movie', 'Roots Test', 'matched', '{}'::text[])`,
|
||||
contentID); err != nil {
|
||||
t.Fatalf("seed item: %v", err)
|
||||
}
|
||||
for _, path := range []string{"/media/movies-a", "/media/movies-b"} {
|
||||
if _, err := pool.Exec(ctx,
|
||||
`INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2)`,
|
||||
folderID, fmt.Sprintf("%s-%d", path, suffix)); err != nil {
|
||||
t.Fatalf("seed folder path: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := pool.Exec(ctx,
|
||||
`INSERT INTO media_item_libraries (content_id, media_folder_id) VALUES ($1, $2)`,
|
||||
contentID, folderID); err != nil {
|
||||
t.Fatalf("seed membership: %v", err)
|
||||
}
|
||||
|
||||
repo := NewFolderRepository(pool)
|
||||
roots, err := repo.LibraryRootsForContent(ctx, contentID)
|
||||
if err != nil {
|
||||
t.Fatalf("LibraryRootsForContent: %v", err)
|
||||
}
|
||||
sort.Strings(roots)
|
||||
want := []string{
|
||||
fmt.Sprintf("/media/movies-a-%d", suffix),
|
||||
fmt.Sprintf("/media/movies-b-%d", suffix),
|
||||
}
|
||||
if len(roots) != 2 || roots[0] != want[0] || roots[1] != want[1] {
|
||||
t.Fatalf("roots = %v, want %v", roots, want)
|
||||
}
|
||||
|
||||
// Unknown content has no roots (and no error).
|
||||
none, err := repo.LibraryRootsForContent(ctx, contentID+"-missing")
|
||||
if err != nil || len(none) != 0 {
|
||||
t.Fatalf("missing content: roots=%v err=%v", none, err)
|
||||
}
|
||||
}
|
||||
+442
-17
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -131,6 +132,40 @@ func (r *ItemRepository) GetPosterPath(ctx context.Context, contentID string) (s
|
||||
// incidental one-mention hits that flooded results before.
|
||||
const overviewMatchFloor = 0.15
|
||||
|
||||
// fuzzyFallbackThreshold is the FTS result count below which SearchPage reaches
|
||||
// for the trigram fuzzy fallback (buildFuzzySearchSQL). At or above it the FTS
|
||||
// result set is considered rich enough that a "did you mean" pass would only add
|
||||
// noise, so common searches stay on the unchanged FTS path and the fuzzy query
|
||||
// fires only in the sparse/misspelling case.
|
||||
const fuzzyFallbackThreshold = 5
|
||||
|
||||
// fuzzyMaxResults caps how many trigram fuzzy matches the fallback contributes.
|
||||
// The combined result the fuzzy path serves is therefore bounded at
|
||||
// (fuzzyFallbackThreshold-1) FTS rows + fuzzyMaxResults fuzzy rows, small enough
|
||||
// to materialize once and slice per page. A "did you mean" fallback has no need
|
||||
// to paginate hundreds of low-similarity typo matches; when the true match count
|
||||
// exceeds this, the overflow is logged and the total is reported as inexact.
|
||||
const fuzzyMaxResults = 50
|
||||
|
||||
// trgmWordSimilarityThreshold pins pg_trgm.strict_word_similarity_threshold
|
||||
// for the fuzzy query via SET LOCAL. The <<% operator's selectivity is
|
||||
// otherwise governed by a cluster-wide GUC a DBA (or another application) can
|
||||
// change, silently altering both match quality and scan cost per deployment —
|
||||
// and its server default (0.6) would reject ordinary one-edit typos
|
||||
// ("avegners" vs "avengers" scores ~0.38), so pinning is load-bearing, not
|
||||
// just hygiene.
|
||||
const trgmWordSimilarityThreshold = 0.30
|
||||
|
||||
// fuzzyAugmentSimilarityFloor is the minimum WHOLE-TITLE similarity demanded of
|
||||
// fuzzy rows when the sparse FTS block is non-empty. A correctly spelled query
|
||||
// with a few genuine hits ("coraline") should only gain near-identical titles,
|
||||
// not every title clearing the permissive base threshold — word similarity is
|
||||
// deliberately not used here, since it scores embedded prefix words far too
|
||||
// high ("coral" is 0.5 to "coraline"). A zero-hit query (a likely misspelling)
|
||||
// skips the floor entirely so typo recall, including word-extent matches into
|
||||
// long titles, stays high.
|
||||
const fuzzyAugmentSimilarityFloor = 0.45
|
||||
|
||||
// itemColumnNames lists, in scan order, every column selected by media_items
|
||||
// item queries. Shared by itemColumns, qualifiedItemColumns, and
|
||||
// qualifiedListItemColumns so the select lists can never drift from each
|
||||
@@ -194,6 +229,17 @@ func qualifiedItemColumns(alias string) string {
|
||||
return joinItemColumns(alias)
|
||||
}
|
||||
|
||||
// qualifiedItemColumnRefs renders plain alias-qualified column references
|
||||
// without COALESCE or AS aliases, for contexts like GROUP BY where output
|
||||
// aliases are invalid.
|
||||
func qualifiedItemColumnRefs(alias string) string {
|
||||
refs := make([]string, len(itemColumnNames))
|
||||
for i, col := range itemColumnNames {
|
||||
refs[i] = alias + "." + col
|
||||
}
|
||||
return strings.Join(refs, ", ")
|
||||
}
|
||||
|
||||
func qualifiedListItemColumns(alias string) string {
|
||||
exprs := make([]string, len(itemColumnNames))
|
||||
for i, col := range itemColumnNames {
|
||||
@@ -823,7 +869,7 @@ func (r *ItemRepository) Delete(ctx context.Context, contentID string) ([]string
|
||||
}
|
||||
for _, p := range []string{p1, p2, p3} {
|
||||
if p != "" && !strings.Contains(p, "://") {
|
||||
if dir := pathDir(p); dir != "" {
|
||||
if dir := imageDeletePrefix(p); dir != "" {
|
||||
imageDirs = append(imageDirs, dir)
|
||||
}
|
||||
}
|
||||
@@ -873,26 +919,241 @@ func (r *ItemRepository) SearchPage(
|
||||
if filter.AllowedLibraryIDs != nil && len(filter.AllowedLibraryIDs) == 0 {
|
||||
return []*models.MediaItem{}, 0, false, includeTotal, nil
|
||||
}
|
||||
|
||||
parsed := parseSearchQuery(query)
|
||||
|
||||
// FTS block (the common path). queryLimit fetches one extra row in cursor
|
||||
// mode so execSearchBlock can derive hasMore without a separate count. When
|
||||
// the fuzzy fallback below could fire (first page of a fuzzy-eligible
|
||||
// query) the probe is additionally floored at fuzzyFallbackThreshold rows:
|
||||
// a tiny caller limit (e.g. an autocomplete asking for 2) with a few
|
||||
// incidental exact hits would otherwise report hasMore and hide the FTS
|
||||
// block's true size, so the sparse test below could never fire. Fetching up
|
||||
// to the threshold reveals a genuinely sparse block regardless of limit;
|
||||
// the returned page is still trimmed to limit. Where the fallback cannot
|
||||
// fire, fetching past limit+1 would be pure waste, so the floor is skipped.
|
||||
queryLimit := limit
|
||||
if !includeTotal {
|
||||
queryLimit = limit + 1
|
||||
if offset == 0 && eligibleForFuzzy(parsed) && queryLimit < fuzzyFallbackThreshold {
|
||||
queryLimit = fuzzyFallbackThreshold
|
||||
}
|
||||
}
|
||||
sql, countSQL, args := r.buildSearchSQLWithTotal(query, itemTypes, queryLimit, offset, filter, includeTotal)
|
||||
sql, countSQL, args := r.buildSearchSQLFromParsed(parsed, itemTypes, queryLimit, offset, filter, includeTotal)
|
||||
if sql == "" {
|
||||
return []*models.MediaItem{}, 0, false, includeTotal, nil
|
||||
}
|
||||
|
||||
rows, err := r.pool.Query(ctx, sql, args...)
|
||||
ftsItems, ftsTotal, ftsHasMore, ftsUntrimmed, err := r.execSearchBlock(ctx, r.pool, sql, countSQL, args, limit, offset, includeTotal)
|
||||
if err != nil {
|
||||
return nil, 0, false, includeTotal, fmt.Errorf("searching media items: %w", err)
|
||||
return nil, 0, false, includeTotal, err
|
||||
}
|
||||
|
||||
// Fuzzy fallback trigger. The trigram (typo-tolerant) arm is deliberately NOT
|
||||
// part of the FTS query: OR-ing the pg_trgm % operator into that WHERE forces
|
||||
// a lossy bitmap heap recheck that rebuilds the three title tsvectors for
|
||||
// every one of the thousands of near-miss rows the % operator surfaces,
|
||||
// making *every* search take seconds. Instead we reach for the trigram index
|
||||
// only when the FTS result set is fully known and sparse (a likely
|
||||
// misspelling or word-boundary miss).
|
||||
//
|
||||
// Exact mode judges sparsity by the window count (page-independent), so every
|
||||
// page of a sparse query agrees and the combined result paginates fully.
|
||||
//
|
||||
// Cursor mode has no window count, so it uses the pre-trim size of a probe
|
||||
// floored at fuzzyFallbackThreshold rows: fewer than the threshold came
|
||||
// back ⇒ the whole FTS block is smaller than the threshold ⇒ sparse,
|
||||
// independent of the caller's page size. It only trusts this on the first
|
||||
// page (offset 0), where searchWithFuzzyFallback serves fuzzy as a terminal
|
||||
// augmentation (no phantom next page a bare offset cursor couldn't locate),
|
||||
// and only when the whole FTS block fits inside the caller's page: a
|
||||
// terminal page must never hide exact matches the plain cursor path would
|
||||
// have surfaced via hasMore. Past offset 0 an empty FTS page can't be told
|
||||
// from a rich one, so sparse stays false.
|
||||
var sparse bool
|
||||
if includeTotal {
|
||||
sparse = ftsTotal < fuzzyFallbackThreshold
|
||||
} else if offset == 0 {
|
||||
sparse = len(ftsUntrimmed) < fuzzyFallbackThreshold && len(ftsUntrimmed) <= limit
|
||||
}
|
||||
if !sparse || !eligibleForFuzzy(parsed) {
|
||||
return ftsItems, ftsTotal, ftsHasMore, includeTotal, nil
|
||||
}
|
||||
|
||||
// Hand the fallback the FTS block when the rows just fetched provably form
|
||||
// the complete block, sparing it an identical second FTS query. Cursor mode
|
||||
// always qualifies here (the floored probe returned fewer rows than it
|
||||
// asked for); exact mode qualifies on the first page when the window count
|
||||
// fit within the page.
|
||||
ftsBlock, haveFTSBlock := []*models.MediaItem(nil), false
|
||||
if !includeTotal {
|
||||
ftsBlock, haveFTSBlock = ftsUntrimmed, true
|
||||
} else if offset == 0 && ftsTotal <= len(ftsItems) {
|
||||
ftsBlock, haveFTSBlock = ftsItems, true
|
||||
}
|
||||
return r.searchWithFuzzyFallback(ctx, parsed, itemTypes, limit, offset, filter, includeTotal, ftsBlock, haveFTSBlock)
|
||||
}
|
||||
|
||||
// searchWithFuzzyFallback serves the combined [FTS block][fuzzy block] result
|
||||
// for a sparse, fuzzy-eligible query. Because fuzzy only fires when the FTS
|
||||
// block is sparse (< fuzzyFallbackThreshold rows) and the fuzzy contribution is
|
||||
// capped at fuzzyMaxResults, the ENTIRE combined result is small
|
||||
// (< fuzzyFallbackThreshold + fuzzyMaxResults rows). We materialize it once and
|
||||
// slice the requested page in memory. That keeps two-block pagination correct on
|
||||
// every page — stable total, no cross-page duplicates, no offset arithmetic
|
||||
// straddling the block boundary — for both exact and cursor callers. The
|
||||
// FTS-only path in SearchPage is untouched and still streams strictly per page.
|
||||
//
|
||||
// When haveFTSBlock is true, ftsBlock is the complete sparse FTS block the
|
||||
// caller already fetched and no second FTS query is issued; otherwise (exact
|
||||
// mode past the first page, or a first page smaller than the block) the block
|
||||
// is re-fetched here at offset 0 — its ids drive fuzzy dedup below.
|
||||
func (r *ItemRepository) searchWithFuzzyFallback(
|
||||
ctx context.Context,
|
||||
parsed parsedSearchQuery,
|
||||
itemTypes []string,
|
||||
limit, offset int,
|
||||
filter AccessFilter,
|
||||
includeTotal bool,
|
||||
ftsBlock []*models.MediaItem,
|
||||
haveFTSBlock bool,
|
||||
) ([]*models.MediaItem, int, bool, bool, error) {
|
||||
if !haveFTSBlock {
|
||||
ftsSQL, ftsCountSQL, ftsArgs := r.buildSearchSQLFromParsed(parsed, itemTypes, fuzzyFallbackThreshold, 0, filter, true)
|
||||
var err error
|
||||
ftsBlock, _, _, _, err = r.execSearchBlock(ctx, r.pool, ftsSQL, ftsCountSQL, ftsArgs, fuzzyFallbackThreshold, 0, true)
|
||||
if err != nil {
|
||||
return nil, 0, false, includeTotal, err
|
||||
}
|
||||
}
|
||||
|
||||
// Fuzzy block, excluding every FTS-block id so no title appears in both
|
||||
// blocks and the combined total is stable per page. Cursor mode serves the
|
||||
// combined result as a single terminal page, so it never needs more fuzzy
|
||||
// rows than the page has room for; exact mode materializes up to the cap.
|
||||
// Either way one extra row is fetched so truncation is detectable without
|
||||
// paying a COUNT(*) OVER () window count over the whole trigram match set.
|
||||
// When the FTS block found real hits the fuzzy arm demands near-certain
|
||||
// corrections (fuzzyAugmentSimilarityFloor) instead of base-threshold noise.
|
||||
fuzzyLimit := fuzzyMaxResults
|
||||
if !includeTotal && limit-len(ftsBlock) < fuzzyLimit {
|
||||
fuzzyLimit = limit - len(ftsBlock)
|
||||
}
|
||||
minSimilarity := 0.0
|
||||
if len(ftsBlock) > 0 {
|
||||
minSimilarity = fuzzyAugmentSimilarityFloor
|
||||
}
|
||||
var fuzzyBlock []*models.MediaItem
|
||||
fuzzyTruncated := false
|
||||
if fuzzyLimit > 0 {
|
||||
fuzzySQL, _, fuzzyArgs := r.buildFuzzySearchFromParsed(parsed, itemTypes, fuzzyLimit+1, 0, filter, false, contentIDsFromMediaItems(ftsBlock), minSimilarity)
|
||||
if fuzzySQL != "" {
|
||||
var err error
|
||||
fuzzyBlock, fuzzyTruncated, err = r.execFuzzyBlock(ctx, fuzzySQL, fuzzyArgs, fuzzyLimit)
|
||||
if err != nil {
|
||||
return nil, 0, false, includeTotal, err
|
||||
}
|
||||
if fuzzyTruncated {
|
||||
slog.DebugContext(ctx, "fuzzy search fallback truncated to cap",
|
||||
"query", parsed.Text, "cap", fuzzyLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
combined := make([]*models.MediaItem, 0, len(ftsBlock)+len(fuzzyBlock))
|
||||
combined = append(combined, ftsBlock...)
|
||||
combined = append(combined, fuzzyBlock...)
|
||||
total := len(combined)
|
||||
|
||||
// Slice the requested page out of the materialized list, clamped so a
|
||||
// caller passing a negative offset or an overflowing limit (the exported
|
||||
// SearchPage is reachable outside the HTTP parser, which otherwise
|
||||
// guarantees offset>=0 and limit>0) gets a sane page instead of a
|
||||
// reversed-slice panic.
|
||||
lo := min(max(offset, 0), total)
|
||||
hi := lo + max(limit, 0)
|
||||
if hi < lo || hi > total { // hi < lo: lo+limit overflowed; serve the rest
|
||||
hi = total
|
||||
}
|
||||
page := combined[lo:hi]
|
||||
|
||||
if !includeTotal {
|
||||
// Cursor mode reaches here only at offset 0 with the whole FTS block
|
||||
// inside the page (see SearchPage gate), so no exact match is hidden;
|
||||
// only fuzzy augmentation past the page is cut. Serve as a terminal
|
||||
// page: report only what we return and no next page, since a bare
|
||||
// offset cursor can't locate the FTS/fuzzy boundary on a follow-up
|
||||
// request.
|
||||
return page, len(page), false, false, nil
|
||||
}
|
||||
// When the fuzzy arm hit its cap the true match count is larger than the
|
||||
// materialized list, so surface the total as inexact rather than presenting
|
||||
// the cap as an exact count.
|
||||
return page, total, hi < total, !fuzzyTruncated, nil
|
||||
}
|
||||
|
||||
// execFuzzyBlock runs the fuzzy data query inside a transaction that pins
|
||||
// pg_trgm.strict_word_similarity_threshold, so the <<% operator's selectivity
|
||||
// cannot drift with cluster configuration (its 0.6 server default would reject
|
||||
// ordinary typos outright). The query was built with LIMIT fuzzyLimit+1; the
|
||||
// extra row only signals truncation and is trimmed from the returned block.
|
||||
func (r *ItemRepository) execFuzzyBlock(ctx context.Context, dataSQL string, args []any, fuzzyLimit int) ([]*models.MediaItem, bool, error) {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("beginning fuzzy search tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf("SET LOCAL pg_trgm.strict_word_similarity_threshold = %g", trgmWordSimilarityThreshold)); err != nil {
|
||||
return nil, false, fmt.Errorf("pinning trigram word-similarity threshold: %w", err)
|
||||
}
|
||||
block, _, truncated, _, err := r.execSearchBlock(ctx, tx, dataSQL, "", args, fuzzyLimit, 0, false)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, false, fmt.Errorf("committing fuzzy search tx: %w", err)
|
||||
}
|
||||
return block, truncated, nil
|
||||
}
|
||||
|
||||
// searchQuerier is the subset of pgxpool.Pool / pgx.Tx that execSearchBlock
|
||||
// needs, so a block can run either directly on the pool or inside a
|
||||
// transaction (the fuzzy block pins a GUC with SET LOCAL).
|
||||
type searchQuerier interface {
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||
}
|
||||
|
||||
// execSearchBlock runs one search data query (FTS or fuzzy fallback) and its
|
||||
// count-fallback sibling, returning that block's page. It is the shared engine
|
||||
// behind SearchPage's two blocks so both derive hasMore/total identically.
|
||||
//
|
||||
// In cursor mode (includeTotal == false) the caller fetches limit+1 rows so
|
||||
// hasMore is len(items) > limit; the extra row is trimmed here. In exact mode
|
||||
// the data query carries COUNT(*) OVER () for the total, which emits no rows on
|
||||
// an empty page (e.g. OFFSET past the last row); only then, and only past
|
||||
// offset 0, is the count sibling re-run to recover the real total. Both queries
|
||||
// must end with the trailing limit/offset args so the count sibling can drop
|
||||
// them.
|
||||
//
|
||||
// The fourth return value is the untrimmed row set the data query actually
|
||||
// returned (before the cursor-mode +1 row is trimmed; in exact mode it equals
|
||||
// the page). SearchPage uses it in cursor mode to judge FTS-block sparsity
|
||||
// independently of the caller's page size and to hand the fuzzy fallback the
|
||||
// complete block without a second FTS query.
|
||||
func (r *ItemRepository) execSearchBlock(ctx context.Context, q searchQuerier, dataSQL, countSQL string, args []any, limit, offset int, includeTotal bool) ([]*models.MediaItem, int, bool, []*models.MediaItem, error) {
|
||||
rows, err := q.Query(ctx, dataSQL, args...)
|
||||
if err != nil {
|
||||
return nil, 0, false, nil, fmt.Errorf("searching media items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
if !includeTotal {
|
||||
items, err := scanItems(rows)
|
||||
if err != nil {
|
||||
return nil, 0, false, false, err
|
||||
return nil, 0, false, nil, err
|
||||
}
|
||||
untrimmed := items
|
||||
hasMore := len(items) > limit
|
||||
if hasMore {
|
||||
items = items[:limit]
|
||||
@@ -901,28 +1162,23 @@ func (r *ItemRepository) SearchPage(
|
||||
if hasMore {
|
||||
total++
|
||||
}
|
||||
return items, total, hasMore, false, nil
|
||||
return items, total, hasMore, untrimmed, nil
|
||||
}
|
||||
|
||||
items, total, err := scanItemsWithTotal(rows)
|
||||
if err != nil {
|
||||
return nil, 0, false, true, err
|
||||
return nil, 0, false, nil, err
|
||||
}
|
||||
hasMore := total > offset+len(items)
|
||||
// COUNT(*) OVER () emits no rows when the data SELECT is empty, so total
|
||||
// stays 0 even when the broader result set has matching rows (e.g. OFFSET
|
||||
// past the last page). Re-query the count to give callers the real total.
|
||||
// Skip when offset == 0 because in that case an empty page genuinely means
|
||||
// total = 0.
|
||||
if len(items) == 0 && offset > 0 {
|
||||
// Drop the trailing limit/offset args from the data query.
|
||||
countArgs := args[:len(args)-2]
|
||||
if err := r.pool.QueryRow(ctx, countSQL, countArgs...).Scan(&total); err != nil {
|
||||
return nil, 0, false, true, fmt.Errorf("count fallback for empty search page: %w", err)
|
||||
if err := q.QueryRow(ctx, countSQL, countArgs...).Scan(&total); err != nil {
|
||||
return nil, 0, false, nil, fmt.Errorf("count fallback for empty search page: %w", err)
|
||||
}
|
||||
hasMore = total > offset+len(items)
|
||||
}
|
||||
return items, total, hasMore, true, nil
|
||||
return items, total, hasMore, items, nil
|
||||
}
|
||||
|
||||
// buildSearchSQL assembles the unified search query, returning the SQL string
|
||||
@@ -949,12 +1205,181 @@ func (r *ItemRepository) SearchPage(
|
||||
// parsed.Year (or NULL)
|
||||
// parsed.Phrase
|
||||
// limit, offset
|
||||
//
|
||||
// searchTextFromParsed derives the effective search text shared by the FTS and
|
||||
// fuzzy builders: the parsed Text, or (when the query was only quotes/whitespace
|
||||
// that parsed to empty Text) a whitespace-collapsed, quote-stripped fallback off
|
||||
// the raw query. Centralized so the two builders can never search different text.
|
||||
func searchTextFromParsed(parsed parsedSearchQuery) string {
|
||||
if parsed.Text != "" {
|
||||
return parsed.Text
|
||||
}
|
||||
return collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(parsed.Raw), "\"", " "))
|
||||
}
|
||||
|
||||
func (r *ItemRepository) buildSearchSQL(query string, itemTypes []string, limit, offset int, filter AccessFilter) (dataSQL, countSQL string, args []any) {
|
||||
return r.buildSearchSQLWithTotal(query, itemTypes, limit, offset, filter, true)
|
||||
}
|
||||
|
||||
func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) {
|
||||
return r.buildMixedSearchSQLWithTotal(query, itemTypes, limit, offset, filter, includeTotal)
|
||||
return r.buildSearchSQLFromParsed(parseSearchQuery(query), itemTypes, limit, offset, filter, includeTotal)
|
||||
}
|
||||
|
||||
// buildSearchSQLFromParsed is the FTS query builder proper; the string-taking
|
||||
// wrappers above parse first. SearchPage parses once and calls this directly so
|
||||
// the same parsedSearchQuery feeds the FTS builder, the fuzzy builder, and the
|
||||
// eligibility gate without re-parsing. The mixed builder unions media_items and
|
||||
// episode candidates into one ranked page.
|
||||
func (r *ItemRepository) buildSearchSQLFromParsed(parsed parsedSearchQuery, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) {
|
||||
return r.buildMixedSearchSQLFromParsed(parsed, itemTypes, limit, offset, filter, includeTotal)
|
||||
}
|
||||
|
||||
// appendSearchScopeFilters appends the scope predicates shared by the FTS search
|
||||
// (buildSearchSQLWithTotal) and the trigram fuzzy fallback (buildFuzzySearchSQL):
|
||||
// item type, allowed/disabled libraries, the access filter, and the
|
||||
// manga-chapter exclusion. It mutates conditions/args/argIdx in place and
|
||||
// returns the FROM clause (always "media_items mi"; library scoping is enforced
|
||||
// with independent EXISTS/NOT EXISTS subqueries rather than a JOIN). Both callers
|
||||
// append these in the same order so a single helper keeps the two queries'
|
||||
// filtering provably identical.
|
||||
func appendSearchScopeFilters(itemTypes []string, filter AccessFilter, conditions *[]string, args *[]any, argIdx *int) (fromClause string) {
|
||||
fromClause = "media_items mi"
|
||||
|
||||
if len(itemTypes) > 0 {
|
||||
placeholders := make([]string, 0, len(itemTypes))
|
||||
for _, itemType := range itemTypes {
|
||||
if strings.TrimSpace(itemType) == "" {
|
||||
continue
|
||||
}
|
||||
placeholders = append(placeholders, fmt.Sprintf("$%d", *argIdx))
|
||||
*args = append(*args, strings.ToLower(strings.TrimSpace(itemType)))
|
||||
*argIdx++
|
||||
}
|
||||
if len(placeholders) > 0 {
|
||||
*conditions = append(*conditions, fmt.Sprintf("mi.type IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
// Library allow/deny via the shared leak-safe helper: an item linked to both a
|
||||
// passing and a disabled library must not slip through. The old JOIN +
|
||||
// NOT(mil.media_folder_id = ANY(...)) form let the passing membership row
|
||||
// satisfy the deny check (audit 2026-05-01 §3.3), so both the FTS search and
|
||||
// the fuzzy fallback that share this helper used it. appendLibraryAccessConditions
|
||||
// emits independent EXISTS/NOT EXISTS subqueries keyed on mi.content_id and
|
||||
// needs no JOIN.
|
||||
appendLibraryAccessConditions("mi.content_id", filter, conditions, args, argIdx)
|
||||
|
||||
applyAccessFilter("mi", AccessFilter{MaxContentRating: filter.MaxContentRating, ExcludedMediaTypes: filter.ExcludedMediaTypes}, conditions, args, argIdx)
|
||||
|
||||
// Manga chapters (type='ebook' rows linked into a manga series) are internal
|
||||
// sub-units and must never surface as standalone search results.
|
||||
*conditions = append(*conditions, MangaChapterExclusionWhere("mi"))
|
||||
return fromClause
|
||||
}
|
||||
|
||||
// buildFuzzySearchSQL assembles the trigram fuzzy-fallback query invoked by
|
||||
// SearchPage only when the FTS query is sparse. Unlike buildSearchSQLWithTotal,
|
||||
// the sole match predicate is the pg_trgm strict-word-similarity operator
|
||||
// (<<%) against the indexed title_normalized generated column (migration 105's
|
||||
// gin_trgm_ops index serves it), and the ranking signals are
|
||||
// strict_word_similarity()/similarity() on that same column. Crucially it
|
||||
// never references the title tsvectors, so the low-similarity rows the
|
||||
// operator can surface never pay a per-row tsvector rebuild — that rebuild
|
||||
// over the fuzzy candidate set was the entire cause of the multi-second search
|
||||
// regression. The operator's base selectivity is pinned by the caller
|
||||
// (execFuzzyBlock sets pg_trgm.strict_word_similarity_threshold via SET LOCAL)
|
||||
// so it cannot drift with cluster configuration.
|
||||
//
|
||||
// Scope note: only title_normalized — a generated column over `title` — is
|
||||
// fuzzy-matched, because it is the only column with a gin_trgm_ops index
|
||||
// (migration 105). Typos of original_title or sort_title, which the FTS arm
|
||||
// covers exactly, are deliberately out of the fuzzy arm's scope; extending it
|
||||
// would need matching normalized columns + trigram indexes (follow-up).
|
||||
//
|
||||
// includeTotal toggles the COUNT(*) OVER () window total the same way
|
||||
// buildSearchSQLWithTotal does, so the fuzzy block honors the caller's cursor
|
||||
// vs. exact-count mode. excludeContentIDs drops rows already returned by the FTS
|
||||
// block so a title matching both is not shown twice. minSimilarity > 0 adds an
|
||||
// explicit similarity floor above the base threshold — used when the FTS block
|
||||
// had real hits so augmentation only admits near-certain corrections. Argument
|
||||
// order: $1 searchText, then the similarity floor (if any), then the shared
|
||||
// scope placeholders, then the exclusion array, then limit/offset.
|
||||
func (r *ItemRepository) buildFuzzySearchSQL(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool, excludeContentIDs []string, minSimilarity float64) (dataSQL, countSQL string, args []any) {
|
||||
return r.buildFuzzySearchFromParsed(parseSearchQuery(query), itemTypes, limit, offset, filter, includeTotal, excludeContentIDs, minSimilarity)
|
||||
}
|
||||
|
||||
// buildFuzzySearchFromParsed is the fuzzy query builder proper; the string-taking
|
||||
// wrapper above parses first. SearchPage parses once and calls this directly.
|
||||
func (r *ItemRepository) buildFuzzySearchFromParsed(parsed parsedSearchQuery, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool, excludeContentIDs []string, minSimilarity float64) (dataSQL, countSQL string, args []any) {
|
||||
searchText := searchTextFromParsed(parsed)
|
||||
if searchText == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
args = []any{searchText}
|
||||
argIdx := 2
|
||||
// Strict word similarity (<<%) rather than full-string similarity (%): a
|
||||
// typo of one word must still reach long titles ("avegners" → "Avengers:
|
||||
// Endgame"), where full-string similarity is diluted by every extra trigram
|
||||
// the rest of the title contributes. <<% scores the query against the best
|
||||
// word-boundary extent of the title instead, and at equal thresholds it is
|
||||
// a strict superset of % (the whole string is itself a valid extent). The
|
||||
// same gin_trgm_ops index serves both operators.
|
||||
conditions := []string{"public.normalize_search_text($1) <<% mi.title_normalized"}
|
||||
if minSimilarity > 0 {
|
||||
// The floor is whole-title similarity, NOT word similarity: augmenting
|
||||
// a query that already has hits must only admit near-identical titles,
|
||||
// and word similarity rates embedded prefix words far too high (see
|
||||
// fuzzyAugmentSimilarityFloor).
|
||||
conditions = append(conditions, fmt.Sprintf("similarity(public.normalize_search_text($1), mi.title_normalized) >= $%d", argIdx))
|
||||
args = append(args, minSimilarity)
|
||||
argIdx++
|
||||
}
|
||||
|
||||
fromClause := appendSearchScopeFilters(itemTypes, filter, &conditions, &args, &argIdx)
|
||||
|
||||
if len(excludeContentIDs) > 0 {
|
||||
conditions = append(conditions, fmt.Sprintf("NOT (mi.content_id = ANY($%d))", argIdx))
|
||||
args = append(args, excludeContentIDs)
|
||||
argIdx++
|
||||
}
|
||||
|
||||
whereClause := "WHERE " + strings.Join(conditions, " AND ")
|
||||
|
||||
// GROUP BY is required so the MAX(similarity(...)) ranking aggregate is legal,
|
||||
// mirroring buildSearchSQLWithTotal; COUNT(*) OVER () then counts distinct
|
||||
// content_ids. qualifiedItemColumns aliases the coalesced columns so the outer
|
||||
// SELECT can re-reference them; GROUP BY uses the raw refs.
|
||||
qualifiedCols := qualifiedItemColumns("mi")
|
||||
groupByCols := qualifiedItemColumnRefs("mi")
|
||||
// fuzzy_rank orders by how well the query matches SOME word extent of the
|
||||
// title; fuzzy_full_rank tie-breaks by whole-title closeness so "The
|
||||
// Avengers" sorts above "Avengers: Endgame" for the typo "avegners". Both
|
||||
// are computed only over the matched candidate set.
|
||||
scoredCTE := fmt.Sprintf(`
|
||||
WITH scored AS (
|
||||
SELECT
|
||||
%s,
|
||||
MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank,
|
||||
MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank
|
||||
FROM %s
|
||||
%s
|
||||
GROUP BY %s
|
||||
)
|
||||
`, qualifiedCols, fromClause, whereClause, groupByCols)
|
||||
|
||||
totalColumn := ""
|
||||
if includeTotal {
|
||||
totalColumn = ", COUNT(*) OVER () AS total_count"
|
||||
}
|
||||
dataSQL = scoredCTE + fmt.Sprintf(`
|
||||
SELECT %s%s
|
||||
FROM scored
|
||||
ORDER BY fuzzy_rank DESC, fuzzy_full_rank DESC, LOWER(title) ASC, content_id ASC
|
||||
LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, argIdx, argIdx+1)
|
||||
countSQL = scoredCTE + `SELECT COUNT(*) FROM scored`
|
||||
args = append(args, limit, offset)
|
||||
return dataSQL, countSQL, args
|
||||
}
|
||||
|
||||
// ListUnmatchedByFolderAndPathPrefix returns content IDs for unmatched-style
|
||||
|
||||
@@ -382,6 +382,223 @@ func TestItemRepo_Search_EmptyQueryReturnsEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEligibleForFuzzy pins the min-token gate that guards the fuzzy title
|
||||
// fallback: the longest normalized token must clear fuzzyMinTokenLen, so short,
|
||||
// non-selective queries stay on the exact FTS/prefix path.
|
||||
func TestEligibleForFuzzy(t *testing.T) {
|
||||
cases := []struct {
|
||||
query string
|
||||
want bool
|
||||
}{
|
||||
{"avegners", true}, // 8-char typo
|
||||
{"sponge bob", true}, // longest token "sponge" (6)
|
||||
{"a vengers", true}, // judged on "vengers" (7), not "a"
|
||||
{"dune", true}, // exactly at the floor
|
||||
{"the", false}, // 3 chars
|
||||
{"a b c", false}, // all short
|
||||
{"and the", false}, // "and" normalized away, "the" is 3
|
||||
{" ", false}, // empty
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := eligibleForFuzzy(parseSearchQuery(tc.query)); got != tc.want {
|
||||
t.Errorf("eligibleForFuzzy(%q) = %v, want %v", tc.query, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchTextFromParsed pins the shared searchText derivation used by both
|
||||
// the FTS and fuzzy builders: parsed Text when present, else a quote-stripped
|
||||
// fallback off the raw query. Both builders must agree so they search the same
|
||||
// text.
|
||||
func TestSearchTextFromParsed(t *testing.T) {
|
||||
cases := []struct {
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{"interstellar", "interstellar"},
|
||||
{`"the matrix"`, "the matrix"},
|
||||
{" spaced out ", "spaced out"},
|
||||
{`"`, ""},
|
||||
{" ", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := searchTextFromParsed(parseSearchQuery(tc.raw)); got != tc.want {
|
||||
t.Errorf("searchTextFromParsed(%q) = %q, want %q", tc.raw, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseSearchQuery_NormalizedText asserts parseSearchQuery precomputes the
|
||||
// full normalized text that eligibleForFuzzy's token gate reads (so the gate
|
||||
// does not re-normalize on every sparse search).
|
||||
func TestParseSearchQuery_NormalizedText(t *testing.T) {
|
||||
if got := parseSearchQuery("The Avengers").NormalizedText; got != "the avengers" {
|
||||
t.Fatalf("NormalizedText = %q, want %q", got, "the avengers")
|
||||
}
|
||||
// "and" is dropped and folded, matching normalizeTitleForComparison.
|
||||
if got := parseSearchQuery("law and order").NormalizedText; got != "law order" {
|
||||
t.Fatalf("NormalizedText = %q, want %q", got, "law order")
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_Search_FTSQueryHasNoFuzzyArm asserts that the FTS query never
|
||||
// carries the trigram % arm or its similarity ranking. Fusing them forced a
|
||||
// lossy bitmap recheck that rebuilt the title tsvectors for thousands of
|
||||
// near-miss rows on every search; the fuzzy path is now a separate query
|
||||
// (buildFuzzySearchSQL) invoked only as a fallback by SearchPage.
|
||||
func TestItemRepo_Search_FTSQueryHasNoFuzzyArm(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
dataSQL, countSQL, _ := repo.buildSearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{})
|
||||
for _, sql := range []string{dataSQL, countSQL} {
|
||||
if strings.Contains(sql, "% mi.title_normalized") {
|
||||
t.Fatalf("FTS query must not include the trigram %% arm; got:\n%s", sql)
|
||||
}
|
||||
if strings.Contains(sql, "fuzzy_rank") {
|
||||
t.Fatalf("FTS query must not include fuzzy_rank; got:\n%s", sql)
|
||||
}
|
||||
if strings.Contains(sql, "similarity(") {
|
||||
t.Fatalf("FTS query must not compute similarity(); got:\n%s", sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_BuildFuzzySearchSQL asserts that the fuzzy fallback query scores
|
||||
// only on the indexed title_normalized column (no title tsvector rebuild),
|
||||
// matches via strict word similarity so long titles stay reachable, ranks by
|
||||
// descending word similarity with whole-title closeness as tie-break, excludes
|
||||
// already-seen content_ids, and applies the same scope filters (type, manga
|
||||
// exclusion) as the FTS query.
|
||||
func TestItemRepo_BuildFuzzySearchSQL(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
dataSQL, countSQL, args := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, []string{"abc", "def"}, 0)
|
||||
|
||||
// <<%, not %: full-string similarity is diluted by long titles ("avegners"
|
||||
// must reach "Avengers: Endgame"); the same gin_trgm_ops index serves both.
|
||||
if !strings.Contains(dataSQL, "public.normalize_search_text($1) <<% mi.title_normalized") {
|
||||
t.Fatalf("expected strict-word-similarity arm against title_normalized; got:\n%s", dataSQL)
|
||||
}
|
||||
if !strings.Contains(dataSQL, "MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank") {
|
||||
t.Fatalf("expected strict word similarity ranking on title_normalized; got:\n%s", dataSQL)
|
||||
}
|
||||
if !strings.Contains(dataSQL, "MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank") {
|
||||
t.Fatalf("expected whole-title similarity tie-break rank; got:\n%s", dataSQL)
|
||||
}
|
||||
// The whole point of the separate query: it must never rebuild the title
|
||||
// tsvectors that made the fused query slow.
|
||||
if strings.Contains(dataSQL, "to_tsvector") || strings.Contains(dataSQL, "ts_rank_cd") {
|
||||
t.Fatalf("fuzzy query must not rebuild title tsvectors; got:\n%s", dataSQL)
|
||||
}
|
||||
if !strings.Contains(dataSQL, "ORDER BY fuzzy_rank DESC, fuzzy_full_rank DESC, LOWER(title) ASC, content_id ASC") {
|
||||
t.Fatalf("expected similarity-ordered results; got:\n%s", dataSQL)
|
||||
}
|
||||
if !strings.Contains(dataSQL, "NOT (mi.content_id = ANY($") {
|
||||
t.Fatalf("expected exclusion of already-seen content_ids; got:\n%s", dataSQL)
|
||||
}
|
||||
if !strings.Contains(dataSQL, "mi.type IN ($") {
|
||||
t.Fatalf("expected shared type scope filter; got:\n%s", dataSQL)
|
||||
}
|
||||
if !strings.Contains(countSQL, "SELECT COUNT(*) FROM scored") {
|
||||
t.Fatalf("expected count sibling over the scored CTE; got:\n%s", countSQL)
|
||||
}
|
||||
// Arg order: $1 searchText, $2 type, $3 exclusion array, $4 limit, $5 offset.
|
||||
if len(args) != 5 {
|
||||
t.Fatalf("expected 5 args, got %d: %#v", len(args), args)
|
||||
}
|
||||
if args[0] != "avegners" {
|
||||
t.Fatalf("expected $1=searchText; got %#v", args[0])
|
||||
}
|
||||
if args[len(args)-2] != 20 || args[len(args)-1] != 0 {
|
||||
t.Fatalf("expected trailing limit/offset args; got %#v", args[len(args)-2:])
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_BuildFuzzySearchSQL_CursorModeOmitsWindowCount asserts that the
|
||||
// fuzzy fallback honors cursor mode: with includeTotal=false it must not carry
|
||||
// COUNT(*) OVER (), matching buildSearchSQLWithTotal's skip-total contract so
|
||||
// SearchPage's cursor path stays a pure keyset page.
|
||||
func TestItemRepo_BuildFuzzySearchSQL_CursorModeOmitsWindowCount(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
dataSQL, _, _ := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 21, 0, AccessFilter{}, false, nil, 0)
|
||||
if strings.Contains(dataSQL, "COUNT(*) OVER") {
|
||||
t.Fatalf("cursor-mode fuzzy query must omit window count; got:\n%s", dataSQL)
|
||||
}
|
||||
if strings.Contains(dataSQL, "total_count") {
|
||||
t.Fatalf("cursor-mode fuzzy query must not select total_count; got:\n%s", dataSQL)
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor asserts the augment floor
|
||||
// contract: minSimilarity > 0 adds an explicit similarity() >= $n predicate
|
||||
// (used when the FTS block had real hits, so augmentation only admits
|
||||
// near-certain corrections), and minSimilarity == 0 omits it so zero-hit typo
|
||||
// queries keep the base pinned threshold's recall.
|
||||
func TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
|
||||
floorSQL, _, floorArgs := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, fuzzyAugmentSimilarityFloor)
|
||||
// Whole-title similarity, not word similarity: the augment floor must not
|
||||
// admit embedded prefix words ("coral" scores 0.5 word-similarity to
|
||||
// "coraline").
|
||||
if !strings.Contains(floorSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") {
|
||||
t.Fatalf("expected explicit whole-title similarity floor predicate as $2; got:\n%s", floorSQL)
|
||||
}
|
||||
if len(floorArgs) < 2 || floorArgs[1] != fuzzyAugmentSimilarityFloor {
|
||||
t.Fatalf("expected $2 = fuzzyAugmentSimilarityFloor; got args %#v", floorArgs)
|
||||
}
|
||||
// Trailing limit/offset must still close the arg list for the count sibling.
|
||||
if floorArgs[len(floorArgs)-2] != 20 || floorArgs[len(floorArgs)-1] != 0 {
|
||||
t.Fatalf("expected trailing limit/offset args; got %#v", floorArgs[len(floorArgs)-2:])
|
||||
}
|
||||
|
||||
baseSQL, _, baseArgs := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, 0)
|
||||
if strings.Contains(baseSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") {
|
||||
t.Fatalf("zero floor must not add a similarity predicate; got:\n%s", baseSQL)
|
||||
}
|
||||
if len(baseArgs) != len(floorArgs)-1 {
|
||||
t.Fatalf("zero floor should bind one fewer arg: base %d vs floor %d", len(baseArgs), len(floorArgs))
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_BuildFuzzySearchSQL_EmptyQueryReturnsEmpty guards the same
|
||||
// empty-input contract as buildSearchSQL.
|
||||
func TestItemRepo_BuildFuzzySearchSQL_EmptyQueryReturnsEmpty(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
dataSQL, countSQL, args := repo.buildFuzzySearchSQL(" ", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, 0)
|
||||
if dataSQL != "" || countSQL != "" || args != nil {
|
||||
t.Fatalf("expected empty result for blank query; got dataSQL=%q countSQL=%q args=%#v", dataSQL, countSQL, args)
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_Search_LibraryScopeUsesIndependentExistsPredicates pins the
|
||||
// leak-safe library scoping shared by the FTS search and the fuzzy fallback via
|
||||
// appendSearchScopeFilters. The prior JOIN + NOT(mil.media_folder_id = ANY(...))
|
||||
// form let an item linked to BOTH a disabled and a non-disabled library survive
|
||||
// the deny check on the passing membership row, so a typo (fuzzy) or exact
|
||||
// search could surface items from a disabled library. Both builders must instead
|
||||
// emit independent EXISTS/NOT EXISTS subqueries and no membership JOIN.
|
||||
func TestItemRepo_Search_LibraryScopeUsesIndependentExistsPredicates(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
filter := AccessFilter{DisabledLibraryIDs: []int{9}}
|
||||
|
||||
ftsSQL, _, _ := repo.buildSearchSQL("avatar", []string{"movie"}, 20, 0, filter)
|
||||
fuzzySQL, _, _ := repo.buildFuzzySearchSQL("avatar", []string{"movie"}, 20, 0, filter, true, nil, 0)
|
||||
|
||||
for name, sql := range map[string]string{"fts": ftsSQL, "fuzzy": fuzzySQL} {
|
||||
if strings.Contains(sql, "JOIN media_item_libraries") {
|
||||
t.Fatalf("%s query must not JOIN media_item_libraries for scoping; got:\n%s", name, sql)
|
||||
}
|
||||
// Disabled-only path requires positive membership (argument-free EXISTS)
|
||||
// so orphan items don't slip through the vacuous NOT EXISTS...
|
||||
if !strings.Contains(sql, "EXISTS (SELECT 1 FROM media_item_libraries mil WHERE mil.content_id = mi.content_id)") {
|
||||
t.Fatalf("%s query must require library membership via an argument-free EXISTS; got:\n%s", name, sql)
|
||||
}
|
||||
// ...plus the disabled NOT EXISTS.
|
||||
if !strings.Contains(sql, "NOT EXISTS (SELECT 1 FROM media_item_libraries mil WHERE mil.content_id = mi.content_id AND mil.media_folder_id = ANY($") {
|
||||
t.Fatalf("%s query must exclude disabled libraries via NOT EXISTS; got:\n%s", name, sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_Search_UsesTitleNormalizedColumn asserts that buildSearchSQL
|
||||
// reads the mi.title_normalized stored generated column for the title rank
|
||||
// arms (exact_title_match and contiguous_title_match), so the LIKE
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/pathscope"
|
||||
)
|
||||
|
||||
// LibraryItemRepository provides CRUD operations for the media_item_libraries
|
||||
@@ -421,7 +422,19 @@ func (r *LibraryItemRepository) Delete(ctx context.Context, contentID string, fo
|
||||
// longer has any non-missing files in the given folder. It also deletes orphaned
|
||||
// media items once they no longer belong to any library. Returns removed
|
||||
// membership count, deleted item count, orphaned S3 image dirs, and any error.
|
||||
func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, folderID int) (int, int, []string, error) {
|
||||
//
|
||||
// protectedPathPrefixes lists library roots that are currently unreachable
|
||||
// (dead drive, lost mount). Membership removal proceeds regardless — browse
|
||||
// and home queries hide items via media_item_libraries, so removal is what
|
||||
// keeps a title with no playable files out of the catalog — but items whose
|
||||
// files in this folder sit under a protected prefix are exempt from the
|
||||
// orphan delete. Deleting them is not losslessly recoverable (user
|
||||
// collections cascade via library_collection_items, manual metadata edits and
|
||||
// cached artwork are lost), whereas a hidden membership-less item restores
|
||||
// automatically: when the root returns, the scanner clears missing_since on
|
||||
// its surviving media_files rows and syncPresentLibraryState re-inserts the
|
||||
// membership from those rows.
|
||||
func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, folderID int, protectedPathPrefixes []string) (int, int, []string, error) {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, nil, fmt.Errorf("beginning membership reconciliation transaction: %w", err)
|
||||
@@ -470,11 +483,28 @@ func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, f
|
||||
|
||||
deletedItems := 0
|
||||
var orphanedImageDirs []string
|
||||
if len(removedContentIDs) > 0 {
|
||||
// Find items that will become orphaned (no remaining library memberships).
|
||||
orphanIDs, err := collectOrphanIDs(ctx, tx, removedContentIDs)
|
||||
if err != nil {
|
||||
return 0, 0, nil, err
|
||||
// Find both newly orphaned items and items preserved by an earlier
|
||||
// protected-root pass. The latter no longer have a membership to return from
|
||||
// the DELETE above, but their surviving media_files row still ties them to
|
||||
// this folder so they can be reconsidered after the root recovers.
|
||||
orphanIDs, err := collectOrphanIDs(ctx, tx, removedContentIDs)
|
||||
if err != nil {
|
||||
return 0, 0, nil, err
|
||||
}
|
||||
previouslyProtected, err := collectFolderFileOrphanIDs(ctx, tx, folderID)
|
||||
if err != nil {
|
||||
return 0, 0, nil, err
|
||||
}
|
||||
orphanIDs = appendUniqueStrings(orphanIDs, previouslyProtected...)
|
||||
if len(orphanIDs) > 0 {
|
||||
|
||||
// Exempt orphans whose files sit under an unreachable root: the files
|
||||
// still exist, the root is just offline. See the doc comment above.
|
||||
if len(orphanIDs) > 0 && len(protectedPathPrefixes) > 0 {
|
||||
orphanIDs, err = excludeOrphansUnderProtectedPrefixes(ctx, tx, orphanIDs, folderID, protectedPathPrefixes)
|
||||
if err != nil {
|
||||
return 0, 0, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Collect image paths before deletion.
|
||||
@@ -516,3 +546,85 @@ func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, f
|
||||
|
||||
return len(removedContentIDs), deletedItems, orphanedImageDirs, nil
|
||||
}
|
||||
|
||||
func collectFolderFileOrphanIDs(ctx context.Context, tx pgx.Tx, folderID int) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT DISTINCT mf.content_id
|
||||
FROM media_files mf
|
||||
WHERE mf.media_folder_id = $1
|
||||
AND mf.content_id IS NOT NULL
|
||||
AND mf.content_id <> ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil WHERE mil.content_id = mf.content_id
|
||||
)
|
||||
`, folderID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("finding previously protected folder orphans: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
ids, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("collecting previously protected folder orphans: %w", err)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func appendUniqueStrings(values []string, additions ...string) []string {
|
||||
seen := make(map[string]struct{}, len(values)+len(additions))
|
||||
for _, value := range values {
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
for _, value := range additions {
|
||||
if _, ok := seen[value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
values = append(values, value)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// excludeOrphansUnderProtectedPrefixes returns the subset of orphanIDs that
|
||||
// have no media_files row in the folder at or under any protected prefix.
|
||||
// Matching uses the exact-path + escaped prefix-LIKE shape shared with the
|
||||
// scanner's root matching, so a sibling root that merely shares a string
|
||||
// prefix (/mnt/movies2 vs /mnt/movies) is never protected by accident.
|
||||
//
|
||||
// Known limitation: the check is scoped to the reconciled folder's own files
|
||||
// (mf.media_folder_id = folderID). An item shared across two libraries whose
|
||||
// only surviving files sit under ANOTHER folder's currently-unreachable root
|
||||
// is not exempted here — if its membership in this folder is genuinely
|
||||
// removed while the other folder's root is offline, the item is purged even
|
||||
// though its files under the dead root would have resurrected. Closing this
|
||||
// would require probing every enabled folder's roots (or persisting per-
|
||||
// folder unreachable state) on each reconcile; accepted for now given how
|
||||
// narrow the window is.
|
||||
func excludeOrphansUnderProtectedPrefixes(ctx context.Context, tx pgx.Tx, orphanIDs []string, folderID int, prefixes []string) ([]string, error) {
|
||||
conds, condArgs := pathscope.CoverageClauses("mf.file_path", prefixes, 3)
|
||||
args := append([]any{orphanIDs, folderID}, condArgs...)
|
||||
|
||||
rows, err := tx.Query(ctx, fmt.Sprintf(`
|
||||
SELECT cid FROM unnest($1::text[]) AS cid
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM media_files mf
|
||||
WHERE mf.media_folder_id = $2
|
||||
AND mf.content_id = cid
|
||||
AND (%s)
|
||||
)
|
||||
`, strings.Join(conds, " OR ")), args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("filtering orphans under protected prefixes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
ids := make([]string, 0, len(orphanIDs))
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("scanning unprotected orphan id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// TestReconcileFolderMembershipProtectsUnreachableRoots verifies the
|
||||
// dead-root exemption in the orphan purge: memberships are still removed for
|
||||
// content with no non-missing files (that is what hides the item from
|
||||
// browse/home, which gate on media_item_libraries), but the media_items row
|
||||
// of an item whose files sit under a protected (unreachable) root must
|
||||
// survive so user collections, metadata edits, and artwork are not destroyed
|
||||
// by a temporary outage. An orphan with no protected files is still purged.
|
||||
func TestReconcileFolderMembershipProtectsUnreachableRoots(t *testing.T) {
|
||||
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("SILO_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
|
||||
suffix := time.Now().UnixNano()
|
||||
deadRoot := fmt.Sprintf("/drp-dead-%d", suffix)
|
||||
goneItem := fmt.Sprintf("drp-gone-%d", suffix)
|
||||
protectedItem := fmt.Sprintf("drp-protected-%d", suffix)
|
||||
|
||||
var folderID int
|
||||
if err := pool.QueryRow(ctx,
|
||||
`INSERT INTO media_folders (type, name, enabled) VALUES ('movies', 'DRP Test', true) RETURNING id`,
|
||||
).Scan(&folderID); err != nil {
|
||||
t.Fatalf("seed folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_files WHERE media_folder_id = $1`, folderID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{goneItem, protectedItem})
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID)
|
||||
})
|
||||
|
||||
for _, contentID := range []string{goneItem, protectedItem} {
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO media_items (content_id, type, title, status, genres)
|
||||
VALUES ($1, 'movie', 'DRP Item', 'matched', '{}'::text[])
|
||||
`, contentID); err != nil {
|
||||
t.Fatalf("seed item %s: %v", contentID, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||||
VALUES ($1, $2, NOW())
|
||||
`, contentID, folderID); err != nil {
|
||||
t.Fatalf("seed membership %s: %v", contentID, err)
|
||||
}
|
||||
}
|
||||
missingSince := time.Now().UTC().Add(-72 * time.Hour)
|
||||
seedFile := func(contentID, path string) {
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO media_files (content_id, media_folder_id, file_path, file_size, missing_since)
|
||||
VALUES ($1, $2, $3, 1024, $4)
|
||||
`, contentID, folderID, path, missingSince); err != nil {
|
||||
t.Fatalf("seed file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
// The protected item's only file lives under the unreachable root; the
|
||||
// other item's file was under a root that is reachable (its absence is a
|
||||
// genuine deletion).
|
||||
seedFile(protectedItem, deadRoot+"/Alpha (2020)/Alpha (2020).mkv")
|
||||
seedFile(goneItem, fmt.Sprintf("/drp-live-%d/Beta (2021)/Beta (2021).mkv", suffix))
|
||||
|
||||
repo := NewLibraryItemRepository(pool)
|
||||
removed, deletedItems, _, err := repo.ReconcileFolderMembership(ctx, folderID, []string{deadRoot})
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcileFolderMembership: %v", err)
|
||||
}
|
||||
if removed != 2 {
|
||||
t.Fatalf("removed memberships = %d, want 2 (hiding still requires removal)", removed)
|
||||
}
|
||||
if deletedItems != 1 {
|
||||
t.Fatalf("deleted items = %d, want 1 (only the unprotected orphan)", deletedItems)
|
||||
}
|
||||
|
||||
var exists bool
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM media_items WHERE content_id = $1)`, protectedItem,
|
||||
).Scan(&exists); err != nil {
|
||||
t.Fatalf("check protected item: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatal("protected item was purged despite its files sitting under an unreachable root")
|
||||
}
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM media_items WHERE content_id = $1)`, goneItem,
|
||||
).Scan(&exists); err != nil {
|
||||
t.Fatalf("check gone item: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("unprotected orphan item survived; purge regressed")
|
||||
}
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM media_item_libraries WHERE media_folder_id = $1)`, folderID,
|
||||
).Scan(&exists); err != nil {
|
||||
t.Fatalf("check memberships: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("memberships remain; items with only missing files must be hidden")
|
||||
}
|
||||
|
||||
// Once the root is reachable again, reconciliation must revisit the item
|
||||
// even though its membership was removed during the protected pass.
|
||||
removed, deletedItems, _, err = repo.ReconcileFolderMembership(ctx, folderID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcileFolderMembership after recovery: %v", err)
|
||||
}
|
||||
if removed != 0 || deletedItems != 1 {
|
||||
t.Fatalf("recovery reconciliation removed/deleted = %d/%d, want 0/1", removed, deletedItems)
|
||||
}
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM media_items WHERE content_id = $1)`, protectedItem,
|
||||
).Scan(&exists); err != nil {
|
||||
t.Fatalf("check recovered orphan: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("previously protected orphan survived after its root recovered")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package catalog
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestImageTypeFromCachedPathLocalHashedLayout(t *testing.T) {
|
||||
// Local keys interpose a content-hash segment BEFORE the image type
|
||||
// (local/{contentType}/{contentID}/{hash8}/{imageType}/{variant}.{ext})
|
||||
// so the variant's parent directory stays the image type.
|
||||
cases := map[string]string{
|
||||
"local/movies/movie-1/deadbeef/poster/original.webp": "poster",
|
||||
"local/movies/movie-1/deadbeef/backdrop/original.webp": "backdrop",
|
||||
"local/series/show-1/cafef00d/logo/w500.webp": "logo",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := imageTypeFromCachedPath(path); got != want {
|
||||
t.Errorf("imageTypeFromCachedPath(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageDeletePrefixTrimsLocalPathsToContentRoot(t *testing.T) {
|
||||
// On item deletion, local paths trim to local/{contentType}/{contentID}
|
||||
// so every stale hash prefix is swept, not just the live one.
|
||||
cases := map[string]string{
|
||||
"local/movies/movie-1/deadbeef/poster/original.webp": "local/movies/movie-1/",
|
||||
"local/series/show-1/cafef00d/logo/w500.webp": "local/series/show-1/",
|
||||
// Remote cached keys keep the existing per-image-type prefix.
|
||||
"tmdb/movies/550/poster/original.webp": "tmdb/movies/550/poster/",
|
||||
// Legacy scanner keys without a hash segment stay whole-path dirs.
|
||||
"local/ebooks/book-1/poster/original.webp": "local/ebooks/book-1/",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := imageDeletePrefix(path); got != want {
|
||||
t.Errorf("imageDeletePrefix(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,22 +69,18 @@ const mixedSearchOrder = `exact_title_match DESC, contiguous_title_match DESC, y
|
||||
phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC,
|
||||
LOWER(title) ASC, content_id ASC`
|
||||
|
||||
// buildMixedSearchSQLWithTotal builds one ranked candidate set from the two
|
||||
// buildMixedSearchSQLFromParsed builds one ranked candidate set from the two
|
||||
// physical catalog sources. The scored CTE deliberately carries only ranking
|
||||
// fields; the wide MediaItem projection is hydrated after LIMIT/OFFSET so a
|
||||
// broad match never sorts posters, arrays, or metadata blobs for every hit.
|
||||
func (r *ItemRepository) buildMixedSearchSQLWithTotal(
|
||||
query string,
|
||||
func (r *ItemRepository) buildMixedSearchSQLFromParsed(
|
||||
parsed parsedSearchQuery,
|
||||
itemTypes []string,
|
||||
limit, offset int,
|
||||
filter AccessFilter,
|
||||
includeTotal bool,
|
||||
) (dataSQL, countSQL string, args []any) {
|
||||
parsed := parseSearchQuery(query)
|
||||
searchText := parsed.Text
|
||||
if searchText == "" {
|
||||
searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " "))
|
||||
}
|
||||
searchText := searchTextFromParsed(parsed)
|
||||
if searchText == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
@@ -11,6 +11,11 @@ type parsedSearchQuery struct {
|
||||
Text string
|
||||
Phrase string
|
||||
ExactTitleHint string
|
||||
// NormalizedText is normalizeTitleForComparison(Text) computed once at parse
|
||||
// time. Text already folds phrase + remainder together, so this is the full
|
||||
// normalized query used by eligibleForFuzzy's token gate (which would
|
||||
// otherwise re-normalize on every sparse search).
|
||||
NormalizedText string
|
||||
Year *int
|
||||
}
|
||||
|
||||
@@ -37,10 +42,31 @@ func parseSearchQuery(raw string) parsedSearchQuery {
|
||||
Text: text,
|
||||
Phrase: phrase,
|
||||
ExactTitleHint: normalizeTitleForComparison(firstNonEmptySearchValue(phrase, text)),
|
||||
NormalizedText: normalizeTitleForComparison(text),
|
||||
Year: year,
|
||||
}
|
||||
}
|
||||
|
||||
// fuzzyMinTokenLen is the shortest normalized token that may enable the trigram
|
||||
// fuzzy title fallback. A token shorter than this forms too few trigrams to use
|
||||
// the gin_trgm_ops index selectively (and a 1-2 char token can't use it at
|
||||
// all), so for short queries the fuzzy fallback is skipped and search stays on
|
||||
// the exact FTS/prefix path. The gate is applied to the longest token so a stray
|
||||
// short token ("a vengers") is judged on "vengers", not "a".
|
||||
const fuzzyMinTokenLen = 4
|
||||
|
||||
// eligibleForFuzzy reports whether a parsed query clears the min-token gate for
|
||||
// the trigram fuzzy title fallback.
|
||||
func eligibleForFuzzy(parsed parsedSearchQuery) bool {
|
||||
longest := 0
|
||||
for _, tok := range strings.Fields(parsed.NormalizedText) {
|
||||
if n := len([]rune(tok)); n > longest {
|
||||
longest = n
|
||||
}
|
||||
}
|
||||
return longest >= fuzzyMinTokenLen
|
||||
}
|
||||
|
||||
func extractBalancedPhrase(input string) (string, string) {
|
||||
start := strings.Index(input, "\"")
|
||||
if start == -1 {
|
||||
|
||||
@@ -2580,6 +2580,7 @@ func toVideoTrackRecords(tracks []models.VideoTrack) []VideoTrackRecord {
|
||||
Bitrate: track.Bitrate,
|
||||
VideoRange: track.VideoRange,
|
||||
VideoRangeType: track.VideoRangeType,
|
||||
ColorRange: track.ColorRange,
|
||||
ColorPrimaries: track.ColorPrimaries,
|
||||
ColorSpace: track.ColorSpace,
|
||||
ColorTransfer: track.ColorTransfer,
|
||||
|
||||
@@ -3,8 +3,30 @@ package catalogseed
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestToVideoTrackRecordsPreservesColorRange(t *testing.T) {
|
||||
got := toVideoTrackRecords([]models.VideoTrack{
|
||||
{ColorRange: "tv"},
|
||||
{ColorRange: "pc"},
|
||||
{ColorRange: "unknown"},
|
||||
})
|
||||
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("records length = %d, want 3", len(got))
|
||||
}
|
||||
if got[0].ColorRange != "tv" || got[1].ColorRange != "pc" || got[2].ColorRange != "unknown" {
|
||||
t.Fatalf(
|
||||
"ColorRange values = [%q, %q, %q], want [tv, pc, unknown]",
|
||||
got[0].ColorRange,
|
||||
got[1].ColorRange,
|
||||
got[2].ColorRange,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSeedSearchUpsertIDsIncludesChangedItemsAndEmbeddings(t *testing.T) {
|
||||
itemStates := map[string]bool{
|
||||
"movie-1": true,
|
||||
|
||||
@@ -156,6 +156,7 @@ type VideoTrackRecord struct {
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
VideoRange string `json:"video_range,omitempty"`
|
||||
VideoRangeType string `json:"video_range_type,omitempty"`
|
||||
ColorRange string `json:"color_range,omitempty"`
|
||||
ColorPrimaries string `json:"color_primaries,omitempty"`
|
||||
ColorSpace string `json:"color_space,omitempty"`
|
||||
ColorTransfer string `json:"color_transfer,omitempty"`
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const ReportDownloadContentType = BundleContentType
|
||||
|
||||
var ErrReportNotReady = errors.New("diagnostic report is not ready")
|
||||
|
||||
func (s *Service) ListForAdmin(ctx context.Context, filters ListFilters) (ListResult, error) {
|
||||
if s.reports == nil {
|
||||
return ListResult{}, ErrReportStoreUnavailable
|
||||
}
|
||||
return s.reports.ListForAdmin(ctx, filters)
|
||||
}
|
||||
|
||||
func (s *Service) GetReport(ctx context.Context, id string) (*Report, error) {
|
||||
if s.reports == nil {
|
||||
return nil, ErrReportStoreUnavailable
|
||||
}
|
||||
return s.reports.GetByID(ctx, id)
|
||||
}
|
||||
|
||||
func (s *Service) PresignReportDownload(ctx context.Context, report *Report, expiry time.Duration) (string, error) {
|
||||
bucket, key, err := s.readyReportBlobLocation(report)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if expiry <= 0 {
|
||||
expiry = 15 * time.Minute
|
||||
}
|
||||
return s.store.PresignGetURL(ctx, bucket, key, expiry)
|
||||
}
|
||||
|
||||
func (s *Service) EffectiveReportDownloadTTL(requested time.Duration) time.Duration {
|
||||
if requested <= 0 {
|
||||
return requested
|
||||
}
|
||||
effectiveStore, ok := s.store.(interface {
|
||||
EffectivePresignTTL(time.Duration) time.Duration
|
||||
})
|
||||
if !ok {
|
||||
return requested
|
||||
}
|
||||
effective := effectiveStore.EffectivePresignTTL(requested)
|
||||
if effective <= 0 {
|
||||
return requested
|
||||
}
|
||||
return effective
|
||||
}
|
||||
|
||||
func (s *Service) OpenReportDownload(ctx context.Context, report *Report) (io.ReadCloser, error) {
|
||||
bucket, key, err := s.readyReportBlobLocation(report)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.store.GetObject(ctx, bucket, key)
|
||||
}
|
||||
|
||||
func (s *Service) DeleteReport(ctx context.Context, id string) (*Report, error) {
|
||||
if s.reports == nil {
|
||||
return nil, ErrReportStoreUnavailable
|
||||
}
|
||||
// Delete the row before the blob: a DB failure after the object is gone
|
||||
// would otherwise leave a visible report whose bundle can no longer be
|
||||
// downloaded. DeleteByID returns the deleted row (and ErrNotFound when
|
||||
// absent), so the blob location is captured from it. If the blob delete
|
||||
// fails the row is already gone, so log the bucket/key for an operator (and
|
||||
// the orphan reconciler) to reap instead of failing the delete.
|
||||
deleted, err := s.reports.DeleteByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := deleteReportObjects(ctx, s.store, deleted, s.logger); err != nil {
|
||||
logDeferredBlobDeletion(ctx, s.logger, s.store, deleted, err)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func reportBlobBucket(report *Report, store ObjectStore) string {
|
||||
bucket := stringValue(report.BlobBucket)
|
||||
if bucket == "" && store != nil {
|
||||
bucket = store.Bucket()
|
||||
}
|
||||
return bucket
|
||||
}
|
||||
|
||||
func (s *Service) readyReportBlobLocation(report *Report) (string, string, error) {
|
||||
if report == nil {
|
||||
return "", "", ErrNotFound
|
||||
}
|
||||
if report.State != StateReady {
|
||||
return "", "", ErrReportNotReady
|
||||
}
|
||||
if s.store == nil || strings.TrimSpace(s.store.Bucket()) == "" {
|
||||
return "", "", ErrStorageUnavailable
|
||||
}
|
||||
|
||||
bucket := stringValue(report.BlobBucket)
|
||||
if bucket == "" {
|
||||
bucket = s.store.Bucket()
|
||||
}
|
||||
key := stringValue(report.BlobKey)
|
||||
if key == "" {
|
||||
return "", "", fmt.Errorf("%w: diagnostic report has no blob key", ErrStorageUnavailable)
|
||||
}
|
||||
return bucket, key, nil
|
||||
}
|
||||
|
||||
func stringValue(value *string) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(*value)
|
||||
}
|
||||
@@ -0,0 +1,581 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"math"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics/contract"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultMaxBundleEntries = 16
|
||||
DefaultMaxCompressionRatio = int64(200)
|
||||
defaultBundleReadBufferSize = 32 * 1024
|
||||
// Standard tar writers (GNU tar, Python tarfile, Apache Commons Compress)
|
||||
// pad the archive with zero blocks to a record boundary, 10240 bytes by
|
||||
// default; 64 KiB also covers writers configured with larger blocking
|
||||
// factors.
|
||||
maxTarTrailingPaddingBytes = 64 * 1024
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCompressedTooLarge = errors.New("diagnostics bundle compressed size exceeds limit")
|
||||
ErrUncompressedTooLarge = errors.New("diagnostics bundle uncompressed size exceeds limit")
|
||||
ErrEntryTooLarge = errors.New("diagnostics bundle entry size exceeds limit")
|
||||
ErrTooManyEntries = errors.New("diagnostics bundle has too many entries")
|
||||
ErrInvalidBundle = errors.New("invalid diagnostics bundle")
|
||||
ErrCompressionRatio = errors.New("diagnostics bundle compression ratio exceeds limit")
|
||||
)
|
||||
|
||||
type BundleLimits struct {
|
||||
MaxCompressedBytes int64
|
||||
MaxUncompressedBytes int64
|
||||
MaxEntryBytes int64
|
||||
MaxEntries int
|
||||
MaxCompressionRatio int64
|
||||
}
|
||||
|
||||
type BundleInfo struct {
|
||||
CompressedBytes int64
|
||||
// UncompressedBytes is the total size of the decompressed tar stream —
|
||||
// headers, entry payloads, end-of-archive marker, and padding — matching
|
||||
// what a client counts on the way into its gzip writer (and `gzip -l`).
|
||||
UncompressedBytes int64
|
||||
SHA256 string
|
||||
Entries []string
|
||||
// EmbeddedManifest is the raw bytes of the archive's first entry
|
||||
// (manifest.json). Per the bundle contract this is the part-1 manifest with
|
||||
// the `archive` object removed; Ingest compares the two so a stored archive
|
||||
// can never carry a manifest that disagrees with the accepted report.
|
||||
EmbeddedManifest []byte
|
||||
}
|
||||
|
||||
var allowedBundleEntries = allowlistMap(contract.ArchiveEntryAllowlist)
|
||||
|
||||
func ValidateBundle(r io.Reader, limits BundleLimits) (BundleInfo, error) {
|
||||
limits = normalizeBundleLimits(limits)
|
||||
metered := newCompressedMeter(r, limits.MaxCompressedBytes)
|
||||
|
||||
gz, err := gzip.NewReader(metered)
|
||||
if err != nil {
|
||||
if isBundleUploadAbortError(err) {
|
||||
return BundleInfo{}, err
|
||||
}
|
||||
if errors.Is(err, ErrCompressedTooLarge) {
|
||||
return BundleInfo{}, ErrCompressedTooLarge
|
||||
}
|
||||
return BundleInfo{}, fmt.Errorf("%w: open gzip: %v", ErrInvalidBundle, err)
|
||||
}
|
||||
gz.Multistream(false)
|
||||
gzipClosed := false
|
||||
defer func() {
|
||||
if !gzipClosed {
|
||||
_ = gz.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
uncompressed := &uncompressedCounter{r: gz}
|
||||
tr := tar.NewReader(uncompressed)
|
||||
info := BundleInfo{
|
||||
Entries: make([]string, 0, len(allowedBundleEntries)),
|
||||
}
|
||||
buffer := make([]byte, defaultBundleReadBufferSize)
|
||||
var payloadBytes int64
|
||||
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return BundleInfo{}, classifyBundleReadError(err)
|
||||
}
|
||||
|
||||
if len(info.Entries) >= limits.MaxEntries {
|
||||
return BundleInfo{}, ErrTooManyEntries
|
||||
}
|
||||
|
||||
name, err := validateEntryHeader(hdr)
|
||||
if err != nil {
|
||||
return BundleInfo{}, err
|
||||
}
|
||||
isManifestEntry := len(info.Entries) == 0
|
||||
if isManifestEntry && name != "manifest.json" {
|
||||
return BundleInfo{}, fmt.Errorf("%w: first entry must be manifest.json", ErrInvalidBundle)
|
||||
}
|
||||
info.Entries = append(info.Entries, name)
|
||||
|
||||
if hdr.Size > limits.MaxEntryBytes {
|
||||
return BundleInfo{}, ErrEntryTooLarge
|
||||
}
|
||||
if isManifestEntry && hdr.Size > MaxManifestBytes {
|
||||
return BundleInfo{}, fmt.Errorf("%w: embedded manifest exceeds %d bytes", ErrInvalidBundle, MaxManifestBytes)
|
||||
}
|
||||
if payloadBytes+hdr.Size > limits.MaxUncompressedBytes {
|
||||
return BundleInfo{}, ErrUncompressedTooLarge
|
||||
}
|
||||
|
||||
entry := &tarEntryReader{tr: tr, limits: limits, metered: metered, total: &payloadBytes}
|
||||
var manifestCapture *bytes.Buffer
|
||||
var consumeErr error
|
||||
switch {
|
||||
case isManifestEntry:
|
||||
// Capture the embedded manifest so Ingest can compare it against the
|
||||
// part-1 manifest; it is the only entry buffered whole.
|
||||
manifestCapture = &bytes.Buffer{}
|
||||
manifestCapture.Grow(int(hdr.Size))
|
||||
_, consumeErr = io.Copy(manifestCapture, entry)
|
||||
case isNDJSONBundleEntry(name):
|
||||
// logs.jsonl / breadcrumbs.jsonl: newline-delimited JSON objects,
|
||||
// validated line-by-line so memory stays bounded to one capped line.
|
||||
consumeErr = validateNDJSONObjectEntry(entry)
|
||||
case isJSONObjectBundleEntry(name):
|
||||
// device.json / crash/*.json: a single JSON object, validated with a
|
||||
// streaming token decoder (no whole-entry buffering).
|
||||
consumeErr = validateJSONObjectEntry(entry)
|
||||
default:
|
||||
// Opaque members (crash/stack.txt, crash/tombstone.pb) carry no JSON
|
||||
// contract; drain them so their bytes still count toward the size,
|
||||
// ratio, and count limits.
|
||||
_, consumeErr = io.Copy(io.Discard, entry)
|
||||
}
|
||||
if consumeErr != nil {
|
||||
return BundleInfo{}, consumeErr
|
||||
}
|
||||
if entry.entryBytes != hdr.Size {
|
||||
return BundleInfo{}, fmt.Errorf("%w: entry size mismatch for %s", ErrInvalidBundle, name)
|
||||
}
|
||||
if manifestCapture != nil {
|
||||
info.EmbeddedManifest = manifestCapture.Bytes()
|
||||
}
|
||||
}
|
||||
|
||||
if len(info.Entries) == 0 {
|
||||
return BundleInfo{}, fmt.Errorf("%w: empty archive", ErrInvalidBundle)
|
||||
}
|
||||
if err := rejectPostTarData(uncompressed, buffer); err != nil {
|
||||
return BundleInfo{}, err
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return BundleInfo{}, classifyBundleReadError(err)
|
||||
}
|
||||
gzipClosed = true
|
||||
if err := rejectTrailingCompressedData(metered); err != nil {
|
||||
return BundleInfo{}, err
|
||||
}
|
||||
if uncompressed.count > limits.MaxUncompressedBytes {
|
||||
return BundleInfo{}, ErrUncompressedTooLarge
|
||||
}
|
||||
if ratioExceeded(uncompressed.count, metered.Count(), limits.MaxCompressionRatio) {
|
||||
return BundleInfo{}, ErrCompressionRatio
|
||||
}
|
||||
|
||||
info.UncompressedBytes = uncompressed.count
|
||||
info.CompressedBytes = metered.Count()
|
||||
info.SHA256 = hex.EncodeToString(metered.Sum())
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func normalizeBundleLimits(limits BundleLimits) BundleLimits {
|
||||
if limits.MaxCompressedBytes <= 0 {
|
||||
limits.MaxCompressedBytes = DefaultMaxBundleBytes
|
||||
}
|
||||
if limits.MaxUncompressedBytes <= 0 {
|
||||
limits.MaxUncompressedBytes = DefaultMaxUncompressed
|
||||
}
|
||||
if limits.MaxEntryBytes <= 0 || limits.MaxEntryBytes > limits.MaxUncompressedBytes {
|
||||
limits.MaxEntryBytes = limits.MaxUncompressedBytes
|
||||
}
|
||||
if limits.MaxEntries <= 0 {
|
||||
limits.MaxEntries = DefaultMaxBundleEntries
|
||||
}
|
||||
if limits.MaxCompressionRatio <= 0 {
|
||||
limits.MaxCompressionRatio = DefaultMaxCompressionRatio
|
||||
}
|
||||
return limits
|
||||
}
|
||||
|
||||
func validateEntryHeader(hdr *tar.Header) (string, error) {
|
||||
if hdr == nil {
|
||||
return "", fmt.Errorf("%w: missing tar header", ErrInvalidBundle)
|
||||
}
|
||||
// PAX/GNU extension records are consumed by archive/tar before the next
|
||||
// header surfaces, so bytes carried in them never reach the allowlist and
|
||||
// count checks. The contract fixtures are plain USTAR; reject anything else
|
||||
// rather than let extension records smuggle data past validation.
|
||||
if hdr.Format == tar.FormatPAX || hdr.Format == tar.FormatGNU {
|
||||
return "", fmt.Errorf("%w: unsupported tar format", ErrInvalidBundle)
|
||||
}
|
||||
if len(hdr.PAXRecords) > 0 {
|
||||
return "", fmt.Errorf("%w: unsupported tar extension records", ErrInvalidBundle)
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
|
||||
return "", fmt.Errorf("%w: unsupported tar entry type", ErrInvalidBundle)
|
||||
}
|
||||
if hdr.Size < 0 {
|
||||
return "", fmt.Errorf("%w: negative tar entry size", ErrInvalidBundle)
|
||||
}
|
||||
// Reject names that are not already trimmed instead of normalizing them:
|
||||
// otherwise a padded name like "manifest.json " would be recorded as the
|
||||
// allowlisted entry while the archive stores a different literal member.
|
||||
name := hdr.Name
|
||||
if name != strings.TrimSpace(name) {
|
||||
return "", fmt.Errorf("%w: unsafe entry name", ErrInvalidBundle)
|
||||
}
|
||||
if name == "" || strings.Contains(name, "\\") || strings.HasPrefix(name, "/") || path.IsAbs(name) {
|
||||
return "", fmt.Errorf("%w: unsafe entry name", ErrInvalidBundle)
|
||||
}
|
||||
clean := path.Clean(name)
|
||||
if clean == "." || clean != name || strings.HasPrefix(clean, "../") || strings.Contains(clean, "/../") {
|
||||
return "", fmt.Errorf("%w: unsafe entry name", ErrInvalidBundle)
|
||||
}
|
||||
if _, ok := allowedBundleEntries[name]; !ok {
|
||||
return "", fmt.Errorf("%w: disallowed entry %q", ErrInvalidBundle, name)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// tarEntryReader streams one tar entry's payload while charging every read
|
||||
// against the bundle's entry-size, total-size, and compression-ratio limits —
|
||||
// the same accounting the opaque drain path applied — so structural validators
|
||||
// can pull the payload without letting a member slip past the guards. It also
|
||||
// tracks entryBytes so the caller can compare it against the declared header
|
||||
// size.
|
||||
type tarEntryReader struct {
|
||||
tr *tar.Reader
|
||||
limits BundleLimits
|
||||
metered *compressedMeter
|
||||
total *int64
|
||||
entryBytes int64
|
||||
}
|
||||
|
||||
func (er *tarEntryReader) Read(p []byte) (int, error) {
|
||||
n, err := er.tr.Read(p)
|
||||
if n > 0 {
|
||||
er.entryBytes += int64(n)
|
||||
if er.entryBytes > er.limits.MaxEntryBytes {
|
||||
return n, ErrEntryTooLarge
|
||||
}
|
||||
*er.total += int64(n)
|
||||
if *er.total > er.limits.MaxUncompressedBytes {
|
||||
return n, ErrUncompressedTooLarge
|
||||
}
|
||||
if ratioExceeded(*er.total, er.metered.Count(), er.limits.MaxCompressionRatio) {
|
||||
return n, ErrCompressionRatio
|
||||
}
|
||||
}
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return n, classifyBundleReadError(err)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func isNDJSONBundleEntry(name string) bool {
|
||||
return name == "logs.jsonl" || name == "breadcrumbs.jsonl"
|
||||
}
|
||||
|
||||
func isJSONObjectBundleEntry(name string) bool {
|
||||
return name == "device.json" ||
|
||||
(strings.HasPrefix(name, "crash/") && strings.HasSuffix(name, ".json"))
|
||||
}
|
||||
|
||||
// validateJSONObjectEntry streams a single JSON object from r, rejecting
|
||||
// anything that is not exactly one top-level object. It uses the token decoder
|
||||
// so memory stays bounded to the nesting depth rather than the entry size.
|
||||
func validateJSONObjectEntry(r io.Reader) error {
|
||||
dec := json.NewDecoder(r)
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return classifyEntryContentError(err)
|
||||
}
|
||||
if delim, ok := tok.(json.Delim); !ok || delim != '{' {
|
||||
return fmt.Errorf("%w: entry is not a JSON object", ErrInvalidBundle)
|
||||
}
|
||||
depth := 1
|
||||
for depth > 0 {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return classifyEntryContentError(err)
|
||||
}
|
||||
if delim, ok := tok.(json.Delim); ok {
|
||||
switch delim {
|
||||
case '{', '[':
|
||||
depth++
|
||||
case '}', ']':
|
||||
depth--
|
||||
}
|
||||
}
|
||||
}
|
||||
// A well-formed entry ends here; anything after the top-level object is
|
||||
// smuggled content.
|
||||
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
|
||||
if err == nil {
|
||||
return fmt.Errorf("%w: trailing data after JSON object", ErrInvalidBundle)
|
||||
}
|
||||
return classifyEntryContentError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateNDJSONObjectEntry streams newline-delimited JSON, requiring each
|
||||
// non-blank line to be a single JSON object. Lines are validated one at a time
|
||||
// with a byte cap so a pathological entry can never buffer unbounded memory.
|
||||
func validateNDJSONObjectEntry(r io.Reader) error {
|
||||
buffer := make([]byte, defaultBundleReadBufferSize)
|
||||
line := make([]byte, 0, 512)
|
||||
for {
|
||||
n, err := r.Read(buffer)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return classifyEntryContentError(err)
|
||||
}
|
||||
chunk := buffer[:n]
|
||||
for len(chunk) > 0 {
|
||||
idx := bytes.IndexByte(chunk, '\n')
|
||||
if idx < 0 {
|
||||
line = append(line, chunk...)
|
||||
if len(line) > contract.MaxLogLineBytes {
|
||||
return errLogLineTooLarge()
|
||||
}
|
||||
break
|
||||
}
|
||||
line = append(line, chunk[:idx]...)
|
||||
if len(line) > contract.MaxLogLineBytes {
|
||||
return errLogLineTooLarge()
|
||||
}
|
||||
if verr := validateJSONObjectLine(line); verr != nil {
|
||||
return verr
|
||||
}
|
||||
line = line[:0]
|
||||
chunk = chunk[idx+1:]
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Handle a trailing line without a closing newline.
|
||||
if len(bytes.TrimSpace(line)) > 0 {
|
||||
if len(line) > contract.MaxLogLineBytes {
|
||||
return errLogLineTooLarge()
|
||||
}
|
||||
return validateJSONObjectLine(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateJSONObjectLine(line []byte) error {
|
||||
trimmed := bytes.TrimSpace(line)
|
||||
if len(trimmed) == 0 {
|
||||
return nil
|
||||
}
|
||||
if trimmed[0] != '{' || !json.Valid(trimmed) {
|
||||
return fmt.Errorf("%w: entry contains a line that is not a JSON object", ErrInvalidBundle)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func errLogLineTooLarge() error {
|
||||
return fmt.Errorf("%w: entry line exceeds %d bytes", ErrInvalidBundle, contract.MaxLogLineBytes)
|
||||
}
|
||||
|
||||
// classifyEntryContentError maps a decode-time error to the right bundle
|
||||
// sentinel: the streaming limit sentinels and existing bundle errors pass
|
||||
// through unchanged; a truncated/empty stream or malformed JSON becomes
|
||||
// ErrInvalidBundle.
|
||||
func classifyEntryContentError(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, ErrEntryTooLarge),
|
||||
errors.Is(err, ErrUncompressedTooLarge),
|
||||
errors.Is(err, ErrCompressionRatio),
|
||||
errors.Is(err, ErrCompressedTooLarge),
|
||||
errors.Is(err, ErrInvalidBundle):
|
||||
return err
|
||||
case isBundleUploadAbortError(err):
|
||||
return err
|
||||
case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
|
||||
return fmt.Errorf("%w: truncated or empty JSON entry", ErrInvalidBundle)
|
||||
default:
|
||||
return fmt.Errorf("%w: %v", ErrInvalidBundle, err)
|
||||
}
|
||||
}
|
||||
|
||||
func rejectPostTarData(r io.Reader, buffer []byte) error {
|
||||
var padding int64
|
||||
for {
|
||||
n, err := r.Read(buffer)
|
||||
if errors.Is(err, ErrCompressedTooLarge) {
|
||||
return ErrCompressedTooLarge
|
||||
}
|
||||
if n > 0 {
|
||||
for _, b := range buffer[:n] {
|
||||
if b != 0 {
|
||||
return fmt.Errorf("%w: trailing data after tar archive", ErrInvalidBundle)
|
||||
}
|
||||
}
|
||||
padding += int64(n)
|
||||
if padding > maxTarTrailingPaddingBytes {
|
||||
return fmt.Errorf("%w: excessive padding after tar archive", ErrInvalidBundle)
|
||||
}
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return classifyBundleReadError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rejectTrailingCompressedData(metered *compressedMeter) error {
|
||||
var one [1]byte
|
||||
n, err := metered.Read(one[:])
|
||||
if errors.Is(err, ErrCompressedTooLarge) {
|
||||
return ErrCompressedTooLarge
|
||||
}
|
||||
if n > 0 {
|
||||
return fmt.Errorf("%w: trailing data after gzip stream", ErrInvalidBundle)
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return classifyBundleReadError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ratioExceeded(uncompressed, compressed, maxRatio int64) bool {
|
||||
if compressed <= 0 || maxRatio <= 0 {
|
||||
return false
|
||||
}
|
||||
if compressed > math.MaxInt64/maxRatio {
|
||||
return false
|
||||
}
|
||||
return uncompressed > compressed*maxRatio
|
||||
}
|
||||
|
||||
func classifyBundleReadError(err error) error {
|
||||
if isBundleUploadAbortError(err) {
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrCompressedTooLarge) {
|
||||
return ErrCompressedTooLarge
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return fmt.Errorf("%w: truncated archive", ErrInvalidBundle)
|
||||
}
|
||||
return fmt.Errorf("%w: %v", ErrInvalidBundle, err)
|
||||
}
|
||||
|
||||
type bundleUploadAbortError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *bundleUploadAbortError) Error() string {
|
||||
if e == nil || e.err == nil {
|
||||
return "diagnostics bundle upload aborted"
|
||||
}
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e *bundleUploadAbortError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.err
|
||||
}
|
||||
|
||||
func isBundleUploadAbortError(err error) bool {
|
||||
var abortErr *bundleUploadAbortError
|
||||
return errors.As(err, &abortErr)
|
||||
}
|
||||
|
||||
func allowlistMap(entries []string) map[string]struct{} {
|
||||
allowed := make(map[string]struct{}, len(entries))
|
||||
for _, entry := range entries {
|
||||
allowed[entry] = struct{}{}
|
||||
}
|
||||
return allowed
|
||||
}
|
||||
|
||||
// uncompressedCounter tracks the total number of decompressed bytes read out
|
||||
// of the gzip stream, including tar headers, end-of-archive blocks, and
|
||||
// record padding.
|
||||
type uncompressedCounter struct {
|
||||
r io.Reader
|
||||
count int64
|
||||
}
|
||||
|
||||
func (c *uncompressedCounter) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.count += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
type compressedMeter struct {
|
||||
r io.Reader
|
||||
max int64
|
||||
n int64
|
||||
hash hash.Hash
|
||||
limit bool
|
||||
}
|
||||
|
||||
func newCompressedMeter(r io.Reader, max int64) *compressedMeter {
|
||||
return &compressedMeter{
|
||||
r: r,
|
||||
max: max,
|
||||
hash: sha256.New(),
|
||||
limit: max > 0,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *compressedMeter) Read(p []byte) (int, error) {
|
||||
if m.limit {
|
||||
remaining := m.max - m.n
|
||||
if remaining < 0 {
|
||||
return 0, ErrCompressedTooLarge
|
||||
}
|
||||
if int64(len(p)) > remaining+1 {
|
||||
p = p[:int(remaining+1)]
|
||||
}
|
||||
}
|
||||
n, err := m.r.Read(p)
|
||||
if n > 0 {
|
||||
m.n += int64(n)
|
||||
_, _ = m.hash.Write(p[:n])
|
||||
if m.limit && m.n > m.max {
|
||||
return n, ErrCompressedTooLarge
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (m *compressedMeter) ReadByte() (byte, error) {
|
||||
var one [1]byte
|
||||
n, err := m.Read(one[:])
|
||||
if n == 1 {
|
||||
if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return one[0], err
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
func (m *compressedMeter) Count() int64 {
|
||||
return m.n
|
||||
}
|
||||
|
||||
func (m *compressedMeter) Sum() []byte {
|
||||
return m.hash.Sum(nil)
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testArchiveEntry struct {
|
||||
name string
|
||||
body []byte
|
||||
typeflag byte
|
||||
}
|
||||
|
||||
func TestValidateBundle(t *testing.T) {
|
||||
validEntries := []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{"schema_version":1}`)},
|
||||
{name: "logs.jsonl", body: []byte("{}\n")},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
entries []testArchiveEntry
|
||||
limits BundleLimits
|
||||
mutate func([]byte) []byte
|
||||
wantErr error
|
||||
wantEntries string
|
||||
}{
|
||||
{
|
||||
name: "happy path",
|
||||
entries: validEntries,
|
||||
},
|
||||
{
|
||||
name: "wrong first entry",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "device.json", body: []byte(`{}`)},
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "disallowed name",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "extra.txt", body: []byte("nope")},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "traversal name",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "../logs.jsonl", body: []byte("nope")},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "ratio bomb",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
// Valid newline-delimited JSON objects that still compress far
|
||||
// beyond the ratio limit, so the ratio guard trips before the
|
||||
// content validator can pass judgement on every line.
|
||||
{name: "logs.jsonl", body: bytes.Repeat([]byte("{}\n"), 400*1024)},
|
||||
},
|
||||
limits: BundleLimits{
|
||||
MaxUncompressedBytes: 4 * 1024 * 1024,
|
||||
},
|
||||
wantErr: ErrCompressionRatio,
|
||||
},
|
||||
{
|
||||
name: "rejects malformed device.json",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "device.json", body: []byte(`{"identity": }`)},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects non-object device.json",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "device.json", body: []byte(`[1,2,3]`)},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects trailing data in device.json",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "device.json", body: []byte(`{}{}`)},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects malformed logs.jsonl line",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "logs.jsonl", body: []byte("{}\nnot json\n")},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects non-object logs.jsonl line",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "logs.jsonl", body: []byte("{}\n[1,2]\n")},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "accepts well-formed device.json and crash json",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "device.json", body: []byte(`{"identity":{"model":"Pixel"}}` + "\n")},
|
||||
{name: "logs.jsonl", body: []byte(`{"lvl":"I","msg":"hi"}` + "\n")},
|
||||
{name: "crash/summary.json", body: []byte(`{"summary":"boom"}`)},
|
||||
{name: "crash/tombstone.pb", body: []byte{0x00, 0x01, 0x02, 0x03}},
|
||||
},
|
||||
wantEntries: "manifest.json,device.json,logs.jsonl,crash/summary.json,crash/tombstone.pb",
|
||||
},
|
||||
{
|
||||
name: "over count",
|
||||
entries: append([]testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
}, repeatedEntries("logs.jsonl", 16)...),
|
||||
wantErr: ErrTooManyEntries,
|
||||
},
|
||||
{
|
||||
name: "oversize",
|
||||
entries: validEntries,
|
||||
limits: BundleLimits{MaxCompressedBytes: 64},
|
||||
wantErr: ErrCompressedTooLarge,
|
||||
},
|
||||
{
|
||||
name: "truncated gzip",
|
||||
entries: validEntries,
|
||||
mutate: func(raw []byte) []byte {
|
||||
return raw[:len(raw)-8]
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects trailing tar data inside gzip stream",
|
||||
entries: validEntries,
|
||||
mutate: func([]byte) []byte {
|
||||
return buildTestArchiveWithPostTarPayload(t, validEntries, []byte("hidden"))
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "accepts tar record padding",
|
||||
entries: validEntries,
|
||||
mutate: func([]byte) []byte {
|
||||
return buildTestArchiveWithPostTarPayload(t, validEntries, make([]byte, 8*1024))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "rejects nonzero byte inside trailing padding",
|
||||
entries: validEntries,
|
||||
mutate: func([]byte) []byte {
|
||||
padding := make([]byte, 8*1024)
|
||||
padding[len(padding)-1] = 1
|
||||
return buildTestArchiveWithPostTarPayload(t, validEntries, padding)
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects excessive trailing padding",
|
||||
entries: validEntries,
|
||||
mutate: func([]byte) []byte {
|
||||
return buildTestArchiveWithPostTarPayload(t, validEntries, make([]byte, maxTarTrailingPaddingBytes+1))
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects concatenated gzip member",
|
||||
entries: validEntries,
|
||||
mutate: func(raw []byte) []byte {
|
||||
return append(raw, buildTestGzipMember(t, bytes.Repeat([]byte{0}, 1024))...)
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects symlinks",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json", body: []byte(`{}`)},
|
||||
{name: "logs.jsonl", typeflag: tar.TypeSymlink},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects padded entry name",
|
||||
entries: []testArchiveEntry{
|
||||
{name: "manifest.json ", body: []byte(`{}`)},
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
{
|
||||
name: "rejects pax extension records",
|
||||
entries: validEntries,
|
||||
mutate: func([]byte) []byte {
|
||||
return buildTestArchivePAX(t)
|
||||
},
|
||||
wantErr: ErrInvalidBundle,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
raw := buildTestArchive(t, tt.entries)
|
||||
if tt.mutate != nil {
|
||||
raw = tt.mutate(raw)
|
||||
}
|
||||
info, err := ValidateBundle(bytes.NewReader(raw), tt.limits)
|
||||
if tt.wantErr != nil {
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("ValidateBundle error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateBundle error = %v", err)
|
||||
}
|
||||
if info.CompressedBytes != int64(len(raw)) {
|
||||
t.Fatalf("compressed bytes = %d, want %d", info.CompressedBytes, len(raw))
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if info.SHA256 != fmtHex(sum[:]) {
|
||||
t.Fatalf("sha256 = %s, want %s", info.SHA256, fmtHex(sum[:]))
|
||||
}
|
||||
wantEntries := tt.wantEntries
|
||||
if wantEntries == "" {
|
||||
wantEntries = "manifest.json,logs.jsonl"
|
||||
}
|
||||
if got := strings.Join(info.Entries, ","); got != wantEntries {
|
||||
t.Fatalf("entries = %s, want %s", got, wantEntries)
|
||||
}
|
||||
gzr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("open gzip for uncompressed size: %v", err)
|
||||
}
|
||||
wantUncompressed, err := io.Copy(io.Discard, gzr)
|
||||
if err != nil {
|
||||
t.Fatalf("measure uncompressed size: %v", err)
|
||||
}
|
||||
if info.UncompressedBytes != wantUncompressed {
|
||||
t.Fatalf("uncompressed bytes = %d, want %d", info.UncompressedBytes, wantUncompressed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func repeatedEntries(name string, count int) []testArchiveEntry {
|
||||
entries := make([]testArchiveEntry, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
entries = append(entries, testArchiveEntry{name: name, body: []byte("{}\n")})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func buildTestArchive(t *testing.T, entries []testArchiveEntry) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
for _, entry := range entries {
|
||||
typeflag := entry.typeflag
|
||||
if typeflag == 0 {
|
||||
typeflag = tar.TypeReg
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
Name: entry.name,
|
||||
Mode: 0o600,
|
||||
Size: int64(len(entry.body)),
|
||||
Typeflag: typeflag,
|
||||
}
|
||||
if typeflag != tar.TypeReg && typeflag != tar.TypeRegA {
|
||||
hdr.Size = 0
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatalf("write header: %v", err)
|
||||
}
|
||||
if hdr.Size > 0 {
|
||||
if _, err := tw.Write(entry.body); err != nil {
|
||||
t.Fatalf("write body: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("close tar: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("close gzip: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func buildTestArchiveWithPostTarPayload(t *testing.T, entries []testArchiveEntry, trailing []byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
for _, entry := range entries {
|
||||
typeflag := entry.typeflag
|
||||
if typeflag == 0 {
|
||||
typeflag = tar.TypeReg
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
Name: entry.name,
|
||||
Mode: 0o600,
|
||||
Size: int64(len(entry.body)),
|
||||
Typeflag: typeflag,
|
||||
}
|
||||
if typeflag != tar.TypeReg && typeflag != tar.TypeRegA {
|
||||
hdr.Size = 0
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatalf("write header: %v", err)
|
||||
}
|
||||
if hdr.Size > 0 {
|
||||
if _, err := tw.Write(entry.body); err != nil {
|
||||
t.Fatalf("write body: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("close tar: %v", err)
|
||||
}
|
||||
if _, err := gz.Write(trailing); err != nil {
|
||||
t.Fatalf("write trailing gzip payload: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("close gzip: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// buildTestArchivePAX writes an archive whose first entry uses PAX extended
|
||||
// headers (forced by a custom PAX record). archive/tar consumes those records
|
||||
// before the entry header surfaces, so validation must reject the format even
|
||||
// though the visible name is allowlisted.
|
||||
func buildTestArchivePAX(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
body := []byte(`{"schema_version":1}`)
|
||||
hdr := &tar.Header{
|
||||
Name: "manifest.json",
|
||||
Mode: 0o600,
|
||||
Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg,
|
||||
Format: tar.FormatPAX,
|
||||
PAXRecords: map[string]string{"SILO.smuggled": "payload"},
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatalf("write pax header: %v", err)
|
||||
}
|
||||
if _, err := tw.Write(body); err != nil {
|
||||
t.Fatalf("write pax body: %v", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("close tar: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("close gzip: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func buildTestGzipMember(t *testing.T, payload []byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
if _, err := gz.Write(payload); err != nil {
|
||||
t.Fatalf("write gzip payload: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("close gzip: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func fmtHex(raw []byte) string {
|
||||
const table = "0123456789abcdef"
|
||||
out := make([]byte, len(raw)*2)
|
||||
for i, b := range raw {
|
||||
out[i*2] = table[b>>4]
|
||||
out[i*2+1] = table[b&0x0f]
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ObjectPrefix = diagnosticObjectPrefix + "/"
|
||||
DefaultReceivingGrace = time.Hour
|
||||
)
|
||||
|
||||
type CleanupRepository interface {
|
||||
DeleteByID(ctx context.Context, id string) (*Report, error)
|
||||
MarkFailed(ctx context.Context, id string) error
|
||||
RetentionCandidates(ctx context.Context, olderThan time.Time, perUserByteCap int64) ([]Report, error)
|
||||
StaleReceiving(ctx context.Context, grace time.Duration) ([]Report, error)
|
||||
LiveBlobKeys(ctx context.Context, keys []string) (map[string]ReportState, error)
|
||||
}
|
||||
|
||||
type CleanupOptions struct {
|
||||
Now func() time.Time
|
||||
StaleReceivingGrace time.Duration
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type CleanupResult struct {
|
||||
RetentionReportsDeleted int `json:"retention_reports_deleted"`
|
||||
StaleReportsDeleted int `json:"stale_reports_deleted"`
|
||||
OrphanObjectsDeleted int `json:"orphan_objects_deleted"`
|
||||
}
|
||||
|
||||
func (r CleanupResult) ReportsDeleted() int {
|
||||
return r.RetentionReportsDeleted + r.StaleReportsDeleted
|
||||
}
|
||||
|
||||
func CleanupOnce(
|
||||
ctx context.Context,
|
||||
repo CleanupRepository,
|
||||
settingsStore SettingsStore,
|
||||
store ObjectStore,
|
||||
logger *slog.Logger,
|
||||
) (CleanupResult, error) {
|
||||
settings, err := LoadSettings(ctx, settingsStore)
|
||||
if err != nil {
|
||||
return CleanupResult{}, fmt.Errorf("load diagnostics cleanup settings: %w", err)
|
||||
}
|
||||
return CleanupReports(ctx, repo, store, settings, CleanupOptions{Logger: logger})
|
||||
}
|
||||
|
||||
func CleanupReports(
|
||||
ctx context.Context,
|
||||
repo CleanupRepository,
|
||||
store ObjectStore,
|
||||
settings Settings,
|
||||
opts CleanupOptions,
|
||||
) (CleanupResult, error) {
|
||||
if repo == nil {
|
||||
return CleanupResult{}, nil
|
||||
}
|
||||
now := func() time.Time { return time.Now().UTC() }
|
||||
if opts.Now != nil {
|
||||
now = opts.Now
|
||||
}
|
||||
logger := opts.Logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
grace := opts.StaleReceivingGrace
|
||||
if grace <= 0 {
|
||||
grace = DefaultReceivingGrace
|
||||
}
|
||||
|
||||
cutoff := time.Time{}
|
||||
if settings.RetentionDays > 0 {
|
||||
cutoff = now().Add(-time.Duration(settings.RetentionDays) * 24 * time.Hour)
|
||||
}
|
||||
|
||||
// A single poisoned report (e.g. a permission error on one S3 object) must
|
||||
// not abort the whole run, or every scheduled pass re-hits it first and
|
||||
// blocks retention, stale reconciliation, and orphan cleanup indefinitely.
|
||||
// Log-and-continue per report; aggregate per-item failures and still surface
|
||||
// genuine query/iteration errors.
|
||||
var result CleanupResult
|
||||
var errs []error
|
||||
|
||||
candidates, err := repo.RetentionCandidates(ctx, cutoff, settings.MaxBytesPerUser)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("load diagnostic retention candidates: %w", err))
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
for _, report := range candidates {
|
||||
// Row first: a DB failure after the object is gone would leave a visible
|
||||
// ready report whose bundle 404s on download. If the blob delete then
|
||||
// fails the row is already gone, so log it for the orphan pass below to
|
||||
// reap rather than aborting the run (see the admin DeleteReport path).
|
||||
if _, err := repo.DeleteByID(ctx, report.ID); err != nil && !IsReportNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("delete diagnostic retention row %s: %w", report.ID, err))
|
||||
continue
|
||||
}
|
||||
if err := deleteReportObjects(ctx, store, &report, logger); err != nil {
|
||||
logDeferredBlobDeletion(ctx, logger, store, &report, err)
|
||||
}
|
||||
result.RetentionReportsDeleted++
|
||||
}
|
||||
|
||||
stale, err := repo.StaleReceiving(ctx, grace)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("load stale diagnostic reports: %w", err))
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
for _, report := range stale {
|
||||
// Same rows-first ordering as retention: mark/delete the row before the
|
||||
// blob so a mid-cleanup DB failure never leaves a row pointing at a
|
||||
// missing object, and let orphan cleanup reap a blob that fails to delete.
|
||||
if report.State == StateReceiving {
|
||||
if err := repo.MarkFailed(ctx, report.ID); err != nil && !IsReportNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("mark stale diagnostic report failed %s: %w", report.ID, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, err := repo.DeleteByID(ctx, report.ID); err != nil && !IsReportNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("delete stale diagnostic row %s: %w", report.ID, err))
|
||||
continue
|
||||
}
|
||||
if err := deleteReportObjects(ctx, store, &report, logger); err != nil {
|
||||
logDeferredBlobDeletion(ctx, logger, store, &report, err)
|
||||
}
|
||||
result.StaleReportsDeleted++
|
||||
}
|
||||
|
||||
if store == nil || strings.TrimSpace(store.Bucket()) == "" {
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
keys, err := store.ListObjects(ctx, ObjectPrefix)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("list diagnostic objects: %w", err))
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
liveKeys, err := repo.LiveBlobKeys(ctx, keys)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("load live diagnostic blob keys: %w", err))
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if _, ok := liveKeys[key]; ok {
|
||||
continue
|
||||
}
|
||||
if err := deleteObjectIfPresent(ctx, store, store.Bucket(), key); err != nil {
|
||||
errs = append(errs, fmt.Errorf("delete orphan diagnostic object %s: %w", key, err))
|
||||
continue
|
||||
}
|
||||
logger.InfoContext(ctx, "diagnostic orphan object deleted",
|
||||
"component", "diagnostics",
|
||||
"key", key,
|
||||
)
|
||||
result.OrphanObjectsDeleted++
|
||||
}
|
||||
return result, errors.Join(errs...)
|
||||
}
|
||||
|
||||
func deleteReportObjects(ctx context.Context, store ObjectStore, report *Report, logger *slog.Logger) error {
|
||||
if report == nil {
|
||||
return nil
|
||||
}
|
||||
keys := reportObjectKeys(*report)
|
||||
if store == nil || strings.TrimSpace(store.Bucket()) == "" {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
logSkippedObjectDeletion(ctx, logger, report.ID, keys)
|
||||
return nil
|
||||
}
|
||||
bucket := stringValue(report.BlobBucket)
|
||||
if bucket == "" {
|
||||
bucket = store.Bucket()
|
||||
}
|
||||
for _, key := range keys {
|
||||
if err := deleteObjectIfPresent(ctx, store, bucket, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func logSkippedObjectDeletion(ctx context.Context, logger *slog.Logger, reportID string, keys []string) {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
logger.WarnContext(ctx, "diagnostic report blob deletion skipped because storage is unavailable",
|
||||
"component", "diagnostics",
|
||||
"report_id", reportID,
|
||||
"keys", keys,
|
||||
)
|
||||
}
|
||||
|
||||
// logDeferredBlobDeletion records a report blob that could not be deleted after
|
||||
// its DB row was already removed. The row is the source of truth, so the object
|
||||
// is now an orphan for the orphan-cleanup pass (this run or a later one) to
|
||||
// reap; we log the bucket and keys rather than fail so one unreachable object
|
||||
// can't block row deletion. Shared by the retention/stale loops and the admin
|
||||
// DeleteReport path.
|
||||
func logDeferredBlobDeletion(ctx context.Context, logger *slog.Logger, store ObjectStore, report *Report, err error) {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
logger.ErrorContext(ctx, "diagnostic report blob deletion failed after row deletion",
|
||||
"component", "diagnostics",
|
||||
"report_id", report.ID,
|
||||
"bucket", reportBlobBucket(report, store),
|
||||
"keys", reportObjectKeys(*report),
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
|
||||
func deleteObjectIfPresent(ctx context.Context, store ObjectStore, bucket, key string) error {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return nil
|
||||
}
|
||||
if err := store.DeleteObject(ctx, bucket, key); err != nil && !IsObjectNotFound(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportObjectKeys(report Report) []string {
|
||||
keys := make([]string, 0, 2)
|
||||
if key := stringValue(report.BlobKey); key != "" {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
if report.ID != "" && report.UserID > 0 {
|
||||
deterministic := reportObjectKey(report.UserID, report.ID)
|
||||
seen := false
|
||||
for _, key := range keys {
|
||||
if key == deterministic {
|
||||
seen = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !seen {
|
||||
keys = append(keys, deterministic)
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func IsReportNotFound(err error) bool {
|
||||
return errors.Is(err, ErrNotFound)
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCleanupReportsDeletesRowBeforeBlobAndToleratesMissingObject(t *testing.T) {
|
||||
ops := []string{}
|
||||
repo := &fakeCleanupRepo{
|
||||
retention: []Report{
|
||||
testCleanupReport("r1", 7, StateReady, "diagnostics/7/r1.tar.gz"),
|
||||
testCleanupReport("r2", 7, StateReady, "diagnostics/7/r2.tar.gz"),
|
||||
},
|
||||
ops: &ops,
|
||||
}
|
||||
store := &fakeCleanupStore{
|
||||
bucket: "private",
|
||||
missingKeys: map[string]bool{"diagnostics/7/r1.tar.gz": true},
|
||||
ops: &ops,
|
||||
}
|
||||
|
||||
result, err := CleanupReports(context.Background(), repo, store, Settings{
|
||||
RetentionDays: 30,
|
||||
MaxBytesPerUser: DefaultMaxBytesPerUser,
|
||||
}, CleanupOptions{
|
||||
Now: func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) },
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupReports: %v", err)
|
||||
}
|
||||
if result.RetentionReportsDeleted != 2 {
|
||||
t.Fatalf("RetentionReportsDeleted = %d, want 2", result.RetentionReportsDeleted)
|
||||
}
|
||||
wantOps := []string{
|
||||
"delete-row:r1",
|
||||
"delete-object:diagnostics/7/r1.tar.gz",
|
||||
"delete-row:r2",
|
||||
"delete-object:diagnostics/7/r2.tar.gz",
|
||||
}
|
||||
assertStrings(t, ops, wantOps)
|
||||
}
|
||||
|
||||
func TestCleanupReportsTreatsBlobFailureAsNonFatal(t *testing.T) {
|
||||
ops := []string{}
|
||||
repo := &fakeCleanupRepo{
|
||||
retention: []Report{
|
||||
testCleanupReport("r1", 7, StateReady, "diagnostics/7/r1.tar.gz"),
|
||||
testCleanupReport("r2", 7, StateReady, "diagnostics/7/r2.tar.gz"),
|
||||
},
|
||||
ops: &ops,
|
||||
}
|
||||
store := &fakeCleanupStore{
|
||||
bucket: "private",
|
||||
deleteErrs: map[string]error{"diagnostics/7/r1.tar.gz": errors.New("s3 access denied")},
|
||||
ops: &ops,
|
||||
}
|
||||
|
||||
result, err := CleanupReports(context.Background(), repo, store, Settings{
|
||||
RetentionDays: 30,
|
||||
MaxBytesPerUser: DefaultMaxBytesPerUser,
|
||||
}, CleanupOptions{
|
||||
Now: func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) },
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
// r1's row is deleted first, so its blob delete failing does not abort the
|
||||
// run or roll back the row: it is logged for orphan cleanup to reap, both
|
||||
// rows count as deleted, and no error is surfaced.
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupReports: %v", err)
|
||||
}
|
||||
if result.RetentionReportsDeleted != 2 {
|
||||
t.Fatalf("RetentionReportsDeleted = %d, want 2", result.RetentionReportsDeleted)
|
||||
}
|
||||
assertStrings(t, ops, []string{
|
||||
"delete-row:r1",
|
||||
"delete-object:diagnostics/7/r1.tar.gz",
|
||||
"delete-row:r2",
|
||||
"delete-object:diagnostics/7/r2.tar.gz",
|
||||
})
|
||||
}
|
||||
|
||||
func TestCleanupReportsCleansStaleReceiving(t *testing.T) {
|
||||
ops := []string{}
|
||||
repo := &fakeCleanupRepo{
|
||||
stale: []Report{
|
||||
testCleanupReport("r1", 7, StateReceiving, ""),
|
||||
},
|
||||
ops: &ops,
|
||||
}
|
||||
store := &fakeCleanupStore{bucket: "private", ops: &ops}
|
||||
|
||||
result, err := CleanupReports(context.Background(), repo, store, Settings{
|
||||
RetentionDays: 30,
|
||||
MaxBytesPerUser: DefaultMaxBytesPerUser,
|
||||
}, CleanupOptions{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))})
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupReports: %v", err)
|
||||
}
|
||||
if result.StaleReportsDeleted != 1 {
|
||||
t.Fatalf("StaleReportsDeleted = %d, want 1", result.StaleReportsDeleted)
|
||||
}
|
||||
assertStrings(t, ops, []string{
|
||||
"mark-failed:r1",
|
||||
"delete-row:r1",
|
||||
"delete-object:diagnostics/7/r1.tar.gz",
|
||||
})
|
||||
}
|
||||
|
||||
func TestCleanupReportsDeletesRowsWhenStorageUnavailable(t *testing.T) {
|
||||
ops := []string{}
|
||||
repo := &fakeCleanupRepo{
|
||||
retention: []Report{
|
||||
testCleanupReport("r1", 7, StateReady, "diagnostics/7/r1.tar.gz"),
|
||||
},
|
||||
stale: []Report{
|
||||
testCleanupReport("r2", 7, StateReceiving, "diagnostics/7/r2.tar.gz"),
|
||||
},
|
||||
ops: &ops,
|
||||
}
|
||||
|
||||
result, err := CleanupReports(context.Background(), repo, nil, Settings{
|
||||
RetentionDays: 30,
|
||||
MaxBytesPerUser: DefaultMaxBytesPerUser,
|
||||
}, CleanupOptions{
|
||||
Now: func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) },
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupReports: %v", err)
|
||||
}
|
||||
if result.RetentionReportsDeleted != 1 || result.StaleReportsDeleted != 1 {
|
||||
t.Fatalf("result = %#v, want one retention and one stale deletion", result)
|
||||
}
|
||||
assertStrings(t, ops, []string{
|
||||
"delete-row:r1",
|
||||
"mark-failed:r2",
|
||||
"delete-row:r2",
|
||||
})
|
||||
}
|
||||
|
||||
func TestCleanupReportsDeletesUnmatchedObjects(t *testing.T) {
|
||||
ops := []string{}
|
||||
repo := &fakeCleanupRepo{
|
||||
live: map[string]ReportState{
|
||||
"diagnostics/7/ready.tar.gz": StateReady,
|
||||
"diagnostics/7/receiving.tar.gz": StateReceiving,
|
||||
},
|
||||
ops: &ops,
|
||||
}
|
||||
store := &fakeCleanupStore{
|
||||
bucket: "private",
|
||||
list: []string{
|
||||
"diagnostics/7/ready.tar.gz",
|
||||
"diagnostics/7/receiving.tar.gz",
|
||||
"diagnostics/7/orphan.tar.gz",
|
||||
"diagnostics/7/failed.tar.gz",
|
||||
},
|
||||
ops: &ops,
|
||||
}
|
||||
|
||||
result, err := CleanupReports(context.Background(), repo, store, Settings{
|
||||
RetentionDays: 30,
|
||||
MaxBytesPerUser: DefaultMaxBytesPerUser,
|
||||
}, CleanupOptions{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))})
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupReports: %v", err)
|
||||
}
|
||||
if result.OrphanObjectsDeleted != 2 {
|
||||
t.Fatalf("OrphanObjectsDeleted = %d, want 2", result.OrphanObjectsDeleted)
|
||||
}
|
||||
wantDeleted := []string{"diagnostics/7/orphan.tar.gz", "diagnostics/7/failed.tar.gz"}
|
||||
if !sameStrings(store.deleted, wantDeleted) {
|
||||
t.Fatalf("deleted objects = %v, want %v", store.deleted, wantDeleted)
|
||||
}
|
||||
}
|
||||
|
||||
func testCleanupReport(id string, userID int, state ReportState, blobKey string) Report {
|
||||
report := Report{ID: id, UserID: userID, State: state}
|
||||
if blobKey != "" {
|
||||
report.BlobKey = &blobKey
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
type fakeCleanupRepo struct {
|
||||
retention []Report
|
||||
stale []Report
|
||||
live map[string]ReportState
|
||||
ops *[]string
|
||||
}
|
||||
|
||||
func (f *fakeCleanupRepo) DeleteByID(_ context.Context, id string) (*Report, error) {
|
||||
f.record("delete-row:" + id)
|
||||
return &Report{ID: id}, nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupRepo) MarkFailed(_ context.Context, id string) error {
|
||||
f.record("mark-failed:" + id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupRepo) RetentionCandidates(context.Context, time.Time, int64) ([]Report, error) {
|
||||
return append([]Report(nil), f.retention...), nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupRepo) StaleReceiving(context.Context, time.Duration) ([]Report, error) {
|
||||
return append([]Report(nil), f.stale...), nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupRepo) LiveBlobKeys(_ context.Context, keys []string) (map[string]ReportState, error) {
|
||||
live := make(map[string]ReportState)
|
||||
for _, key := range keys {
|
||||
if state, ok := f.live[key]; ok {
|
||||
live[key] = state
|
||||
}
|
||||
}
|
||||
return live, nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupRepo) record(op string) {
|
||||
if f.ops != nil {
|
||||
*f.ops = append(*f.ops, op)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeCleanupStore struct {
|
||||
bucket string
|
||||
list []string
|
||||
missingKeys map[string]bool
|
||||
deleteErrs map[string]error
|
||||
deleted []string
|
||||
ops *[]string
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) PutStream(context.Context, string, string, io.Reader, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) GetObject(context.Context, string, string) (io.ReadCloser, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) DeleteObject(_ context.Context, _ string, key string) error {
|
||||
f.record("delete-object:" + key)
|
||||
f.deleted = append(f.deleted, key)
|
||||
if f.missingKeys[key] {
|
||||
return ErrObjectNotFound
|
||||
}
|
||||
if err := f.deleteErrs[key]; err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) ListObjects(context.Context, string) ([]string, error) {
|
||||
return append([]string(nil), f.list...), nil
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) PresignGetURL(context.Context, string, string, time.Duration) (string, error) {
|
||||
return "", errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) Bucket() string {
|
||||
return f.bucket
|
||||
}
|
||||
|
||||
func (f *fakeCleanupStore) record(op string) {
|
||||
if f.ops != nil {
|
||||
*f.ops = append(*f.ops, op)
|
||||
}
|
||||
}
|
||||
|
||||
func assertStrings(t *testing.T, got, want []string) {
|
||||
t.Helper()
|
||||
if !sameStrings(got, want) {
|
||||
t.Fatalf("ops = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sameStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,323 @@
|
||||
package contract
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const contractSchemaRoot = "../../../docs/design/schemas/client-diagnostics"
|
||||
|
||||
func TestValidFixtures(t *testing.T) {
|
||||
fixtures := mustGlob(t, "v1/fixtures/valid/*")
|
||||
if len(fixtures) == 0 {
|
||||
t.Fatal("valid fixtures missing")
|
||||
}
|
||||
|
||||
for _, fixture := range fixtures {
|
||||
fixture := fixture
|
||||
t.Run(filepath.Base(fixture), func(t *testing.T) {
|
||||
data := mustReadFixture(t, fixture)
|
||||
switch {
|
||||
case strings.HasSuffix(fixture, "device.json"):
|
||||
if _, err := ValidateDevice(data); err != nil {
|
||||
t.Fatalf("ValidateDevice() error = %v", err)
|
||||
}
|
||||
case strings.HasSuffix(fixture, ".jsonl"):
|
||||
validateLogLinesFixture(t, data)
|
||||
case strings.HasSuffix(fixture, ".json"):
|
||||
if _, err := ValidateManifest(data); err != nil {
|
||||
t.Fatalf("ValidateManifest() error = %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected fixture extension: %s", fixture)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidManifestFixtures(t *testing.T) {
|
||||
fixtures := mustGlob(t, "v1/fixtures/invalid/*.json")
|
||||
if len(fixtures) < 6 {
|
||||
t.Fatalf("invalid fixtures = %d, want at least 6", len(fixtures))
|
||||
}
|
||||
|
||||
seenErrors := map[string]string{}
|
||||
for _, fixture := range fixtures {
|
||||
fixture := fixture
|
||||
t.Run(filepath.Base(fixture), func(t *testing.T) {
|
||||
_, err := ValidateManifest(mustReadFixture(t, fixture))
|
||||
if err == nil {
|
||||
t.Fatal("ValidateManifest() error = nil, want failure")
|
||||
}
|
||||
msg := err.Error()
|
||||
if prior := seenErrors[msg]; prior != "" {
|
||||
t.Fatalf("error %q also used by %s", msg, prior)
|
||||
}
|
||||
seenErrors[msg] = filepath.Base(fixture)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateManifestRequiresCrashForEventReports(t *testing.T) {
|
||||
// A non-manual report with the crash object stripped must be rejected so
|
||||
// event reports always carry crash summary/source/occurred_at.
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(mustReadFixture(t, "v1/fixtures/valid/android-tv-crash-ueh.json"), &m); err != nil {
|
||||
t.Fatalf("parse crash fixture: %v", err)
|
||||
}
|
||||
delete(m, "crash")
|
||||
stripped, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal stripped manifest: %v", err)
|
||||
}
|
||||
_, err = ValidateManifest(stripped)
|
||||
if err == nil || !strings.Contains(err.Error(), "crash") {
|
||||
t.Fatalf("ValidateManifest() error = %v, want crash required for event report", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateManifestRejectsCrashOnManualReports(t *testing.T) {
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(mustReadFixture(t, "v1/fixtures/valid/android-tv-crash-ueh.json"), &m); err != nil {
|
||||
t.Fatalf("parse crash fixture: %v", err)
|
||||
}
|
||||
var report map[string]json.RawMessage
|
||||
if err := json.Unmarshal(m["report"], &report); err != nil {
|
||||
t.Fatalf("parse report: %v", err)
|
||||
}
|
||||
report["type"] = json.RawMessage(`"manual"`)
|
||||
reportJSON, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal report: %v", err)
|
||||
}
|
||||
m["report"] = reportJSON
|
||||
withCrash, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal manifest: %v", err)
|
||||
}
|
||||
_, err = ValidateManifest(withCrash)
|
||||
if err == nil || !strings.Contains(err.Error(), "crash") {
|
||||
t.Fatalf("ValidateManifest() error = %v, want crash rejected for manual report", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemaEnumsAndRequiredFieldsStayInSync(t *testing.T) {
|
||||
manifest := mustReadObject(t, "v1/manifest.schema.json")
|
||||
assertStringsEqual(t, "manifest.required", schemaStrings(t, manifest, "required"), manifestRequiredFields)
|
||||
assertConstInt(t, "manifest.schema_version.const", schemaValue(t, manifest, "properties", "schema_version", "const"), SchemaVersion)
|
||||
assertStringsEqual(t, "manifest.report.required", schemaStrings(t, manifest, "properties", "report", "required"), manifestReportRequiredFields)
|
||||
assertStringsEqual(t, "manifest.destination.required", schemaStrings(t, manifest, "properties", "destination", "required"), manifestDestinationRequiredFields)
|
||||
assertStringsEqual(t, "manifest.consent.required", schemaStrings(t, manifest, "properties", "consent", "required"), manifestConsentRequiredFields)
|
||||
assertStringsEqual(t, "manifest.crash.required", schemaStrings(t, manifest, "properties", "crash", "required"), manifestCrashRequiredFields)
|
||||
assertStringsEqual(t, "manifest.device_summary.required", schemaStrings(t, manifest, "properties", "device_summary", "required"), manifestDeviceSummaryFields)
|
||||
assertStringsEqual(t, "manifest.log_summary.required", schemaStrings(t, manifest, "properties", "log_summary", "required"), manifestLogSummaryRequiredFields)
|
||||
assertStringsEqual(t, "manifest.archive.required", schemaStrings(t, manifest, "properties", "archive", "required"), manifestArchiveRequiredFields)
|
||||
assertStringsEqual(t, "manifest.report.type.enum", schemaStrings(t, manifest, "properties", "report", "properties", "type", "enum"), reportTypes)
|
||||
assertStringsEqual(t, "manifest.report.platform.enum", schemaStrings(t, manifest, "properties", "report", "properties", "platform", "enum"), platforms)
|
||||
assertStringsEqual(t, "manifest.consent.mode.enum", schemaStrings(t, manifest, "properties", "consent", "properties", "mode", "enum"), consentModes)
|
||||
assertStringsEqual(t, "manifest.crash.source.enum", schemaStrings(t, manifest, "properties", "crash", "properties", "source", "enum"), crashSources)
|
||||
assertStringsEqual(t, "manifest.crash.provenance.enum", schemaStrings(t, manifest, "properties", "crash", "properties", "provenance", "enum"), crashProvenances)
|
||||
assertStringsEqual(t, "manifest.log_summary.categories.enum", schemaStrings(t, manifest, "properties", "log_summary", "properties", "categories", "items", "enum"), logCategories)
|
||||
assertStringsEqual(t, "manifest.archive.entries.enum", schemaStrings(t, manifest, "properties", "archive", "properties", "entries", "items", "enum"), ArchiveEntryAllowlist)
|
||||
|
||||
logline := mustReadObject(t, "v1/logline.schema.json")
|
||||
assertStringsEqual(t, "logline.required", schemaStrings(t, logline, "required"), []string{"ts", "run", "lvl", "cat", "tag", "msg"})
|
||||
assertStringsEqual(t, "logline.lvl.enum", schemaStrings(t, logline, "properties", "lvl", "enum"), logLevels)
|
||||
assertStringsEqual(t, "logline.cat.enum", schemaStrings(t, logline, "properties", "cat", "enum"), logCategories)
|
||||
|
||||
device := mustReadObject(t, "v1/device.schema.json")
|
||||
assertStringsEqual(t, "device.required", schemaStrings(t, device, "required"), deviceRequiredFields)
|
||||
assertStringsEqual(t, "device.provenance.enum", schemaStrings(t, device, "properties", "provenance", "enum"), deviceProvenances)
|
||||
}
|
||||
|
||||
func TestAttrRegistryStaysInSync(t *testing.T) {
|
||||
var registry struct {
|
||||
Categories map[string]map[string]struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"categories"`
|
||||
}
|
||||
if err := json.Unmarshal(mustReadFixture(t, "v1/attr-registry.json"), ®istry); err != nil {
|
||||
t.Fatalf("parse attr-registry.json: %v", err)
|
||||
}
|
||||
|
||||
got := make(map[string]map[string]string, len(registry.Categories))
|
||||
for category, keys := range registry.Categories {
|
||||
got[category] = make(map[string]string, len(keys))
|
||||
for key, spec := range keys {
|
||||
got[category][key] = spec.Type
|
||||
}
|
||||
}
|
||||
want := make(map[string]map[string]string, len(attrRegistry))
|
||||
for category, keys := range attrRegistry {
|
||||
want[category] = make(map[string]string, len(keys))
|
||||
for key, valueType := range keys {
|
||||
want[category][key] = string(valueType)
|
||||
}
|
||||
}
|
||||
|
||||
if !mapsEqual(got, want) {
|
||||
t.Fatalf("attr registry mismatch\ngot: %#v\nwant: %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLogLineDropsUnregisteredAttrs(t *testing.T) {
|
||||
line := []byte(`{"ts":"2026-07-19T18:22:29Z","run":"run_1","lvl":"I","cat":"playback","tag":"Player","msg":"started","attrs":{"sink":"HDMI","unregistered":"drop-me"}}`)
|
||||
got, err := ValidateLogLine(line)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateLogLine() error = %v", err)
|
||||
}
|
||||
if _, ok := got.Attrs["sink"]; !ok {
|
||||
t.Fatalf("registered attr missing: %#v", got.Attrs)
|
||||
}
|
||||
if _, ok := got.Attrs["unregistered"]; ok {
|
||||
t.Fatalf("unregistered attr was not dropped: %#v", got.Attrs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLogLineRejectsRegisteredAttrTypeMismatch(t *testing.T) {
|
||||
line := []byte(`{"ts":"2026-07-19T18:22:29Z","run":"run_1","lvl":"I","cat":"network","tag":"HTTP","msg":"done","attrs":{"status":"200"}}`)
|
||||
if _, err := ValidateLogLine(line); err == nil {
|
||||
t.Fatal("ValidateLogLine() error = nil, want type mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func validateLogLinesFixture(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
scanner.Buffer(make([]byte, 0, 1024), 1024*1024)
|
||||
lineCount := 0
|
||||
for scanner.Scan() {
|
||||
line := bytes.TrimSpace(scanner.Bytes())
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
lineCount++
|
||||
if _, err := ValidateLogLine(line); err != nil {
|
||||
t.Fatalf("line %d: ValidateLogLine() error = %v", lineCount, err)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatalf("scan loglines fixture: %v", err)
|
||||
}
|
||||
if lineCount == 0 {
|
||||
t.Fatal("loglines fixture has no log lines")
|
||||
}
|
||||
}
|
||||
|
||||
func mustGlob(t *testing.T, pattern string) []string {
|
||||
t.Helper()
|
||||
matches, err := filepath.Glob(filepath.Join(contractSchemaRoot, filepath.FromSlash(pattern)))
|
||||
if err != nil {
|
||||
t.Fatalf("glob %s: %v", pattern, err)
|
||||
}
|
||||
for i, match := range matches {
|
||||
rel, err := filepath.Rel(contractSchemaRoot, match)
|
||||
if err != nil {
|
||||
t.Fatalf("rel %s: %v", match, err)
|
||||
}
|
||||
matches[i] = filepath.ToSlash(rel)
|
||||
}
|
||||
sort.Strings(matches)
|
||||
return matches
|
||||
}
|
||||
|
||||
func mustReadFixture(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(contractSchemaRoot, filepath.FromSlash(path)))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func mustReadObject(t *testing.T, path string) map[string]any {
|
||||
t.Helper()
|
||||
var obj map[string]any
|
||||
if err := json.Unmarshal(mustReadFixture(t, path), &obj); err != nil {
|
||||
t.Fatalf("parse %s: %v", path, err)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
func schemaStrings(t *testing.T, root any, path ...string) []string {
|
||||
t.Helper()
|
||||
value := schemaValue(t, root, path...)
|
||||
items, ok := value.([]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s is %T, want array", strings.Join(path, "."), value)
|
||||
}
|
||||
out := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
s, ok := item.(string)
|
||||
if !ok {
|
||||
t.Fatalf("%s item is %T, want string", strings.Join(path, "."), item)
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func schemaValue(t *testing.T, root any, path ...string) any {
|
||||
t.Helper()
|
||||
current := root
|
||||
for _, key := range path {
|
||||
obj, ok := current.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s parent is %T, want object", key, current)
|
||||
}
|
||||
next, ok := obj[key]
|
||||
if !ok {
|
||||
t.Fatalf("missing schema path %s", strings.Join(path, "."))
|
||||
}
|
||||
current = next
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func assertConstInt(t *testing.T, label string, got any, want int) {
|
||||
t.Helper()
|
||||
gotFloat, ok := got.(float64)
|
||||
if !ok {
|
||||
t.Fatalf("%s = %T, want JSON number", label, got)
|
||||
}
|
||||
if int(gotFloat) != want || gotFloat != float64(want) {
|
||||
t.Fatalf("%s = %v, want %d", label, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertStringsEqual(t *testing.T, label string, got, want []string) {
|
||||
t.Helper()
|
||||
got = append([]string(nil), got...)
|
||||
want = append([]string(nil), want...)
|
||||
sort.Strings(got)
|
||||
sort.Strings(want)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("%s mismatch\ngot: %v\nwant: %v", label, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func mapsEqual(got, want map[string]map[string]string) bool {
|
||||
if len(got) != len(want) {
|
||||
return false
|
||||
}
|
||||
for category, gotKeys := range got {
|
||||
wantKeys, ok := want[category]
|
||||
if !ok || len(gotKeys) != len(wantKeys) {
|
||||
return false
|
||||
}
|
||||
for key, gotType := range gotKeys {
|
||||
if wantKeys[key] != gotType {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,672 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
diagnosticQuotaLockNamespace = 173
|
||||
defaultShortIDRetries = 8
|
||||
defaultAdminListLimit = 50
|
||||
maxAdminListLimit = 200
|
||||
)
|
||||
|
||||
type PostgresRepository struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewPostgresRepository(pool *pgxpool.Pool) *PostgresRepository {
|
||||
return &PostgresRepository{pool: pool}
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) InsertReceiving(ctx context.Context, input InsertReceivingInput) (InsertReceivingResult, error) {
|
||||
if err := validateInsertReceivingInput(input); err != nil {
|
||||
return InsertReceivingResult{}, err
|
||||
}
|
||||
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("begin diagnostic report insert: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::int4, $2::int4)`,
|
||||
diagnosticQuotaLockNamespace, input.UserID); err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("acquire diagnostic quota lock: %w", err)
|
||||
}
|
||||
|
||||
now := input.Now
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
if err := r.checkQuotas(ctx, tx, input, now); err != nil {
|
||||
return InsertReceivingResult{}, err
|
||||
}
|
||||
|
||||
shortIDGenerator := input.ShortIDGenerator
|
||||
if shortIDGenerator == nil {
|
||||
shortIDGenerator = NewShortID
|
||||
}
|
||||
reportIDGenerator := input.ReportIDGenerator
|
||||
if reportIDGenerator == nil {
|
||||
reportIDGenerator = uuid.NewString
|
||||
}
|
||||
retries := input.ShortIDCollisionRetries
|
||||
if retries <= 0 {
|
||||
retries = defaultShortIDRetries
|
||||
}
|
||||
|
||||
crashSummary := truncateCrashSummary(input.CrashSummary)
|
||||
sessionIDs := input.PlaybackSessionIDs
|
||||
if sessionIDs == nil {
|
||||
// pgx binds a nil slice as SQL NULL, which bypasses the column's
|
||||
// '{}' default and violates its NOT NULL constraint.
|
||||
sessionIDs = []string{}
|
||||
}
|
||||
for attempt := 0; attempt < retries; attempt++ {
|
||||
id := reportIDGenerator()
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("%w: report id must be a UUID", ErrInvalidReportInput)
|
||||
}
|
||||
shortID, err := shortIDGenerator()
|
||||
if err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("generate diagnostic short id: %w", err)
|
||||
}
|
||||
shortID, err = ParseShortID(shortID)
|
||||
if err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("%w: generated short id is invalid", err)
|
||||
}
|
||||
|
||||
var insertedID, insertedShortID string
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO client_diagnostic_reports (
|
||||
id, short_id, user_id, profile_id, state, captured_at, report_type,
|
||||
platform, app_version, crash_summary, manifest, playback_session_ids,
|
||||
blob_bytes
|
||||
)
|
||||
VALUES ($1::uuid, $2, $3, $4, 'receiving', $5, $6, $7, $8, $9, $10::jsonb, $11, $12)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING id::text, short_id
|
||||
`, id, shortID, input.UserID, nullableString(input.ProfileID), input.CapturedAt,
|
||||
strings.TrimSpace(input.ReportType), strings.TrimSpace(input.Platform),
|
||||
strings.TrimSpace(input.AppVersion), nullableString(crashSummary),
|
||||
string(input.Manifest), sessionIDs, input.ExpectedBlobBytes).Scan(&insertedID, &insertedShortID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("insert diagnostic report: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return InsertReceivingResult{}, fmt.Errorf("commit diagnostic report insert: %w", err)
|
||||
}
|
||||
return InsertReceivingResult{ID: insertedID, ShortID: insertedShortID}, nil
|
||||
}
|
||||
|
||||
return InsertReceivingResult{}, ErrShortIDExhausted
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) checkQuotas(ctx context.Context, tx pgx.Tx, input InsertReceivingInput, now time.Time) error {
|
||||
if input.MaxReportsPerUserDay > 0 {
|
||||
var count int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM client_diagnostic_reports
|
||||
WHERE user_id = $1
|
||||
AND state IN ('receiving', 'ready')
|
||||
AND received_at >= date_trunc('day', $2::timestamptz)
|
||||
`, input.UserID, now).Scan(&count); err != nil {
|
||||
return fmt.Errorf("count diagnostic reports for quota: %w", err)
|
||||
}
|
||||
if count >= input.MaxReportsPerUserDay {
|
||||
return &QuotaError{Kind: QuotaKindReportsPerDay, Limit: int64(input.MaxReportsPerUserDay)}
|
||||
}
|
||||
}
|
||||
|
||||
if input.MaxBytesPerUser > 0 {
|
||||
// Count 'receiving' rows too: a new upload reserves its expected size in
|
||||
// blob_bytes before the bundle streams in, so concurrent or multi-node
|
||||
// uploads can't each pass a near-limit check and both mark ready. Ready
|
||||
// rows overwrite the reservation with their actual size on completion.
|
||||
var used int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(blob_bytes), 0)
|
||||
FROM client_diagnostic_reports
|
||||
WHERE user_id = $1
|
||||
AND state IN ('receiving', 'ready')
|
||||
`, input.UserID).Scan(&used); err != nil {
|
||||
return fmt.Errorf("sum diagnostic bytes for quota: %w", err)
|
||||
}
|
||||
if input.ExpectedBlobBytes > input.MaxBytesPerUser || used+input.ExpectedBlobBytes > input.MaxBytesPerUser {
|
||||
return &QuotaError{Kind: QuotaKindBytesPerUser, Limit: input.MaxBytesPerUser}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) MarkReady(ctx context.Context, id string, blob BlobInfo) error {
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE client_diagnostic_reports
|
||||
SET state = 'ready',
|
||||
blob_bucket = $2,
|
||||
blob_key = $3,
|
||||
blob_bytes = $4,
|
||||
uncompressed_bytes = $5,
|
||||
blob_sha256 = $6
|
||||
WHERE id = $1::uuid
|
||||
`, id, strings.TrimSpace(blob.Bucket), strings.TrimSpace(blob.Key), blob.Bytes, blob.UncompressedBytes, strings.TrimSpace(blob.SHA256))
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark diagnostic report ready: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) MarkFailed(ctx context.Context, id string) error {
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE client_diagnostic_reports
|
||||
SET state = 'failed'
|
||||
WHERE id = $1::uuid
|
||||
`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark diagnostic report failed: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) GetByID(ctx context.Context, id string) (*Report, error) {
|
||||
row := r.pool.QueryRow(ctx, reportSelectSQL()+`
|
||||
WHERE id = $1::uuid
|
||||
`, id)
|
||||
report, err := scanReport(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) ListForAdmin(ctx context.Context, filters ListFilters) (ListResult, error) {
|
||||
limit := filters.Limit
|
||||
if limit <= 0 || limit > maxAdminListLimit {
|
||||
limit = defaultAdminListLimit
|
||||
}
|
||||
|
||||
conditions := []string{"1=1"}
|
||||
args := make([]any, 0, 10)
|
||||
argIdx := 1
|
||||
|
||||
if filters.UserID != nil {
|
||||
conditions = append(conditions, fmt.Sprintf("user_id = $%d", argIdx))
|
||||
args = append(args, *filters.UserID)
|
||||
argIdx++
|
||||
}
|
||||
if strings.TrimSpace(filters.Platform) != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("platform = $%d", argIdx))
|
||||
args = append(args, strings.TrimSpace(filters.Platform))
|
||||
argIdx++
|
||||
}
|
||||
if strings.TrimSpace(filters.ReportType) != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("report_type = $%d", argIdx))
|
||||
args = append(args, strings.TrimSpace(filters.ReportType))
|
||||
argIdx++
|
||||
}
|
||||
if filters.From != nil {
|
||||
conditions = append(conditions, fmt.Sprintf("received_at >= $%d", argIdx))
|
||||
args = append(args, *filters.From)
|
||||
argIdx++
|
||||
}
|
||||
if filters.To != nil {
|
||||
conditions = append(conditions, fmt.Sprintf("received_at <= $%d", argIdx))
|
||||
args = append(args, *filters.To)
|
||||
argIdx++
|
||||
}
|
||||
if strings.TrimSpace(filters.ShortID) != "" {
|
||||
shortID, err := ParseShortID(filters.ShortID)
|
||||
if err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("lower(short_id) = lower($%d)", argIdx))
|
||||
args = append(args, shortID)
|
||||
argIdx++
|
||||
}
|
||||
if strings.TrimSpace(filters.Cursor) != "" {
|
||||
cursorReceivedAt, cursorID, err := decodeReportCursor(filters.Cursor)
|
||||
if err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("(received_at, id) < ($%d, $%d::uuid)", argIdx, argIdx+1))
|
||||
args = append(args, cursorReceivedAt, cursorID)
|
||||
argIdx += 2
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("%s\n\t\tWHERE %s\n\t\tORDER BY received_at DESC, id DESC\n\t\tLIMIT $%d",
|
||||
reportListSelectSQL(), strings.Join(conditions, " AND "), argIdx)
|
||||
args = append(args, limit+1)
|
||||
|
||||
rows, err := r.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return ListResult{}, fmt.Errorf("list diagnostic reports: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
reports := make([]Report, 0, limit+1)
|
||||
for rows.Next() {
|
||||
report, err := scanReportSummary(rows)
|
||||
if err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
reports = append(reports, *report)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return ListResult{}, fmt.Errorf("iterate diagnostic reports: %w", err)
|
||||
}
|
||||
|
||||
result := ListResult{}
|
||||
if len(reports) > limit {
|
||||
last := reports[limit-1]
|
||||
result.NextCursor = encodeReportCursor(last.ReceivedAt, last.ID)
|
||||
reports = reports[:limit]
|
||||
}
|
||||
result.Reports = reports
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) DeleteByID(ctx context.Context, id string) (*Report, error) {
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
DELETE FROM client_diagnostic_reports
|
||||
WHERE id = $1::uuid
|
||||
RETURNING id::text, short_id, user_id, profile_id, state, captured_at, received_at,
|
||||
report_type, platform, app_version, crash_summary, manifest, playback_session_ids,
|
||||
blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256,
|
||||
COALESCE(manifest->'report'->>'app_build', '')
|
||||
`, id)
|
||||
report, err := scanReport(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) RetentionCandidates(ctx context.Context, olderThan time.Time, perUserByteCap int64) ([]Report, error) {
|
||||
if olderThan.IsZero() && perUserByteCap <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
query := reportCleanupSelectSQL() + `
|
||||
WHERE received_at < $1
|
||||
ORDER BY received_at ASC, id ASC
|
||||
`
|
||||
args := []any{olderThan}
|
||||
if olderThan.IsZero() {
|
||||
query = reportCleanupSelectSQL() + `
|
||||
WHERE false
|
||||
`
|
||||
args = nil
|
||||
}
|
||||
|
||||
if perUserByteCap > 0 {
|
||||
byteCapQuery := `
|
||||
WITH quota_candidates AS (
|
||||
SELECT id
|
||||
FROM (
|
||||
SELECT id,
|
||||
SUM(COALESCE(blob_bytes, 0)) OVER (
|
||||
PARTITION BY user_id
|
||||
ORDER BY received_at DESC, id DESC
|
||||
) AS retained_bytes
|
||||
FROM client_diagnostic_reports
|
||||
WHERE state = 'ready'
|
||||
) ranked
|
||||
WHERE retained_bytes > $1
|
||||
)
|
||||
` + reportCleanupSelectSQL() + `
|
||||
WHERE id IN (SELECT id FROM quota_candidates)
|
||||
ORDER BY received_at ASC, id ASC
|
||||
`
|
||||
if olderThan.IsZero() {
|
||||
return r.queryReportCleanup(ctx, byteCapQuery, perUserByteCap)
|
||||
}
|
||||
|
||||
query = `
|
||||
WITH quota_candidates AS (
|
||||
SELECT id
|
||||
FROM (
|
||||
SELECT id,
|
||||
SUM(COALESCE(blob_bytes, 0)) OVER (
|
||||
PARTITION BY user_id
|
||||
ORDER BY received_at DESC, id DESC
|
||||
) AS retained_bytes
|
||||
FROM client_diagnostic_reports
|
||||
WHERE state = 'ready'
|
||||
) ranked
|
||||
WHERE retained_bytes > $2
|
||||
)
|
||||
` + reportCleanupSelectSQL() + `
|
||||
WHERE received_at < $1 OR id IN (SELECT id FROM quota_candidates)
|
||||
ORDER BY received_at ASC, id ASC
|
||||
`
|
||||
args = []any{olderThan, perUserByteCap}
|
||||
}
|
||||
|
||||
return r.queryReportCleanup(ctx, query, args...)
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) StaleReceiving(ctx context.Context, grace time.Duration) ([]Report, error) {
|
||||
if grace <= 0 {
|
||||
grace = time.Hour
|
||||
}
|
||||
cutoff := time.Now().UTC().Add(-grace)
|
||||
return r.queryReportCleanup(ctx, reportCleanupSelectSQL()+`
|
||||
WHERE state IN ('receiving', 'failed')
|
||||
AND received_at < $1
|
||||
ORDER BY received_at ASC, id ASC
|
||||
`, cutoff)
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) LiveBlobKeys(ctx context.Context, keys []string) (map[string]ReportState, error) {
|
||||
live := make(map[string]ReportState, len(keys))
|
||||
if len(keys) == 0 {
|
||||
return live, nil
|
||||
}
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT candidate.key, report.state
|
||||
FROM unnest($1::text[]) AS candidate(key)
|
||||
JOIN client_diagnostic_reports report
|
||||
ON report.blob_key = candidate.key
|
||||
OR ($2 || report.user_id::text || '/' || report.id::text || '.tar.gz') = candidate.key
|
||||
WHERE report.state IN ('ready', 'receiving')
|
||||
`, keys, ObjectPrefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load live diagnostic blob keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var key string
|
||||
var state ReportState
|
||||
if err := rows.Scan(&key, &state); err != nil {
|
||||
return nil, fmt.Errorf("scan live diagnostic blob key: %w", err)
|
||||
}
|
||||
live[key] = state
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate live diagnostic blob keys: %w", err)
|
||||
}
|
||||
return live, nil
|
||||
}
|
||||
|
||||
// queryReportCleanup runs a reportCleanupSelectSQL-shaped query; the returned
|
||||
// reports carry no manifest and no app_build. Only the retention and
|
||||
// stale-cleanup batches use it.
|
||||
func (r *PostgresRepository) queryReportCleanup(ctx context.Context, query string, args ...any) ([]Report, error) {
|
||||
rows, err := r.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query diagnostic reports: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
reports := []Report{}
|
||||
for rows.Next() {
|
||||
report, err := scanReportCleanup(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports = append(reports, *report)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate diagnostic reports: %w", err)
|
||||
}
|
||||
return reports, nil
|
||||
}
|
||||
|
||||
func validateInsertReceivingInput(input InsertReceivingInput) error {
|
||||
if input.UserID <= 0 {
|
||||
return fmt.Errorf("%w: user id is required", ErrInvalidReportInput)
|
||||
}
|
||||
if input.CapturedAt.IsZero() {
|
||||
return fmt.Errorf("%w: captured_at is required", ErrInvalidReportInput)
|
||||
}
|
||||
if strings.TrimSpace(input.ReportType) == "" {
|
||||
return fmt.Errorf("%w: report type is required", ErrInvalidReportInput)
|
||||
}
|
||||
if strings.TrimSpace(input.Platform) == "" {
|
||||
return fmt.Errorf("%w: platform is required", ErrInvalidReportInput)
|
||||
}
|
||||
if strings.TrimSpace(input.AppVersion) == "" {
|
||||
return fmt.Errorf("%w: app version is required", ErrInvalidReportInput)
|
||||
}
|
||||
if len(input.Manifest) == 0 || !json.Valid(input.Manifest) {
|
||||
return fmt.Errorf("%w: manifest must be valid JSON", ErrInvalidReportInput)
|
||||
}
|
||||
if input.ExpectedBlobBytes < 0 {
|
||||
return fmt.Errorf("%w: expected blob bytes cannot be negative", ErrInvalidReportInput)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reportSelectSQL is the full projection including the manifest JSONB (up to
|
||||
// MaxManifestBytes per row). Reserve it for single-report detail/download paths;
|
||||
// the admin list uses reportListSelectSQL and retention/stale batches use
|
||||
// reportCleanupSelectSQL so browsing a page or running a cleanup pass doesn't
|
||||
// drag every report's manifest through the DB.
|
||||
func reportSelectSQL() string {
|
||||
return `SELECT id::text, short_id, user_id, profile_id, state, captured_at, received_at,
|
||||
report_type, platform, app_version, crash_summary, manifest, playback_session_ids,
|
||||
blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256,
|
||||
COALESCE(manifest->'report'->>'app_build', '')
|
||||
FROM client_diagnostic_reports`
|
||||
}
|
||||
|
||||
// reportListSelectSQL is the admin-list projection: it omits the full manifest
|
||||
// column but still extracts app_build from the manifest JSONB (a cheap text
|
||||
// extraction) so list rows can show the build number without shipping the whole
|
||||
// manifest. Reports scanned with scanReportSummary therefore have a nil
|
||||
// Manifest. Cleanup batches must not pay for the JSONB extraction, so they use
|
||||
// reportCleanupSelectSQL instead.
|
||||
func reportListSelectSQL() string {
|
||||
return `SELECT id::text, short_id, user_id, profile_id, state, captured_at, received_at,
|
||||
report_type, platform, app_version, crash_summary, playback_session_ids,
|
||||
blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256,
|
||||
COALESCE(manifest->'report'->>'app_build', '')
|
||||
FROM client_diagnostic_reports`
|
||||
}
|
||||
|
||||
// reportCleanupSelectSQL is the retention/stale-cleanup projection. It omits
|
||||
// both the manifest column and the app_build extraction because cleanup only
|
||||
// needs id/state/blob fields to delete a row and its object; touching the JSONB
|
||||
// per candidate would turn a scheduled pass over many large manifests into
|
||||
// needless manifest I/O. Reports scanned with scanReportCleanup therefore have
|
||||
// a nil Manifest and an empty AppBuild.
|
||||
func reportCleanupSelectSQL() string {
|
||||
return `SELECT id::text, short_id, user_id, profile_id, state, captured_at, received_at,
|
||||
report_type, platform, app_version, crash_summary, playback_session_ids,
|
||||
blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256
|
||||
FROM client_diagnostic_reports`
|
||||
}
|
||||
|
||||
type reportScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
// reportNulls holds the nullable columns shared by the full and summary
|
||||
// projections so both scanners apply them the same way.
|
||||
type reportNulls struct {
|
||||
profileID sql.NullString
|
||||
crashSummary sql.NullString
|
||||
blobBucket sql.NullString
|
||||
blobKey sql.NullString
|
||||
blobSHA256 sql.NullString
|
||||
blobBytes sql.NullInt64
|
||||
uncompressedBytes sql.NullInt64
|
||||
}
|
||||
|
||||
func (n *reportNulls) apply(report *Report) {
|
||||
if n.profileID.Valid {
|
||||
report.ProfileID = &n.profileID.String
|
||||
}
|
||||
if n.crashSummary.Valid {
|
||||
report.CrashSummary = &n.crashSummary.String
|
||||
}
|
||||
if report.PlaybackSessionIDs == nil {
|
||||
report.PlaybackSessionIDs = []string{}
|
||||
}
|
||||
if n.blobBucket.Valid {
|
||||
report.BlobBucket = &n.blobBucket.String
|
||||
}
|
||||
if n.blobKey.Valid {
|
||||
report.BlobKey = &n.blobKey.String
|
||||
}
|
||||
if n.blobBytes.Valid {
|
||||
report.BlobBytes = &n.blobBytes.Int64
|
||||
}
|
||||
if n.uncompressedBytes.Valid {
|
||||
report.UncompressedBytes = &n.uncompressedBytes.Int64
|
||||
}
|
||||
if n.blobSHA256.Valid {
|
||||
report.BlobSHA256 = &n.blobSHA256.String
|
||||
}
|
||||
}
|
||||
|
||||
// scanReport scans the full reportSelectSQL projection, including the manifest.
|
||||
func scanReport(row reportScanner) (*Report, error) {
|
||||
var report Report
|
||||
var nulls reportNulls
|
||||
var manifest []byte
|
||||
if err := row.Scan(
|
||||
&report.ID,
|
||||
&report.ShortID,
|
||||
&report.UserID,
|
||||
&nulls.profileID,
|
||||
&report.State,
|
||||
&report.CapturedAt,
|
||||
&report.ReceivedAt,
|
||||
&report.ReportType,
|
||||
&report.Platform,
|
||||
&report.AppVersion,
|
||||
&nulls.crashSummary,
|
||||
&manifest,
|
||||
&report.PlaybackSessionIDs,
|
||||
&nulls.blobBucket,
|
||||
&nulls.blobKey,
|
||||
&nulls.blobBytes,
|
||||
&nulls.uncompressedBytes,
|
||||
&nulls.blobSHA256,
|
||||
&report.AppBuild,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan diagnostic report: %w", err)
|
||||
}
|
||||
if len(manifest) > 0 {
|
||||
report.Manifest = append(json.RawMessage(nil), manifest...)
|
||||
}
|
||||
nulls.apply(&report)
|
||||
return &report, nil
|
||||
}
|
||||
|
||||
// summaryScanTargets returns the scan destinations shared by the admin-list and
|
||||
// cleanup projections, in column order. The two scanners differ only in whether
|
||||
// app_build is appended, so keeping the common prefix in one place stops the
|
||||
// long Scan lists from drifting apart.
|
||||
func summaryScanTargets(report *Report, nulls *reportNulls) []any {
|
||||
return []any{
|
||||
&report.ID,
|
||||
&report.ShortID,
|
||||
&report.UserID,
|
||||
&nulls.profileID,
|
||||
&report.State,
|
||||
&report.CapturedAt,
|
||||
&report.ReceivedAt,
|
||||
&report.ReportType,
|
||||
&report.Platform,
|
||||
&report.AppVersion,
|
||||
&nulls.crashSummary,
|
||||
&report.PlaybackSessionIDs,
|
||||
&nulls.blobBucket,
|
||||
&nulls.blobKey,
|
||||
&nulls.blobBytes,
|
||||
&nulls.uncompressedBytes,
|
||||
&nulls.blobSHA256,
|
||||
}
|
||||
}
|
||||
|
||||
// scanReportSummary scans the reportListSelectSQL projection, which omits the
|
||||
// manifest but includes app_build; the returned report's Manifest is left nil.
|
||||
func scanReportSummary(row reportScanner) (*Report, error) {
|
||||
var report Report
|
||||
var nulls reportNulls
|
||||
targets := append(summaryScanTargets(&report, &nulls), &report.AppBuild)
|
||||
if err := row.Scan(targets...); err != nil {
|
||||
return nil, fmt.Errorf("scan diagnostic report: %w", err)
|
||||
}
|
||||
nulls.apply(&report)
|
||||
return &report, nil
|
||||
}
|
||||
|
||||
// scanReportCleanup scans the reportCleanupSelectSQL projection, which omits
|
||||
// both the manifest and app_build; the returned report's Manifest is nil and
|
||||
// AppBuild is empty. Cleanup only reads id/state/blob fields.
|
||||
func scanReportCleanup(row reportScanner) (*Report, error) {
|
||||
var report Report
|
||||
var nulls reportNulls
|
||||
if err := row.Scan(summaryScanTargets(&report, &nulls)...); err != nil {
|
||||
return nil, fmt.Errorf("scan diagnostic report: %w", err)
|
||||
}
|
||||
nulls.apply(&report)
|
||||
return &report, nil
|
||||
}
|
||||
|
||||
func nullableString(s *string) any {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(*s) == "" {
|
||||
return nil
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func encodeReportCursor(receivedAt time.Time, id string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%d|%s", receivedAt.UnixNano(), id)))
|
||||
}
|
||||
|
||||
func decodeReportCursor(cursor string) (time.Time, string, error) {
|
||||
raw, err := base64.RawURLEncoding.DecodeString(cursor)
|
||||
if err != nil {
|
||||
return time.Time{}, "", fmt.Errorf("%w: decode: %v", ErrInvalidCursor, err)
|
||||
}
|
||||
parts := strings.Split(string(raw), "|")
|
||||
if len(parts) != 2 {
|
||||
return time.Time{}, "", ErrInvalidCursor
|
||||
}
|
||||
nanos, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, "", fmt.Errorf("%w: timestamp: %v", ErrInvalidCursor, err)
|
||||
}
|
||||
id := parts[1]
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return time.Time{}, "", fmt.Errorf("%w: id: %v", ErrInvalidCursor, err)
|
||||
}
|
||||
return time.Unix(0, nanos).UTC(), id, nil
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics/contract"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxManifestBytes = int64(64 * 1024)
|
||||
BundleContentType = "application/gzip"
|
||||
diagnosticObjectPrefix = "diagnostics"
|
||||
|
||||
failedUploadCompensationTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
ErrDisabled = errors.New("diagnostics disabled")
|
||||
ErrStorageUnavailable = errors.New("diagnostics storage unavailable")
|
||||
ErrTooLarge = errors.New("diagnostics upload too large")
|
||||
ErrUnsupportedSchema = errors.New("diagnostics schema version unsupported")
|
||||
ErrDestinationMismatch = errors.New("diagnostics destination mismatch")
|
||||
ErrStaleConsent = errors.New("diagnostics consent notice is stale")
|
||||
ErrArchiveMismatch = errors.New("diagnostics archive metadata mismatch")
|
||||
ErrReportStoreUnavailable = errors.New("diagnostics report store unavailable")
|
||||
ErrProfileMismatch = errors.New("diagnostics profile id mismatch")
|
||||
ErrChildProfileForbidden = errors.New("diagnostics child profile attribution forbidden")
|
||||
)
|
||||
|
||||
type AvailabilityStatus string
|
||||
|
||||
const (
|
||||
StatusAvailable AvailabilityStatus = "available"
|
||||
StatusDisabled AvailabilityStatus = "disabled"
|
||||
StatusStorageUnavailable AvailabilityStatus = "storage_unavailable"
|
||||
)
|
||||
|
||||
type Status struct {
|
||||
Status AvailabilityStatus `json:"status"`
|
||||
ServerInstanceID string `json:"server_instance_id"`
|
||||
AcceptedSchemaVersions []int `json:"accepted_schema_versions"`
|
||||
MaxBundleBytes int64 `json:"max_bundle_bytes"`
|
||||
MaxManifestBytes int64 `json:"max_manifest_bytes"`
|
||||
RetentionDays int `json:"retention_days"`
|
||||
ConsentNoticeVersion int `json:"consent_notice_version"`
|
||||
}
|
||||
|
||||
type IngestResult struct {
|
||||
ReportID string `json:"report_id"`
|
||||
ShortID string `json:"short_id"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
reports ReportStore
|
||||
settings SettingsStore
|
||||
store ObjectStore
|
||||
profileValidator ProfileAttributionValidator
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewService(reports ReportStore, settings SettingsStore, store ObjectStore, logger *slog.Logger) *Service {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Service{
|
||||
reports: reports,
|
||||
settings: settings,
|
||||
store: store,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
}
|
||||
|
||||
type ProfileAttributionValidator interface {
|
||||
ProfileBelongsToUser(ctx context.Context, userID int, profileID string) (bool, error)
|
||||
}
|
||||
|
||||
type ProfileAttributionValidatorFunc func(ctx context.Context, userID int, profileID string) (bool, error)
|
||||
|
||||
func (f ProfileAttributionValidatorFunc) ProfileBelongsToUser(ctx context.Context, userID int, profileID string) (bool, error) {
|
||||
return f(ctx, userID, profileID)
|
||||
}
|
||||
|
||||
// ProfileLookup resolves whether profileID exists for userID and whether it is
|
||||
// a child profile. It backs NewProfileAttributionValidator.
|
||||
type ProfileLookup func(ctx context.Context, userID int, profileID string) (found bool, isChild bool, err error)
|
||||
|
||||
// NewProfileAttributionValidator builds a ProfileAttributionValidator that
|
||||
// attributes a report to a profile only when it belongs to the user and is not
|
||||
// a child profile. The client diagnostics design forbids child profiles from
|
||||
// performing diagnostics actions, so a nonconforming client that sends a child
|
||||
// profile's ID (via X-Profile-Id or a manifest profile_id) has the upload
|
||||
// rejected with ErrChildProfileForbidden rather than silently losing the
|
||||
// attribution. A profile that is simply not the user's (found == false) is not
|
||||
// an error: attribution is dropped and the upload proceeds unattributed.
|
||||
func NewProfileAttributionValidator(lookup ProfileLookup) ProfileAttributionValidator {
|
||||
return ProfileAttributionValidatorFunc(func(ctx context.Context, userID int, profileID string) (bool, error) {
|
||||
found, isChild, err := lookup(ctx, userID, profileID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if found && isChild {
|
||||
return false, ErrChildProfileForbidden
|
||||
}
|
||||
return found, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) SetProfileAttributionValidator(validator ProfileAttributionValidator) {
|
||||
s.profileValidator = validator
|
||||
}
|
||||
|
||||
func (s *Service) Status(ctx context.Context, userID int) (Status, error) {
|
||||
settings, err := s.currentSettings(ctx)
|
||||
if err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
return s.statusFromSettings(settings), nil
|
||||
}
|
||||
|
||||
func (s *Service) Ingest(ctx context.Context, userID int, profileID *string, manifestJSON []byte, bundle io.Reader) (IngestResult, error) {
|
||||
settings, err := s.currentSettings(ctx)
|
||||
if err != nil {
|
||||
s.logRejected(ctx, "", userID, "", "", 0, "settings_error")
|
||||
return IngestResult{}, err
|
||||
}
|
||||
status := s.statusFromSettings(settings).Status
|
||||
switch status {
|
||||
case StatusDisabled:
|
||||
s.logRejected(ctx, "", userID, "", "", 0, "disabled")
|
||||
return IngestResult{}, ErrDisabled
|
||||
case StatusStorageUnavailable:
|
||||
s.logRejected(ctx, "", userID, "", "", 0, "storage_unavailable")
|
||||
return IngestResult{}, ErrStorageUnavailable
|
||||
}
|
||||
if s.reports == nil {
|
||||
s.logRejected(ctx, "", userID, "", "", 0, "report_store_unavailable")
|
||||
return IngestResult{}, ErrReportStoreUnavailable
|
||||
}
|
||||
if len(manifestJSON) > int(MaxManifestBytes) {
|
||||
s.logRejected(ctx, "", userID, "", "", int64(len(manifestJSON)), "too_large")
|
||||
return IngestResult{}, ErrTooLarge
|
||||
}
|
||||
if manifestHasUnsupportedSchema(manifestJSON) {
|
||||
s.logRejected(ctx, "", userID, "", "", int64(len(manifestJSON)), "unsupported_schema")
|
||||
return IngestResult{}, ErrUnsupportedSchema
|
||||
}
|
||||
|
||||
manifest, err := contract.ValidateManifest(manifestJSON)
|
||||
if err != nil {
|
||||
s.logRejected(ctx, "", userID, "", "", int64(len(manifestJSON)), "invalid_manifest")
|
||||
return IngestResult{}, fmt.Errorf("%w: invalid manifest: %v", ErrInvalidBundle, err)
|
||||
}
|
||||
if manifest.Archive.Bytes > settings.MaxBundleBytes {
|
||||
s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "too_large")
|
||||
return IngestResult{}, ErrTooLarge
|
||||
}
|
||||
if manifest.Destination.ServerInstanceID != settings.ServerInstanceID {
|
||||
s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "destination_mismatch")
|
||||
return IngestResult{}, ErrDestinationMismatch
|
||||
}
|
||||
if manifest.Consent.NoticeVersion != settings.ConsentNoticeVersion {
|
||||
s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "stale_consent")
|
||||
return IngestResult{}, ErrStaleConsent
|
||||
}
|
||||
|
||||
reportProfileID, err := s.validatedProfileID(ctx, userID, profileID, manifest.Report.ProfileID)
|
||||
if err != nil {
|
||||
s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "profile_validation_error")
|
||||
return IngestResult{}, err
|
||||
}
|
||||
crashSummary := crashSummaryFromManifest(manifest)
|
||||
// ExpectedBlobBytes reserves the client-claimed archive size against the
|
||||
// per-user byte quota before the bundle streams in. Trusting the claim here is
|
||||
// safe: a report only reaches MarkReady (which rewrites blob_bytes to the
|
||||
// measured size) after archiveMatches confirms the claimed bytes/sha/entries
|
||||
// exactly equal the streamed bundle, so no stored report can ever occupy more
|
||||
// bytes than it reserved. A client that under-claims to under-reserve then
|
||||
// uploads a larger bundle fails archiveMatches and is compensated (blob
|
||||
// deleted, row marked failed) before it counts, gaining no storage. The
|
||||
// residual cost of such rejected churn is bandwidth, not storage — bounded by
|
||||
// the per-account single in-flight upload cap, not by max_bytes_per_user,
|
||||
// which is a storage cap enforced oldest-first by retention.
|
||||
reserved, err := s.reports.InsertReceiving(ctx, InsertReceivingInput{
|
||||
UserID: userID,
|
||||
ProfileID: reportProfileID,
|
||||
CapturedAt: manifest.Report.CapturedAt,
|
||||
ReportType: manifest.Report.Type,
|
||||
Platform: manifest.Report.Platform,
|
||||
AppVersion: manifest.Report.AppVersion,
|
||||
CrashSummary: crashSummary,
|
||||
Manifest: append(json.RawMessage(nil), manifestJSON...),
|
||||
PlaybackSessionIDs: manifest.PlaybackSessionIDs,
|
||||
ExpectedBlobBytes: manifest.Archive.Bytes,
|
||||
MaxReportsPerUserDay: settings.MaxReportsPerUserDay,
|
||||
MaxBytesPerUser: settings.MaxBytesPerUser,
|
||||
Now: s.now(),
|
||||
})
|
||||
if err != nil {
|
||||
reason := "insert_failed"
|
||||
if errors.Is(err, ErrQuotaExceeded) {
|
||||
reason = "quota_exceeded"
|
||||
} else {
|
||||
s.logger.ErrorContext(ctx, "diagnostic report insert failed",
|
||||
"component", "diagnostics",
|
||||
"user_id", userID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, reason)
|
||||
return IngestResult{}, err
|
||||
}
|
||||
|
||||
key := reportObjectKey(userID, reserved.ID)
|
||||
info, err := s.putValidatedBundle(ctx, key, bundle, settings)
|
||||
if err != nil {
|
||||
s.compensateFailedUpload(ctx, reserved.ID, key)
|
||||
rejectErr := classifyBundleUploadError(err)
|
||||
s.logger.ErrorContext(ctx, "diagnostic bundle validation failed",
|
||||
"component", "diagnostics",
|
||||
"report_id", reserved.ID,
|
||||
"user_id", userID,
|
||||
"error", err,
|
||||
)
|
||||
s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, rejectReason(rejectErr))
|
||||
return IngestResult{}, rejectErr
|
||||
}
|
||||
if !archiveMatches(manifest.Archive, info) {
|
||||
s.compensateFailedUpload(ctx, reserved.ID, key)
|
||||
s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, info.CompressedBytes, "archive_mismatch")
|
||||
return IngestResult{}, ErrArchiveMismatch
|
||||
}
|
||||
if !embeddedManifestMatches(manifestJSON, info.EmbeddedManifest) {
|
||||
s.compensateFailedUpload(ctx, reserved.ID, key)
|
||||
s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, info.CompressedBytes, "manifest_mismatch")
|
||||
return IngestResult{}, ErrArchiveMismatch
|
||||
}
|
||||
|
||||
if err := s.reports.MarkReady(ctx, reserved.ID, BlobInfo{
|
||||
Bucket: s.store.Bucket(),
|
||||
Key: key,
|
||||
Bytes: info.CompressedBytes,
|
||||
UncompressedBytes: info.UncompressedBytes,
|
||||
SHA256: info.SHA256,
|
||||
}); err != nil {
|
||||
s.compensateFailedUpload(ctx, reserved.ID, key)
|
||||
s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, info.CompressedBytes, "mark_ready_failed")
|
||||
return IngestResult{}, err
|
||||
}
|
||||
|
||||
s.logger.InfoContext(ctx, "diagnostic report accepted",
|
||||
"component", "diagnostics",
|
||||
"report_id", reserved.ID,
|
||||
"user_id", userID,
|
||||
"platform", manifest.Report.Platform,
|
||||
"type", manifest.Report.Type,
|
||||
"bytes", info.CompressedBytes,
|
||||
"result", "accepted",
|
||||
)
|
||||
return IngestResult{ReportID: reserved.ID, ShortID: reserved.ShortID}, nil
|
||||
}
|
||||
|
||||
func (s *Service) currentSettings(ctx context.Context) (Settings, error) {
|
||||
settings, err := LoadSettings(ctx, s.settings)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
if strings.TrimSpace(settings.ServerInstanceID) != "" || s.settings == nil {
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
instanceID, err := ensureServerInstanceID(ctx, s.settings)
|
||||
if err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
settings.ServerInstanceID = instanceID
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (s *Service) statusFromSettings(settings Settings) Status {
|
||||
availability := StatusDisabled
|
||||
if settings.UploadsEnabled {
|
||||
availability = StatusAvailable
|
||||
if s.store == nil || strings.TrimSpace(s.store.Bucket()) == "" {
|
||||
availability = StatusStorageUnavailable
|
||||
}
|
||||
}
|
||||
|
||||
return Status{
|
||||
Status: availability,
|
||||
ServerInstanceID: settings.ServerInstanceID,
|
||||
AcceptedSchemaVersions: []int{contract.SchemaVersion},
|
||||
MaxBundleBytes: settings.MaxBundleBytes,
|
||||
MaxManifestBytes: MaxManifestBytes,
|
||||
RetentionDays: settings.RetentionDays,
|
||||
ConsentNoticeVersion: settings.ConsentNoticeVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) putValidatedBundle(ctx context.Context, key string, bundle io.Reader, settings Settings) (BundleInfo, error) {
|
||||
pr, pw := io.Pipe()
|
||||
resultCh := make(chan bundleValidationResult, 1)
|
||||
go func() {
|
||||
info, err := ValidateBundle(pr, BundleLimits{
|
||||
MaxCompressedBytes: settings.MaxBundleBytes,
|
||||
MaxUncompressedBytes: settings.MaxUncompressedBytes,
|
||||
})
|
||||
if err != nil {
|
||||
_ = pr.CloseWithError(err)
|
||||
}
|
||||
resultCh <- bundleValidationResult{info: info, err: err}
|
||||
}()
|
||||
|
||||
reader := &validatingUploadReader{src: bundle, pipe: pw}
|
||||
putErr := s.store.PutStream(ctx, s.store.Bucket(), key, reader, BundleContentType)
|
||||
if putErr != nil {
|
||||
reader.abort(&bundleUploadAbortError{err: putErr})
|
||||
}
|
||||
result := <-resultCh
|
||||
if putErr != nil {
|
||||
if result.err != nil && !isBundleUploadAbortError(result.err) {
|
||||
return result.info, result.err
|
||||
}
|
||||
return BundleInfo{}, fmt.Errorf("%w: store diagnostic bundle: %w", ErrStorageUnavailable, putErr)
|
||||
}
|
||||
if result.err != nil {
|
||||
return result.info, result.err
|
||||
}
|
||||
return result.info, nil
|
||||
}
|
||||
|
||||
func (s *Service) compensateFailedUpload(ctx context.Context, reportID, key string) {
|
||||
compensationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), failedUploadCompensationTimeout)
|
||||
defer cancel()
|
||||
|
||||
if s.store != nil && key != "" {
|
||||
if err := s.store.DeleteObject(compensationCtx, s.store.Bucket(), key); err != nil {
|
||||
s.logger.WarnContext(compensationCtx, "diagnostic report blob compensation failed",
|
||||
"component", "diagnostics",
|
||||
"report_id", reportID,
|
||||
"key", key,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
if s.reports != nil && reportID != "" {
|
||||
if err := s.reports.MarkFailed(compensationCtx, reportID); err != nil {
|
||||
s.logger.WarnContext(compensationCtx, "diagnostic report failed-state compensation failed",
|
||||
"component", "diagnostics",
|
||||
"report_id", reportID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) logRejected(ctx context.Context, reportID string, userID int, platform, reportType string, bytes int64, reason string) {
|
||||
s.logger.InfoContext(ctx, "diagnostic report rejected",
|
||||
"component", "diagnostics",
|
||||
"report_id", reportID,
|
||||
"user_id", userID,
|
||||
"platform", platform,
|
||||
"type", reportType,
|
||||
"bytes", bytes,
|
||||
"result", "rejected",
|
||||
"reason", reason,
|
||||
)
|
||||
}
|
||||
|
||||
type bundleValidationResult struct {
|
||||
info BundleInfo
|
||||
err error
|
||||
}
|
||||
|
||||
type validatingUploadReader struct {
|
||||
src io.Reader
|
||||
pipe *io.PipeWriter
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (r *validatingUploadReader) Read(p []byte) (int, error) {
|
||||
n, readErr := r.src.Read(p)
|
||||
if n > 0 {
|
||||
if _, err := r.pipe.Write(p[:n]); err != nil {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
r.close(readErr)
|
||||
}
|
||||
return n, readErr
|
||||
}
|
||||
|
||||
func (r *validatingUploadReader) abort(err error) {
|
||||
r.close(err)
|
||||
}
|
||||
|
||||
func (r *validatingUploadReader) close(err error) {
|
||||
if r.closed {
|
||||
return
|
||||
}
|
||||
r.closed = true
|
||||
if errors.Is(err, io.EOF) {
|
||||
_ = r.pipe.Close()
|
||||
return
|
||||
}
|
||||
_ = r.pipe.CloseWithError(err)
|
||||
}
|
||||
|
||||
func classifyBundleUploadError(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, ErrCompressedTooLarge),
|
||||
errors.Is(err, ErrUncompressedTooLarge),
|
||||
errors.Is(err, ErrEntryTooLarge):
|
||||
return ErrTooLarge
|
||||
case errors.Is(err, ErrInvalidBundle),
|
||||
errors.Is(err, ErrTooManyEntries),
|
||||
errors.Is(err, ErrCompressionRatio):
|
||||
return fmt.Errorf("%w: %v", ErrInvalidBundle, err)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func rejectReason(err error) string {
|
||||
switch {
|
||||
case errors.Is(err, ErrTooLarge):
|
||||
return "too_large"
|
||||
case errors.Is(err, ErrUnsupportedSchema):
|
||||
return "unsupported_schema"
|
||||
case errors.Is(err, ErrInvalidBundle):
|
||||
return "invalid_bundle"
|
||||
default:
|
||||
return "storage_error"
|
||||
}
|
||||
}
|
||||
|
||||
func manifestHasUnsupportedSchema(data []byte) bool {
|
||||
var header struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &header); err != nil {
|
||||
return false
|
||||
}
|
||||
return header.SchemaVersion != nil && *header.SchemaVersion != contract.SchemaVersion
|
||||
}
|
||||
|
||||
// resolveProfileID chooses the profile a report is attributed to from the
|
||||
// X-Profile-Id header and the manifest's report.profile_id fallback. When both
|
||||
// are present they must agree: a header captured under a different profile than
|
||||
// the manifest means the client switched profiles between capture and upload,
|
||||
// and silently preferring either source would attribute the DB row to one
|
||||
// profile while the stored manifest/download still names the other. Reject that
|
||||
// as ErrProfileMismatch. Single-source cases (only one present, or both equal)
|
||||
// resolve to that value; when neither is present the report is unattributed.
|
||||
func resolveProfileID(profileID *string, fallback string) (*string, error) {
|
||||
header := ""
|
||||
if profileID != nil {
|
||||
header = strings.TrimSpace(*profileID)
|
||||
}
|
||||
manifestID := strings.TrimSpace(fallback)
|
||||
|
||||
switch {
|
||||
case header != "" && manifestID != "" && header != manifestID:
|
||||
return nil, fmt.Errorf("%w: header %q, manifest %q", ErrProfileMismatch, header, manifestID)
|
||||
case header != "":
|
||||
return &header, nil
|
||||
case manifestID != "":
|
||||
return &manifestID, nil
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) validatedProfileID(ctx context.Context, userID int, profileID *string, fallback string) (*string, error) {
|
||||
candidate, err := resolveProfileID(profileID, fallback)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if candidate == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if s.profileValidator == nil {
|
||||
return nil, nil
|
||||
}
|
||||
found, err := s.profileValidator.ProfileBelongsToUser(ctx, userID, *candidate)
|
||||
if err != nil {
|
||||
// A child-profile attribution is a client-facing rejection, not an
|
||||
// internal failure: surface it verbatim so the handler can return a
|
||||
// clear error instead of a generic 500.
|
||||
if errors.Is(err, ErrChildProfileForbidden) {
|
||||
return nil, err
|
||||
}
|
||||
return nil, fmt.Errorf("validate diagnostic profile attribution: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return nil, nil
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func crashSummaryFromManifest(manifest contract.Manifest) *string {
|
||||
if manifest.Crash == nil {
|
||||
return nil
|
||||
}
|
||||
summary := manifest.Crash.Summary
|
||||
return &summary
|
||||
}
|
||||
|
||||
func reportObjectKey(userID int, reportID string) string {
|
||||
return diagnosticObjectPrefix + "/" + strconv.Itoa(userID) + "/" + reportID + ".tar.gz"
|
||||
}
|
||||
|
||||
func archiveMatches(archive contract.Archive, info BundleInfo) bool {
|
||||
return archive.Bytes == info.CompressedBytes &&
|
||||
archive.UncompressedBytes == info.UncompressedBytes &&
|
||||
strings.EqualFold(archive.SHA256, info.SHA256) &&
|
||||
slices.Equal(archive.Entries, info.Entries)
|
||||
}
|
||||
|
||||
// embeddedManifestMatches reports whether the manifest.json embedded as the
|
||||
// archive's first entry equals the received part-1 manifest with its `archive`
|
||||
// object removed, per the bundle contract. Both are decoded to JSON before
|
||||
// comparison so formatting or key-ordering differences don't matter.
|
||||
func embeddedManifestMatches(received, embedded []byte) bool {
|
||||
receivedMap, err := decodeJSONObject(received)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
delete(receivedMap, "archive")
|
||||
embeddedMap, err := decodeJSONObject(embedded)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(receivedMap, embeddedMap)
|
||||
}
|
||||
|
||||
func decodeJSONObject(data []byte) (map[string]any, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
// UseNumber keeps every JSON number as its exact decimal text (json.Number)
|
||||
// instead of float64. Without it, two manifests that differ only in a large
|
||||
// integer above 2^53 (e.g. log_summary.lines) both round to the same float
|
||||
// and DeepEqual reports a false match, letting a tampered bundle through.
|
||||
dec.UseNumber()
|
||||
var obj map[string]any
|
||||
if err := dec.Decode(&obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// json.Unmarshal rejects trailing bytes after the value; Decoder.Decode does
|
||||
// not, and Decoder.More() misses a stray closing delimiter (e.g. `{...}}`)
|
||||
// because it only reports iteration inside an array/object. Re-assert that
|
||||
// strictness by requiring the stream to be exhausted: a second Decode must
|
||||
// hit io.EOF, so a manifest with any trailing token — junk or an extra
|
||||
// `}`/`]` — is rejected and cannot compare equal. Trailing whitespace is
|
||||
// fine; the decoder skips it and still returns io.EOF.
|
||||
if err := dec.Decode(new(json.RawMessage)); err != io.EOF {
|
||||
return nil, errors.New("unexpected trailing data after JSON object")
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
@@ -0,0 +1,782 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/diagnostics/contract"
|
||||
)
|
||||
|
||||
func TestServiceIngestStoresReadyReport(t *testing.T) {
|
||||
bundle, manifest, info := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_1")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
result, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if err != nil {
|
||||
t.Fatalf("Ingest: %v", err)
|
||||
}
|
||||
if result.ReportID != "11111111-1111-1111-1111-111111111111" {
|
||||
t.Fatalf("ReportID = %q, want reserved report ID", result.ReportID)
|
||||
}
|
||||
if result.ShortID != "SILO-ABCDEF123456" {
|
||||
t.Fatalf("ShortID = %q, want SILO-ABCDEF123456", result.ShortID)
|
||||
}
|
||||
if len(repo.ready) != 1 || repo.ready[0] != result.ReportID {
|
||||
t.Fatalf("ready reports = %v, want [%s]", repo.ready, result.ReportID)
|
||||
}
|
||||
if len(repo.readyBlobs) != 1 {
|
||||
t.Fatalf("ready blobs = %v, want one", repo.readyBlobs)
|
||||
}
|
||||
wantKey := "diagnostics/42/11111111-1111-1111-1111-111111111111.tar.gz"
|
||||
readyBlob := repo.readyBlobs[0]
|
||||
if readyBlob.Bucket != "private" || readyBlob.Key != wantKey {
|
||||
t.Fatalf("ready blob location = %s/%s, want private/%s", readyBlob.Bucket, readyBlob.Key, wantKey)
|
||||
}
|
||||
if readyBlob.SHA256 != info.SHA256 {
|
||||
t.Fatalf("ready blob sha256 = %q, want %q", readyBlob.SHA256, info.SHA256)
|
||||
}
|
||||
if len(store.puts) != 1 || store.puts[0] != wantKey {
|
||||
t.Fatalf("stored keys = %v, want [%s]", store.puts, wantKey)
|
||||
}
|
||||
if !bytes.Equal(store.data, bundle) {
|
||||
t.Fatal("stored bundle does not match uploaded bundle")
|
||||
}
|
||||
if len(repo.failed) != 0 || len(store.deleted) != 0 {
|
||||
t.Fatalf("unexpected compensation: failed=%v deleted=%v", repo.failed, store.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestRejectsDestinationMismatch(t *testing.T) {
|
||||
bundle, info := testDiagnosticsBundle(t)
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
manifest := testManifestJSON(t, "other-server", DefaultConsentNoticeVer, info)
|
||||
_, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrDestinationMismatch) {
|
||||
t.Fatalf("Ingest error = %v, want ErrDestinationMismatch", err)
|
||||
}
|
||||
if repo.insertCalls != 0 {
|
||||
t.Fatalf("InsertReceiving calls = %d, want 0", repo.insertCalls)
|
||||
}
|
||||
if len(store.puts) != 0 {
|
||||
t.Fatalf("PutStream calls = %d, want 0", len(store.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestRejectsStaleConsent(t *testing.T) {
|
||||
bundle, info := testDiagnosticsBundle(t)
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer+1, info)
|
||||
_, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrStaleConsent) {
|
||||
t.Fatalf("Ingest error = %v, want ErrStaleConsent", err)
|
||||
}
|
||||
if repo.insertCalls != 0 {
|
||||
t.Fatalf("InsertReceiving calls = %d, want 0", repo.insertCalls)
|
||||
}
|
||||
if len(store.puts) != 0 {
|
||||
t.Fatalf("PutStream calls = %d, want 0", len(store.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestArchiveMismatchCompensates(t *testing.T) {
|
||||
bundle, info := testDiagnosticsBundle(t)
|
||||
info.SHA256 = strings.Repeat("0", 64)
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer, info)
|
||||
_, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrArchiveMismatch) {
|
||||
t.Fatalf("Ingest error = %v, want ErrArchiveMismatch", err)
|
||||
}
|
||||
if repo.insertCalls != 1 {
|
||||
t.Fatalf("InsertReceiving calls = %d, want 1", repo.insertCalls)
|
||||
}
|
||||
if len(store.puts) != 1 {
|
||||
t.Fatalf("PutStream calls = %d, want 1", len(store.puts))
|
||||
}
|
||||
wantKey := "diagnostics/42/11111111-1111-1111-1111-111111111111.tar.gz"
|
||||
if len(store.deleted) != 1 || store.deleted[0] != wantKey {
|
||||
t.Fatalf("deleted keys = %v, want [%s]", store.deleted, wantKey)
|
||||
}
|
||||
if len(repo.failed) != 1 || repo.failed[0] != "11111111-1111-1111-1111-111111111111" {
|
||||
t.Fatalf("failed reports = %v, want reserved report id", repo.failed)
|
||||
}
|
||||
if len(repo.ready) != 0 {
|
||||
t.Fatalf("ready reports = %v, want none", repo.ready)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestRejectsEmbeddedManifestMismatch(t *testing.T) {
|
||||
// The tar embeds a manifest.json that disagrees with the part-1 manifest
|
||||
// (different os_version) while the archive size/sha fields still match the
|
||||
// stored bundle, so this is caught only by the embedded-manifest check.
|
||||
embedded := diagnosticsManifestBody("server-1", DefaultConsentNoticeVer, "prof_1", "15", nil)
|
||||
bundle := buildManifestBundle(t, embedded)
|
||||
info, err := ValidateBundle(bytes.NewReader(bundle), BundleLimits{})
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateBundle: %v", err)
|
||||
}
|
||||
manifest := diagnosticsManifestBody("server-1", DefaultConsentNoticeVer, "prof_1", "99", &info)
|
||||
if _, err := contract.ValidateManifest(manifest); err != nil {
|
||||
t.Fatalf("test manifest invalid: %v\n%s", err, manifest)
|
||||
}
|
||||
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
_, err = svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrArchiveMismatch) {
|
||||
t.Fatalf("Ingest error = %v, want ErrArchiveMismatch", err)
|
||||
}
|
||||
wantReportID := "11111111-1111-1111-1111-111111111111"
|
||||
if len(repo.failed) != 1 || repo.failed[0] != wantReportID {
|
||||
t.Fatalf("failed reports = %v, want [%s]", repo.failed, wantReportID)
|
||||
}
|
||||
if len(store.deleted) != 1 {
|
||||
t.Fatalf("deleted keys = %v, want compensation delete", store.deleted)
|
||||
}
|
||||
if len(repo.ready) != 0 {
|
||||
t.Fatalf("ready reports = %v, want none", repo.ready)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedManifestMatchesPreservesLargeIntegers(t *testing.T) {
|
||||
// The received (part-1) and embedded manifests differ only in
|
||||
// log_summary.lines by 1, at a magnitude above 2^53 where float64 cannot
|
||||
// represent adjacent integers. Decoding with UseNumber must keep them distinct
|
||||
// so a bundle whose embedded manifest was tampered is rejected, not accepted.
|
||||
received := []byte(`{"log_summary":{"lines":9007199254740993},"archive":{"bytes":1}}`)
|
||||
embedded := []byte(`{"log_summary":{"lines":9007199254740992}}`)
|
||||
if embeddedManifestMatches(received, embedded) {
|
||||
t.Fatal("embeddedManifestMatches = true for manifests differing by a large integer, want false")
|
||||
}
|
||||
|
||||
// Sanity: with the archive object stripped from the received manifest, an
|
||||
// otherwise identical large integer still compares equal.
|
||||
sameReceived := []byte(`{"log_summary":{"lines":9007199254740993},"archive":{"bytes":1}}`)
|
||||
sameEmbedded := []byte(`{"log_summary":{"lines":9007199254740993}}`)
|
||||
if !embeddedManifestMatches(sameReceived, sameEmbedded) {
|
||||
t.Fatal("embeddedManifestMatches = false for identical manifests, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedManifestMatchesRejectsTrailingData(t *testing.T) {
|
||||
// A stray closing delimiter after an otherwise-matching object must be
|
||||
// rejected. Decoder.More() returns false here (it only tracks array/object
|
||||
// iteration), so the decode-to-EOF check is what catches it; without it a
|
||||
// bundle whose embedded manifest is not a single clean JSON object would
|
||||
// still compare equal to the accepted part-1 manifest.
|
||||
received := []byte(`{"log_summary":{"lines":1},"archive":{"bytes":1}}`)
|
||||
for _, trailer := range []string{"}", "]", "garbage", `{"extra":1}`} {
|
||||
embedded := []byte(`{"log_summary":{"lines":1}}` + trailer)
|
||||
if embeddedManifestMatches(received, embedded) {
|
||||
t.Fatalf("embeddedManifestMatches = true for embedded manifest with trailing %q, want false", trailer)
|
||||
}
|
||||
}
|
||||
|
||||
// Trailing whitespace after the object is not trailing data; it must still
|
||||
// compare equal.
|
||||
embeddedWithWhitespace := []byte("{\"log_summary\":{\"lines\":1}}\n ")
|
||||
if !embeddedManifestMatches(received, embeddedWithWhitespace) {
|
||||
t.Fatal("embeddedManifestMatches = false for embedded manifest with trailing whitespace, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestReturnsStorageErrorForMidStreamPutFailure(t *testing.T) {
|
||||
bundle, info := testDiagnosticsBundle(t)
|
||||
putErr := errors.New("s3: connection reset by peer")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{
|
||||
bucket: "private",
|
||||
putErr: putErr,
|
||||
putReadBytes: 8,
|
||||
}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer, info)
|
||||
_, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrStorageUnavailable) || !errors.Is(err, putErr) {
|
||||
t.Fatalf("Ingest error = %v, want retryable storage error wrapping put failure", err)
|
||||
}
|
||||
if errors.Is(err, ErrInvalidBundle) {
|
||||
t.Fatalf("Ingest error = %v, must not be classified as ErrInvalidBundle", err)
|
||||
}
|
||||
wantReportID := "11111111-1111-1111-1111-111111111111"
|
||||
if len(repo.failed) != 1 || repo.failed[0] != wantReportID {
|
||||
t.Fatalf("failed reports = %v, want [%s]", repo.failed, wantReportID)
|
||||
}
|
||||
wantKey := "diagnostics/42/" + wantReportID + ".tar.gz"
|
||||
if len(store.deleted) != 1 || store.deleted[0] != wantKey {
|
||||
t.Fatalf("deleted keys = %v, want [%s]", store.deleted, wantKey)
|
||||
}
|
||||
if len(repo.ready) != 0 {
|
||||
t.Fatalf("ready reports = %v, want none", repo.ready)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestCompensatesWithDetachedContext(t *testing.T) {
|
||||
bundle, info := testDiagnosticsBundle(t)
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{
|
||||
bucket: "private",
|
||||
putErr: errors.New("s3: connection reset by peer"),
|
||||
putReadBytes: 8,
|
||||
}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer, info)
|
||||
_, err := svc.Ingest(ctx, 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if err == nil {
|
||||
t.Fatal("Ingest error = nil, want storage error")
|
||||
}
|
||||
if len(store.deleteCtxErrs) != 1 || store.deleteCtxErrs[0] != nil {
|
||||
t.Fatalf("delete ctx errors = %v, want [nil]", store.deleteCtxErrs)
|
||||
}
|
||||
if len(repo.markFailedCtxErrs) != 1 || repo.markFailedCtxErrs[0] != nil {
|
||||
t.Fatalf("mark failed ctx errors = %v, want [nil]", repo.markFailedCtxErrs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestStoresOnlyValidatedProfileAttribution(t *testing.T) {
|
||||
// Header-only attribution (manifest carries no profile_id): the header is
|
||||
// the single source, so it is validated and stored.
|
||||
bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(_ context.Context, userID int, profileID string) (bool, error) {
|
||||
return userID == 42 && profileID == "prof_header", nil
|
||||
}))
|
||||
|
||||
headerProfileID := "prof_header"
|
||||
if _, err := svc.Ingest(context.Background(), 42, &headerProfileID, manifest, bytes.NewReader(bundle)); err != nil {
|
||||
t.Fatalf("Ingest: %v", err)
|
||||
}
|
||||
if repo.insertInput.ProfileID == nil || *repo.insertInput.ProfileID != "prof_header" {
|
||||
t.Fatalf("ProfileID = %v, want prof_header", repo.insertInput.ProfileID)
|
||||
}
|
||||
|
||||
// Manifest-only attribution that fails validation is dropped, not stored.
|
||||
bundle, manifest, _ = testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_1")
|
||||
repo = &fakeDiagnosticReportStore{}
|
||||
store = &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc = newTestDiagnosticsService(repo, store)
|
||||
svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(context.Context, int, string) (bool, error) {
|
||||
return false, nil
|
||||
}))
|
||||
if _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)); err != nil {
|
||||
t.Fatalf("Ingest with unvalidated profile: %v", err)
|
||||
}
|
||||
if repo.insertInput.ProfileID != nil {
|
||||
t.Fatalf("ProfileID = %v, want nil for unvalidated attribution", *repo.insertInput.ProfileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestRejectsMismatchedProfileID(t *testing.T) {
|
||||
// The manifest was captured under prof_manifest but the request carries a
|
||||
// different X-Profile-Id (prof_header) because the client switched profiles
|
||||
// between capture and upload. The two sources must not be silently
|
||||
// reconciled; the upload is rejected and no report row is created.
|
||||
bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_manifest")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
// Even a validator that would accept any profile must not paper over the
|
||||
// mismatch: resolution fails before attribution validation runs.
|
||||
svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(context.Context, int, string) (bool, error) {
|
||||
return true, nil
|
||||
}))
|
||||
|
||||
headerProfileID := "prof_header"
|
||||
_, err := svc.Ingest(context.Background(), 42, &headerProfileID, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrProfileMismatch) {
|
||||
t.Fatalf("Ingest error = %v, want ErrProfileMismatch", err)
|
||||
}
|
||||
if repo.insertInput.ProfileID != nil {
|
||||
t.Fatalf("ProfileID = %v, want no insert on profile mismatch", *repo.insertInput.ProfileID)
|
||||
}
|
||||
if len(repo.ready) != 0 || len(store.puts) != 0 {
|
||||
t.Fatalf("no report should be stored on mismatch: ready=%v puts=%v", repo.ready, store.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestAcceptsMatchingProfileID(t *testing.T) {
|
||||
// Header and manifest agree: attribution resolves to that single value.
|
||||
bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_1")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(_ context.Context, userID int, profileID string) (bool, error) {
|
||||
return userID == 42 && profileID == "prof_1", nil
|
||||
}))
|
||||
|
||||
headerProfileID := "prof_1"
|
||||
if _, err := svc.Ingest(context.Background(), 42, &headerProfileID, manifest, bytes.NewReader(bundle)); err != nil {
|
||||
t.Fatalf("Ingest: %v", err)
|
||||
}
|
||||
if repo.insertInput.ProfileID == nil || *repo.insertInput.ProfileID != "prof_1" {
|
||||
t.Fatalf("ProfileID = %v, want prof_1", repo.insertInput.ProfileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewProfileAttributionValidatorRejectsChildProfiles(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
found bool
|
||||
isChild bool
|
||||
lookErr error
|
||||
want bool
|
||||
wantErr bool
|
||||
wantChild bool
|
||||
}{
|
||||
{name: "non-child profile attributed", found: true, isChild: false, want: true},
|
||||
{name: "child profile forbidden", found: true, isChild: true, wantErr: true, wantChild: true},
|
||||
{name: "missing profile dropped", found: false, isChild: false, want: false},
|
||||
{name: "lookup error propagated", lookErr: errors.New("boom"), wantErr: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
validator := NewProfileAttributionValidator(func(_ context.Context, userID int, profileID string) (bool, bool, error) {
|
||||
if userID != 42 || profileID != "prof_1" {
|
||||
t.Fatalf("lookup called with userID=%d profileID=%q", userID, profileID)
|
||||
}
|
||||
return tc.found, tc.isChild, tc.lookErr
|
||||
})
|
||||
got, err := validator.ProfileBelongsToUser(context.Background(), 42, "prof_1")
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("ProfileBelongsToUser error = nil, want error")
|
||||
}
|
||||
if tc.wantChild && !errors.Is(err, ErrChildProfileForbidden) {
|
||||
t.Fatalf("ProfileBelongsToUser error = %v, want ErrChildProfileForbidden", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("ProfileBelongsToUser: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("ProfileBelongsToUser = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestRejectsChildProfileAttribution(t *testing.T) {
|
||||
// A nonconforming client attributes the report to a child profile. The
|
||||
// validator rejects it with ErrChildProfileForbidden, so the upload is
|
||||
// refused outright rather than silently stored without attribution.
|
||||
bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_child")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(context.Context, int, string) (bool, error) {
|
||||
return false, ErrChildProfileForbidden
|
||||
}))
|
||||
|
||||
_, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle))
|
||||
if !errors.Is(err, ErrChildProfileForbidden) {
|
||||
t.Fatalf("Ingest error = %v, want ErrChildProfileForbidden", err)
|
||||
}
|
||||
if repo.insertInput.ProfileID != nil {
|
||||
t.Fatalf("ProfileID = %v, want no insert on child profile", *repo.insertInput.ProfileID)
|
||||
}
|
||||
if len(repo.ready) != 0 || len(store.puts) != 0 {
|
||||
t.Fatalf("no report should be stored on child rejection: ready=%v puts=%v", repo.ready, store.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceIngestAcceptsManifestWithoutProfileID(t *testing.T) {
|
||||
bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "")
|
||||
repo := &fakeDiagnosticReportStore{}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
if _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)); err != nil {
|
||||
t.Fatalf("Ingest: %v", err)
|
||||
}
|
||||
if repo.insertInput.ProfileID != nil {
|
||||
t.Fatalf("ProfileID = %v, want nil", *repo.insertInput.ProfileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceDeleteReportDeletesRowWhenStorageUnavailable(t *testing.T) {
|
||||
report := testReadyDiagnosticReport("22222222-2222-2222-2222-222222222222", 42)
|
||||
repo := &fakeDiagnosticReportStore{
|
||||
getReport: &report,
|
||||
deleteReport: &report,
|
||||
}
|
||||
svc := newTestDiagnosticsService(repo, nil)
|
||||
|
||||
deleted, err := svc.DeleteReport(context.Background(), report.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteReport: %v", err)
|
||||
}
|
||||
if deleted == nil || deleted.ID != report.ID {
|
||||
t.Fatalf("deleted report = %#v, want %s", deleted, report.ID)
|
||||
}
|
||||
if len(repo.deleted) != 1 || repo.deleted[0] != report.ID {
|
||||
t.Fatalf("deleted rows = %v, want [%s]", repo.deleted, report.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceDeleteReportKeepsBlobWhenRowDeletionFails(t *testing.T) {
|
||||
report := testReadyDiagnosticReport("22222222-2222-2222-2222-222222222222", 42)
|
||||
repo := &fakeDiagnosticReportStore{
|
||||
getReport: &report,
|
||||
deleteErr: errors.New("db down"),
|
||||
}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private"}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
if _, err := svc.DeleteReport(context.Background(), report.ID); err == nil {
|
||||
t.Fatal("DeleteReport error = nil, want row deletion failure")
|
||||
}
|
||||
// The blob must survive a failed row deletion so a retry can still recover
|
||||
// the original artifact.
|
||||
if len(store.deleted) != 0 {
|
||||
t.Fatalf("blob deleted despite row deletion failure: %v", store.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceDeleteReportSucceedsWhenBlobDeletionFails(t *testing.T) {
|
||||
report := testReadyDiagnosticReport("22222222-2222-2222-2222-222222222222", 42)
|
||||
repo := &fakeDiagnosticReportStore{
|
||||
getReport: &report,
|
||||
deleteReport: &report,
|
||||
}
|
||||
store := &fakeDiagnosticObjectStore{bucket: "private", deleteErr: errors.New("s3 down")}
|
||||
svc := newTestDiagnosticsService(repo, store)
|
||||
|
||||
deleted, err := svc.DeleteReport(context.Background(), report.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteReport error = %v, want success after row deletion", err)
|
||||
}
|
||||
if deleted == nil || deleted.ID != report.ID {
|
||||
t.Fatalf("deleted report = %#v, want %s", deleted, report.ID)
|
||||
}
|
||||
if len(repo.deleted) != 1 || repo.deleted[0] != report.ID {
|
||||
t.Fatalf("deleted rows = %v, want [%s]", repo.deleted, report.ID)
|
||||
}
|
||||
// The row is gone; the orphaned blob delete was attempted (and failed), and
|
||||
// the reconciler will reap it later.
|
||||
if len(store.deleted) == 0 {
|
||||
t.Fatal("blob deletion was not attempted after row removal")
|
||||
}
|
||||
}
|
||||
|
||||
func newTestDiagnosticsService(repo ReportStore, store ObjectStore) *Service {
|
||||
settings := newMemorySettingsStore(map[string]string{
|
||||
KeyUploadsEnabled: "true",
|
||||
KeyMaxBundleBytes: "10485760",
|
||||
KeyMaxUncompressedBytes: "67108864",
|
||||
KeyMaxReportsPerUserDay: "20",
|
||||
KeyRetentionDays: "30",
|
||||
KeyMaxBytesPerUser: "209715200",
|
||||
KeyConsentNoticeVersion: "1",
|
||||
KeyServerInstanceID: "server-1",
|
||||
})
|
||||
svc := NewService(repo, settings, store, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
svc.now = func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) }
|
||||
return svc
|
||||
}
|
||||
|
||||
func testDiagnosticsBundle(t *testing.T) ([]byte, BundleInfo) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
payload := []byte(`{}`)
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: "manifest.json",
|
||||
Mode: 0o600,
|
||||
Size: int64(len(payload)),
|
||||
}); err != nil {
|
||||
t.Fatalf("write tar header: %v", err)
|
||||
}
|
||||
if _, err := tw.Write(payload); err != nil {
|
||||
t.Fatalf("write tar payload: %v", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("close tar: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("close gzip: %v", err)
|
||||
}
|
||||
data := buf.Bytes()
|
||||
info, err := ValidateBundle(bytes.NewReader(data), BundleLimits{})
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateBundle: %v", err)
|
||||
}
|
||||
return data, info
|
||||
}
|
||||
|
||||
func testManifestJSON(t *testing.T, serverID string, noticeVersion int, archive BundleInfo) []byte {
|
||||
t.Helper()
|
||||
return testManifestJSONWithProfile(t, serverID, noticeVersion, archive, "prof_1")
|
||||
}
|
||||
|
||||
func testManifestJSONWithProfile(t *testing.T, serverID string, noticeVersion int, archive BundleInfo, profileID string) []byte {
|
||||
t.Helper()
|
||||
data := diagnosticsManifestBody(serverID, noticeVersion, profileID, "15", &archive)
|
||||
if _, err := contract.ValidateManifest(data); err != nil {
|
||||
t.Fatalf("test manifest invalid: %v\n%s", err, string(data))
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// diagnosticsManifestBody builds a manifest JSON body. When info is nil the
|
||||
// `archive` object is omitted, producing the embedded manifest.json contract
|
||||
// form (part-1 manifest minus `archive`).
|
||||
func diagnosticsManifestBody(serverID string, noticeVersion int, profileID, osVersion string, info *BundleInfo) []byte {
|
||||
profileField := ""
|
||||
if profileID != "" {
|
||||
profileField = `,
|
||||
"profile_id": "` + profileID + `"`
|
||||
}
|
||||
archiveField := ""
|
||||
if info != nil {
|
||||
archiveField = `,
|
||||
"archive": {
|
||||
"entries": ["manifest.json"],
|
||||
"bytes": ` + strconv.FormatInt(info.CompressedBytes, 10) + `,
|
||||
"uncompressed_bytes": ` + strconv.FormatInt(info.UncompressedBytes, 10) + `,
|
||||
"sha256": "` + info.SHA256 + `"
|
||||
}`
|
||||
}
|
||||
return []byte(`{
|
||||
"schema_version": 1,
|
||||
"report": {
|
||||
"type": "manual",
|
||||
"captured_at": "2026-07-20T12:00:00Z",
|
||||
"capture_session_id": "run_test",
|
||||
"app_version": "1.0.0",
|
||||
"app_build": "100",
|
||||
"platform": "android",
|
||||
"os_version": "` + osVersion + `"` + profileField + `
|
||||
},
|
||||
"destination": { "server_instance_id": "` + serverID + `" },
|
||||
"consent": { "mode": "manual", "notice_version": ` + strconv.Itoa(noticeVersion) + ` },
|
||||
"device_summary": {
|
||||
"manufacturer": "Google",
|
||||
"model": "Pixel",
|
||||
"os": "15",
|
||||
"form_factor": "phone"
|
||||
},
|
||||
"playback_session_ids": ["ps_1"],
|
||||
"log_summary": {
|
||||
"lines": 1,
|
||||
"bytes_gz": 1,
|
||||
"dropped_lines": 0,
|
||||
"categories": ["crash"],
|
||||
"debug_logging": true
|
||||
}` + archiveField + `
|
||||
}`)
|
||||
}
|
||||
|
||||
// testDiagnosticsUpload builds a coordinated (bundle, manifest) pair: the tar's
|
||||
// embedded manifest.json is the part-1 manifest minus `archive`, and the part-1
|
||||
// manifest's archive fields match the built bundle — the shape Ingest accepts.
|
||||
func testDiagnosticsUpload(t *testing.T, serverID string, noticeVersion int, profileID string) ([]byte, []byte, BundleInfo) {
|
||||
t.Helper()
|
||||
embedded := diagnosticsManifestBody(serverID, noticeVersion, profileID, "15", nil)
|
||||
bundle := buildManifestBundle(t, embedded)
|
||||
info, err := ValidateBundle(bytes.NewReader(bundle), BundleLimits{})
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateBundle: %v", err)
|
||||
}
|
||||
manifest := diagnosticsManifestBody(serverID, noticeVersion, profileID, "15", &info)
|
||||
if _, err := contract.ValidateManifest(manifest); err != nil {
|
||||
t.Fatalf("test manifest invalid: %v\n%s", err, manifest)
|
||||
}
|
||||
return bundle, manifest, info
|
||||
}
|
||||
|
||||
func buildManifestBundle(t *testing.T, manifestBody []byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: "manifest.json",
|
||||
Mode: 0o600,
|
||||
Size: int64(len(manifestBody)),
|
||||
}); err != nil {
|
||||
t.Fatalf("write tar header: %v", err)
|
||||
}
|
||||
if _, err := tw.Write(manifestBody); err != nil {
|
||||
t.Fatalf("write tar payload: %v", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("close tar: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("close gzip: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type fakeDiagnosticReportStore struct {
|
||||
insertCalls int
|
||||
insertInput InsertReceivingInput
|
||||
insertErr error
|
||||
ready []string
|
||||
readyBlobs []BlobInfo
|
||||
failed []string
|
||||
markFailedCtxErrs []error
|
||||
getReport *Report
|
||||
getErr error
|
||||
deleteReport *Report
|
||||
deleteErr error
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) InsertReceiving(_ context.Context, input InsertReceivingInput) (InsertReceivingResult, error) {
|
||||
f.insertCalls++
|
||||
f.insertInput = input
|
||||
if f.insertErr != nil {
|
||||
return InsertReceivingResult{}, f.insertErr
|
||||
}
|
||||
return InsertReceivingResult{
|
||||
ID: "11111111-1111-1111-1111-111111111111",
|
||||
ShortID: "SILO-ABCDEF123456",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) MarkReady(_ context.Context, id string, blob BlobInfo) error {
|
||||
f.ready = append(f.ready, id)
|
||||
f.readyBlobs = append(f.readyBlobs, blob)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) MarkFailed(ctx context.Context, id string) error {
|
||||
f.failed = append(f.failed, id)
|
||||
f.markFailedCtxErrs = append(f.markFailedCtxErrs, ctx.Err())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) GetByID(context.Context, string) (*Report, error) {
|
||||
if f.getErr != nil {
|
||||
return nil, f.getErr
|
||||
}
|
||||
if f.getReport != nil {
|
||||
report := *f.getReport
|
||||
return &report, nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) ListForAdmin(context.Context, ListFilters) (ListResult, error) {
|
||||
return ListResult{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) DeleteByID(_ context.Context, id string) (*Report, error) {
|
||||
f.deleted = append(f.deleted, id)
|
||||
if f.deleteErr != nil {
|
||||
return nil, f.deleteErr
|
||||
}
|
||||
if f.deleteReport != nil {
|
||||
report := *f.deleteReport
|
||||
return &report, nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) RetentionCandidates(context.Context, time.Time, int64) ([]Report, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticReportStore) StaleReceiving(context.Context, time.Duration) ([]Report, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type fakeDiagnosticObjectStore struct {
|
||||
bucket string
|
||||
puts []string
|
||||
deleted []string
|
||||
deleteCtxErrs []error
|
||||
deleteErr error
|
||||
data []byte
|
||||
putErr error
|
||||
putReadBytes int64
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticObjectStore) PutStream(_ context.Context, _ string, key string, r io.Reader, _ string) error {
|
||||
if f.putErr != nil {
|
||||
if f.putReadBytes > 0 {
|
||||
_, _ = io.CopyN(io.Discard, r, f.putReadBytes)
|
||||
}
|
||||
f.puts = append(f.puts, key)
|
||||
return f.putErr
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.puts = append(f.puts, key)
|
||||
f.data = append(f.data[:0], data...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticObjectStore) GetObject(context.Context, string, string) (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticObjectStore) DeleteObject(ctx context.Context, _ string, key string) error {
|
||||
f.deleted = append(f.deleted, key)
|
||||
f.deleteCtxErrs = append(f.deleteCtxErrs, ctx.Err())
|
||||
return f.deleteErr
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticObjectStore) ListObjects(context.Context, string) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticObjectStore) PresignGetURL(context.Context, string, string, time.Duration) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticObjectStore) Bucket() string {
|
||||
return f.bucket
|
||||
}
|
||||
|
||||
func testReadyDiagnosticReport(id string, userID int) Report {
|
||||
bucket := "private"
|
||||
key := reportObjectKey(userID, id)
|
||||
bytes := int64(128)
|
||||
uncompressed := int64(256)
|
||||
sha := strings.Repeat("a", 64)
|
||||
return Report{
|
||||
ID: id,
|
||||
ShortID: "SILO-ABCDEF123456",
|
||||
UserID: userID,
|
||||
State: StateReady,
|
||||
BlobBucket: &bucket,
|
||||
BlobKey: &key,
|
||||
BlobBytes: &bytes,
|
||||
UncompressedBytes: &uncompressed,
|
||||
BlobSHA256: &sha,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
KeyUploadsEnabled = "diagnostics.uploads_enabled"
|
||||
KeyMaxBundleBytes = "diagnostics.max_bundle_bytes"
|
||||
KeyMaxUncompressedBytes = "diagnostics.max_uncompressed_bytes"
|
||||
KeyMaxReportsPerUserDay = "diagnostics.max_reports_per_user_per_day"
|
||||
KeyRetentionDays = "diagnostics.retention_days"
|
||||
KeyMaxBytesPerUser = "diagnostics.max_bytes_per_user"
|
||||
KeyConsentNoticeVersion = "diagnostics.consent_notice_version"
|
||||
KeyCleanupIntervalMinutes = "diagnostics.cleanup_interval_minutes"
|
||||
KeyServerInstanceID = "diagnostics.server_instance_id"
|
||||
DefaultUploadsEnabled = false
|
||||
DefaultMaxBundleBytes = int64(10 * 1024 * 1024)
|
||||
DefaultMaxUncompressed = int64(64 * 1024 * 1024)
|
||||
DefaultMaxReportsPerDay = 20
|
||||
DefaultRetentionDays = 30
|
||||
DefaultMaxBytesPerUser = int64(200 * 1024 * 1024)
|
||||
DefaultConsentNoticeVer = 1
|
||||
// DefaultCleanupIntervalMinutes matches the prior opslog-shared cadence so
|
||||
// splitting diagnostics onto its own key preserves current behavior.
|
||||
DefaultCleanupIntervalMinutes = 15
|
||||
// maxCleanupIntervalMinutes caps the parsed interval well below the point
|
||||
// where minutes * time.Minute overflows int64 nanoseconds (~1.5e8 minutes,
|
||||
// which would wrap a huge configured value into a tiny — or negative —
|
||||
// duration). 7 days is far longer than any sane cleanup cadence.
|
||||
maxCleanupIntervalMinutes = 7 * 24 * 60
|
||||
defaultUploadsEnabledStr = "false"
|
||||
defaultMaxBundleStr = "10485760"
|
||||
defaultMaxUncompressed = "67108864"
|
||||
defaultMaxReportsStr = "20"
|
||||
defaultRetentionDaysStr = "30"
|
||||
defaultMaxBytesUserStr = "209715200"
|
||||
defaultConsentNoticeStr = "1"
|
||||
defaultCleanupIntervalStr = "15"
|
||||
)
|
||||
|
||||
// SettingsStore is the read/write surface over server_settings used by the
|
||||
// diagnostics feature gate. catalog.ServerSettingsRepo satisfies it.
|
||||
type SettingsStore interface {
|
||||
Get(ctx context.Context, key string) (string, error)
|
||||
Set(ctx context.Context, key, value string) error
|
||||
}
|
||||
|
||||
// conditionalSettingsStore is the optional insert-if-absent surface used to seed
|
||||
// generated singletons (the server instance ID) without racing concurrent
|
||||
// nodes. catalog.ServerSettingsRepo satisfies it.
|
||||
type conditionalSettingsStore interface {
|
||||
SetIfAbsent(ctx context.Context, key, value string) (bool, error)
|
||||
}
|
||||
|
||||
type Settings struct {
|
||||
UploadsEnabled bool
|
||||
MaxBundleBytes int64
|
||||
MaxUncompressedBytes int64
|
||||
MaxReportsPerUserDay int
|
||||
RetentionDays int
|
||||
MaxBytesPerUser int64
|
||||
ConsentNoticeVersion int
|
||||
ServerInstanceID string
|
||||
}
|
||||
|
||||
func SeedDefaults(ctx context.Context, store SettingsStore) error {
|
||||
defaults := map[string]string{
|
||||
KeyUploadsEnabled: defaultUploadsEnabledStr,
|
||||
KeyMaxBundleBytes: defaultMaxBundleStr,
|
||||
KeyMaxUncompressedBytes: defaultMaxUncompressed,
|
||||
KeyMaxReportsPerUserDay: defaultMaxReportsStr,
|
||||
KeyRetentionDays: defaultRetentionDaysStr,
|
||||
KeyMaxBytesPerUser: defaultMaxBytesUserStr,
|
||||
KeyConsentNoticeVersion: defaultConsentNoticeStr,
|
||||
KeyCleanupIntervalMinutes: defaultCleanupIntervalStr,
|
||||
}
|
||||
for key, value := range defaults {
|
||||
existing, err := store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("seed diagnostics defaults for %s: %w", key, err)
|
||||
}
|
||||
if existing != "" {
|
||||
continue
|
||||
}
|
||||
if err := store.Set(ctx, key, value); err != nil {
|
||||
return fmt.Errorf("seed diagnostics default %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := ensureServerInstanceID(ctx, store); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureServerInstanceID returns the diagnostics server instance ID, generating
|
||||
// and persisting one when absent. Seeding is atomic when the store supports
|
||||
// insert-if-absent: concurrent nodes converge on the single winning value
|
||||
// instead of overwriting each other (which would reject already-issued
|
||||
// destination IDs as destination_mismatch).
|
||||
func ensureServerInstanceID(ctx context.Context, store SettingsStore) (string, error) {
|
||||
existing, err := store.Get(ctx, KeyServerInstanceID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("seed diagnostics server instance id: %w", err)
|
||||
}
|
||||
if id := strings.TrimSpace(existing); id != "" {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
instanceID, err := newServerInstanceID()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
conditional, ok := store.(conditionalSettingsStore)
|
||||
if !ok {
|
||||
if err := store.Set(ctx, KeyServerInstanceID, instanceID); err != nil {
|
||||
return "", fmt.Errorf("seed diagnostics server instance id: %w", err)
|
||||
}
|
||||
return instanceID, nil
|
||||
}
|
||||
|
||||
if _, err := conditional.SetIfAbsent(ctx, KeyServerInstanceID, instanceID); err != nil {
|
||||
return "", fmt.Errorf("seed diagnostics server instance id: %w", err)
|
||||
}
|
||||
// Re-read so a node that lost the insert race adopts the winning value.
|
||||
winner, err := store.Get(ctx, KeyServerInstanceID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("seed diagnostics server instance id: %w", err)
|
||||
}
|
||||
if id := strings.TrimSpace(winner); id != "" {
|
||||
return id, nil
|
||||
}
|
||||
return instanceID, nil
|
||||
}
|
||||
|
||||
// LoadCleanupInterval returns the configured diagnostics cleanup interval. It is
|
||||
// intentionally independent of the opslog cleanup cadence so tuning one does not
|
||||
// silently move the other.
|
||||
func LoadCleanupInterval(ctx context.Context, store SettingsStore) time.Duration {
|
||||
minutes := DefaultCleanupIntervalMinutes
|
||||
if store == nil {
|
||||
return time.Duration(minutes) * time.Minute
|
||||
}
|
||||
if raw, err := store.Get(ctx, KeyCleanupIntervalMinutes); err == nil && strings.TrimSpace(raw) != "" {
|
||||
if parsed := parseInt(raw); parsed > 0 {
|
||||
minutes = parsed
|
||||
if minutes > maxCleanupIntervalMinutes {
|
||||
minutes = maxCleanupIntervalMinutes
|
||||
}
|
||||
}
|
||||
}
|
||||
return time.Duration(minutes) * time.Minute
|
||||
}
|
||||
|
||||
func DefaultSettings() Settings {
|
||||
return Settings{
|
||||
UploadsEnabled: DefaultUploadsEnabled,
|
||||
MaxBundleBytes: DefaultMaxBundleBytes,
|
||||
MaxUncompressedBytes: DefaultMaxUncompressed,
|
||||
MaxReportsPerUserDay: DefaultMaxReportsPerDay,
|
||||
RetentionDays: DefaultRetentionDays,
|
||||
MaxBytesPerUser: DefaultMaxBytesPerUser,
|
||||
ConsentNoticeVersion: DefaultConsentNoticeVer,
|
||||
}
|
||||
}
|
||||
|
||||
func LoadSettings(ctx context.Context, store SettingsStore) (Settings, error) {
|
||||
settings := DefaultSettings()
|
||||
if store == nil {
|
||||
return settings, nil
|
||||
}
|
||||
// get distinguishes a genuine read failure — propagated so callers can
|
||||
// surface a retryable error — from a missing or empty value, which falls
|
||||
// back to the default. Silently defaulting on a transient DB error could
|
||||
// report uploads disabled or the wrong quota limits to clients making
|
||||
// decisions from /diagnostics/status.
|
||||
get := func(key string) (string, error) {
|
||||
raw, err := store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("load diagnostics setting %s: %w", key, err)
|
||||
}
|
||||
return strings.TrimSpace(raw), nil
|
||||
}
|
||||
|
||||
if raw, err := get(KeyUploadsEnabled); err != nil {
|
||||
return Settings{}, err
|
||||
} else if raw != "" {
|
||||
if parsed, parseErr := strconv.ParseBool(raw); parseErr == nil {
|
||||
settings.UploadsEnabled = parsed
|
||||
}
|
||||
}
|
||||
if raw, err := get(KeyMaxBundleBytes); err != nil {
|
||||
return Settings{}, err
|
||||
} else if parsed := parseInt64(raw); parsed > 0 {
|
||||
settings.MaxBundleBytes = parsed
|
||||
}
|
||||
if raw, err := get(KeyMaxUncompressedBytes); err != nil {
|
||||
return Settings{}, err
|
||||
} else if parsed := parseInt64(raw); parsed > 0 {
|
||||
settings.MaxUncompressedBytes = parsed
|
||||
}
|
||||
if raw, err := get(KeyMaxReportsPerUserDay); err != nil {
|
||||
return Settings{}, err
|
||||
} else if parsed := parseInt(raw); parsed > 0 {
|
||||
settings.MaxReportsPerUserDay = parsed
|
||||
}
|
||||
if raw, err := get(KeyRetentionDays); err != nil {
|
||||
return Settings{}, err
|
||||
} else if parsed := parseInt(raw); parsed > 0 {
|
||||
settings.RetentionDays = parsed
|
||||
}
|
||||
if raw, err := get(KeyMaxBytesPerUser); err != nil {
|
||||
return Settings{}, err
|
||||
} else if parsed := parseInt64(raw); parsed > 0 {
|
||||
settings.MaxBytesPerUser = parsed
|
||||
}
|
||||
if raw, err := get(KeyConsentNoticeVersion); err != nil {
|
||||
return Settings{}, err
|
||||
} else if parsed := parseInt(raw); parsed > 0 {
|
||||
settings.ConsentNoticeVersion = parsed
|
||||
}
|
||||
if raw, err := get(KeyServerInstanceID); err != nil {
|
||||
return Settings{}, err
|
||||
} else {
|
||||
settings.ServerInstanceID = raw
|
||||
}
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func newServerInstanceID() (string, error) {
|
||||
return uuid.NewString(), nil
|
||||
}
|
||||
|
||||
func parseInt(s string) int {
|
||||
v, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func parseInt64(s string) int64 {
|
||||
v, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSeedDefaultsPreservesExistingServerInstanceID(t *testing.T) {
|
||||
store := newMemorySettingsStore(map[string]string{
|
||||
KeyServerInstanceID: "existing-instance",
|
||||
})
|
||||
if err := SeedDefaults(context.Background(), store); err != nil {
|
||||
t.Fatalf("SeedDefaults: %v", err)
|
||||
}
|
||||
if got := store.values[KeyServerInstanceID]; got != "existing-instance" {
|
||||
t.Fatalf("server instance id = %q, want existing-instance", got)
|
||||
}
|
||||
if got := store.values[KeyUploadsEnabled]; got != defaultUploadsEnabledStr {
|
||||
t.Fatalf("uploads enabled default = %q, want %q", got, defaultUploadsEnabledStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedDefaultsGeneratesDedicatedServerInstanceIDWhenMissing(t *testing.T) {
|
||||
const jellyfinServerIDKey = "jellyfin_compat.server_id"
|
||||
store := newMemorySettingsStore(map[string]string{
|
||||
jellyfinServerIDKey: "constant-jellyfin-id",
|
||||
})
|
||||
if err := SeedDefaults(context.Background(), store); err != nil {
|
||||
t.Fatalf("SeedDefaults: %v", err)
|
||||
}
|
||||
got := store.values[KeyServerInstanceID]
|
||||
if len(got) != 36 {
|
||||
t.Fatalf("server instance id length = %d, want UUID string", len(got))
|
||||
}
|
||||
if got == store.values[jellyfinServerIDKey] {
|
||||
t.Fatal("diagnostics server instance id reused jellyfin_compat.server_id")
|
||||
}
|
||||
if store.values[jellyfinServerIDKey] != "constant-jellyfin-id" {
|
||||
t.Fatalf("jellyfin server id changed to %q", store.values[jellyfinServerIDKey])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedDefaultsAdoptsConcurrentInstanceIDWinner(t *testing.T) {
|
||||
// A store that supports insert-if-absent seeds the instance ID atomically:
|
||||
// when another node wins the race, this node adopts the winning value
|
||||
// instead of overwriting it.
|
||||
store := &racingSettingsStore{
|
||||
memorySettingsStore: memorySettingsStore{values: map[string]string{}},
|
||||
concurrentWinner: "winner-instance",
|
||||
}
|
||||
if err := SeedDefaults(context.Background(), store); err != nil {
|
||||
t.Fatalf("SeedDefaults: %v", err)
|
||||
}
|
||||
if got := store.values[KeyServerInstanceID]; got != "winner-instance" {
|
||||
t.Fatalf("server instance id = %q, want winner-instance", got)
|
||||
}
|
||||
if store.setIfAbsentCalls != 1 {
|
||||
t.Fatalf("SetIfAbsent calls = %d, want 1", store.setIfAbsentCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCleanupIntervalUsesDiagnosticsKey(t *testing.T) {
|
||||
store := newMemorySettingsStore(map[string]string{
|
||||
KeyCleanupIntervalMinutes: "45",
|
||||
})
|
||||
if got := LoadCleanupInterval(context.Background(), store); got != 45*time.Minute {
|
||||
t.Fatalf("LoadCleanupInterval = %v, want 45m", got)
|
||||
}
|
||||
if got := LoadCleanupInterval(context.Background(), newMemorySettingsStore(nil)); got != DefaultCleanupIntervalMinutes*time.Minute {
|
||||
t.Fatalf("LoadCleanupInterval default = %v, want %d minutes", got, DefaultCleanupIntervalMinutes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCleanupIntervalCapsHugeValues(t *testing.T) {
|
||||
// The boundary: the largest value that still fits under the cap is accepted
|
||||
// verbatim, while anything larger (including values that would overflow
|
||||
// int64 nanoseconds when multiplied by time.Minute) clamps to the cap
|
||||
// instead of wrapping into a tiny or negative duration.
|
||||
atCap := newMemorySettingsStore(map[string]string{
|
||||
KeyCleanupIntervalMinutes: strconv.Itoa(maxCleanupIntervalMinutes),
|
||||
})
|
||||
if got := LoadCleanupInterval(context.Background(), atCap); got != maxCleanupIntervalMinutes*time.Minute {
|
||||
t.Fatalf("LoadCleanupInterval at cap = %v, want %v", got, maxCleanupIntervalMinutes*time.Minute)
|
||||
}
|
||||
|
||||
for _, raw := range []string{
|
||||
strconv.Itoa(maxCleanupIntervalMinutes + 1),
|
||||
"153722868", // wraps int64 nanoseconds without the cap
|
||||
"9223372036854775807", // math.MaxInt64 minutes
|
||||
} {
|
||||
store := newMemorySettingsStore(map[string]string{KeyCleanupIntervalMinutes: raw})
|
||||
got := LoadCleanupInterval(context.Background(), store)
|
||||
if got != maxCleanupIntervalMinutes*time.Minute {
|
||||
t.Fatalf("LoadCleanupInterval(%s) = %v, want capped %v", raw, got, maxCleanupIntervalMinutes*time.Minute)
|
||||
}
|
||||
if got <= 0 {
|
||||
t.Fatalf("LoadCleanupInterval(%s) = %v, want a positive duration", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSettingsPropagatesReadErrors(t *testing.T) {
|
||||
// A genuine read failure must surface as an error so callers can return a
|
||||
// retryable failure instead of silently reporting defaults (e.g. uploads
|
||||
// disabled) to clients.
|
||||
store := &failingSettingsStore{
|
||||
memorySettingsStore: memorySettingsStore{values: map[string]string{KeyUploadsEnabled: "true"}},
|
||||
failKey: KeyUploadsEnabled,
|
||||
}
|
||||
if _, err := LoadSettings(context.Background(), store); err == nil {
|
||||
t.Fatal("LoadSettings did not propagate a store read error")
|
||||
}
|
||||
|
||||
// A later key failing must also propagate, not be swallowed as a default.
|
||||
store = &failingSettingsStore{
|
||||
memorySettingsStore: memorySettingsStore{values: map[string]string{KeyUploadsEnabled: "true"}},
|
||||
failKey: KeyMaxBytesPerUser,
|
||||
}
|
||||
if _, err := LoadSettings(context.Background(), store); err == nil {
|
||||
t.Fatal("LoadSettings did not propagate a late store read error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSettingsParsesTypedValues(t *testing.T) {
|
||||
store := newMemorySettingsStore(map[string]string{
|
||||
KeyUploadsEnabled: "true",
|
||||
KeyMaxBundleBytes: "123",
|
||||
KeyMaxUncompressedBytes: "456",
|
||||
KeyMaxReportsPerUserDay: "7",
|
||||
KeyRetentionDays: "8",
|
||||
KeyMaxBytesPerUser: "900",
|
||||
KeyConsentNoticeVersion: "2",
|
||||
KeyServerInstanceID: "server-1",
|
||||
})
|
||||
|
||||
got, err := LoadSettings(context.Background(), store)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSettings: %v", err)
|
||||
}
|
||||
if !got.UploadsEnabled ||
|
||||
got.MaxBundleBytes != 123 ||
|
||||
got.MaxUncompressedBytes != 456 ||
|
||||
got.MaxReportsPerUserDay != 7 ||
|
||||
got.RetentionDays != 8 ||
|
||||
got.MaxBytesPerUser != 900 ||
|
||||
got.ConsentNoticeVersion != 2 ||
|
||||
got.ServerInstanceID != "server-1" {
|
||||
t.Fatalf("LoadSettings parsed unexpected values: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSettingsFallsBackForInvalidNumbers(t *testing.T) {
|
||||
store := newMemorySettingsStore(map[string]string{
|
||||
KeyMaxBundleBytes: "-1",
|
||||
KeyMaxUncompressedBytes: "not-a-number",
|
||||
KeyMaxReportsPerUserDay: "0",
|
||||
KeyRetentionDays: "0",
|
||||
KeyMaxBytesPerUser: "-2",
|
||||
KeyConsentNoticeVersion: "0",
|
||||
})
|
||||
|
||||
got, err := LoadSettings(context.Background(), store)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSettings: %v", err)
|
||||
}
|
||||
want := DefaultSettings()
|
||||
if got.MaxBundleBytes != want.MaxBundleBytes ||
|
||||
got.MaxUncompressedBytes != want.MaxUncompressedBytes ||
|
||||
got.MaxReportsPerUserDay != want.MaxReportsPerUserDay ||
|
||||
got.RetentionDays != want.RetentionDays ||
|
||||
got.MaxBytesPerUser != want.MaxBytesPerUser ||
|
||||
got.ConsentNoticeVersion != want.ConsentNoticeVersion {
|
||||
t.Fatalf("LoadSettings = %+v, want defaults %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
type memorySettingsStore struct {
|
||||
values map[string]string
|
||||
}
|
||||
|
||||
func newMemorySettingsStore(values map[string]string) *memorySettingsStore {
|
||||
store := &memorySettingsStore{values: map[string]string{}}
|
||||
for k, v := range values {
|
||||
store.values[k] = v
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func (s *memorySettingsStore) Get(_ context.Context, key string) (string, error) {
|
||||
return s.values[key], nil
|
||||
}
|
||||
|
||||
func (s *memorySettingsStore) Set(_ context.Context, key, value string) error {
|
||||
s.values[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
// failingSettingsStore returns an error when Get is called for failKey,
|
||||
// simulating a transient database read failure for one setting.
|
||||
type failingSettingsStore struct {
|
||||
memorySettingsStore
|
||||
failKey string
|
||||
}
|
||||
|
||||
func (s *failingSettingsStore) Get(ctx context.Context, key string) (string, error) {
|
||||
if key == s.failKey {
|
||||
return "", errors.New("transient read failure")
|
||||
}
|
||||
return s.memorySettingsStore.Get(ctx, key)
|
||||
}
|
||||
|
||||
// racingSettingsStore adds insert-if-absent semantics and simulates another node
|
||||
// winning the instance-ID seed race via concurrentWinner.
|
||||
type racingSettingsStore struct {
|
||||
memorySettingsStore
|
||||
concurrentWinner string
|
||||
setIfAbsentCalls int
|
||||
}
|
||||
|
||||
func (s *racingSettingsStore) SetIfAbsent(_ context.Context, key, value string) (bool, error) {
|
||||
s.setIfAbsentCalls++
|
||||
if existing := s.values[key]; existing != "" {
|
||||
return false, nil
|
||||
}
|
||||
if s.concurrentWinner != "" {
|
||||
s.values[key] = s.concurrentWinner
|
||||
return false, nil
|
||||
}
|
||||
s.values[key] = value
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
shortIDPrefix = "SILO-"
|
||||
shortIDPayloadLength = 12
|
||||
shortIDChars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
)
|
||||
|
||||
var ErrInvalidShortID = errors.New("invalid diagnostics short id")
|
||||
|
||||
func NewShortID() (string, error) {
|
||||
var random [shortIDPayloadLength]byte
|
||||
if _, err := rand.Read(random[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out [shortIDPayloadLength]byte
|
||||
for i, b := range random {
|
||||
out[i] = shortIDChars[int(b)&31]
|
||||
}
|
||||
return shortIDPrefix + string(out[:]), nil
|
||||
}
|
||||
|
||||
func ParseShortID(raw string) (string, error) {
|
||||
id := strings.ToUpper(strings.TrimSpace(raw))
|
||||
payload := strings.TrimPrefix(id, shortIDPrefix)
|
||||
if len(payload) != shortIDPayloadLength {
|
||||
return "", ErrInvalidShortID
|
||||
}
|
||||
for i := 0; i < len(payload); i++ {
|
||||
if !isShortIDChar(payload[i]) {
|
||||
return "", ErrInvalidShortID
|
||||
}
|
||||
}
|
||||
return shortIDPrefix + payload, nil
|
||||
}
|
||||
|
||||
func isShortIDChar(ch byte) bool {
|
||||
for i := 0; i < len(shortIDChars); i++ {
|
||||
if shortIDChars[i] == ch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewShortIDIsParseable(t *testing.T) {
|
||||
id, err := NewShortID()
|
||||
if err != nil {
|
||||
t.Fatalf("NewShortID: %v", err)
|
||||
}
|
||||
if len(id) != len(shortIDPrefix)+shortIDPayloadLength {
|
||||
t.Fatalf("length = %d, want %d", len(id), len(shortIDPrefix)+shortIDPayloadLength)
|
||||
}
|
||||
if !strings.HasPrefix(id, shortIDPrefix) {
|
||||
t.Fatalf("id = %q, want %s prefix", id, shortIDPrefix)
|
||||
}
|
||||
if _, err := ParseShortID(id); err != nil {
|
||||
t.Fatalf("ParseShortID(%q): %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShortIDNormalizesCaseAndOptionalPrefix(t *testing.T) {
|
||||
for _, raw := range []string{"abcdef123456", "silo-abcdef123456", " SILO-ABCDEF123456 "} {
|
||||
got, err := ParseShortID(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseShortID(%q): %v", raw, err)
|
||||
}
|
||||
if got != "SILO-ABCDEF123456" {
|
||||
t.Fatalf("ParseShortID(%q) = %q, want SILO-ABCDEF123456", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShortIDRejectsAmbiguousCharacters(t *testing.T) {
|
||||
for _, raw := range []string{"ABCDEF12345I", "ABCDEF12345L", "ABCDEF12345O", "ABCDEF12345U"} {
|
||||
if _, err := ParseShortID(raw); !errors.Is(err, ErrInvalidShortID) {
|
||||
t.Fatalf("ParseShortID(%q) error = %v, want ErrInvalidShortID", raw, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/s3client"
|
||||
)
|
||||
|
||||
var ErrObjectNotFound = errors.New("diagnostics object not found")
|
||||
|
||||
// ObjectStore is the diagnostics-owned object storage surface. It is narrow
|
||||
// enough for fake-backed ingest/admin tests and wraps the private S3 bucket in
|
||||
// production.
|
||||
type ObjectStore interface {
|
||||
PutStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error
|
||||
GetObject(ctx context.Context, bucket, key string) (io.ReadCloser, error)
|
||||
DeleteObject(ctx context.Context, bucket, key string) error
|
||||
ListObjects(ctx context.Context, prefix string) ([]string, error)
|
||||
PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error)
|
||||
Bucket() string
|
||||
}
|
||||
|
||||
type s3ObjectStore struct {
|
||||
client *s3client.Client
|
||||
}
|
||||
|
||||
// NewS3ObjectStore adapts the private S3 client to the diagnostics store
|
||||
// interface. A nil client returns nil so interface-nil checks remain reliable.
|
||||
func NewS3ObjectStore(client *s3client.Client) ObjectStore {
|
||||
if client == nil {
|
||||
return nil
|
||||
}
|
||||
return &s3ObjectStore{client: client}
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) PutStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error {
|
||||
return normalizeObjectStoreError(s.client.PutObjectStream(ctx, bucket, key, r, contentType))
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) GetObject(ctx context.Context, bucket, key string) (io.ReadCloser, error) {
|
||||
body, err := s.client.GetObjectStream(ctx, bucket, key)
|
||||
if err != nil {
|
||||
return nil, normalizeObjectStoreError(err)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) DeleteObject(ctx context.Context, bucket, key string) error {
|
||||
return normalizeObjectStoreError(s.client.DeleteObject(ctx, bucket, key))
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) ListObjects(ctx context.Context, prefix string) ([]string, error) {
|
||||
keys, err := s.client.ListObjects(ctx, s.client.Bucket(), prefix)
|
||||
if err != nil {
|
||||
return nil, normalizeObjectStoreError(err)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error) {
|
||||
url, err := s.client.PresignGetURL(ctx, bucket, key, expiry)
|
||||
if err != nil {
|
||||
return "", normalizeObjectStoreError(err)
|
||||
}
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) EffectivePresignTTL(requested time.Duration) time.Duration {
|
||||
return s.client.EffectivePresignTTL(requested)
|
||||
}
|
||||
|
||||
func (s *s3ObjectStore) Bucket() string {
|
||||
return s.client.Bucket()
|
||||
}
|
||||
|
||||
func normalizeObjectStoreError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, s3client.ErrNotFound) {
|
||||
return ErrObjectNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func IsObjectNotFound(err error) bool {
|
||||
return errors.Is(err, ErrObjectNotFound) || errors.Is(err, s3client.ErrNotFound)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user