Files
silo-server/internal/metadata/person_refresh.go
T
14ffc91dfb [codex] Expand provider image cache queue (#176)
* feat(metadata): expand provider image cache queue

* fix(metadata): harden provider image cache queue

Addresses bug-review feedback from Codex/CodeRabbit on the metadata image
cache pipeline. All findings validated against the code before fixing;
false positives (rows/connection deadlock, PhotoSourcePath merge coupling)
were confirmed non-issues and left unchanged.

- Honor metadata.cache_images for the background processor. The
  cache_metadata_images task was registered whenever S3 was configured,
  so merely enabling object storage downloaded the entire provider-artwork
  catalog even with caching disabled. Add ImageCacheProcessor.SetEnabled,
  gate RunOnce/RunUntilIdle on it, and wire it (with hot reload) from
  cfg.Metadata.CacheImages in main.go.
- Guard terminal job updates with lease ownership. EnqueueBatch can
  repurpose a running row with a new source; MarkSucceeded/MarkFailed
  keyed on id alone let a stale worker finalize the replacement job and
  drop the new artwork. Thread locked_by through and add
  status='running' AND locked_by=$n guards.
- Avoid uploading stale jobs onto the live artwork key. Verify the
  target still references the job's source (CurrentTargetSourcePath)
  before CacheImage, so a job whose source an admin/refresh already
  replaced cannot overwrite the deterministic storage object.
- COALESCE nullable external IDs in EnqueueExistingProviderArtwork. A
  NULL tmdb_id/tvdb_id/imdb_id on any candidate failed the scan and
  aborted the whole cache run; matches the existing item_repo pattern.
- Stop re-downloading the catalog every 30 days. Discovery now skips
  targets whose *_path is already a cached relative path, making the
  cached row the durable dedup marker instead of the prunable job row.
- Decouple catalog sweeps from queue draining. RunOnce no longer runs
  discovery per batch; RunUntilIdle sweeps only when the queue drains and
  throttles full sweeps to every 15m, so idle installs stop full-scanning
  every entity table each minute.
- Requeue claimed-but-unstarted jobs on cancellation. Acquire the
  semaphore before spawning workers and RequeueClaimed any jobs not yet
  started, instead of leaving them locked until the 15m lease expires.
- Skip the backoff sleep after the final upload attempt in
  putObjectWithRetry (saves ~1.5s on permanent failures).
- Add the s3/file/local/upload/generated exclusion to the seasons and
  episodes backfill in migration 20260617184537 for consistency with the
  later migration (the bad backfill was inert downstream, but the
  asymmetry is removed).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-18 10:07:58 -04:00

379 lines
10 KiB
Go

package metadata
import (
"context"
"errors"
"fmt"
"log/slog"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/models"
)
var (
ErrPersonNotFound = errors.New("person not found")
ErrPersonMetadataNotFound = errors.New("no person metadata found from any provider")
)
type personRefreshRepo interface {
Get(ctx context.Context, id int64) (*models.Person, error)
Update(ctx context.Context, person models.Person) error
FindRefreshCandidates(ctx context.Context, staleAfter time.Duration, limit int) ([]int64, error)
}
type PersonRefreshService struct {
pool *pgxpool.Pool
pluginResolver pluginMetadataResolver
repo personRefreshRepo
imageCacher ImageCacher
imageCacheJobs ImageCacheJobEnqueuer
imageResolver interface {
ResolveImageURL(ctx context.Context, path string, variant string) string
}
}
func NewPersonRefreshService(
pool *pgxpool.Pool,
pluginResolver pluginMetadataResolver,
repo *catalog.PersonRepository,
) *PersonRefreshService {
return &PersonRefreshService{
pool: pool,
pluginResolver: pluginResolver,
repo: repo,
}
}
func (s *PersonRefreshService) SetImageCacher(cacher ImageCacher) {
s.imageCacher = cacher
}
func (s *PersonRefreshService) SetImageCacheJobEnqueuer(enqueuer ImageCacheJobEnqueuer) {
s.imageCacheJobs = enqueuer
}
func (s *PersonRefreshService) SetImageResolver(resolver interface {
ResolveImageURL(ctx context.Context, path string, variant string) string
}) {
s.imageResolver = resolver
}
func (s *PersonRefreshService) RefreshPerson(ctx context.Context, id int64) (*models.Person, error) {
if s.repo == nil {
return nil, fmt.Errorf("person refresh repository is not configured")
}
if s.pluginResolver == nil || s.pool == nil {
return nil, fmt.Errorf("person refresh providers are not configured")
}
providers, err := resolveEnabledProviders(ctx, s.pluginResolver, s.pool)
if err != nil {
return nil, fmt.Errorf("resolve person providers: %w", err)
}
return s.refreshPersonWithProviders(ctx, id, providers)
}
func (s *PersonRefreshService) FindCandidates(
ctx context.Context,
staleAfter time.Duration,
limit int,
) ([]int64, error) {
if s.repo == nil {
return nil, fmt.Errorf("person refresh repository is not configured")
}
return s.repo.FindRefreshCandidates(ctx, staleAfter, limit)
}
func (s *PersonRefreshService) refreshPersonWithProviders(
ctx context.Context,
id int64,
providers []Provider,
) (*models.Person, error) {
person, err := s.repo.Get(ctx, id)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrPersonNotFound
}
return nil, fmt.Errorf("load person %d: %w", id, err)
}
if person == nil {
return nil, ErrPersonNotFound
}
accumulator := PersonDetailResult{
ProviderIDs: copyMap(personProviderIDs(*person)),
}
photoProviderID := ""
hasMetadata := false
for _, provider := range providers {
personProvider, ok := provider.(PersonProvider)
if !ok {
continue
}
result, err := personProvider.GetPersonDetail(ctx, PersonDetailRequest{
ProviderIDs: accumulator.ProviderIDs,
Language: "en",
})
if err != nil {
slog.Warn("person refresh: provider detail lookup failed",
"provider", provider.Slug(),
"person_id", id,
"error", err,
)
continue
}
if result == nil {
continue
}
hasMetadata = true
if photoProviderID == "" && strings.TrimSpace(result.PhotoPath) != "" {
photoProviderID = provider.Slug()
}
MergePersonDetail(result, &accumulator, MergeFillEmpty)
}
if !hasMetadata {
return nil, ErrPersonMetadataNotFound
}
accumulator.PhotoSourcePath = providerImageSourcePath(accumulator.PhotoPath)
if accumulator.PhotoSourcePath != "" {
accumulator.PhotoPath, accumulator.PhotoThumbhash, accumulator.PhotoSourcePath = preserveCachedArtwork(
accumulator.PhotoPath,
accumulator.PhotoThumbhash,
person.PhotoPath,
person.PhotoSourcePath,
person.PhotoThumbhash,
)
}
existingDetail := personToPersonDetailResult(*person)
MergePersonDetail(&accumulator, &existingDetail, MergeReplaceUnlocked)
accumulator = existingDetail
refreshed, err := mergePersonIntoRecord(*person, accumulator)
if err != nil {
return nil, err
}
if err := s.repo.Update(ctx, refreshed); err != nil {
return nil, fmt.Errorf("update person %d: %w", id, err)
}
s.enqueuePersonPhoto(ctx, refreshed, accumulator.ProviderIDs, photoProviderID)
return &refreshed, nil
}
func (s *PersonRefreshService) enqueuePersonPhoto(ctx context.Context, person models.Person, providerIDs map[string]string, photoProviderID string) {
if s == nil || s.imageCacheJobs == nil || !isRemoteImageSourcePath(person.PhotoSourcePath) {
return
}
providerID := providerIDFromPluginURL(person.PhotoSourcePath)
if providerID == "" {
providerID = photoProviderID
}
if providerID == "" {
providerID = primaryPersonProviderID(providerIDs)
}
if providerID == "" {
providerID = "remote"
}
contentID := personCacheContentID(person, providerIDs, providerID)
enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if err := s.imageCacheJobs.Enqueue(enqueueCtx, EnqueueImageCacheJobInput{
TargetType: ImageCacheTargetPerson,
TargetContentID: strconv.FormatInt(person.ID, 10),
SourcePath: person.PhotoSourcePath,
ProviderID: providerID,
ProviderContentID: contentID,
ContentType: "people",
ImageType: ImageCacheImageProfile,
}); err != nil {
slog.Warn("person refresh: failed to enqueue photo cache job",
"person_id", person.ID,
"provider", providerID,
"error", err,
)
}
}
func (s *PersonRefreshService) cachePersonPhoto(
ctx context.Context,
person models.Person,
detail PersonDetailResult,
photoProviderID string,
) (string, string, error) {
if s.imageCacher == nil {
return detail.PhotoPath, detail.PhotoThumbhash, nil
}
if strings.TrimSpace(detail.PhotoPath) == "" || detail.PhotoPath == "-" {
return detail.PhotoPath, detail.PhotoThumbhash, nil
}
if !strings.Contains(detail.PhotoPath, "://") {
return detail.PhotoPath, detail.PhotoThumbhash, nil
}
downloadURL := detail.PhotoPath
if !strings.HasPrefix(downloadURL, "http://") && !strings.HasPrefix(downloadURL, "https://") {
if s.imageResolver == nil {
return "", "", fmt.Errorf("plugin image resolver is not configured")
}
downloadURL = s.imageResolver.ResolveImageURL(ctx, detail.PhotoPath, "original")
if downloadURL == "" {
return "", "", fmt.Errorf("resolved empty URL for %q", detail.PhotoPath)
}
}
providerID := photoProviderID
if providerID == "" {
providerID = primaryPersonProviderID(detail.ProviderIDs)
}
if providerID == "" {
providerID = "unknown"
}
contentID := personCacheContentID(person, detail.ProviderIDs, providerID)
result, err := s.imageCacher.CacheImage(ctx, CacheImageRequest{
SourceURL: downloadURL,
ProviderID: providerID,
ContentType: "people",
ContentID: contentID,
ImageType: ImagePoster,
})
if err != nil {
return "", "", err
}
return cachedOriginalImagePath(result.BasePath, result.Ext), result.Thumbhash, nil
}
func personProviderIDs(person models.Person) map[string]string {
ids := map[string]string{}
if person.TmdbID != "" {
ids["tmdb"] = person.TmdbID
}
if person.ImdbID != "" {
ids["imdb"] = person.ImdbID
}
if person.TvdbID != "" {
ids["tvdb"] = person.TvdbID
}
if person.PlexGUID != "" {
ids["plex"] = person.PlexGUID
}
return ids
}
func personToPersonDetailResult(person models.Person) PersonDetailResult {
result := PersonDetailResult{
Name: person.Name,
SortName: person.SortName,
Bio: person.Bio,
Birthplace: person.Birthplace,
Homepage: person.Homepage,
PhotoPath: person.PhotoPath,
PhotoSourcePath: person.PhotoSourcePath,
PhotoThumbhash: person.PhotoThumbhash,
ProviderIDs: copyMap(personProviderIDs(person)),
}
if person.BirthDate != nil {
result.BirthDate = person.BirthDate.Format("2006-01-02")
}
if person.DeathDate != nil {
result.DeathDate = person.DeathDate.Format("2006-01-02")
}
return result
}
func mergePersonIntoRecord(person models.Person, detail PersonDetailResult) (models.Person, error) {
birthDate, err := parseOptionalPersonDate(detail.BirthDate)
if err != nil {
return person, fmt.Errorf("parse birth date for person %d: %w", person.ID, err)
}
deathDate, err := parseOptionalPersonDate(detail.DeathDate)
if err != nil {
return person, fmt.Errorf("parse death date for person %d: %w", person.ID, err)
}
person.Name = detail.Name
person.SortName = detail.SortName
person.Bio = detail.Bio
person.BirthDate = birthDate
person.DeathDate = deathDate
person.Birthplace = detail.Birthplace
person.Homepage = detail.Homepage
person.PhotoPath = detail.PhotoPath
person.PhotoSourcePath = detail.PhotoSourcePath
person.PhotoThumbhash = detail.PhotoThumbhash
person.TmdbID = detail.ProviderIDs["tmdb"]
person.ImdbID = detail.ProviderIDs["imdb"]
person.TvdbID = detail.ProviderIDs["tvdb"]
if plexID := detail.ProviderIDs["plex"]; plexID != "" {
person.PlexGUID = plexID
}
return person, nil
}
func parseOptionalPersonDate(value string) (*time.Time, error) {
value = strings.TrimSpace(value)
if value == "" {
return nil, nil
}
parsed, err := time.Parse("2006-01-02", value)
if err != nil {
return nil, err
}
return &parsed, nil
}
func primaryPersonProviderID(providerIDs map[string]string) string {
for _, key := range []string{"tmdb", "tvdb", "imdb", "metadb"} {
if providerIDs[key] != "" {
return key
}
}
return ""
}
func personCacheContentID(
person models.Person,
providerIDs map[string]string,
providerID string,
) string {
if providerID != "" && providerIDs[providerID] != "" {
return providerIDs[providerID]
}
for _, key := range []string{"tmdb", "tvdb", "imdb", "metadb"} {
if providerIDs[key] != "" {
return providerIDs[key]
}
}
return strconv.FormatInt(person.ID, 10)
}
func cachedOriginalImagePath(basePath, ext string) string {
if basePath == "" {
return ""
}
if strings.Contains(basePath, "/original.") {
return basePath
}
if ext == "" {
ext = ".jpg"
}
return strings.TrimRight(basePath, "/") + "/original" + ext
}