perf(literaryworks): cut Postgres load on ebook↔audiobook auto-linking (#253)

* perf(literaryworks): cut Postgres load on ebook↔audiobook auto-linking Rescans and candidate matching were driving high Postgres CPU on book libraries. - AutoLinkContent now checks literary_work_items with a cheap indexed lookup before GetMatchItem, so unchanged already-linked books skip the heavy triple-lateral query on every rescan. - ListMatchCandidates is split into an indexable ID-selection phase (title / provider EXISTS / series EXISTS) and a hydration phase, so the per-row lateral aggregates run only for the LIMIT candidates kept rather than for every opposite-format book. - Add a partial LOWER(title) index scoped to ebook/audiobook so the OR of title/provider/series filters can be driven entirely by indexes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(migrations): drop invalid leftover index before concurrent rebuild An interrupted CREATE INDEX CONCURRENTLY (cancel/restart) can leave an INVALID idx_media_items_books_title_lower; IF NOT EXISTS would then skip the rebuild while Goose records success. Add the preflight invalid-index drop used by the repo's other concurrent-index migrations. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-01 23:21:25 -04:00
parent 9cae868a27
commit 57fbc1e0b4
3 changed files with 134 additions and 38 deletions
@@ -160,48 +160,20 @@ func (r *Repository) ListMatchCandidates(ctx context.Context, source MatchItem,
 	if limit <= 0 {
 		limit = 20
 	}
-	args := []any{source.ContentID, source.Type}
-	matchFilters := make([]string, 0, 3)
-	if strings.TrimSpace(source.Title) != "" {
-		args = append(args, source.Title)
-		matchFilters = append(matchFilters, fmt.Sprintf("LOWER(mi.title) = LOWER($%d)", len(args)))
+	// Two phases: pick candidate IDs with indexable base-table predicates, then
+	// hydrate only those rows. This keeps the per-row lateral aggregates in
+	// queryMatchItems off every opposite-format book and on the LIMIT rows we keep.
+	contentIDs, err := r.listMatchCandidateIDs(ctx, source, limit)
+	if err != nil {
+		return nil, err
 	}
-	for provider, providerID := range source.ExternalIDs {
-		if provider == "" || provider == "asin" || providerID == "" {
-			continue
-		}
-		args = append(args, provider, providerID)
-		matchFilters = append(matchFilters, fmt.Sprintf("provider_ids.external_ids ->> $%d = $%d", len(args)-1, len(args)))
+	if len(contentIDs) == 0 {
+		return nil, nil
 	}
-	if strings.TrimSpace(source.SeriesName) != "" && source.SeriesIndex != nil {
-		args = append(args, source.SeriesName, *source.SeriesIndex)
-		matchFilters = append(matchFilters, fmt.Sprintf(
-			"(LOWER(COALESCE(book_series.series_name, '')) = LOWER($%d) AND book_series.series_index = $%d)",
-			len(args)-1,
-			len(args),
-		))
-	}
-	matchWhere := ""
-	if len(matchFilters) > 0 {
-		matchWhere = " AND (" + strings.Join(matchFilters, " OR ") + ")"
-	}
-	args = append(args, limit)
 	rows, err := r.queryMatchItems(ctx, `
-		WHERE mi.content_id <> $1
-		  AND mi.type IN ('ebook', 'audiobook')
-		  AND mi.type <> $2
-		  AND NOT EXISTS (
-			SELECT 1 FROM literary_work_match_decisions d
-			WHERE d.decision = 'ignored'
-			  AND (
-				(d.source_content_id = $1 AND d.target_content_id = mi.content_id)
-				OR (d.source_content_id = mi.content_id AND d.target_content_id = $1)
-			  )
-		  )
-		`+matchWhere+`
+		WHERE mi.content_id = ANY($1)
 		ORDER BY mi.title ASC, mi.content_id ASC
-		LIMIT $`+fmt.Sprint(len(args))+`
-	`, args...)
+	`, contentIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -217,6 +189,83 @@ func (r *Repository) ListMatchCandidates(ctx context.Context, source MatchItem,
 	return items, rows.Err()
 }

+// listMatchCandidateIDs selects opposite-format candidate content IDs using only
+// predicates that can ride base-table indexes (title on media_items, provider IDs
+// on media_item_provider_ids, series on the format-specific series table) instead
+// of filtering on lateral aggregate output. The opposite format is fixed by the
+// source type, so the series lookup targets a single concrete table.
+func (r *Repository) listMatchCandidateIDs(ctx context.Context, source MatchItem, limit int) ([]string, error) {
+	if r == nil || r.pool == nil {
+		return nil, fmt.Errorf("literary works repository requires a database pool")
+	}
+	// Candidates are the opposite format of the source; match their own series table.
+	seriesTable := "ebook_series"
+	if source.Type == "ebook" {
+		seriesTable = "audiobook_series"
+	}
+	args := []any{source.ContentID, source.Type}
+	matchFilters := make([]string, 0, 3)
+	if strings.TrimSpace(source.Title) != "" {
+		args = append(args, source.Title)
+		matchFilters = append(matchFilters, fmt.Sprintf("LOWER(mi.title) = LOWER($%d)", len(args)))
+	}
+	for provider, providerID := range source.ExternalIDs {
+		if provider == "" || provider == "asin" || providerID == "" {
+			continue
+		}
+		args = append(args, provider, providerID)
+		matchFilters = append(matchFilters, fmt.Sprintf(
+			"EXISTS (SELECT 1 FROM media_item_provider_ids mip WHERE mip.content_id = mi.content_id AND mip.provider = $%d AND mip.provider_id = $%d)",
+			len(args)-1,
+			len(args),
+		))
+	}
+	if strings.TrimSpace(source.SeriesName) != "" && source.SeriesIndex != nil {
+		args = append(args, source.SeriesName, *source.SeriesIndex)
+		matchFilters = append(matchFilters, fmt.Sprintf(
+			"EXISTS (SELECT 1 FROM %s bs WHERE bs.content_id = mi.content_id AND LOWER(bs.series_name) = LOWER($%d) AND bs.series_index = $%d)",
+			seriesTable,
+			len(args)-1,
+			len(args),
+		))
+	}
+	matchWhere := ""
+	if len(matchFilters) > 0 {
+		matchWhere = " AND (" + strings.Join(matchFilters, " OR ") + ")"
+	}
+	args = append(args, limit)
+	rows, err := r.pool.Query(ctx, `
+		SELECT mi.content_id
+		FROM media_items mi
+		WHERE mi.content_id <> $1
+		  AND mi.type IN ('ebook', 'audiobook')
+		  AND mi.type <> $2
+		  AND NOT EXISTS (
+			SELECT 1 FROM literary_work_match_decisions d
+			WHERE d.decision = 'ignored'
+			  AND (
+				(d.source_content_id = $1 AND d.target_content_id = mi.content_id)
+				OR (d.source_content_id = mi.content_id AND d.target_content_id = $1)
+			  )
+		  )`+matchWhere+`
+		ORDER BY mi.title ASC, mi.content_id ASC
+		LIMIT $`+fmt.Sprint(len(args))+`
+	`, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var ids []string
+	for rows.Next() {
+		var id string
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		ids = append(ids, id)
+	}
+	return ids, rows.Err()
+}
+
 func (r *Repository) queryMatchItems(ctx context.Context, suffix string, args ...any) (pgx.Rows, error) {
 	if r == nil || r.pool == nil {
 		return nil, fmt.Errorf("literary works repository requires a database pool")
@@ -139,6 +139,15 @@ func (s *Service) AutoLinkContent(ctx context.Context, contentID string) (string
 	if s == nil || s.repo == nil {
 		return "", false, ErrWorkNotFound
 	}
+	// Cheap guard first: an item already linked to a work needs no candidate
+	// scan. Rescans re-visit every unchanged book, so skipping the expensive
+	// GetMatchItem + ListMatchCandidates path here is what keeps Postgres idle
+	// on steady-state rescans instead of rebuilding lateral aggregates per book.
+	if workID, err := s.repo.GetFirstWorkIDForContentIDs(ctx, []string{contentID}); err != nil {
+		return "", false, err
+	} else if strings.TrimSpace(workID) != "" {
+		return workID, false, nil
+	}
 	source, err := s.repo.GetMatchItem(ctx, contentID)
 	if err != nil {
 		return "", false, err
@@ -0,0 +1,38 @@
+-- +goose NO TRANSACTION
+
+-- +goose Up
+-- literaryworks.ListMatchCandidates matches ebooks against audiobooks (and vice
+-- versa) by exact case-insensitive title. Combined with the provider-id and
+-- series EXISTS predicates it forms an OR, so the planner can only bitmap-OR the
+-- whole filter through indexes when every branch is indexable; without a
+-- LOWER(title) index the title branch forces a full media_items scan and negates
+-- the provider/series indexes. Scoped to books so it stays small on large
+-- movie/show catalogs.
+--
+-- Preflight: a canceled build or server restart can leave an INVALID index of
+-- this name. IF NOT EXISTS would then skip the rebuild while Goose records
+-- success, so drop any invalid leftover before retrying.
+-- +goose StatementBegin
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        JOIN pg_index i ON i.indexrelid = c.oid
+        WHERE n.nspname = 'public'
+          AND c.relname = 'idx_media_items_books_title_lower'
+          AND NOT i.indisvalid
+    ) THEN
+        DROP INDEX public.idx_media_items_books_title_lower;
+    END IF;
+END;
+$$;
+-- +goose StatementEnd
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_media_items_books_title_lower
+    ON media_items (LOWER(title))
+    WHERE type IN ('ebook', 'audiobook');
+
+-- +goose Down
+DROP INDEX CONCURRENTLY IF EXISTS idx_media_items_books_title_lower;