package historyimport import ( "context" "errors" "fmt" "log/slog" "net/http" "net/url" "strconv" "sync" "time" "github.com/google/uuid" "github.com/Silo-Server/silo-server/internal/userstore" "github.com/Silo-Server/silo-server/internal/watchstate" ) // maxConcurrentRuns limits how many import runs execute simultaneously. // Additional runs stay queued until a slot opens. This prevents overwhelming // the external server and database when bulk-importing many users at once. const maxConcurrentRuns = 5 type Service struct { repo *Repository matcher *Matcher emby *EmbyClient jellyfin *JellyfinClient plex *PlexClient watchState *watchstate.Service bgContext context.Context // runSemaphore limits concurrent run goroutines to maxConcurrentRuns. runSemaphore chan struct{} // runCancels allows in-process cancellation of running import goroutines. runCancels map[string]context.CancelFunc runCancelsMu sync.Mutex observers []Observer } func NewService(bgContext context.Context, repo *Repository, storeProvider userstore.UserStoreProvider) *Service { if bgContext == nil { bgContext = context.Background() } service := &Service{ repo: repo, matcher: NewMatcher(repo), emby: NewEmbyClient(), jellyfin: NewJellyfinClient(), plex: NewPlexClient(), watchState: watchstate.NewService(storeProvider), bgContext: bgContext, runSemaphore: make(chan struct{}, maxConcurrentRuns), runCancels: make(map[string]context.CancelFunc), } service.startStaleRunMonitor() return service } func (s *Service) SetStableIdentityResolver(identity *watchstate.StableIdentityResolver) { if s == nil || s.watchState == nil { return } s.watchState = s.watchState.WithStableIdentityResolver(identity) } func (s *Service) registerRunCancel(runID string, cancel context.CancelFunc) { s.runCancelsMu.Lock() s.runCancels[runID] = cancel s.runCancelsMu.Unlock() } func (s *Service) deregisterRunCancel(runID string) { s.runCancelsMu.Lock() delete(s.runCancels, runID) s.runCancelsMu.Unlock() } func (s *Service) cancelRunInProcess(runID string) { s.runCancelsMu.Lock() cancel, ok := s.runCancels[runID] s.runCancelsMu.Unlock() if ok { cancel() } } func (s *Service) ListUserSources(ctx context.Context) ([]Source, error) { if err := s.repo.DeleteExpiredConnectSessions(ctx); err != nil { return nil, err } return s.repo.ListEnabledSources(ctx) } func (s *Service) LoginConnect(ctx context.Context, userID int, input LoginConnectInput) (*ConnectSessionLoginResult, error) { if err := s.repo.DeleteExpiredConnectSessions(ctx); err != nil { return nil, err } authResp, err := s.emby.ConnectAuthenticate(ctx, input.Username, input.Password) if err != nil { return nil, err } servers, err := s.emby.ConnectServers(ctx, authResp.ConnectUserID, authResp.ConnectAccessToken) if err != nil { return nil, err } session, err := s.repo.CreateConnectSession(ctx, ConnectSession{ ID: uuid.NewString(), UserID: userID, ConnectUserID: authResp.ConnectUserID, ConnectAccessToken: authResp.ConnectAccessToken, Servers: servers, ExpiresAt: time.Now().UTC().Add(connectSessionTTL), }) if err != nil { return nil, err } return &ConnectSessionLoginResult{ ConnectSessionID: session.ID, Servers: toConnectServerResponses(session.Servers), ExpiresAt: session.ExpiresAt, }, nil } func (s *Service) CreatePlexPin(ctx context.Context, userID int) (*PlexPinResponse, error) { if err := s.repo.DeleteExpiredPlexSessions(ctx); err != nil { return nil, err } pinID, pinCode, err := s.plex.CreatePin(ctx) if err != nil { return nil, err } session, err := s.repo.CreatePlexSession(ctx, PlexSession{ ID: uuid.NewString(), UserID: userID, PinID: fmt.Sprintf("%d", pinID), PinCode: pinCode, ExpiresAt: time.Now().UTC().Add(connectSessionTTL), }) if err != nil { return nil, err } authURL := fmt.Sprintf("https://app.plex.tv/auth#?clientID=%s&code=%s&context%%5Bdevice%%5D%%5Bproduct%%5D=%s", url.QueryEscape(plexClientIdentifier), url.QueryEscape(pinCode), url.QueryEscape(plexProduct), ) return &PlexPinResponse{ SessionID: session.ID, PinCode: pinCode, AuthURL: authURL, ExpiresAt: session.ExpiresAt, }, nil } func (s *Service) CheckPlexPin(ctx context.Context, userID int, sessionID string) (*PlexCheckResponse, error) { session, err := s.repo.GetPlexSession(ctx, userID, sessionID) if err != nil { return nil, err } if session.AuthToken != "" { return &PlexCheckResponse{ Authenticated: true, Servers: toPlexServerPublicList(session.Servers), }, nil } pinID, err := strconv.Atoi(session.PinID) if err != nil { return nil, fmt.Errorf("invalid pin ID in session: %w", err) } authToken, err := s.plex.CheckPin(ctx, pinID) if err != nil { return nil, err } if authToken == "" { return &PlexCheckResponse{Authenticated: false}, nil } servers, err := s.plex.GetResources(ctx, authToken) if err != nil { return nil, fmt.Errorf("discovering Plex servers: %w", err) } if err := s.repo.UpdatePlexSessionAuth(ctx, session.ID, authToken, servers); err != nil { return nil, err } return &PlexCheckResponse{ Authenticated: true, Servers: toPlexServerPublicList(servers), }, nil } func toPlexServerPublicList(servers []PlexServer) []PlexServerPublic { result := make([]PlexServerPublic, 0, len(servers)) for _, s := range servers { result = append(result, PlexServerPublic{ Name: s.Name, ClientIdentifier: s.ClientIdentifier, Owned: s.Owned, HasRemoteURL: s.HasRemoteURL, HasLocalURL: s.HasLocalURL, }) } return result } func (s *Service) CreateRun(ctx context.Context, userID int, input CreateRunInput) (*Run, error) { if input.ProfileID == "" { return nil, fmt.Errorf("profile_id is required") } var sourceType, connectionMode string var provider Provider switch input.Source { case SourceTypeEmby: mode, err := resolveConnectionMode(input) if err != nil { return nil, err } if err := s.repo.DeleteExpiredConnectSessions(ctx); err != nil { return nil, err } auth, err := s.resolveAuth(ctx, userID, mode, input) if err != nil { return nil, err } sourceType = SourceTypeEmby connectionMode = mode provider = NewEmbyProvider(s.emby, *auth) case SourceTypeJellyfin: if input.JellyfinBaseURL == "" || input.JellyfinUsername == "" || input.JellyfinPassword == "" { return nil, fmt.Errorf("jellyfin_base_url, jellyfin_username, and jellyfin_password are required") } auth, err := s.jellyfin.AuthenticateServerUser(ctx, input.JellyfinBaseURL, input.JellyfinUsername, input.JellyfinPassword) if err != nil { return nil, err } sourceType = SourceTypeJellyfin connectionMode = ConnectionModeCustom provider = NewJellyfinProvider(s.jellyfin, *auth) case SourceTypePlex: auth, mode, err := s.resolvePlexAuth(ctx, userID, input) if err != nil { return nil, err } sourceType = SourceTypePlex connectionMode = mode provider = NewPlexServerProvider(s.plex, auth.BaseURL, auth.Token) default: return nil, fmt.Errorf("unsupported source type") } exists, err := s.repo.ProfileExistsForUser(ctx, userID, input.ProfileID) if err != nil { return nil, err } if !exists { return nil, ErrProfileNotFound } run := Run{ ID: uuid.NewString(), UserID: userID, ProfileID: input.ProfileID, SourceType: sourceType, ConnectionMode: connectionMode, Status: RunStatusQueued, Warnings: []string{}, UnmatchedSamples: []UnmatchedSample{}, } created, err := s.repo.CreateRun(ctx, run) if err != nil { return nil, err } s.notifyRun(created) go s.executeRun(created, provider) return created, nil } func (s *Service) resolveAuth(ctx context.Context, userID int, connectionMode string, input CreateRunInput) (*embyLocalAuth, error) { switch connectionMode { case ConnectionModeConnect: session, err := s.repo.GetConnectSession(ctx, userID, input.ConnectSessionID) if err != nil { return nil, err } var server *ConnectServer for i := range session.Servers { if session.Servers[i].ID == input.ServerID { server = &session.Servers[i] break } } if server == nil { return nil, fmt.Errorf("selected server not found in connect session") } baseURL := firstNonEmpty(server.URL, server.LocalAddress) if baseURL == "" { return nil, fmt.Errorf("selected server does not expose a usable address") } auth, err := s.emby.ConnectExchange(ctx, baseURL, session.ConnectUserID, server.AccessKey) if err != nil { return nil, err } if err := s.repo.ConsumeConnectSession(ctx, session.ID); err != nil { return nil, err } return auth, nil case ConnectionModePredefined: if input.SourceID <= 0 { return nil, fmt.Errorf("source_id is required") } if input.Username == "" || input.Password == "" { return nil, fmt.Errorf("username and password are required") } source, err := s.repo.GetSourceByID(ctx, input.SourceID) if err != nil { return nil, err } if !source.Enabled { return nil, fmt.Errorf("selected source is disabled") } return s.emby.AuthenticateServerUser(ctx, source.BaseURL, input.Username, input.Password) default: return nil, fmt.Errorf("invalid connection mode") } } func (s *Service) resolvePlexAuth(ctx context.Context, userID int, input CreateRunInput) (*plexAuth, string, error) { if err := s.repo.DeleteExpiredPlexSessions(ctx); err != nil { return nil, "", err } if input.PlexSessionID != "" { session, err := s.repo.GetPlexSession(ctx, userID, input.PlexSessionID) if err != nil { return nil, "", err } if session.AuthToken == "" { return nil, "", fmt.Errorf("plex OAuth not completed yet") } var server *PlexServer for i := range session.Servers { if session.Servers[i].ClientIdentifier == input.PlexServerID { server = &session.Servers[i] break } } if server == nil { return nil, "", fmt.Errorf("selected Plex server not found in session") } baseURL := firstNonEmpty(server.RemoteURL, server.LocalURL) if baseURL == "" { return nil, "", fmt.Errorf("selected Plex server has no usable address") } if err := s.repo.ConsumePlexSession(ctx, session.ID); err != nil { return nil, "", err } return &plexAuth{BaseURL: baseURL, Token: server.AccessToken}, ConnectionModePlexOAuth, nil } if input.PlexBaseURL != "" { if input.PlexToken == "" { return nil, "", fmt.Errorf("plex_token is required for browser Plex imports") } return &plexAuth{BaseURL: input.PlexBaseURL, Token: input.PlexToken}, ConnectionModePlexOAuth, nil } if input.PlexToken != "" { return nil, "", fmt.Errorf("plex_base_url is required for browser Plex imports") } if input.SourceID > 0 { source, err := s.repo.GetSourceByID(ctx, input.SourceID) if err != nil { return nil, "", err } if !source.Enabled { return nil, "", fmt.Errorf("selected source is disabled") } if source.SourceType != SourceTypePlex { return nil, "", fmt.Errorf("source is not a Plex server") } if input.PlexToken == "" { return nil, "", fmt.Errorf("plex_token is required for predefined Plex sources") } return &plexAuth{BaseURL: source.BaseURL, Token: input.PlexToken}, ConnectionModePredefined, nil } return nil, "", fmt.Errorf("plex_session_id, plex_base_url, or source_id is required for Plex imports") } func (s *Service) executeRun(run *Run, provider Provider) { ctx, cancel := newRunContext(s.bgContext) s.registerRunCancel(run.ID, cancel) defer s.deregisterRunCancel(run.ID) defer cancel() // Wait for a concurrency slot. The run stays in "queued" status until // a slot opens. If the context is cancelled (server shutdown or admin // cancel), the goroutine exits without executing. select { case s.runSemaphore <- struct{}{}: defer func() { <-s.runSemaphore }() case <-ctx.Done(): slog.Info("history import: run cancelled while queued", "run_id", run.ID) return } summary := ExecutionSummary{ Warnings: []string{}, UnmatchedSamples: []UnmatchedSample{}, } if err := s.repo.MarkRunStarted(ctx, run.ID); err != nil { slog.Error("history import: failed to mark run started", "run_id", run.ID, "error", err) return } s.notifyRunByID(ctx, run.ID) stopHeartbeat := s.startRunHeartbeat(ctx, run.ID) defer stopHeartbeat() slog.Info( "history import: started", "run_id", run.ID, "user_id", run.UserID, "profile_id", run.ProfileID, "source_type", run.SourceType, "connection_mode", run.ConnectionMode, ) records, warnings, err := provider.Fetch(ctx) if err != nil { summary.Warnings = append(summary.Warnings, warnings...) s.failRun(ctx, run.ID, summary, err) return } summary.Warnings = append(summary.Warnings, warnings...) summary.Fetched = len(records) slog.Info( "history import: fetched source records", "run_id", run.ID, "count", len(records), ) s.persistProgress(ctx, run.ID, summary) for i, record := range records { match, reason, err := s.matcher.Match(ctx, record) if err != nil { summary.Warnings = append(summary.Warnings, err.Error()) s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) continue } if match == nil { summary.Unmatched++ if reason != "" { if summary.UnmatchedReasonCounts == nil { summary.UnmatchedReasonCounts = make(map[string]int) } summary.UnmatchedReasonCounts[reason]++ } if len(summary.UnmatchedSamples) < maxUnmatchedSamples { summary.UnmatchedSamples = append(summary.UnmatchedSamples, UnmatchedSample{ Kind: record.Kind, Title: recordTitle(record), Year: record.Year, Reason: reason, }) } if summary.Unmatched <= maxUnmatchedLogSamples { s.logUnmatched(run.ID, i+1, len(records), record, reason) } s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) continue } summary.Matched++ shouldWriteProgress := true localProgress, err := s.repo.GetProgress(ctx, run.UserID, run.ProfileID, match.MediaItemID) if err != nil { summary.Warnings = append(summary.Warnings, err.Error()) s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) continue } if !shouldWriteImportedProgress(record, localProgress) { shouldWriteProgress = false summary.Skipped++ } if shouldWriteProgress { created, err := s.watchState.RecordImportedWatch( ctx, run.UserID, run.ProfileID, match.MediaItemID, record.DurationSeconds, importedPosition(record), record.Played, record.UpdatedAt, record.LastPlayedAt, ) if err != nil { summary.Warnings = append(summary.Warnings, err.Error()) s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) continue } summary.ProgressUpdated++ if created { summary.HistoryCreated++ } } else if record.LastPlayedAt != nil { created, err := s.watchState.RecordImportedHistory( ctx, run.UserID, run.ProfileID, match.MediaItemID, record.DurationSeconds, record.Played, record.LastPlayedAt, ) if err != nil { summary.Warnings = append(summary.Warnings, err.Error()) s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) continue } if created { summary.HistoryCreated++ } } s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) } if err := s.repo.CompleteRun(ctx, run.ID, summary); err != nil { slog.Error("history import: failed to complete run", "run_id", run.ID, "error", err) return } s.notifyRunByID(ctx, run.ID) slog.Info( "history import: completed", "run_id", run.ID, "fetched", summary.Fetched, "matched", summary.Matched, "unmatched", summary.Unmatched, "progress_updated", summary.ProgressUpdated, "history_created", summary.HistoryCreated, "skipped", summary.Skipped, "warnings", len(summary.Warnings), ) // Update the mapping's last_imported_at timestamp for admin-initiated runs. if run.MappingID != nil { if err := s.repo.TouchMappingLastImported(s.bgContext, *run.MappingID); err != nil { slog.Warn("history import: failed to touch mapping last_imported_at", "mapping_id", *run.MappingID, "error", err) } } } func (s *Service) failRun(ctx context.Context, runID string, summary ExecutionSummary, err error) { message := userFacingRunError(summary, err) if updateErr := s.repo.FailRun(ctx, runID, summary, message); updateErr != nil { slog.Error("history import: failed to mark run failed", "run_id", runID, "error", updateErr, "root_error", err) return } s.notifyRunByID(ctx, runID) slog.Error( "history import: failed", "run_id", runID, "error", message, "root_error", err, "fetched", summary.Fetched, "matched", summary.Matched, "unmatched", summary.Unmatched, "progress_updated", summary.ProgressUpdated, "history_created", summary.HistoryCreated, "skipped", summary.Skipped, ) } func userFacingRunError(summary ExecutionSummary, err error) string { if UpstreamHTTPStatus(err) == http.StatusUnauthorized { return "Couldn't connect to that server. Check the URL, username, and password and try again." } if summary.Fetched > 0 || summary.Matched > 0 || summary.ProgressUpdated > 0 || summary.HistoryCreated > 0 { return "Import stopped early. Some history may already be imported." } return "Import couldn't be completed. Please try again." } func (s *Service) ListRuns(ctx context.Context, userID, limit int) ([]Run, error) { if limit <= 0 { limit = 10 } if limit > 50 { limit = 50 } return s.repo.ListRunsForUser(ctx, userID, limit) } func (s *Service) ListActiveRuns(ctx context.Context, userID int) ([]Run, error) { return s.repo.ListActiveRunsForUser(ctx, userID) } func (s *Service) GetRun(ctx context.Context, userID int, runID string) (*Run, error) { return s.repo.GetRunForUser(ctx, userID, runID) } func (s *Service) ListAdminSources(ctx context.Context) ([]Source, error) { return s.repo.ListAdminSources(ctx) } func (s *Service) CreateSource(ctx context.Context, input CreateSourceInput) (*Source, error) { if input.Name == "" || input.BaseURL == "" { return nil, fmt.Errorf("name and base_url are required") } switch input.SourceType { case SourceTypeEmby, SourceTypeJellyfin, SourceTypePlex: default: return nil, fmt.Errorf("source_type must be emby, jellyfin, or plex") } return s.repo.CreateSource(ctx, input) } func (s *Service) UpdateSource(ctx context.Context, id int, input UpdateSourceInput) (*Source, error) { return s.repo.UpdateSource(ctx, id, input) } func (s *Service) DeleteSource(ctx context.Context, id int) error { return s.repo.DeleteSource(ctx, id) } func resolveConnectionMode(input CreateRunInput) (string, error) { if input.ServerURL != "" { return "", fmt.Errorf("direct server_url imports are no longer supported") } hasConnect := input.ConnectSessionID != "" hasSource := input.SourceID > 0 if hasConnect == hasSource { return "", fmt.Errorf("exactly one of connect_session_id or source_id is required") } if hasConnect { return ConnectionModeConnect, nil } return ConnectionModePredefined, nil } func recordTitle(record Record) string { if record.Kind == KindEpisode && record.SeriesTitle != "" { return record.SeriesTitle } return record.Title } func importedPosition(record Record) float64 { if record.Played && record.DurationSeconds > 0 { return record.DurationSeconds } return record.PositionSeconds } func (s *Service) logUnmatched(runID string, processed, total int, record Record, reason string) { slog.Info( "history import: unmatched item", "run_id", runID, "processed", processed, "total", total, "kind", record.Kind, "title", record.Title, "year", record.Year, "series_title", record.SeriesTitle, "series_year", record.SeriesYear, "season_number", record.SeasonNumber, "episode_number", record.EpisodeNumber, "tmdb_id", record.TMDBID, "imdb_id", record.IMDbID, "tvdb_id", record.TVDBID, "series_tmdb_id", record.SeriesTMDBID, "series_imdb_id", record.SeriesIMDbID, "series_tvdb_id", record.SeriesTVDBID, "reason", reason, ) } func IsNotFoundError(err error) bool { return errors.Is(err, ErrRunNotFound) || errors.Is(err, ErrSourceNotFound) || errors.Is(err, ErrProfileNotFound) || errors.Is(err, ErrConnectSessionNotFound) || errors.Is(err, ErrPlexSessionNotFound) } func shouldWriteImportedProgress(record Record, localProgress *localProgressRow) bool { if localProgress == nil { return true } if record.UpdatedAt.IsZero() { return false } return record.UpdatedAt.After(localProgress.UpdatedAt) } func toConnectServerResponses(servers []ConnectServer) []ConnectServerResponse { resp := make([]ConnectServerResponse, 0, len(servers)) for _, server := range servers { resp = append(resp, ConnectServerResponse{ ServerID: server.ID, Name: server.Name, SystemID: server.SystemID, HasRemoteURL: server.HasRemoteURL, HasLocalAddress: server.HasLocalURL, }) } return resp } func (s *Service) persistProgress(ctx context.Context, runID string, summary ExecutionSummary) { if err := s.repo.UpdateRunProgress(ctx, runID, summary); err != nil && !errors.Is(err, ErrRunNotFound) { slog.Warn("history import: failed to persist progress", "run_id", runID, "error", err) return } s.notifyRunByID(ctx, runID) } func (s *Service) persistProgressMaybe(ctx context.Context, runID string, summary ExecutionSummary, processed, total int) { if processed == total || processed%25 == 0 { s.persistProgress(ctx, runID, summary) slog.Info( "history import: progress", "run_id", runID, "processed", processed, "total", total, "matched", summary.Matched, "unmatched", summary.Unmatched, "progress_updated", summary.ProgressUpdated, "history_created", summary.HistoryCreated, "skipped", summary.Skipped, ) } }