package scanner import ( "context" "errors" "seanime/internal/api/anilist" "seanime/internal/api/mal" "seanime/internal/api/metadata" "seanime/internal/hook" "seanime/internal/library/anime" "seanime/internal/platforms/platform" "seanime/internal/util" "seanime/internal/util/limiter" "seanime/internal/util/parallel" "time" "github.com/davecgh/go-spew/spew" "github.com/rs/zerolog" "github.com/samber/lo" lop "github.com/samber/lo/parallel" ) // MediaFetcher holds all anilist.BaseAnime that will be used for the comparison process type MediaFetcher struct { AllMedia []*anilist.CompleteAnime CollectionMediaIds []int UnknownMediaIds []int // Media IDs that are not in the user's collection AnimeCollectionWithRelations *anilist.AnimeCollectionWithRelations ScanLogger *ScanLogger } type MediaFetcherOptions struct { Enhanced bool Platform platform.Platform MetadataProvider metadata.Provider LocalFiles []*anime.LocalFile CompleteAnimeCache *anilist.CompleteAnimeCache Logger *zerolog.Logger AnilistRateLimiter *limiter.Limiter DisableAnimeCollection bool ScanLogger *ScanLogger } // NewMediaFetcher // Calling this method will kickstart the fetch process // When enhancing is false, MediaFetcher.AllMedia will be all anilist.BaseAnime from the user's AniList collection. // When enhancing is true, MediaFetcher.AllMedia will be anilist.BaseAnime for each unique, parsed anime title and their relations. func NewMediaFetcher(ctx context.Context, opts *MediaFetcherOptions) (ret *MediaFetcher, retErr error) { defer util.HandlePanicInModuleWithError("library/scanner/NewMediaFetcher", &retErr) if opts.Platform == nil || opts.LocalFiles == nil || opts.CompleteAnimeCache == nil || opts.MetadataProvider == nil || opts.Logger == nil || opts.AnilistRateLimiter == nil { return nil, errors.New("missing options") } mf := new(MediaFetcher) mf.ScanLogger = opts.ScanLogger opts.Logger.Debug(). Any("enhanced", opts.Enhanced). Msg("media fetcher: Creating media fetcher") if mf.ScanLogger != nil { mf.ScanLogger.LogMediaFetcher(zerolog.InfoLevel). Msg("Creating media fetcher") } // Invoke ScanMediaFetcherStarted hook event := &ScanMediaFetcherStartedEvent{ Enhanced: opts.Enhanced, } hook.GlobalHookManager.OnScanMediaFetcherStarted().Trigger(event) opts.Enhanced = event.Enhanced // +---------------------+ // | All media | // +---------------------+ // Fetch latest user's AniList collection animeCollectionWithRelations, err := opts.Platform.GetAnimeCollectionWithRelations(ctx) if err != nil { return nil, err } mf.AnimeCollectionWithRelations = animeCollectionWithRelations mf.AllMedia = make([]*anilist.CompleteAnime, 0) if !opts.DisableAnimeCollection { // For each collection entry, append the media to AllMedia for _, list := range animeCollectionWithRelations.GetMediaListCollection().GetLists() { for _, entry := range list.GetEntries() { mf.AllMedia = append(mf.AllMedia, entry.GetMedia()) // +---------------------+ // | Cache | // +---------------------+ // We assume the CompleteAnimeCache is empty. Add media to cache. opts.CompleteAnimeCache.Set(entry.GetMedia().ID, entry.GetMedia()) } } } if mf.ScanLogger != nil { mf.ScanLogger.LogMediaFetcher(zerolog.DebugLevel). Int("count", len(mf.AllMedia)). Msg("Fetched media from AniList collection") } //-------------------------------------------- // Get the media IDs from the collection mf.CollectionMediaIds = lop.Map(mf.AllMedia, func(m *anilist.CompleteAnime, index int) int { return m.ID }) //-------------------------------------------- // +---------------------+ // | Enhanced | // +---------------------+ // If enhancing is on, scan media from local files and get their relations if opts.Enhanced { _, ok := FetchMediaFromLocalFiles( ctx, opts.Platform, opts.LocalFiles, opts.CompleteAnimeCache, // CompleteAnimeCache will be populated on success opts.MetadataProvider, opts.AnilistRateLimiter, mf.ScanLogger, ) if ok { // We assume the CompleteAnimeCache is populated. We overwrite AllMedia with the cache content. // This is because the cache will contain all media from the user's collection AND scanned ones mf.AllMedia = make([]*anilist.CompleteAnime, 0) opts.CompleteAnimeCache.Range(func(key int, value *anilist.CompleteAnime) bool { mf.AllMedia = append(mf.AllMedia, value) return true }) } } // +---------------------+ // | Unknown media | // +---------------------+ // Media that are not in the user's collection // Get the media that are not in the user's collection unknownMedia := lo.Filter(mf.AllMedia, func(m *anilist.CompleteAnime, _ int) bool { return !lo.Contains(mf.CollectionMediaIds, m.ID) }) // Get the media IDs that are not in the user's collection mf.UnknownMediaIds = lop.Map(unknownMedia, func(m *anilist.CompleteAnime, _ int) int { return m.ID }) if mf.ScanLogger != nil { mf.ScanLogger.LogMediaFetcher(zerolog.DebugLevel). Int("unknownMediaCount", len(mf.UnknownMediaIds)). Int("allMediaCount", len(mf.AllMedia)). Msg("Finished creating media fetcher") } // Invoke ScanMediaFetcherCompleted hook completedEvent := &ScanMediaFetcherCompletedEvent{ AllMedia: mf.AllMedia, UnknownMediaIds: mf.UnknownMediaIds, } _ = hook.GlobalHookManager.OnScanMediaFetcherCompleted().Trigger(completedEvent) mf.AllMedia = completedEvent.AllMedia mf.UnknownMediaIds = completedEvent.UnknownMediaIds return mf, nil } //---------------------------------------------------------------------------------------------------------------------- // FetchMediaFromLocalFiles gets media and their relations from local file titles. // It retrieves unique titles from local files, // fetches mal.SearchResultAnime from MAL, // uses these search results to get AniList IDs using metadata.AnimeMetadata mappings, // queries AniList to retrieve all anilist.BaseAnime using anilist.GetBaseAnimeById and their relations using anilist.FetchMediaTree. // It does not return an error if one of the steps fails. // It returns the scanned media and a boolean indicating whether the process was successful. func FetchMediaFromLocalFiles( ctx context.Context, platform platform.Platform, localFiles []*anime.LocalFile, completeAnime *anilist.CompleteAnimeCache, metadataProvider metadata.Provider, anilistRateLimiter *limiter.Limiter, scanLogger *ScanLogger, ) (ret []*anilist.CompleteAnime, ok bool) { defer util.HandlePanicInModuleThen("library/scanner/FetchMediaFromLocalFiles", func() { ok = false }) if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.DebugLevel). Str("module", "Enhanced"). Msg("Fetching media from local files") } rateLimiter := limiter.NewLimiter(time.Second, 20) rateLimiter2 := limiter.NewLimiter(time.Second, 20) // Get titles titles := anime.GetUniqueAnimeTitlesFromLocalFiles(localFiles) if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.DebugLevel). Str("module", "Enhanced"). Str("context", spew.Sprint(titles)). Msg("Parsed titles from local files") } // +---------------------+ // | MyAnimeList | // +---------------------+ // Get MAL media from titles malSR := parallel.NewSettledResults[string, *mal.SearchResultAnime](titles) malSR.AllSettled(func(title string, index int) (*mal.SearchResultAnime, error) { rateLimiter.Wait() return mal.AdvancedSearchWithMAL(title) }) malRes, ok := malSR.GetFulfilledResults() if !ok { return nil, false } // Get duplicate-free version of MAL media malMedia := lo.UniqBy(*malRes, func(res *mal.SearchResultAnime) int { return res.ID }) // Get the MAL media IDs malIds := lop.Map(malMedia, func(n *mal.SearchResultAnime, index int) int { return n.ID }) if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.DebugLevel). Str("module", "Enhanced"). Str("context", spew.Sprint(lo.Map(malMedia, func(n *mal.SearchResultAnime, _ int) string { return n.Name }))). Msg("Fetched MAL media from titles") } // +---------------------+ // | Animap | // +---------------------+ // Get Animap mappings for each MAL ID and store them in `metadataProvider` // This step is necessary because MAL doesn't provide AniList IDs and some MAL media don't exist on AniList lop.ForEach(malIds, func(id int, index int) { rateLimiter2.Wait() //_, _ = metadataProvider.GetAnimeMetadata(metadata.MalPlatform, id) _, _ = metadataProvider.GetCache().GetOrSet(metadata.GetAnimeMetadataCacheKey(metadata.MalPlatform, id), func() (*metadata.AnimeMetadata, error) { res, err := metadataProvider.GetAnimeMetadata(metadata.MalPlatform, id) return res, err }) }) // +---------------------+ // | AniList | // +---------------------+ // Retrieve the AniList IDs from the Animap mappings stored in the cache anilistIds := make([]int, 0) metadataProvider.GetCache().Range(func(key string, value *metadata.AnimeMetadata) bool { if value != nil { anilistIds = append(anilistIds, value.GetMappings().AnilistId) } return true }) // Fetch all media from the AniList IDs anilistMedia := make([]*anilist.CompleteAnime, 0) lop.ForEach(anilistIds, func(id int, index int) { anilistRateLimiter.Wait() media, err := platform.GetAnimeWithRelations(ctx, id) if err == nil { anilistMedia = append(anilistMedia, media) if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.DebugLevel). Str("module", "Enhanced"). Str("title", media.GetTitleSafe()). Msg("Fetched Anilist media from MAL id") } } else { if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.WarnLevel). Str("module", "Enhanced"). Int("id", id). Msg("Failed to fetch Anilist media from MAL id") } } }) if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.DebugLevel). Str("module", "Enhanced"). Str("context", spew.Sprint(lo.Map(anilistMedia, func(n *anilist.CompleteAnime, _ int) string { return n.GetTitleSafe() }))). Msg("Fetched Anilist media from MAL ids") } // +---------------------+ // | MediaTree | // +---------------------+ // Create a new tree that will hold the fetched relations // /!\ This is redundant because we already have a cache, but `FetchMediaTree` needs its tree := anilist.NewCompleteAnimeRelationTree() start := time.Now() // For each media, fetch its relations // The relations are fetched in parallel and added to `completeAnime` lop.ForEach(anilistMedia, func(m *anilist.CompleteAnime, index int) { // We ignore errors because we want to continue even if one of the media fails _ = m.FetchMediaTree(anilist.FetchMediaTreeAll, platform.GetAnilistClient(), anilistRateLimiter, tree, completeAnime) }) // +---------------------+ // | Cache | // +---------------------+ // Retrieve all media from the cache scanned := make([]*anilist.CompleteAnime, 0) completeAnime.Range(func(key int, value *anilist.CompleteAnime) bool { scanned = append(scanned, value) return true }) if scanLogger != nil { scanLogger.LogMediaFetcher(zerolog.InfoLevel). Str("module", "Enhanced"). Int("ms", int(time.Since(start).Milliseconds())). Int("count", len(scanned)). Str("context", spew.Sprint(lo.Map(scanned, func(n *anilist.CompleteAnime, _ int) string { return n.GetTitleSafe() }))). Msg("Finished fetching media from local files") } return scanned, true }