464 lines
14 KiB
Go
464 lines
14 KiB
Go
package manga
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"seanime/internal/api/anilist"
|
|
"seanime/internal/database/db"
|
|
"seanime/internal/database/models"
|
|
"seanime/internal/events"
|
|
"seanime/internal/hook"
|
|
chapter_downloader "seanime/internal/manga/downloader"
|
|
manga_providers "seanime/internal/manga/providers"
|
|
"seanime/internal/util"
|
|
"seanime/internal/util/filecache"
|
|
"sync"
|
|
|
|
"github.com/rs/zerolog"
|
|
)
|
|
|
|
type (
|
|
Downloader struct {
|
|
logger *zerolog.Logger
|
|
wsEventManager events.WSEventManagerInterface
|
|
database *db.Database
|
|
downloadDir string
|
|
chapterDownloader *chapter_downloader.Downloader
|
|
repository *Repository
|
|
filecacher *filecache.Cacher
|
|
|
|
mediaMap *MediaMap // Refreshed on start and after each download
|
|
mediaMapMu sync.RWMutex
|
|
|
|
chapterDownloadedCh chan chapter_downloader.DownloadID
|
|
readingDownloadDir bool
|
|
isOffline *bool
|
|
}
|
|
|
|
// MediaMap is created after reading the download directory.
|
|
// It is used to store all downloaded chapters for each media.
|
|
// The key is the media ID and the value is a map of provider to a list of chapters.
|
|
//
|
|
// e.g., downloadDir/comick_1234_abc_13/
|
|
// downloadDir/comick_1234_def_13.5/
|
|
// -> { 1234: { "comick": [ { "chapterId": "abc", "chapterNumber": "13" }, { "chapterId": "def", "chapterNumber": "13.5" } ] } }
|
|
MediaMap map[int]ProviderDownloadMap
|
|
|
|
// ProviderDownloadMap is used to store all downloaded chapters for a specific media and provider.
|
|
// The key is the provider and the value is a list of chapters.
|
|
ProviderDownloadMap map[string][]ProviderDownloadMapChapterInfo
|
|
|
|
ProviderDownloadMapChapterInfo struct {
|
|
ChapterID string `json:"chapterId"`
|
|
ChapterNumber string `json:"chapterNumber"`
|
|
}
|
|
|
|
MediaDownloadData struct {
|
|
Downloaded ProviderDownloadMap `json:"downloaded"`
|
|
Queued ProviderDownloadMap `json:"queued"`
|
|
}
|
|
)
|
|
|
|
type (
|
|
NewDownloaderOptions struct {
|
|
Database *db.Database
|
|
Logger *zerolog.Logger
|
|
WSEventManager events.WSEventManagerInterface
|
|
DownloadDir string
|
|
Repository *Repository
|
|
IsOffline *bool
|
|
}
|
|
|
|
DownloadChapterOptions struct {
|
|
Provider string
|
|
MediaId int
|
|
ChapterId string
|
|
StartNow bool
|
|
}
|
|
)
|
|
|
|
func NewDownloader(opts *NewDownloaderOptions) *Downloader {
|
|
_ = os.MkdirAll(opts.DownloadDir, os.ModePerm)
|
|
filecacher, _ := filecache.NewCacher(opts.DownloadDir)
|
|
|
|
d := &Downloader{
|
|
logger: opts.Logger,
|
|
wsEventManager: opts.WSEventManager,
|
|
database: opts.Database,
|
|
downloadDir: opts.DownloadDir,
|
|
repository: opts.Repository,
|
|
mediaMap: new(MediaMap),
|
|
filecacher: filecacher,
|
|
isOffline: opts.IsOffline,
|
|
}
|
|
|
|
d.chapterDownloader = chapter_downloader.NewDownloader(&chapter_downloader.NewDownloaderOptions{
|
|
Logger: opts.Logger,
|
|
WSEventManager: opts.WSEventManager,
|
|
Database: opts.Database,
|
|
DownloadDir: opts.DownloadDir,
|
|
})
|
|
|
|
go d.hydrateMediaMap()
|
|
|
|
return d
|
|
}
|
|
|
|
// Start is called once to start the Chapter downloader 's main goroutine.
|
|
func (d *Downloader) Start() {
|
|
d.chapterDownloader.Start()
|
|
go func() {
|
|
for {
|
|
select {
|
|
// Listen for downloaded chapters
|
|
case downloadId := <-d.chapterDownloader.ChapterDownloaded():
|
|
if d.isOffline != nil && *d.isOffline {
|
|
continue
|
|
}
|
|
|
|
// When a chapter is downloaded, fetch the chapter container from the file cache
|
|
// and store it in the permanent bucket.
|
|
// DEVNOTE: This will be useful to avoid re-fetching the chapter container when the cache expires.
|
|
// This is deleted when a chapter is deleted.
|
|
go func() {
|
|
chapterContainerKey := getMangaChapterContainerCacheKey(downloadId.Provider, downloadId.MediaId)
|
|
chapterContainer, found := d.repository.getChapterContainerFromFilecache(downloadId.Provider, downloadId.MediaId)
|
|
if found {
|
|
// Store the chapter container in the permanent bucket
|
|
permBucket := getPermanentChapterContainerCacheBucket(downloadId.Provider, downloadId.MediaId)
|
|
_ = d.filecacher.SetPerm(permBucket, chapterContainerKey, chapterContainer)
|
|
}
|
|
}()
|
|
|
|
// Refresh the media map when a chapter is downloaded
|
|
d.hydrateMediaMap()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// The bucket for storing downloaded chapter containers.
|
|
// e.g. manga_downloaded_comick_chapters_1234
|
|
// The key is the chapter ID.
|
|
func getPermanentChapterContainerCacheBucket(provider string, mId int) filecache.PermanentBucket {
|
|
return filecache.NewPermanentBucket(fmt.Sprintf("manga_downloaded_%s_chapters_%d", provider, mId))
|
|
}
|
|
|
|
// getChapterContainerFromFilecache returns the chapter container from the temporary file cache.
|
|
func (r *Repository) getChapterContainerFromFilecache(provider string, mId int) (*ChapterContainer, bool) {
|
|
// Find chapter container in the file cache
|
|
chapterBucket := r.getFcProviderBucket(provider, mId, bucketTypeChapter)
|
|
|
|
chapterContainerKey := getMangaChapterContainerCacheKey(provider, mId)
|
|
|
|
var chapterContainer *ChapterContainer
|
|
// Get the key-value pair in the bucket
|
|
if found, _ := r.fileCacher.Get(chapterBucket, chapterContainerKey, &chapterContainer); !found {
|
|
// If the chapter container is not found, return an error
|
|
// since it means that it wasn't fetched (for some reason) -- This shouldn't happen
|
|
return nil, false
|
|
}
|
|
|
|
return chapterContainer, true
|
|
}
|
|
|
|
// getChapterContainerFromPermanentFilecache returns the chapter container from the permanent file cache.
|
|
func (r *Repository) getChapterContainerFromPermanentFilecache(provider string, mId int) (*ChapterContainer, bool) {
|
|
permBucket := getPermanentChapterContainerCacheBucket(provider, mId)
|
|
|
|
chapterContainerKey := getMangaChapterContainerCacheKey(provider, mId)
|
|
|
|
var chapterContainer *ChapterContainer
|
|
// Get the key-value pair in the bucket
|
|
if found, _ := r.fileCacher.GetPerm(permBucket, chapterContainerKey, &chapterContainer); !found {
|
|
// If the chapter container is not found, return an error
|
|
// since it means that it wasn't fetched (for some reason) -- This shouldn't happen
|
|
return nil, false
|
|
}
|
|
|
|
return chapterContainer, true
|
|
}
|
|
|
|
// DownloadChapter is called by the client to download a chapter.
|
|
// It fetches the chapter pages by using Repository.GetMangaPageContainer
|
|
// and invokes the chapter_downloader.Downloader 'Download' method to add the chapter to the download queue.
|
|
func (d *Downloader) DownloadChapter(opts DownloadChapterOptions) error {
|
|
|
|
if d.isOffline != nil && *d.isOffline {
|
|
return errors.New("manga downloader: Manga downloader is in offline mode")
|
|
}
|
|
|
|
chapterContainer, found := d.repository.getChapterContainerFromFilecache(opts.Provider, opts.MediaId)
|
|
if !found {
|
|
return errors.New("chapters not found")
|
|
}
|
|
|
|
// Find the chapter in the chapter container
|
|
// e.g. Wind-Breaker$0062
|
|
chapter, ok := chapterContainer.GetChapter(opts.ChapterId)
|
|
if !ok {
|
|
return errors.New("chapter not found")
|
|
}
|
|
|
|
// Fetch the chapter pages
|
|
pageContainer, err := d.repository.GetMangaPageContainer(opts.Provider, opts.MediaId, opts.ChapterId, false, &[]bool{false}[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Add the chapter to the download queue
|
|
return d.chapterDownloader.AddToQueue(chapter_downloader.DownloadOptions{
|
|
DownloadID: chapter_downloader.DownloadID{
|
|
Provider: opts.Provider,
|
|
MediaId: opts.MediaId,
|
|
ChapterId: opts.ChapterId,
|
|
ChapterNumber: manga_providers.GetNormalizedChapter(chapter.Chapter),
|
|
},
|
|
Pages: pageContainer.Pages,
|
|
})
|
|
}
|
|
|
|
// DeleteChapter is called by the client to delete a downloaded chapter.
|
|
func (d *Downloader) DeleteChapter(provider string, mediaId int, chapterId string, chapterNumber string) (err error) {
|
|
err = d.chapterDownloader.DeleteChapter(chapter_downloader.DownloadID{
|
|
Provider: provider,
|
|
MediaId: mediaId,
|
|
ChapterId: chapterId,
|
|
ChapterNumber: chapterNumber,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
permBucket := getPermanentChapterContainerCacheBucket(provider, mediaId)
|
|
_ = d.filecacher.DeletePerm(permBucket, chapterId)
|
|
|
|
d.hydrateMediaMap()
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteChapters is called by the client to delete downloaded chapters.
|
|
func (d *Downloader) DeleteChapters(ids []chapter_downloader.DownloadID) (err error) {
|
|
for _, id := range ids {
|
|
err = d.chapterDownloader.DeleteChapter(chapter_downloader.DownloadID{
|
|
Provider: id.Provider,
|
|
MediaId: id.MediaId,
|
|
ChapterId: id.ChapterId,
|
|
ChapterNumber: id.ChapterNumber,
|
|
})
|
|
|
|
permBucket := getPermanentChapterContainerCacheBucket(id.Provider, id.MediaId)
|
|
_ = d.filecacher.DeletePerm(permBucket, id.ChapterId)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
d.hydrateMediaMap()
|
|
|
|
return nil
|
|
}
|
|
|
|
func (d *Downloader) GetMediaDownloads(mediaId int, cached bool) (ret MediaDownloadData, err error) {
|
|
defer util.HandlePanicInModuleWithError("manga/GetMediaDownloads", &err)
|
|
|
|
if !cached {
|
|
d.hydrateMediaMap()
|
|
}
|
|
|
|
return d.mediaMap.getMediaDownload(mediaId, d.database)
|
|
}
|
|
|
|
func (d *Downloader) RunChapterDownloadQueue() {
|
|
d.chapterDownloader.Run()
|
|
}
|
|
|
|
func (d *Downloader) StopChapterDownloadQueue() {
|
|
_ = d.database.ResetDownloadingChapterDownloadQueueItems()
|
|
d.chapterDownloader.Stop()
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
type (
|
|
NewDownloadListOptions struct {
|
|
MangaCollection *anilist.MangaCollection
|
|
}
|
|
|
|
DownloadListItem struct {
|
|
MediaId int `json:"mediaId"`
|
|
// Media will be nil if the manga is no longer in the user's collection.
|
|
// The client should handle this case by displaying the download data without the media data.
|
|
Media *anilist.BaseManga `json:"media"`
|
|
DownloadData ProviderDownloadMap `json:"downloadData"`
|
|
}
|
|
)
|
|
|
|
// NewDownloadList returns a list of DownloadListItem for the client to display.
|
|
func (d *Downloader) NewDownloadList(opts *NewDownloadListOptions) (ret []*DownloadListItem, err error) {
|
|
defer util.HandlePanicInModuleWithError("manga/NewDownloadList", &err)
|
|
|
|
mm := d.mediaMap
|
|
|
|
ret = make([]*DownloadListItem, 0)
|
|
|
|
for mId, data := range *mm {
|
|
listEntry, ok := opts.MangaCollection.GetListEntryFromMangaId(mId)
|
|
if !ok {
|
|
ret = append(ret, &DownloadListItem{
|
|
MediaId: mId,
|
|
Media: nil,
|
|
DownloadData: data,
|
|
})
|
|
continue
|
|
}
|
|
|
|
media := listEntry.GetMedia()
|
|
if media == nil {
|
|
ret = append(ret, &DownloadListItem{
|
|
MediaId: mId,
|
|
Media: nil,
|
|
DownloadData: data,
|
|
})
|
|
continue
|
|
}
|
|
|
|
item := &DownloadListItem{
|
|
MediaId: mId,
|
|
Media: media,
|
|
DownloadData: data,
|
|
}
|
|
|
|
ret = append(ret, item)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// Media map
|
|
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
func (mm *MediaMap) getMediaDownload(mediaId int, db *db.Database) (MediaDownloadData, error) {
|
|
|
|
if mm == nil {
|
|
return MediaDownloadData{}, errors.New("could not check downloaded chapters")
|
|
}
|
|
|
|
// Get all downloaded chapters for the media
|
|
downloads, ok := (*mm)[mediaId]
|
|
if !ok {
|
|
downloads = make(map[string][]ProviderDownloadMapChapterInfo)
|
|
}
|
|
|
|
// Get all queued chapters for the media
|
|
queued, err := db.GetMediaQueuedChapters(mediaId)
|
|
if err != nil {
|
|
queued = make([]*models.ChapterDownloadQueueItem, 0)
|
|
}
|
|
|
|
qm := make(ProviderDownloadMap)
|
|
for _, item := range queued {
|
|
if _, ok := qm[item.Provider]; !ok {
|
|
qm[item.Provider] = []ProviderDownloadMapChapterInfo{
|
|
{
|
|
ChapterID: item.ChapterID,
|
|
ChapterNumber: item.ChapterNumber,
|
|
},
|
|
}
|
|
} else {
|
|
qm[item.Provider] = append(qm[item.Provider], ProviderDownloadMapChapterInfo{
|
|
ChapterID: item.ChapterID,
|
|
ChapterNumber: item.ChapterNumber,
|
|
})
|
|
}
|
|
}
|
|
|
|
data := MediaDownloadData{
|
|
Downloaded: downloads,
|
|
Queued: qm,
|
|
}
|
|
|
|
return data, nil
|
|
|
|
}
|
|
|
|
// hydrateMediaMap hydrates the MediaMap by reading the download directory.
|
|
func (d *Downloader) hydrateMediaMap() {
|
|
|
|
if d.readingDownloadDir {
|
|
return
|
|
}
|
|
|
|
d.mediaMapMu.Lock()
|
|
defer d.mediaMapMu.Unlock()
|
|
|
|
d.readingDownloadDir = true
|
|
defer func() {
|
|
d.readingDownloadDir = false
|
|
}()
|
|
|
|
d.logger.Debug().Msg("manga downloader: Reading download directory")
|
|
|
|
ret := make(MediaMap)
|
|
|
|
files, err := os.ReadDir(d.downloadDir)
|
|
if err != nil {
|
|
d.logger.Error().Err(err).Msg("manga downloader: Failed to read download directory")
|
|
}
|
|
|
|
// Hydrate MediaMap by going through all chapter directories
|
|
mu := sync.Mutex{}
|
|
wg := sync.WaitGroup{}
|
|
for _, file := range files {
|
|
wg.Add(1)
|
|
go func(file os.DirEntry) {
|
|
defer wg.Done()
|
|
|
|
if file.IsDir() {
|
|
// e.g. comick_1234_abc_13.5
|
|
id, ok := chapter_downloader.ParseChapterDirName(file.Name())
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
mu.Lock()
|
|
newMapInfo := ProviderDownloadMapChapterInfo{
|
|
ChapterID: id.ChapterId,
|
|
ChapterNumber: id.ChapterNumber,
|
|
}
|
|
|
|
if _, ok := ret[id.MediaId]; !ok {
|
|
ret[id.MediaId] = make(map[string][]ProviderDownloadMapChapterInfo)
|
|
ret[id.MediaId][id.Provider] = []ProviderDownloadMapChapterInfo{newMapInfo}
|
|
} else {
|
|
if _, ok := ret[id.MediaId][id.Provider]; !ok {
|
|
ret[id.MediaId][id.Provider] = []ProviderDownloadMapChapterInfo{newMapInfo}
|
|
} else {
|
|
ret[id.MediaId][id.Provider] = append(ret[id.MediaId][id.Provider], newMapInfo)
|
|
}
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
}(file)
|
|
}
|
|
wg.Wait()
|
|
|
|
// Trigger hook event
|
|
ev := &MangaDownloadMapEvent{
|
|
MediaMap: &ret,
|
|
}
|
|
_ = hook.GlobalHookManager.OnMangaDownloadMap().Trigger(ev) // ignore the error
|
|
// make sure the media map is not nil
|
|
if ev.MediaMap != nil {
|
|
ret = *ev.MediaMap
|
|
}
|
|
|
|
d.mediaMap = &ret
|
|
|
|
// When done refreshing, send a message to the client to refetch the download data
|
|
d.wsEventManager.SendEvent(events.RefreshedMangaDownloadData, nil)
|
|
}
|