From 2ad147a541bb736669f63fd2478775ec09e12758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=92scar=20Casajuana?= Date: Sun, 12 May 2024 16:49:35 +0200 Subject: [PATCH] Adding progress bars - Progress bars for downloading and archiving single chapters - Progress bar for the bundle, although the other ones should be improved in this case - Removed logs and other console outputs here and there. - Errors are still NOT properly managed in the progress bars. A channel decorator should be used for that. refs #24 --- cmd/root.go | 328 ++++++++++++++++++++++++++++--------------- downloader/fetch.go | 37 +++-- go.mod | 11 +- go.sum | 24 +++- grabber/mangadex.go | 7 +- grabber/plainhtml.go | 4 +- grabber/tcb.go | 11 +- packer/cbz.go | 3 +- packer/pack.go | 12 +- 9 files changed, 295 insertions(+), 142 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 65eb22d..e1e3342 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,6 +17,8 @@ import ( "github.com/fatih/color" "github.com/manifoldco/promptui" "github.com/spf13/cobra" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" cc "github.com/ivanpirog/coloredcobra" ) @@ -57,139 +59,219 @@ Note arguments aren't really positional, you can specify them in any order: manga-downloader --language es 10-20 https://mangadex.org/title/e7eabe96-aa17-476f-b431-2497d5e9d060/black-clover --bundle`), Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - s, errs := grabber.NewSite(getUrlArg(args), &settings) - if len(errs) > 0 { - color.Red("Errors testing site (a site may be down):") - for _, err := range errs { - color.Red(err.Error()) - } + Run: Run, +} + +// Run is the main function of the root command, the main downloading cmd +func Run(cmd *cobra.Command, args []string) { + s, errs := grabber.NewSite(getUrlArg(args), &settings) + if len(errs) > 0 { + color.Red("Errors testing site (a site may be down):") + for _, err := range errs { + color.Red(err.Error()) } - if s == nil { - color.Yellow("Site not recognised") - os.Exit(1) + } + if s == nil { + color.Yellow("Site not recognised") + os.Exit(1) + } + s.InitFlags(cmd) + + // fetch series title + _, err := s.FetchTitle() + cerr(err, "Error fetching title: ") + + // fetch all chapters + chapters, errs := s.FetchChapters() + if len(errs) > 0 { + color.Red("Errors fetching chapters:") + for _, err := range errs { + color.Red(err.Error()) } - s.InitFlags(cmd) - - // fetch series title - title, err := s.FetchTitle() - cerr(err, "Error fetching title: %s") - - // fetch all chapters - chapters, errs := s.FetchChapters() - if len(errs) > 0 { - color.Red("Errors fetching chapters:") - for _, err := range errs { - color.Red(err.Error()) - } - os.Exit(1) + os.Exit(1) + } + + chapters = chapters.SortByNumber() + + var rngs []ranges.Range + // ranges argument is not provided + if len(args) == 1 { + lastChapter := chapters[len(chapters)-1].GetNumber() + prompt := promptui.Prompt{ + Label: fmt.Sprintf("Do you want to download all %g chapters", lastChapter), + IsConfirm: true, } - chapters = chapters.SortByNumber() + _, err := prompt.Run() - var rngs []ranges.Range - // ranges argument is not provided - if len(args) == 1 { - lastChapter := chapters[len(chapters)-1].GetNumber() - prompt := promptui.Prompt{ - Label: fmt.Sprintf("Do you want to download all %g chapters", lastChapter), - IsConfirm: true, - } + if err != nil { + color.Yellow("Canceled by user") + os.Exit(0) + } - _, err := prompt.Run() + rngs = []ranges.Range{{Begin: 1, End: int64(lastChapter)}} + } else { + // ranges parsing + settings.Range = getRangesArg(args) + rngs, err = ranges.Parse(settings.Range) + cerr(err, "Error parsing ranges: ") + } - if err != nil { - color.Yellow("Canceled by user") - os.Exit(0) - } + // sort and filter specified ranges + chapters = chapters.FilterRanges(rngs) - rngs = []ranges.Range{{Begin: 1, End: int64(lastChapter)}} - } else { - // ranges parsing - settings.Range = getRangesArg(args) - rngs, err = ranges.Parse(settings.Range) - cerr(err, "Error parsing ranges: %s") - } + if len(chapters) == 0 { + color.Yellow("No chapters found for the specified ranges") + os.Exit(1) + } - // sort and filter specified ranges - chapters = chapters.FilterRanges(rngs) + // download chapters + wg := sync.WaitGroup{} + g := make(chan struct{}, s.GetMaxConcurrency().Chapters) + downloaded := grabber.Filterables{} + // progress bar + p := mpb.New( + mpb.WithWidth(40), + mpb.WithOutput(color.Output), + mpb.WithAutoRefresh(), + ) - if len(chapters) == 0 { - color.Yellow("No chapters found for the specified ranges") - os.Exit(1) - } + green, blue := color.New(color.FgGreen), color.New(color.FgBlue) - // download chapters - wg := sync.WaitGroup{} - g := make(chan struct{}, s.GetMaxConcurrency().Chapters) - downloaded := grabber.Filterables{} - - for _, chap := range chapters { - g <- struct{}{} - wg.Add(1) - go func(chap grabber.Filterable) { - defer wg.Done() - chapter, err := s.FetchChapter(chap) - if err != nil { - color.Red("- error fetching chapter %s: %s", chap.GetTitle(), err.Error()) - <-g - return - } - fmt.Printf("fetched %s %s\n", color.CyanString(title), color.HiBlackString(chapter.GetTitle())) + for _, chap := range chapters { + g <- struct{}{} + wg.Add(1) - files, err := downloader.FetchChapter(s, chapter) - if err != nil { - color.Red("- error downloading chapter %s: %s", chapter.GetTitle(), err.Error()) - <-g - return - } + go func(chap grabber.Filterable) { + defer wg.Done() - d := &packer.DownloadedChapter{ - Chapter: chapter, - Files: files, - } + chapter, err := s.FetchChapter(chap) + if err != nil { + color.Red("- error fetching chapter %s: %s", chap.GetTitle(), err.Error()) + <-g + return + } - if !settings.Bundle { - filename, err := packer.PackSingle(settings.OutputDir, s, d) - if err == nil { - fmt.Printf("- %s %s\n", color.GreenString("saved file"), color.HiBlackString(filename)) - } else { - color.Red(err.Error()) - } + // chapter download progress bar + title := fmt.Sprintf("%s:", truncateString(chap.GetTitle(), 30)) + cdbar := p.AddBar(chapter.PagesCount, + mpb.PrependDecorators( + decor.Name(title, decor.WCSyncWidthR), + decor.Meta(decor.Name("downloading", decor.WC{C: decor.DextraSpace}), toMetaFunc(blue)), + decor.CountersNoUnit("%d / %d", decor.WC{C: decor.DextraSpace}), + ), + mpb.AppendDecorators( + decor.OnCompleteMeta( + decor.OnComplete(decor.Percentage(decor.WC{W: 4}), "dld."), + toMetaFunc(green), + ), + ), + ) + // save chapter progress bar + scbar := p.AddBar(chapter.PagesCount, + mpb.BarQueueAfter(cdbar), + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.Name(title, decor.WCSyncWidthR), + decor.OnCompleteMeta( + decor.OnComplete( + decor.Meta(decor.Name("archiving", decor.WC{C: decor.DextraSpace}), toMetaFunc(blue)), + "done!", + ), + toMetaFunc(green), + ), + decor.OnComplete(decor.CountersNoUnit("%d / %d", decor.WC{C: decor.DextraSpace}), ""), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.Percentage(decor.WC{W: 5}), ""), + ), + ) + + files, err := downloader.FetchChapter(s, chapter, func(page, _ int) { + cdbar.IncrBy(page) + }) + if err != nil { + color.Red("- error downloading chapter %s: %s", chapter.GetTitle(), err.Error()) + <-g + return + } + + d := &packer.DownloadedChapter{ + Chapter: chapter, + Files: files, + } + + if !settings.Bundle { + _, err := packer.PackSingle(settings.OutputDir, s, d, func(page, _ int) { + scbar.IncrBy(page) + }) + // filename, err := packer.PackSingle(settings.OutputDir, s, d) + if err == nil { + // fmt.Printf("- %s %s\n", color.GreenString("saved file"), color.HiBlackString(filename)) } else { - // avoid adding it to memory if we're not gonna use it - downloaded = append(downloaded, d) + color.Red(err.Error()) } + } else { + // avoid adding it to memory if we're not gonna use it + downloaded = append(downloaded, d) + } - // release guard - <-g - }(chap) - } - wg.Wait() - close(g) + // release guard + <-g + }(chap) + } + // wait for all routines to finish + wg.Wait() + close(g) - if !settings.Bundle { - // if we're not bundling, just finish it - os.Exit(0) - } + if !settings.Bundle { + // if we're not bundling, we're done + os.Exit(0) + } - // resort downloaded - downloaded = downloaded.SortByNumber() + // resort downloaded + downloaded = downloaded.SortByNumber() - dc := []*packer.DownloadedChapter{} - // convert slice back to DownloadedChapter - for _, d := range downloaded { - dc = append(dc, d.(*packer.DownloadedChapter)) - } + dc := []*packer.DownloadedChapter{} + tp := 0 + // convert slice back to DownloadedChapter + for _, d := range downloaded { + chapter := d.(*packer.DownloadedChapter) + dc = append(dc, chapter) + tp += int(chapter.PagesCount) + } - filename, err := packer.PackBundle(settings.OutputDir, s, dc, settings.Range) - if err != nil { - color.Red(err.Error()) - os.Exit(1) - } + // bundle progress bar + bbar := p.AddBar(int64(tp), + mpb.PrependDecorators( + decor.Name("Bundle", decor.WCSyncWidthR), + decor.OnCompleteMeta( + decor.OnComplete( + decor.Meta(decor.Name("bundling", decor.WC{C: decor.DextraSpace}), toMetaFunc(blue)), + "done!", + ), + toMetaFunc(green), + ), + decor.OnComplete(decor.CountersNoUnit("%d / %d", decor.WC{C: decor.DextraSpace}), ""), + ), + mpb.AppendDecorators( + decor.OnCompleteMeta( + decor.OnComplete(decor.Percentage(decor.WC{W: 4}), "done"), + toMetaFunc(green), + ), + ), + ) + + filename, err := packer.PackBundle(settings.OutputDir, s, dc, settings.Range, func(page, _ int) { + bbar.IncrBy(page) + }) + + if err != nil { + color.Red(err.Error()) + os.Exit(1) + } - fmt.Printf("- %s %s\n", color.GreenString("saved file"), color.HiBlackString(filename)) - }, + fmt.Printf("- %s %s\n", color.GreenString("saved file"), color.HiBlackString(filename)) } // Execute adds all child commands to the root command and sets flags appropriately. @@ -276,3 +358,27 @@ func getUrlArg(args []string) string { return args[1] } + +// truncateString truncates the input string at a specified maximum length +// without cutting words. It finds the last space within the limit and truncates there. +func truncateString(input string, maxLength int) string { + if len(input) <= maxLength { + return input + } + + // Find the last index of a space before maxLength + truncationPoint := strings.LastIndex(input[:maxLength], " ") + if truncationPoint == -1 { + // No spaces found, force to maxLength (cuts the word) + return input[:maxLength] + "..." + } + + // Return substring up to the last found space + return input[:truncationPoint] + "..." +} + +func toMetaFunc(c *color.Color) func(string) string { + return func(s string) string { + return c.Sprint(s) + } +} diff --git a/downloader/fetch.go b/downloader/fetch.go index 56ead23..59753b2 100644 --- a/downloader/fetch.go +++ b/downloader/fetch.go @@ -7,7 +7,6 @@ import ( "github.com/elboletaire/manga-downloader/grabber" "github.com/elboletaire/manga-downloader/http" - "github.com/fatih/color" ) // File represents a downloaded file @@ -17,17 +16,19 @@ type File struct { } // FetchChapter downloads all the pages of a chapter -func FetchChapter(site grabber.Site, chapter *grabber.Chapter) (files []*File, err error) { +func FetchChapter(site grabber.Site, chapter *grabber.Chapter, onprogress func(page, progress int)) (files []*File, err error) { wg := sync.WaitGroup{} - - color.Blue("- downloading %s pages...", color.HiBlackString(chapter.GetTitle())) guard := make(chan struct{}, site.GetMaxConcurrency().Pages) + errChan := make(chan error, 1) + done := make(chan bool) for _, page := range chapter.Pages { guard <- struct{}{} wg.Add(1) go func(page grabber.Page) { + // release waitgroup and guard when done defer wg.Done() + defer func() { <-guard }() file, err := FetchFile(http.RequestParams{ URL: page.URL, @@ -35,18 +36,35 @@ func FetchChapter(site grabber.Site, chapter *grabber.Chapter) (files []*File, e }, uint(page.Number)) if err != nil { - color.Red("- error downloading page %d of %s", page.Number, chapter.GetTitle()) + select { + case errChan <- err: + default: + } return } files = append(files, file) + pn := int(page.Number) + cp := pn * 100 / len(chapter.Pages) - // release guard - <-guard + onprogress(pn, cp) }(page) } - wg.Wait() - close(guard) + + go func() { + wg.Wait() + // signal that all goroutines have completed + close(done) + }() + + select { + // in case of error, return the very first one + case err := <-errChan: + close(guard) + return nil, err + case <-done: + // all goroutines finished successfully, continue + } // sort files by page number sort.SliceStable(files, func(i, j int) bool { @@ -60,6 +78,7 @@ func FetchChapter(site grabber.Site, chapter *grabber.Chapter) (files []*File, e func FetchFile(params http.RequestParams, page uint) (file *File, err error) { body, err := http.Get(params) if err != nil { + // TODO: should retry at least once (configurable) return } diff --git a/go.mod b/go.mod index d69796e..642c7f8 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,13 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e - golang.org/x/net v0.4.0 + github.com/vbauerster/mpb/v8 v8.7.3 + golang.org/x/net v0.21.0 ) require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/google/go-github v17.0.0+incompatible // indirect @@ -21,6 +24,8 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - golang.org/x/sys v0.3.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/sys v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index 23e45a2..b7e9357 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= @@ -33,8 +37,13 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= @@ -43,9 +52,11 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e h1:IWllFTiDjjLIf2oeKxpIUmtiDV5sn71VgeQgg6vcE7k= github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e/go.mod h1:d7u6HkTYKSv5m6MCKkOQlHwaShTMl3HjqSGW3XtVhXM= +github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0= +github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -53,8 +64,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/grabber/mangadex.go b/grabber/mangadex.go index bc198ab..ae00819 100644 --- a/grabber/mangadex.go +++ b/grabber/mangadex.go @@ -139,17 +139,20 @@ func (m Mangadex) FetchChapter(f Filterable) (*Chapter, error) { return nil, err } + pcount := len(body.Chapter.Data) + chapter := &Chapter{ Title: fmt.Sprintf("Chapter %04d %s", int64(f.GetNumber()), chap.Title), Number: f.GetNumber(), - PagesCount: int64(len(body.Chapter.Data)), + PagesCount: int64(pcount), Language: chap.Language, } // create pages for i, p := range body.Chapter.Data { + num := i + 1 chapter.Pages = append(chapter.Pages, Page{ - Number: int64(i + 1), + Number: int64(num), URL: body.BaseUrl + path.Join("/data", body.Chapter.Hash, p), }) } diff --git a/grabber/plainhtml.go b/grabber/plainhtml.go index 25584bb..020ce60 100644 --- a/grabber/plainhtml.go +++ b/grabber/plainhtml.go @@ -182,11 +182,12 @@ func (m PlainHTML) FetchChapter(f Filterable) (*Chapter, error) { } pimages := getPlainHTMLImageURL(m.site.Image, doc) + pcount := len(pimages) chapter := &Chapter{ Title: f.GetTitle(), Number: f.GetNumber(), - PagesCount: int64(len(pimages)), + PagesCount: int64(pcount), Language: "en", } @@ -199,6 +200,7 @@ func (m PlainHTML) FetchChapter(f Filterable) (*Chapter, error) { if !strings.HasPrefix(img, "http") { img = m.BaseUrl() + img } + page := Page{ Number: int64(i), URL: img, diff --git a/grabber/tcb.go b/grabber/tcb.go index 387e823..abe0c3a 100644 --- a/grabber/tcb.go +++ b/grabber/tcb.go @@ -121,17 +121,19 @@ func (t Tcb) FetchChapter(f Filterable) (*Chapter, error) { } pimages := body.Find("div.reading-content img") + pcount := pimages.Length() + // progress := make(chan int, pcount) chapter := &Chapter{ Title: f.GetTitle(), Number: f.GetNumber(), - PagesCount: int64(pimages.Length()), + PagesCount: int64(pcount), Language: "en", } pages := []Page{} pimages.Each(func(i int, s *goquery.Selection) { u := strings.TrimSpace(s.AttrOr("data-src", s.AttrOr("src", ""))) - n := int64(i + 1) + n := i + 1 if u == "" { // this error is not critical and is not from our side, so just log it out color.Yellow("page %d of %s has no URL to fetch from 😕 (will be ignored)", n, chapter.GetTitle()) @@ -140,13 +142,16 @@ func (t Tcb) FetchChapter(f Filterable) (*Chapter, error) { if !strings.HasPrefix(u, "http") { u = t.BaseUrl() + u } + // progress <- n pages = append(pages, Page{ - Number: n, + Number: int64(n), URL: u, }) }) chapter.Pages = pages + // close(progress) + return chapter, nil } diff --git a/packer/cbz.go b/packer/cbz.go index 0ee4122..9ffbe97 100644 --- a/packer/cbz.go +++ b/packer/cbz.go @@ -10,7 +10,7 @@ import ( ) // ArchiveCBZ archives the given files into a CBZ file -func ArchiveCBZ(filename string, files []*downloader.File) error { +func ArchiveCBZ(filename string, files []*downloader.File, progress func(page, progress int)) error { if len(files) == 0 { return errors.New("no files to pack") } @@ -29,6 +29,7 @@ func ArchiveCBZ(filename string, files []*downloader.File) error { if _, err = f.Write(file.Data); err != nil { return err } + progress(i, i*100/len(files)) } err = w.Close() diff --git a/packer/pack.go b/packer/pack.go index 6d60383..9865220 100644 --- a/packer/pack.go +++ b/packer/pack.go @@ -15,13 +15,13 @@ type DownloadedChapter struct { } // PackSingle packs a single downloaded chapter -func PackSingle(outputdir string, s grabber.Site, chapter *DownloadedChapter) (string, error) { +func PackSingle(outputdir string, s grabber.Site, chapter *DownloadedChapter, progress func(page, progress int)) (string, error) { title, _ := s.FetchTitle() - return pack(outputdir, s.GetFilenameTemplate(), title, NewChapterFileTemplateParts(title, chapter.Chapter), chapter.Files) + return pack(outputdir, s.GetFilenameTemplate(), title, NewChapterFileTemplateParts(title, chapter.Chapter), chapter.Files, progress) } // PackBundle packs a bundle of downloaded chapters -func PackBundle(outputdir string, s grabber.Site, chapters []*DownloadedChapter, rng string) (string, error) { +func PackBundle(outputdir string, s grabber.Site, chapters []*DownloadedChapter, rng string, progress func(page, progress int)) (string, error) { title, _ := s.FetchTitle() files := []*downloader.File{} for _, chapter := range chapters { @@ -32,10 +32,10 @@ func PackBundle(outputdir string, s grabber.Site, chapters []*DownloadedChapter, Series: title, Number: rng, Title: "bundle", - }, files) + }, files, progress) } -func pack(outputdir, template, title string, parts FilenameTemplateParts, files []*downloader.File) (string, error) { +func pack(outputdir, template, title string, parts FilenameTemplateParts, files []*downloader.File, progress func(page, progress int)) (string, error) { filename, err := NewFilenameFromTemplate(template, parts) if err != nil { return "", fmt.Errorf("- error creating filename for chapter %s: %s", title, err.Error()) @@ -43,7 +43,7 @@ func pack(outputdir, template, title string, parts FilenameTemplateParts, files filename += ".cbz" - if err = ArchiveCBZ(filepath.Join(outputdir, filename), files); err != nil { + if err = ArchiveCBZ(filepath.Join(outputdir, filename), files, progress); err != nil { return "", fmt.Errorf("- error saving file %s: %s", filename, err.Error()) }