start implement #706

PixivBrowserFactory.py

@@ -25,6 +25,7 @@
 from PixivModelSketch import SketchArtist, SketchPost
 from PixivOAuth import PixivOAuth
 from PixivTags import PixivTags
+from PixivNovel import NovelSeries, PixivNovel, MAX_LIMIT
 
 defaultCookieJar = None
 defaultConfig = None
@@ -1080,6 +1081,44 @@ def getMangaSeries(self, manga_series_id: int, current_page: int, returnJSON=Fal
 
         return manga_series
 
+    def getNovelPage(self, novel_id) -> PixivNovel:
+        # https://www.pixiv.net/ajax/novel/14521816?lang=en
+        locale = ""
+        if self._locale is not None and len(self._locale) > 0:
+            locale = f"&lang={self._locale}"
+        url = f"https://www.pixiv.net/ajax/novel/{novel_id}?{locale}"
+        response = self.getPixivPage(url, returnParsed=False, enable_cache=True)
+        novel = PixivNovel(novel_id, response)
+        return novel
+
+    def getNovelSeries(self, novel_series_id) -> NovelSeries:
+        locale = ""
+        if self._locale is not None and len(self._locale) > 0:
+            locale = f"&lang={self._locale}"
+
+        # https://www.pixiv.net/ajax/novel/series/1328575?lang=en
+        url = f"https://www.pixiv.net/ajax/novel/series/{novel_series_id}?{locale}"
+        response = self.getPixivPage(url, returnParsed=False, enable_cache=True)
+        novel_series = NovelSeries(novel_series_id, series_json=response)
+        return novel_series
+
+    def getNovelSeriesContent(self, novel_series, limit=MAX_LIMIT, current_page=1, order_by='asc'):
+        locale = ""
+        if self._locale is not None and len(self._locale) > 0:
+            locale = f"&lang={self._locale}"
+        # https://www.pixiv.net/ajax/novel/series_content/1328575?limit=10&last_order=0&order_by=asc&lang=en
+        params = list()
+        params.append(f"limit={limit}")
+        last_order = 10 * (current_page - 1)
+        params.append(f"last_order={last_order}")
+        params.append(f"order_by={order_by}")
+        params_str = "&".join(params)
+        novel_series_id = novel_series.series_id
+        url = f"https://www.pixiv.net/ajax/novel/series_content/{novel_series_id}?{params_str}{locale}"
+        response = self.getPixivPage(url, returnParsed=False, enable_cache=True)
+        novel_series.parse_series_content(response, current_page)
+        return novel_series
+
 
 def getBrowser(config=None, cookieJar=None):
     global defaultCookieJar

PixivConstant.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-PIXIVUTIL_VERSION = '20210108-beta4'
+PIXIVUTIL_VERSION = '20210215-beta1'
 PIXIVUTIL_LINK = 'https://github.com/Nandaka/PixivUtil2/releases'
 PIXIVUTIL_DONATE = 'https://bit.ly/PixivUtilDonation'
 

PixivHelper.py

@@ -291,7 +291,7 @@ def make_filename(nameFormat: str,
     # clean up double space
     while nameFormat.find('  ') > -1:
         nameFormat = nameFormat.replace('  ', ' ')
-        
+
     # clean up double slash
     while nameFormat.find('//') > -1 or nameFormat.find('\\\\') > -1:
         nameFormat = nameFormat.replace('//', '/').replace('\\\\', '\\')

PixivNovel.py

@@ -0,0 +1,80 @@
+import codecs
+import json
+
+import PixivHelper
+from PixivException import PixivException
+
+MAX_LIMIT = 10
+
+
+class PixivNovel:
+    novel_id = 0
+    novel_json_str = ""
+    content = ""
+    title = ""
+
+    def __init__(self, novel_id, novel_json) -> None:
+        self.novel_id = novel_id
+        self.novel_json_str = novel_json
+        self.parse()
+
+    def parse(self):
+        js = json.loads(self.novel_json_str)
+        if js["error"]:
+            raise PixivException("Cannot get novel details",
+                                 errorCode=PixivException.UNKNOWN_IMAGE_ERROR,
+                                 htmlPage=self.novel_json_str)
+
+        self.title = js["body"]["title"]
+        self.content = js["body"]["content"]
+
+    def write_content(self, filename):
+        ft = open("novel_template.html")
+        template_str = ft.read()
+        ft.close()
+
+        fh = None
+        try:
+            PixivHelper.makeSubdirs(filename)
+            fh = codecs.open(filename, 'wb', encoding='utf-8')
+        except IOError:
+            fh = codecs.open(str(self.novel_id) + ".html", 'wb', encoding='utf-8')
+            PixivHelper.get_logger().exception("Error when saving novel: %s, file is saved to: %s.html", filename, str(self.novel_id))
+
+        if fh is not None:
+            content_str = template_str.replace("%title%", self.title)
+            content_str = content_str.replace("%novel_json_str%", self.novel_json_str)
+            fh.write(content_str)
+            fh.close()
+
+
+class NovelSeries:
+    series_id = 0
+    series_str = ""
+    series_list = list()
+    series_list_str = dict()
+    total = 0
+
+    def __init__(self, series_id, series_json) -> None:
+        self.series_id = series_id
+        self.series_str = series_json
+
+        self.parse()
+
+    def parse(self):
+        js = json.loads(self.series_str)
+        if js["error"]:
+            raise PixivException("Cannot get novel series content details",
+                                 errorCode=PixivException.UNKNOWN_IMAGE_ERROR,
+                                 htmlPage=self.series_str)
+        self.total = js["body"]["total"]
+
+    def parse_series_content(self, page_info, current_page):
+        js = json.loads(page_info)
+        if js["error"]:
+            raise PixivException("Cannot get novel series content details",
+                                 errorCode=PixivException.UNKNOWN_IMAGE_ERROR,
+                                 htmlPage=page_info)
+
+        self.series_list.extend(js["body"]["seriesContents"])
+        self.series_list_str[current_page] = page_info

PixivNovelHandler.py

@@ -0,0 +1,49 @@
+import PixivHelper
+import PixivNovel
+
+
+def process_novel(caller,
+                  config,
+                  novel_id,
+                  notifier=None):
+    if notifier is None:
+        notifier = PixivHelper.dummy_notifier
+
+    PixivHelper.print_and_log(None, f"Processing Novel details = {novel_id}")
+    novel = caller.__br__.getNovelPage(novel_id)
+    PixivHelper.print_and_log(None, f"Title = {novel.title}")
+    filename = f"novel-{novel_id}.html"
+    PixivHelper.print_and_log(None, f"Saving Novel details to = {filename}")
+    novel.write_content(filename)
+
+
+def process_novel_series(caller,
+                         config,
+                         series_id,
+                         start_page=1,
+                         end_page=0,
+                         notifier=None):
+    if notifier is None:
+        notifier = PixivHelper.dummy_notifier
+    PixivHelper.print_and_log(None, f"Processing Novel Series = {series_id}")
+
+    novel_series = caller.__br__.getNovelSeries(series_id)
+    page = start_page
+    flag = True
+    while(flag):
+        PixivHelper.print_and_log(None, f"Getting page = {page}")
+        novel_series = caller.__br__.getNovelSeriesContent(novel_series, current_page=page)
+        page = page + 1
+        if end_page > 0 and page > end_page:
+            PixivHelper.print_and_log(None, f"Page limit reached = {end_page}.")
+            flag = False
+        if (page * PixivNovel.MAX_LIMIT) >= novel_series.total:
+            PixivHelper.print_and_log(None, "No more novel.")
+            flag = False
+
+    for novel in novel_series.series_list:
+        print(novel["id"])
+        # process_novel(caller,
+        #               config,
+        #               novel_id=novel["id"],
+        #               notifier=notifier)

PixivUtil2.py

@@ -27,6 +27,7 @@
 import PixivListHandler
 import PixivModelFanbox
 import PixivSketchHandler
+import PixivNovelHandler
 import PixivTagsHandler
 from PixivDBManager import PixivDBManager
 from PixivException import PixivException
@@ -161,6 +162,8 @@ def menu():
     print(' 11. Download Member Bookmark (/bookmark.php?id=)')
     print(' 12. Download by Group Id')
     print(' 13. Download by Manga Series Id')
+    print(' 14. Download by Novel Id')
+    print(' 15. Download by Novel Series Id')
     print(Style.BRIGHT + '── FANBOX '.ljust(PADDING, "─") + Style.RESET_ALL)
     print(' f1. Download from supporting list (FANBOX)')
     print(' f2. Download by artist/creator id (FANBOX)')
@@ -636,6 +639,36 @@ def menu_download_by_manga_series_id(opisvalid, args, options):
                                                end_page=end_page)
 
 
+def menu_download_by_novel_id(opisvalid, args, options):
+    __log__.info('Novel mode (14).')
+    novel_ids = input('Novel IDs: ').rstrip("\r")
+    novel_ids = PixivHelper.get_ids_from_csv(novel_ids)
+    PixivHelper.print_and_log('info', f"Novel IDs: {novel_ids}")
+
+    for novel_id in novel_ids:
+        PixivNovelHandler.process_novel(sys.modules[__name__],
+                                        __config__,
+                                        novel_id)
+
+
+def menu_download_by_novel_series_id(opisvalid, args, options):
+    __log__.info('Novel Series mode (15).')
+    start_page = 1
+    end_page = 0
+
+    novel_series_ids = input('Novel Series IDs: ').rstrip("\r")
+    (start_page, end_page) = PixivHelper.get_start_and_end_number(total_number_of_page=options.number_of_pages)
+    novel_series_ids = PixivHelper.get_ids_from_csv(novel_series_ids)
+    PixivHelper.print_and_log('info', f"Novel Series IDs: {novel_series_ids}")
+
+    for novel_series_id in novel_series_ids:
+        PixivNovelHandler.process_novel_series(sys.modules[__name__],
+                                               __config__,
+                                               novel_series_id,
+                                               start_page=start_page,
+                                               end_page=end_page)
+
+
 def menu_download_by_group_id(opisvalid, args, options):
     __log__.info('Group mode (12).')
     process_external = False
@@ -1067,6 +1100,10 @@ def main_loop(ewd, op_is_valid, selection, np_is_valid_local, args, options):
                 menu_download_by_group_id(op_is_valid, args, options)
             elif selection == '13':
                 menu_download_by_manga_series_id(op_is_valid, args, options)
+            elif selection == '14':
+                menu_download_by_novel_id(op_is_valid, args, options)
+            elif selection == '15':
+                menu_download_by_novel_series_id(op_is_valid, args, options)
             elif selection == 'b':
                 PixivBatchHandler.process_batch_job(sys.modules[__name__], batch_file=options.batch_file)
             elif selection == 'e':

changelog.txt

@@ -1,4 +1,10 @@
-20210108-beta4
+20210215-beta1
+- Start implement #706: add novel support.
+- Merge Update PixivDBManager.py (#910) by amatuerCoder.
+- Merge Update PixivBrowserFactory.py (#908) by amatuerCoder.
+- Merge Fix #889 #899 (#903) by amatuerCoder.
+
+20210128
 - Fix Issue #878: update search by tag parser to handle missing 'isAdContainer'.
 - Fix Issue #877: update fanbox parser for missing embedId.
 - Fix Issue #881: change content_provider.json path resolution.
@@ -12,6 +18,9 @@
 - Fix Issue #892: PixivSketch now load all images, including R-18 (follow your settings in the website).
 - Merge Fix #873 and #886 (#888) by amatuerCoder.
 - Fix Issue #883: update parser for avatar image.
+- Merge Remove double slash (#894) by Yuki.
+- Merge Fix (#897) by amatuerCoder.
+- Update cloudscraper to 1.2.54
 
 20201130
 - Merge Added series JSON and an option to download it, expended %manga_series% to work with all modes, added an option for untouched JSON (#855) by byjtje.