Mokuro 0.2.0 (#91)

* web reader

* remove unused code

* fix

* update version

* fix

* detect broken cache

* update version

* fix

* update favicon

* fix deleting volume

* ignore volumes uploaded with missing files

* fix drag and drop on firefox

* recreate _ocr cache from mokuro file if missing

* update version

* alert user when they upload .mokuro file without corresponding volume data

* fix natural sort for full width numerals

* make drag and drop work on whole catalog popup

* change service worker to network first

* display storage usage

* request persistent storage

* error message on missing paths

* Search for webp files too when constructing img_paths (#86)

* update tests, add help

* bug fix

* delete reader

* change disable_html -> legacy_html; add test for converting legacy _ocr to .mokuro

* update readme

* replace setup.py with pyproject.toml

* exclude ctd from ruff

* format with ruff

* fox mokuro entry point

* update gh workflows

* fix package discovery

* fix missing setuptools dependency (needed for pkg_resources used by CTD)

* add --version flag

---------

Co-authored-by: precondition <[email protected]>

Author: kha-white

.github/workflows/main.yml

@@ -10,14 +10,13 @@ permissions:
   contents: read
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          path: mokuro
           submodules: recursive
 
       - name: Set up Python
@@ -28,9 +27,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest
-          pip install -e mokuro
+          pip install -e ".[dev]"
 
       - name: Test
         run: |
-          pytest -v mokuro/tests
+          pytest

.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,122 @@
+name: Publish package to PyPI and TestPyPI
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]*'
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build:
+    name: Build dist
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/mokuro  # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/[email protected]
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        '${{ github.ref_name }}'
+        --repo '${{ github.repository }}'
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dist/**
+        --repo '${{ github.repository }}'
+
+  publish-to-testpypi:
+    name: Publish to TestPyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/mokuro
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/

.github/workflows/ruff.yml

@@ -0,0 +1,8 @@
+name: Ruff
+on: [ push, pull_request ]
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1

MANIFEST.in

@@ -8,25 +8,32 @@ https://user-images.githubusercontent.com/22717958/164993274-3e8d1650-9be3-457d-
 
 <sup>Demo contains excerpt from [Manga109-s dataset](http://www.manga109.org/en/download_s.html). うちの猫’ず日記 © がぁさん</sup>
 
-mokuro is aimed towards Japanese learners, who want to read manga in Japanese with a pop-up dictionary like [Yomichan](https://github.com/FooSoft/yomichan).
+mokuro is aimed towards Japanese learners, who want to read manga in Japanese with a pop-up dictionary like [Yomitan](https://github.com/themoeway/yomitan).
 It works like this:
 1. Perform text detection and OCR for each page.
-2. After processing a whole volume, generate a HTML file, which you can open in a browser.
-3. All processing is done offline (before reading). You can transfer the resulting HTML file together with manga images to
-another device (e.g. your mobile phone) and read there.
+2. After processing a whole volume, generate a .mokuro file, which contains OCR results and metadata. All processing is done offline (before reading).
+3. Load the .mokuro file together with manga images in [web reader](https://reader.mokuro.app/), which serves both as a manga reader and a catalog for processed series and volumes.
+
+Alternatively, you can still use the old method from mokuro 0.1.*:
+Instead of a .mokuro file, generate an HTML file, which you can open in a browser.
+You can transfer the resulting HTML file together with manga images to another device (e.g. your mobile phone) and read there.
+This method is still supported for backward compatibility, but it is recommended to use the new .mokuro format and the web reader.
+For details, see [Legacy HTML vs. new .mokuro format](#legacy-html-vs-new-mokuro-format).
 
 mokuro uses [comic-text-detector](https://github.com/dmMaze/comic-text-detector) for text detection
 and [manga-ocr](https://github.com/kha-white/manga-ocr) for OCR.
 
 Try running on your manga in Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/kha-white/mokuro/blob/master/notebooks/mokuro_demo.ipynb)
 
 See also:
+- [mokuro-reader](https://github.com/ZXY101/mokuro-reader), a web reader for mokuro, developed by ZXY101
 - [Mokuro2Pdf](https://github.com/Kartoffel0/Mokuro2Pdf), cli Ruby script to generate pdf files with selectable text from Mokuro's html overlay
 - [Xelieu's guide](https://xelieu.github.io/jp-lazy-guide/setupMangaOnPC/), a comprehensive guide on setting up a reading and mining workflow with manga-ocr/mokuro (and many other useful tips)
 
 # Installation
 
-You need Python 3.6 or newer. Please note, that the newest Python release might not be supported due to a PyTorch dependency, which often breaks with new Python releases and needs some time to catch up.
+You need Python 3.6 or newer. Please note, that the newest Python release might not be supported due to a PyTorch dependency, 
+which often breaks with new Python releases and needs some time to catch up.
 Refer to [PyTorch website](https://pytorch.org/get-started/locally/) for a list of supported Python versions.
 
 Some users have reported problems with Python installed from Microsoft Store. If you see an error:
@@ -86,11 +93,32 @@ mokuro --parent_dir manga_title/
 ## Other options
 
 ```
---force_cpu - disable GPU
---as_one_file - generate separate css and js files instead of embedding everything in html
---disable_confirmation - run without asking for confirmation
+--pretrained_model_name_or_path: Name or path of the manga-ocr model.
+--force_cpu: Force the use of CPU even if CUDA is available.
+--disable_confirmation: Disable confirmation prompt. If False, the user will be prompted to confirm the list of volumes to be processed.
+--disable_ocr: Disable OCR processing. Generate mokuro/HTML files without OCR results.
+--ignore_errors: Continue processing volumes even if an error occurs.
+--no_cache: Do not use cached OCR results from previous runs (_ocr directories).
+--unzip: Extract volumes in zip/cbz format in their original location.
+--disable_html: Disable legacy HTML output. If True, acts as if --unzip is True.
+--as_one_file: Applies only to legacy HTML. If False, generate separate CSS and JS files instead of embedding them in the HTML file.
+--version: Print the version of mokuro and exit.
 ```
 
+## Legacy HTML vs. new .mokuro format
+
+Before version 0.2.0, mokuro generated a separate HTML file for each processed volume, which caused some usability issues:
+- HTML files contained both the OCR results and the whole web reader GUI, so in order to update the GUI, all volumes needed to be updated with a new mokuro version
+- images were stored separately and linked in HTML files, so any change in the directory structure could break the links
+- transferring the manga to another device required transferring both the HTML files and the images
+- there was no unified GUI for a whole catalog containing multiple volumes
+- on some mobile devices, some workarounds were needed to open HTML files
+
+Starting from version 0.2.0, a new .mokuro format is introduced, which is generated for each volume and contains only the OCR results and metadata necessary for the web reader GUI.
+Web reader is now a separate web app, which can open manga volumes with their associated .mokuro files.
+
+The old HTML format is still generated for backward compatibility, but it will not be developed further, and it is recommended to use the new .mokuro format and the web reader.
+
 # Contact
 For any inquiries, please feel free to contact me at [email protected]
 

README.md

@@ -1,4 +1,4 @@
-__version__ = '0.1.8'
+from ._version import __version__ as __version__
 
-from mokuro.manga_page_ocr import MangaPageOcr, InvalidImage
-from mokuro.overlay_generator import OverlayGenerator
+from mokuro.manga_page_ocr import MangaPageOcr as MangaPageOcr
+from mokuro.mokuro_generator import MokuroGenerator as MokuroGenerator

mokuro/__init__.py

@@ -7,5 +7,5 @@ def main():
     fire.Fire(run)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

mokuro/__main__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0-beta.8"

mokuro/_version.py

@@ -6,28 +6,28 @@
 
 class cache:
     def __init__(self):
-        self.root = Path.home() / '.cache' / 'manga-ocr'
+        self.root = Path.home() / ".cache" / "manga-ocr"
         self.root.mkdir(parents=True, exist_ok=True)
 
     @property
     def comic_text_detector(self):
-        path = self.root / 'comictextdetector.pt'
-        url = 'https://github.com/zyddnys/manga-image-translator/releases/download/beta-0.2.1/comictextdetector.pt'
+        path = self.root / "comictextdetector.pt"
+        url = "https://github.com/zyddnys/manga-image-translator/releases/download/beta-0.2.1/comictextdetector.pt"
 
         self._download_if_needed(path, url)
         return path
 
     def _download_if_needed(self, path, url):
         if not path.is_file():
-            logger.info(f'Downloading {url}')
+            logger.info(f"Downloading {url}")
             r = requests.get(url, stream=True, verify=True)
             if r.status_code != 200:
-                raise RuntimeError(f'Failed downloading {url}')
-            with path.open('wb') as f:
+                raise RuntimeError(f"Failed downloading {url}")
+            with path.open("wb") as f:
                 for chunk in r.iter_content(1024):
                     if chunk:
                         f.write(chunk)
-            logger.info(f'Finished downloading {url}')
+            logger.info(f"Finished downloading {url}")
 
 
 cache = cache()

mokuro/cache.py

@@ -1,5 +1,5 @@
 from pathlib import Path
 
-ASSETS_PATH = Path(__file__).parent / 'assets'
+ASSETS_PATH = Path(__file__).parent / "assets"
 
-assert ASSETS_PATH.is_dir(), f'{ASSETS_PATH} missing'
+assert ASSETS_PATH.is_dir(), f"{ASSETS_PATH} missing"