Merge pull request #197 from mjuhanne/non_unicode_primitives__refactored

Non-unicode primitive support and major Kanji database update

addon/__init__.py

@@ -21,7 +21,7 @@
 
 # Allow webviews accessing kanjivg svgs, web data for lookups and fonts
 aqt.mw.addonManager.setWebExports(
-    __name__, r"(kanjivg/.*\.svg|web/.*|fonts/.*|user_files/fonts/.*)"
+    __name__, r"(kanjivg/.*\.svg|kanjivg-supplementary/.*\.svg|primitives/.*\.svg|web/.*|fonts/.*|user_files/fonts/.*)"
 )
 
 

addon/card_type.py

@@ -124,9 +124,12 @@ def field_exists(name):
         settings_html = f"""
             <script>
                 var settings = JSON.parse('{json.dumps(settings)}');
+                var primitives_uri = '_primitive_';
+                var cursive_primitives_uri = '_primitive_cursive_';
             </script>
         """
 
+        common_js = "<script>" + read_web_file("common.js") + "</script>\n\n"
         jquery_js = "<script>" + read_web_file("jquery.js") + "</script>\n\n"
         common_back_js = "<script>" + read_web_file("common_back.js") + "</script>\n\n"
         dmak_js = "<script>" + read_web_file("dmak.js") + "</script>\n\n"
@@ -139,6 +142,7 @@ def field_exists(name):
             jquery_js +
             settings_html + "\n\n" +
             japanese_util_js +
+            common_js +
             read_web_file_with_includes(f"front-{self.label}.html")
         )
         template["afmt"] = (
@@ -147,6 +151,7 @@ def field_exists(name):
             dmak_js +
             raphael_js +
             japanese_util_js +
+            common_js +
             read_web_file_with_includes(f"back-{self.label}.html") +
             common_back_js +
             launch_back_js

addon/fonts.py

@@ -23,6 +23,22 @@ def get_col_name(idx):
 def get_col_path(idx):
     return util.col_media_path(get_col_name(idx))
 
+def assure_primitive_col_media(sub_dir=''):
+    primitive_dir = util.addon_path("primitives",sub_dir)
+    primitive_files = os.listdir(primitive_dir)
+    if sub_dir != '':
+        sub_dir_tag = sub_dir + '_'
+    else:
+        sub_dir_tag = ''
+    for f in primitive_files:
+        src_path = util.addon_path("primitives", sub_dir, f)
+        if not os.path.isdir(src_path):
+            dest_path =  util.col_media_path('_primitive_' + sub_dir_tag + f)
+            shutil.copy(src_path, dest_path)
+        else:
+            # subdirectory 
+            assure_primitive_col_media(f)
+
 
 def get_addon_uri(idx):
     user_name = config.get("fonts", config_default)[idx]
@@ -109,3 +125,4 @@ def ui_css():
 def assure_col_media():
     for idx in range(font_num):
         shutil.copy(get_path(idx), get_col_path(idx))
+    assure_primitive_col_media()

addon/kanji.py

@@ -8,7 +8,7 @@
 import anki
 import aqt
 
-from .util import addon_path, user_path, assure_user_dir, unique_characters
+from .util import addon_path, user_path, assure_user_dir, unique_characters, custom_list
 from .errors import InvalidStateError, InvalidDeckError
 from .card_type import CardType
 from . import config
@@ -24,10 +24,12 @@ def clean_character_field(f):
     f = f.lstrip()
     f = text_parser.html_regex.sub("", f)
     if len(f):
+        # Leave [primitive_tag] as it is, otherwise return the single unicode character
+        if f[0] == '[':
+            return f
         return f[0]
     return ""
 
-
 class KanjiDB:
     def __init__(self):
         self.initialize()
@@ -155,17 +157,6 @@ def _new_characters_find(self, card_type, character, out, max_characters=-1):
         if character in out:
             return
 
-        # Check if card already exists for character
-        table = f"usr.{card_type.label}_card_ids"
-        r = self.crs_execute_and_fetch_one(
-            f"SELECT COUNT(*) FROM {table} "
-            "WHERE character == (?) AND card_id NOT NULL",
-            (character,),
-        )
-
-        if r[0] != 0:
-            return
-
         # Get primitives
         primitives_result = self.crs_execute_and_fetch_one(
             "SELECT primitives FROM characters " "WHERE character == (?)", (character,)
@@ -175,6 +166,7 @@ def _new_characters_find(self, card_type, character, out, max_characters=-1):
             print(f"Lookup of primitive {character} failed.")
             return
         primitives = primitives_result[0]
+        primitives = custom_list(primitives)
 
         # Recusivly add primitives that need to be learned if enabled
         if card_type.add_primitives:
@@ -187,6 +179,17 @@ def _new_characters_find(self, card_type, character, out, max_characters=-1):
         if max_characters >= 0 and len(out) >= max_characters:
             return
 
+        # Check if card already exists for character
+        table = f"usr.{card_type.label}_card_ids"
+        r = self.crs_execute_and_fetch_one(
+            f"SELECT COUNT(*) FROM {table} "
+            "WHERE character == (?) AND card_id NOT NULL",
+            (character,),
+        )
+
+        if r[0] != 0:
+            return
+
         out.append(character)
 
     # Recursivley find new characters to learn
@@ -612,23 +615,18 @@ def make_cards_from_characters(self, card_type, new_characters, checkpoint=None)
         deck_id = deck["id"]
         model = aqt.mw.col.models.byName(model_name)
 
-        aqt.mw.requireReset()
-
         for c in characters:
             note = anki.notes.Note(aqt.mw.col, model)
             note["Character"] = c
             self.refresh_note(note)
             add_note_no_hook(aqt.mw.col, note, deck_id)
 
-        aqt.mw.maybeReset()
-
         self.recalc_user_cards(card_type)
 
     def refresh_note(self, note, do_flush=False):
         c = clean_character_field(note["Character"])
         if len(c) < 1:
             return
-        c = c[0]
         note["Character"] = c
 
         r = self.get_kanji_result_data(c, card_ids=False)
@@ -637,10 +635,20 @@ def refresh_note(self, note, do_flush=False):
         data_json_b64 = str(data_json_b64_b, "utf-8")
         note["MigakuData"] = data_json_b64
 
-        svg_name = "%05x.svg" % ord(c)
-        svg_path = addon_path("kanjivg", svg_name)
+        if c[0] == '[':
+            svg_name = c[1:-1] + ".svg"
+        else:
+            svg_name = "%05x.svg" % ord(c)
+
+        # Try to find the KanjiVG file first in supplementary directory and
+        # only then from the main repository
+        svg_path = addon_path("kanjivg-supplementary", svg_name)
+        if not os.path.exists(svg_path):
+            svg_path = addon_path("kanjivg", svg_name)
+            if not os.path.exists(svg_path):
+                svg_path = ''
 
-        if os.path.exists(svg_path):
+        if svg_path != '':
             with open(svg_path, "r", encoding="utf-8") as file:
                 svg_data = file.read()
 
@@ -651,6 +659,31 @@ def refresh_note(self, note, do_flush=False):
         if do_flush:
             note.flush()
 
+    # If the deck has cards that have now references for new primitives 
+    # which are not yet included in the stack, add them.
+    def add_missing_characters(self):
+        new_kanji_for_msg = OrderedDict()
+
+        for ct in CardType:
+            if not ct.add_primitives:
+                continue
+
+            find_filter = f'"note:{ct.model_name}"'
+            note_ids = aqt.mw.col.find_notes(find_filter)
+
+            all_characters_in_the_deck = []
+            for i, note_id in enumerate(note_ids):
+                note = aqt.mw.col.getNote(note_id)
+                c = clean_character_field(note["Character"])
+                all_characters_in_the_deck.append(c)
+
+            new_characters = self.new_characters(ct, all_characters_in_the_deck, -1)
+            if len(new_characters) > 0:
+                new_kanji_for_msg[ct] = new_characters
+
+        if len(new_kanji_for_msg) > 0:
+            KanjiConfirmDialog.show_new_kanji(new_kanji_for_msg, aqt.mw)
+
     # Recalc everything
     def recalc_all(self, callback=None):
         if callback:
@@ -664,6 +697,8 @@ def recalc_all(self, callback=None):
 
         self.recalc_user_words()
 
+        self.add_missing_characters()
+
         find_filter = [f'"note:{ct.model_name}"' for ct in CardType]
         note_ids = aqt.mw.col.find_notes(" OR ".join(find_filter))
         num_notes = len(note_ids)
@@ -701,10 +736,10 @@ def get_kanji_result_data(
             ("grade", _, None),
             ("jlpt", _, None),
             ("kanken", _, None),
-            ("primitives", list, None),
-            ("primitive_of", list, None),
+            ("primitives", custom_list, None),
+            ("primitive_of", custom_list, None),
             ("primitive_keywords", json.loads, None),
-            ("primitive_alternatives", list, None),
+            ("primitive_alternatives", custom_list, None),
             ("heisig_id5", _, None),
             ("heisig_id6", _, None),
             ("heisig_keyword5", _, None),

addon/kanji_confirm_dialog.py

@@ -55,7 +55,11 @@ def data(self, idx, role):
 
         if role == Qt.ItemDataRole.DisplayRole:
             # This should be a QBrush, but we also use the method
-            return str(kanji)
+            if kanji[0] != '[':
+                return str(kanji)
+        if role == Qt.ItemDataRole.DecorationRole:
+            if kanji[0] == '[':
+                return util.get_pixmap_from_tag(kanji, 35)
         if role == Qt.ItemDataRole.BackgroundRole:
             state = self.states[kanji]
             return QBrush(QColor(self.state_colors[state]))
@@ -163,9 +167,10 @@ def __init__(self, parent=None, ct_kanji={}):
 
         self.resize(500, 400)
 
-        button_box = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok)
+        button_box = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
         button_box.accepted.connect(self.accept)
-        lyt.addWidget(button_box)
+        button_box.rejected.connect(self.reject)
+        lyt.addWidget(button_box, Qt.AlignmentFlag.AlignRight)
 
         self.add_kanji(ct_kanji)