Allow create database in python

.gitignore

@@ -0,0 +1,24 @@
+.account
+.account_attr
+.security
+.security_attr
+.security_event
+.security_prop
+.latest_price
+.price
+.watchlist
+.watchlist_security
+.xact
+.xact_unit
+.xact_cross_entry
+.taxonomy
+.taxonomy_category
+.taxonomy_assignment
+.taxonomy_data
+.dashboard
+.property
+.bookmark
+.attribute_type
+.config_set
+.config_entry
+.create-db

README.md

@@ -46,7 +46,13 @@ or next earnings date - possibilities are limitless.
 * `*.sql` - Database schema, one table per file.
 * `ppxml2db.py` - Script to import XML file into a database.
 * `db2ppxml.py` - Script to export database to XML file.
-* `Makefile` - Makefile to create an empty database.
+
+## Installing
+
+The ppxml2db scripts have been packaged with pipx so they can be run without worrying about conflicts.
+
+* Install pipx: https://pipx.pypa.io/stable/installation/ (which requires at least python 3.8)
+* Install ppxml2db eg: `pipx install git+https://github.com/flywire/ppxml2db.git`
 
 ## Example usage
 
@@ -59,17 +65,15 @@ XML variant of PortfolioPerformance, as introduced in PortfolioPerformance
 0.70.3.
 
 1. Start PortfolioPerformance. Make sure you see "Welcome" page.
-2. On the "Welcome" page, click "Open the Kommer sample file".
-3. "kommer.xml" will open in a new tab.
-4. In the application menu, choose: File -> Save as -> XML with "id" attributes.
-4. Copy the file to this project's directory for easy access.
-5. Create an empty database with all the needed tables:
-   `make -B init DB=kommer.db`
-6. Import the XML into the database:
-   `python3 ppxml2db.py kommer.xml kommer.db`
-7. Export the database to a new XML file:
-   `python3 db2ppxml.py kommer.db kommer.xml.out`
-8. Ensure that the new file matches the original character-by-character:
+1. On the "Welcome" page, click "Open the Kommer sample file".
+1. "kommer.xml" will open in a new tab.
+1. In the application menu, choose: File -> Save as -> XML with "id" attributes.
+1. Copy the file to this project's directory for easy access.
+1. Import the XML into the database:
+   `ppxml2db kommer.xml kommer.db`
+1. Export the database to a new XML file:
+   `db2ppxml kommer.db kommer.xml.out`
+1. Ensure that the new file matches the original character-by-character:
    `diff -u kommer.xml kommer.xml.out`
 
 Now let's do something pretty simple, yet useful, and already something
@@ -116,4 +120,4 @@ https://github.com/pfalcon/ppxml2db/issues/ .
 ## References
 
 * ["anybody interested in PP with a database?"](https://github.com/portfolio-performance/portfolio/issues/2216)
-* [Criticism of PP XML format](https://github.com/portfolio-performance/portfolio/issues/3417)
+* [Criticism of PP XML format](https://github.com/portfolio-performance/portfolio/issues/3417)

db2ppxml.py → ppxml2db/db2ppxml.py

@@ -6,8 +6,8 @@
 
 import lxml.etree as ET
 
-from version import __version__
-import dbhelper
+from .version import __version__
+from . import dbhelper
 
 
 # uuid to #
@@ -312,7 +312,7 @@ def make_taxonomy_level(etree, pel, level_r):
                 ET.SubElement(d_e, "string").text = d_r["name"]
                 ET.SubElement(d_e, "string").text = d_r["value"]
 
-def main():
+def main0():
     root = ET.Element("client")
     add_xmlid(root)
     etree = ET.ElementTree(root)
@@ -482,18 +482,23 @@ def main():
     custom_dump(root, out)
 
 
-if __name__ == "__main__":
+def main():
     argp = argparse.ArgumentParser(description="Export Sqlite DB to PortfolioPerformance XML file")
     argp.add_argument("db_file", help="input DB file")
     argp.add_argument("xml_file", nargs="?", help="output XML file (stdout if not provided)")
     argp.add_argument("--sort-events", action="store_true", help="sort events by date (then description)")
     argp.add_argument("--xpath", action="store_true", help="use legacy XPath references")
     argp.add_argument("--debug", action="store_true", help="enable debug logging")
     argp.add_argument("--version", action="version", version="%(prog)s " + __version__)
+    global args
     args = argp.parse_args()
 
     if args.debug:
         logging.basicConfig(level=logging.DEBUG)
 
     dbhelper.init(args.db_file)
-    main()
+    main0()
+
+
+if __name__ == "__main__":
+    main()

dbhelper.py → ppxml2db/dbhelper.py

@@ -1,5 +1,7 @@
 import logging
+import os
 import sqlite3
+from pathlib import Path
 
 
 LOG_SQL_TO_FILE = 0
@@ -12,15 +14,47 @@
 sqllog = None
 
 
-def init(dbname):
+def init(dbname, new_db = False):
     global db
-    db = sqlite3.connect(dbname)
+    if new_db:
+        sql_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                "setup_scripts")
+        sql_files = read_sql_files(sql_path)
+        if os.path.exists(dbname):
+            os.remove(dbname)
+        db = sqlite3.connect(dbname)
+        execute_sql_files(db, sql_files)
+    else:
+        db = sqlite3.connect(dbname)
     db.row_factory = sqlite3.Row
     if LOG_SQL_TO_FILE:
         global sqllog
         sqllog = open(dbname + ".sql", "w")
 
 
+def read_sql_files(sql_path):
+    sql_files = {}
+    for filename in os.listdir(sql_path):
+        if filename.endswith(".sql"):
+            with open(os.path.join(sql_path, filename), "r") as file:
+                sql_files[filename] = file.read()
+    # [print(f"{key}: {value}") for key, value in sql_files.items()]
+    return sql_files
+
+
+def execute_sql_files(db, sql_files):
+    # cursor = db.cursor()
+    for filename, sql_content in sql_files.items():
+        try:
+            # cursor.executescript(sql_content)
+            db.executescript(sql_content)
+            # print(f"Executed {filename} successfully")
+        except sqlite3.Error as e:
+            print(f"Error executing {filename}: {e}")
+    db.commit()
+    # db.close()
+
+
 def execute_insert(sql, values = ()):
     if LOG_SQL_TO_FILE:
         sqllog.write("%s %s\n" % (sql, values))

ppxml2db.py → ppxml2db/ppxml2db.py

@@ -1,16 +1,12 @@
-import sys
 import argparse
-import logging
 from collections import defaultdict
-from pprint import pprint
 import json
 import logging
-import os.path
 
 import lxml.etree as ET
 
-from version import __version__
-import dbhelper
+from .version import __version__
+from . import dbhelper
 
 
 _log = logging.getLogger(__name__)
@@ -588,7 +584,7 @@ def iterparse(self):
                         el.text = el.tail = None
 
 
-if __name__ == "__main__":
+def main():
     argp = argparse.ArgumentParser(description="Import PortfolioPerformance XML file to Sqlite DB")
     argp.add_argument("xml_file", help="input XML file")
     argp.add_argument("db_file", help="output DB file")
@@ -600,11 +596,15 @@ def iterparse(self):
     if args.debug:
         logging.basicConfig(level=logging.DEBUG)
 
-    dbhelper.init(args.db_file)
+    dbhelper.init(args.db_file, True)
 
     with open(args.xml_file, "rb") as f:
         conv = PortfolioPerformanceXML2DB(f)
         conv.iterparse()
 
     if not args.dry_run:
         dbhelper.commit()
+
+
+if __name__ == "__main__":
+    main()

account.sql → ppxml2db/setup_scripts/account.sql

@@ -10,4 +10,4 @@ _xmlid INT NOT NULL,
 _order INT NOT NULL,
 PRIMARY KEY(uuid)
 );
-#CREATE UNIQUE INDEX account__uuid ON account(uuid);
+-- CREATE UNIQUE INDEX account__uuid ON account(uuid);

pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "ppxml2db"
+version = "1.7.1"
+description = "Import Portfolio Performance XML into an SQLite database and vice versa"
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [ "lxml (>=5.3.0,<6.0.0)",]
+
+[build-system]
+requires = [ "poetry-core>=2.0.0,<3.0.0",]
+build-backend = "poetry.core.masonry.api"
+
+[project.scripts]
+ppxml2db = "ppxml2db.ppxml2db:main"
+db2ppxml = "ppxml2db.db2ppxml:main"