diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3291ed3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +.account +.account_attr +.security +.security_attr +.security_event +.security_prop +.latest_price +.price +.watchlist +.watchlist_security +.xact +.xact_unit +.xact_cross_entry +.taxonomy +.taxonomy_category +.taxonomy_assignment +.taxonomy_data +.dashboard +.property +.bookmark +.attribute_type +.config_set +.config_entry +.create-db \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 43e9185..0000000 --- a/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -PYTHON = python3 -DB = pp.db -TABLES = .account .account_attr \ - .security .security_attr .security_event .security_prop .latest_price .price \ - .watchlist .watchlist_security \ - .xact .xact_unit .xact_cross_entry \ - .taxonomy .taxonomy_category .taxonomy_assignment .taxonomy_data \ - .dashboard \ - .property \ - .bookmark .attribute_type .config_set .config_entry - - -all: - -init: tables - -tables: $(TABLES) - -reload-data: - rm -f $(DATA) - make all - -.%: %.sql .create-db - -echo "DROP TABLE IF EXISTS $(patsubst .%,%,$@);" | sqlite3 $(DB) - sqlite3 $(DB) < $< - touch $@ - -.create-db: - -rm $(DB) - touch $@ - -clean: - -rm -f .[a-z]* - - -# It appears that sqlite3 dump CSV by default with CRLF line endings. Use -# ".separator" directive to override that. -dump: - for db in `echo ".tables" | sqlite3 $(DB)`; do \ - echo -e ".mode csv\n.separator , \\\n\n select * from $$db;" | sqlite3 $(DB) >$$db.csv; \ - done diff --git a/README.md b/README.md index c34be47..2995a23 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,13 @@ or next earnings date - possibilities are limitless. * `*.sql` - Database schema, one table per file. * `ppxml2db.py` - Script to import XML file into a database. * `db2ppxml.py` - Script to export database to XML file. -* `Makefile` - Makefile to create an empty database. + +## Installing + +The ppxml2db scripts have been packaged with pipx so they can be run without worrying about conflicts. + +* Install pipx: https://pipx.pypa.io/stable/installation/ (which requires at least python 3.8) +* Install ppxml2db eg: `pipx install git+https://github.com/flywire/ppxml2db.git` ## Example usage @@ -59,17 +65,15 @@ XML variant of PortfolioPerformance, as introduced in PortfolioPerformance 0.70.3. 1. Start PortfolioPerformance. Make sure you see "Welcome" page. -2. On the "Welcome" page, click "Open the Kommer sample file". -3. "kommer.xml" will open in a new tab. -4. In the application menu, choose: File -> Save as -> XML with "id" attributes. -4. Copy the file to this project's directory for easy access. -5. Create an empty database with all the needed tables: - `make -B init DB=kommer.db` -6. Import the XML into the database: - `python3 ppxml2db.py kommer.xml kommer.db` -7. Export the database to a new XML file: - `python3 db2ppxml.py kommer.db kommer.xml.out` -8. Ensure that the new file matches the original character-by-character: +1. On the "Welcome" page, click "Open the Kommer sample file". +1. "kommer.xml" will open in a new tab. +1. In the application menu, choose: File -> Save as -> XML with "id" attributes. +1. Copy the file to this project's directory for easy access. +1. Import the XML into the database: + `ppxml2db kommer.xml kommer.db` +1. Export the database to a new XML file: + `db2ppxml kommer.db kommer.xml.out` +1. Ensure that the new file matches the original character-by-character: `diff -u kommer.xml kommer.xml.out` Now let's do something pretty simple, yet useful, and already something @@ -116,4 +120,4 @@ https://github.com/pfalcon/ppxml2db/issues/ . ## References * ["anybody interested in PP with a database?"](https://github.com/portfolio-performance/portfolio/issues/2216) -* [Criticism of PP XML format](https://github.com/portfolio-performance/portfolio/issues/3417) +* [Criticism of PP XML format](https://github.com/portfolio-performance/portfolio/issues/3417) \ No newline at end of file diff --git a/ppxml2db/__init__.py b/ppxml2db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db2ppxml.py b/ppxml2db/db2ppxml.py similarity index 99% rename from db2ppxml.py rename to ppxml2db/db2ppxml.py index 1a2d595..af3d491 100644 --- a/db2ppxml.py +++ b/ppxml2db/db2ppxml.py @@ -6,8 +6,8 @@ import lxml.etree as ET -from version import __version__ -import dbhelper +from .version import __version__ +from . import dbhelper # uuid to # @@ -312,7 +312,7 @@ def make_taxonomy_level(etree, pel, level_r): ET.SubElement(d_e, "string").text = d_r["name"] ET.SubElement(d_e, "string").text = d_r["value"] -def main(): +def main0(): root = ET.Element("client") add_xmlid(root) etree = ET.ElementTree(root) @@ -482,7 +482,7 @@ def main(): custom_dump(root, out) -if __name__ == "__main__": +def main(): argp = argparse.ArgumentParser(description="Export Sqlite DB to PortfolioPerformance XML file") argp.add_argument("db_file", help="input DB file") argp.add_argument("xml_file", nargs="?", help="output XML file (stdout if not provided)") @@ -490,10 +490,15 @@ def main(): argp.add_argument("--xpath", action="store_true", help="use legacy XPath references") argp.add_argument("--debug", action="store_true", help="enable debug logging") argp.add_argument("--version", action="version", version="%(prog)s " + __version__) + global args args = argp.parse_args() if args.debug: logging.basicConfig(level=logging.DEBUG) dbhelper.init(args.db_file) - main() + main0() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dbhelper.py b/ppxml2db/dbhelper.py similarity index 55% rename from dbhelper.py rename to ppxml2db/dbhelper.py index e04d141..10ff98b 100644 --- a/dbhelper.py +++ b/ppxml2db/dbhelper.py @@ -1,5 +1,7 @@ import logging +import os import sqlite3 +from pathlib import Path LOG_SQL_TO_FILE = 0 @@ -12,15 +14,47 @@ sqllog = None -def init(dbname): +def init(dbname, new_db = False): global db - db = sqlite3.connect(dbname) + if new_db: + sql_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "setup_scripts") + sql_files = read_sql_files(sql_path) + if os.path.exists(dbname): + os.remove(dbname) + db = sqlite3.connect(dbname) + execute_sql_files(db, sql_files) + else: + db = sqlite3.connect(dbname) db.row_factory = sqlite3.Row if LOG_SQL_TO_FILE: global sqllog sqllog = open(dbname + ".sql", "w") +def read_sql_files(sql_path): + sql_files = {} + for filename in os.listdir(sql_path): + if filename.endswith(".sql"): + with open(os.path.join(sql_path, filename), "r") as file: + sql_files[filename] = file.read() + # [print(f"{key}: {value}") for key, value in sql_files.items()] + return sql_files + + +def execute_sql_files(db, sql_files): + # cursor = db.cursor() + for filename, sql_content in sql_files.items(): + try: + # cursor.executescript(sql_content) + db.executescript(sql_content) + # print(f"Executed {filename} successfully") + except sqlite3.Error as e: + print(f"Error executing {filename}: {e}") + db.commit() + # db.close() + + def execute_insert(sql, values = ()): if LOG_SQL_TO_FILE: sqllog.write("%s %s\n" % (sql, values)) diff --git a/ppxml2db.py b/ppxml2db/ppxml2db.py similarity index 99% rename from ppxml2db.py rename to ppxml2db/ppxml2db.py index 860ed12..027fbcd 100644 --- a/ppxml2db.py +++ b/ppxml2db/ppxml2db.py @@ -1,16 +1,12 @@ -import sys import argparse -import logging from collections import defaultdict -from pprint import pprint import json import logging -import os.path import lxml.etree as ET -from version import __version__ -import dbhelper +from .version import __version__ +from . import dbhelper _log = logging.getLogger(__name__) @@ -588,7 +584,7 @@ def iterparse(self): el.text = el.tail = None -if __name__ == "__main__": +def main(): argp = argparse.ArgumentParser(description="Import PortfolioPerformance XML file to Sqlite DB") argp.add_argument("xml_file", help="input XML file") argp.add_argument("db_file", help="output DB file") @@ -600,7 +596,7 @@ def iterparse(self): if args.debug: logging.basicConfig(level=logging.DEBUG) - dbhelper.init(args.db_file) + dbhelper.init(args.db_file, True) with open(args.xml_file, "rb") as f: conv = PortfolioPerformanceXML2DB(f) @@ -608,3 +604,7 @@ def iterparse(self): if not args.dry_run: dbhelper.commit() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/account.sql b/ppxml2db/setup_scripts/account.sql similarity index 84% rename from account.sql rename to ppxml2db/setup_scripts/account.sql index 0962e1e..60741a0 100644 --- a/account.sql +++ b/ppxml2db/setup_scripts/account.sql @@ -10,4 +10,4 @@ _xmlid INT NOT NULL, _order INT NOT NULL, PRIMARY KEY(uuid) ); -#CREATE UNIQUE INDEX account__uuid ON account(uuid); +-- CREATE UNIQUE INDEX account__uuid ON account(uuid); diff --git a/account_attr.sql b/ppxml2db/setup_scripts/account_attr.sql similarity index 100% rename from account_attr.sql rename to ppxml2db/setup_scripts/account_attr.sql diff --git a/attribute_type.sql b/ppxml2db/setup_scripts/attribute_type.sql similarity index 100% rename from attribute_type.sql rename to ppxml2db/setup_scripts/attribute_type.sql diff --git a/bookmark.sql b/ppxml2db/setup_scripts/bookmark.sql similarity index 100% rename from bookmark.sql rename to ppxml2db/setup_scripts/bookmark.sql diff --git a/config_entry.sql b/ppxml2db/setup_scripts/config_entry.sql similarity index 100% rename from config_entry.sql rename to ppxml2db/setup_scripts/config_entry.sql diff --git a/config_set.sql b/ppxml2db/setup_scripts/config_set.sql similarity index 100% rename from config_set.sql rename to ppxml2db/setup_scripts/config_set.sql diff --git a/dashboard.sql b/ppxml2db/setup_scripts/dashboard.sql similarity index 100% rename from dashboard.sql rename to ppxml2db/setup_scripts/dashboard.sql diff --git a/latest_price.sql b/ppxml2db/setup_scripts/latest_price.sql similarity index 100% rename from latest_price.sql rename to ppxml2db/setup_scripts/latest_price.sql diff --git a/price.sql b/ppxml2db/setup_scripts/price.sql similarity index 100% rename from price.sql rename to ppxml2db/setup_scripts/price.sql diff --git a/property.sql b/ppxml2db/setup_scripts/property.sql similarity index 100% rename from property.sql rename to ppxml2db/setup_scripts/property.sql diff --git a/security.sql b/ppxml2db/setup_scripts/security.sql similarity index 100% rename from security.sql rename to ppxml2db/setup_scripts/security.sql diff --git a/security_attr.sql b/ppxml2db/setup_scripts/security_attr.sql similarity index 100% rename from security_attr.sql rename to ppxml2db/setup_scripts/security_attr.sql diff --git a/security_event.sql b/ppxml2db/setup_scripts/security_event.sql similarity index 100% rename from security_event.sql rename to ppxml2db/setup_scripts/security_event.sql diff --git a/security_prop.sql b/ppxml2db/setup_scripts/security_prop.sql similarity index 100% rename from security_prop.sql rename to ppxml2db/setup_scripts/security_prop.sql diff --git a/taxonomy.sql b/ppxml2db/setup_scripts/taxonomy.sql similarity index 100% rename from taxonomy.sql rename to ppxml2db/setup_scripts/taxonomy.sql diff --git a/taxonomy_assignment.sql b/ppxml2db/setup_scripts/taxonomy_assignment.sql similarity index 100% rename from taxonomy_assignment.sql rename to ppxml2db/setup_scripts/taxonomy_assignment.sql diff --git a/taxonomy_category.sql b/ppxml2db/setup_scripts/taxonomy_category.sql similarity index 100% rename from taxonomy_category.sql rename to ppxml2db/setup_scripts/taxonomy_category.sql diff --git a/taxonomy_data.sql b/ppxml2db/setup_scripts/taxonomy_data.sql similarity index 100% rename from taxonomy_data.sql rename to ppxml2db/setup_scripts/taxonomy_data.sql diff --git a/watchlist.sql b/ppxml2db/setup_scripts/watchlist.sql similarity index 100% rename from watchlist.sql rename to ppxml2db/setup_scripts/watchlist.sql diff --git a/watchlist_security.sql b/ppxml2db/setup_scripts/watchlist_security.sql similarity index 100% rename from watchlist_security.sql rename to ppxml2db/setup_scripts/watchlist_security.sql diff --git a/xact.sql b/ppxml2db/setup_scripts/xact.sql similarity index 100% rename from xact.sql rename to ppxml2db/setup_scripts/xact.sql diff --git a/xact_cross_entry.sql b/ppxml2db/setup_scripts/xact_cross_entry.sql similarity index 100% rename from xact_cross_entry.sql rename to ppxml2db/setup_scripts/xact_cross_entry.sql diff --git a/xact_unit.sql b/ppxml2db/setup_scripts/xact_unit.sql similarity index 100% rename from xact_unit.sql rename to ppxml2db/setup_scripts/xact_unit.sql diff --git a/version.py b/ppxml2db/version.py similarity index 100% rename from version.py rename to ppxml2db/version.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e232636 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "ppxml2db" +version = "1.7.1" +description = "Import Portfolio Performance XML into an SQLite database and vice versa" +license = "MIT" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ "lxml (>=5.3.0,<6.0.0)",] + +[build-system] +requires = [ "poetry-core>=2.0.0,<3.0.0",] +build-backend = "poetry.core.masonry.api" + +[project.scripts] +ppxml2db = "ppxml2db.ppxml2db:main" +db2ppxml = "ppxml2db.db2ppxml:main"