From 311f18cd27b56dd71a5c8b02fd68e17818096967 Mon Sep 17 00:00:00 2001 From: David Straub Date: Tue, 8 Oct 2024 08:08:11 +0200 Subject: [PATCH 1/2] Close database after updating media usage --- gramps_webapi/api/media.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gramps_webapi/api/media.py b/gramps_webapi/api/media.py index b18d536f..ba7546ef 100644 --- a/gramps_webapi/api/media.py +++ b/gramps_webapi/api/media.py @@ -363,8 +363,11 @@ def update_usage_media( db_handle = get_db_outside_request( tree=tree, view_private=True, readonly=True, user_id=user_id ) - media_handler = get_media_handler(db_handle, tree=tree) - usage_media = media_handler.get_media_size(db_handle=db_handle) + try: + media_handler = get_media_handler(db_handle, tree=tree) + usage_media = media_handler.get_media_size(db_handle=db_handle) + finally: + db_handle.close() set_tree_usage(tree, usage_media=usage_media) return usage_media From 19b4a25855aec1fd1da3b0e6def2ae8d0160986f Mon Sep 17 00:00:00 2001 From: David Straub Date: Tue, 8 Oct 2024 21:22:06 +0200 Subject: [PATCH 2/2] Close connection to undo log --- gramps_webapi/__main__.py | 6 +++--- gramps_webapi/api/media.py | 3 ++- gramps_webapi/api/tasks.py | 21 +++++++++++---------- gramps_webapi/api/util.py | 12 +++++++++++- gramps_webapi/app.py | 20 ++++++++++---------- gramps_webapi/undodb.py | 2 +- 6 files changed, 38 insertions(+), 26 deletions(-) diff --git a/gramps_webapi/__main__.py b/gramps_webapi/__main__.py index b4cba0b2..0840f20c 100644 --- a/gramps_webapi/__main__.py +++ b/gramps_webapi/__main__.py @@ -31,7 +31,7 @@ import click from .api.search import get_search_indexer -from .api.util import get_db_manager, list_trees +from .api.util import get_db_manager, list_trees, close_db from .app import create_app from .auth import add_user, delete_user, fill_tree, user_db from .const import ENV_CONFIG_FILE, TREE_MULTI @@ -172,7 +172,7 @@ def index_full(ctx): except: LOG.exception("Error during indexing") finally: - db.close() + close_db(db) LOG.info(f"Done building search index in {time.time() - t0:.0f} seconds.") @@ -189,7 +189,7 @@ def index_incremental(ctx): except Exception: LOG.exception("Error during indexing") finally: - db.close() + close_db(db) LOG.info("Done updating search index.") diff --git a/gramps_webapi/api/media.py b/gramps_webapi/api/media.py index ba7546ef..d1270db5 100644 --- a/gramps_webapi/api/media.py +++ b/gramps_webapi/api/media.py @@ -37,6 +37,7 @@ from .s3 import ObjectStorageFileHandler, get_object_keys_size, upload_file_s3 from .util import ( abort_with_message, + close_db, get_db_handle, get_db_outside_request, get_tree_from_jwt, @@ -367,7 +368,7 @@ def update_usage_media( media_handler = get_media_handler(db_handle, tree=tree) usage_media = media_handler.get_media_size(db_handle=db_handle) finally: - db_handle.close() + close_db(db_handle) set_tree_usage(tree, usage_media=usage_media) return usage_media diff --git a/gramps_webapi/api/tasks.py b/gramps_webapi/api/tasks.py index de6d8c36..f4fe54ca 100644 --- a/gramps_webapi/api/tasks.py +++ b/gramps_webapi/api/tasks.py @@ -42,6 +42,7 @@ from .search import get_search_indexer from .util import ( check_quota_people, + close_db, get_config, get_db_outside_request, send_email, @@ -116,7 +117,7 @@ def _search_reindex_full( try: indexer.reindex_full(db, progress_cb=progress_cb) finally: - db.close() + close_db(db) def progress_callback_count(self, title: str = "", message: str = "") -> Callable: @@ -159,7 +160,7 @@ def _search_reindex_incremental( try: indexer.reindex_incremental(db, progress_cb=progress_cb) finally: - db.close() + close_db(db) @shared_task(bind=True) @@ -194,7 +195,7 @@ def import_file( task=self, ) finally: - db_handle.close() + close_db(db_handle) update_usage_people(tree=tree, user_id=user_id) _search_reindex_incremental( tree=tree, @@ -229,7 +230,7 @@ def export_db( db_handle, extension, prepared_options, task=self ) finally: - db_handle.close() + close_db(db_handle) extension = file_type.lstrip(".") return { @@ -260,7 +261,7 @@ def generate_report( language=locale, ) finally: - db_handle.close() + close_db(db_handle) return { "file_name": file_name, @@ -290,7 +291,7 @@ def export_media( progress_cb=progress_callback_count(self), ) finally: - db_handle.close() + close_db(db_handle) file_size = os.path.getsize(zip_filename) return { @@ -318,7 +319,7 @@ def import_media_archive( ) result = importer(progress_cb=progress_callback_count(self)) finally: - db_handle.close() + close_db(db_handle) return result @@ -341,7 +342,7 @@ def media_ocr( ) return handler.get_ocr(lang=lang, output_format=output_format) finally: - db_handle.close() + close_db(db_handle) @shared_task(bind=True) @@ -353,7 +354,7 @@ def check_repair_database(self, tree: str, user_id: str): try: return check_database(db_handle, progress_cb=progress_callback_count(self)) finally: - db_handle.close() + close_db(db_handle) @shared_task(bind=True) @@ -377,7 +378,7 @@ def delete_objects( progress_cb=progress_callback_count(self), ) finally: - db_handle.close() + close_db(db_handle) update_usage_people(tree=tree, user_id=user_id) _search_reindex_incremental( diff --git a/gramps_webapi/api/util.py b/gramps_webapi/api/util.py index 51ab669a..eeddb85b 100644 --- a/gramps_webapi/api/util.py +++ b/gramps_webapi/api/util.py @@ -64,6 +64,7 @@ from gramps.gen.dbstate import DbState from gramps.gen.errors import HandleError from gramps.gen.proxy import PrivateProxyDb +from gramps.gen.proxy.proxybase import ProxyDbBase from gramps.gen.proxy.private import sanitize_media from gramps.gen.user import UserBase from gramps.gen.utils.grampslocale import GrampsLocale @@ -404,6 +405,15 @@ def get_db_outside_request( return dbstate.db +def close_db(db_handle: DbReadBase) -> None: + """Close the connection to the database including the undo log.""" + db_handle.close() + if isinstance(db_handle, ProxyDbBase): + db_handle.basedb.undodb.close() + else: + db_handle.undodb.close() + + def get_db_handle(readonly: bool = True) -> DbReadBase: """Open the database and get the current instance. @@ -624,7 +634,7 @@ def update_usage_people( try: usage_people = db_handle.get_number_of_people() finally: - db_handle.close() + close_db(db_handle) set_tree_usage(tree, usage_people=usage_people) return usage_people diff --git a/gramps_webapi/app.py b/gramps_webapi/app.py index 310f2b72..f091fba4 100644 --- a/gramps_webapi/app.py +++ b/gramps_webapi/app.py @@ -34,6 +34,7 @@ from .api.cache import thumbnail_cache from .api.ratelimiter import limiter from .api.search.embeddings import load_model +from .api.util import close_db from .auth import user_db from .config import DefaultConfig, DefaultConfigJWT from .const import API_PREFIX, ENV_CONFIG_FILE, TREE_MULTI @@ -171,23 +172,22 @@ def send_static(path): # instantiate celery create_celery(app) - # close Gramps DB after every request @app.teardown_appcontext - def close_db(exception) -> None: - """Close the database.""" + def close_db_connection(exception) -> None: + """Close the Gramps database after every request.""" db = g.pop("db", None) - if db and db.is_open(): - db.close() + if db: + close_db(db) db_write = g.pop("db_write", None) - if db_write and db_write.is_open(): - db_write.close() + if db_write: + close_db(db_write) - # close user DB after every request @app.teardown_request - def close_user_db(exception) -> None: - """Close the user database.""" + def close_user_db_connection(exception) -> None: + """Close the user database after every request.""" if exception: user_db.session.rollback() # pylint: disable=no-member + user_db.session.close() # pylint: disable=no-member user_db.session.remove() # pylint: disable=no-member if app.config.get("VECTOR_EMBEDDING_MODEL"): diff --git a/gramps_webapi/undodb.py b/gramps_webapi/undodb.py index 2815aa71..35cfc385 100644 --- a/gramps_webapi/undodb.py +++ b/gramps_webapi/undodb.py @@ -224,7 +224,7 @@ def _make_connection_id(self) -> int: def close(self) -> None: """Close the backing storage.""" - pass + self.engine.dispose() def append(self, value) -> None: """Add a new entry on the end."""