Upgrade SQLAlchemy to version 2.0

Author: bruntib

web/requirements.txt

@@ -7,8 +7,8 @@
 Authlib<2
 requests<3  # Required by Authlib. Not installed automatically for some reason.
 lxml<6
-sqlalchemy<2
-alembic<2
+sqlalchemy~=2.0
+alembic~=1.5
 portalocker<4
 psutil<8
 multiprocess<0.71

web/requirements_py/db_pg8000/requirements.txt

@@ -1,7 +1,7 @@
 lxml<6
-sqlalchemy<2
-alembic<2
-pg8000<=1.31.4
+sqlalchemy~=2.0
+alembic~=1.5
+pg8000~=1.31
 psutil<8
 portalocker<4
 

web/requirements_py/db_psycopg2/requirements.txt

@@ -1,7 +1,7 @@
 lxml<6
-sqlalchemy<2
-alembic<2
-psycopg2-binary<=2.9.10
+sqlalchemy~=2.0
+alembic~=1.5
+psycopg2-binary~=2.9
 psutil<8
 portalocker<4
 

web/server/codechecker_server/api/product_server.py

@@ -13,6 +13,7 @@
 import os
 import random
 
+from sqlalchemy import text
 from sqlalchemy.sql.expression import and_
 
 from sqlalchemy import create_engine, exc
@@ -354,7 +355,7 @@ def __create_product_database(self, product):
         db_pass = convert.from_b64(product_info.password_b64)
         db_name = product_info.database
 
-        engine_url = URL(
+        engine_url = URL.create(
             drivername=db_engine,
             username=db_user,
             password=db_pass,
@@ -365,9 +366,9 @@ def __create_product_database(self, product):
         engine = create_engine(engine_url)
         try:
             with engine.connect() as conn:
-                conn.execute("commit")
+                conn.execute(text("commit"))
                 LOG.info("Creating database '%s'", db_name)
-                conn.execute(f"CREATE DATABASE {db_name}")
+                conn.execute(text(f"CREATE DATABASE {db_name}"))
                 conn.close()
         except exc.ProgrammingError as e:
             LOG.error("ProgrammingError occurred: %s", str(e))

web/server/codechecker_server/api/report_server.py

@@ -265,10 +265,10 @@ def process_report_filter(
     if report_filter.cleanupPlanNames:
         OR = []
         for cleanup_plan_name in report_filter.cleanupPlanNames:
-            q = select([CleanupPlanReportHash.bug_hash]) \
+            q = select(CleanupPlanReportHash.bug_hash) \
                 .where(
                     CleanupPlanReportHash.cleanup_plan_id.in_(
-                        select([CleanupPlan.id])
+                        select(CleanupPlan.id)
                         .where(CleanupPlan.name == cleanup_plan_name)
                         .distinct()
                     )) \
@@ -477,12 +477,12 @@ def get_include_skip_queries(
 
     To get the include and skip lists use the 'get_component_values' function.
     """
-    include_q = select([File.id]) \
+    include_q = select(File.id) \
         .where(or_(*[
             File.filepath.like(conv(fp)) for fp in include])) \
         .distinct()
 
-    skip_q = select([File.id]) \
+    skip_q = select(File.id) \
         .where(or_(*[
             File.filepath.like(conv(fp)) for fp in skip])) \
         .distinct()
@@ -1367,7 +1367,7 @@ def get_is_enabled_case(subquery):
     ))
 
     return case(
-        [(detection_status_filters, False)],
+        (detection_status_filters, False),
         else_=True
     )
 
@@ -1393,7 +1393,7 @@ def get_is_opened_case(subquery):
             review_status_str, review_statuses)))
     ]
     return case(
-        [(and_(*detection_and_review_status_filters), True)],
+        (and_(*detection_and_review_status_filters), True),
         else_=False
     )
 
@@ -1850,9 +1850,9 @@ def getDiffResultsHash(self, run_ids, report_hashes, diff_type,
                         base_hashes, run_ids, tag_ids)
 
                 if self._product.driver_name == 'postgresql':
-                    new_hashes = select([
+                    new_hashes = select(
                         func.unnest(cast(report_hashes, ARRAY(String)))
-                            .label('bug_id')]) \
+                            .label('bug_id')) \
                         .except_(base_hashes).alias('new_bugs')
                     return [res[0] for res in session.query(new_hashes)]
                 else:
@@ -1865,10 +1865,10 @@ def getDiffResultsHash(self, run_ids, report_hashes, diff_type,
                     for chunk in util.chunks(
                             iter(report_hashes), SQLITE_MAX_COMPOUND_SELECT):
                         new_hashes_query = union_all(*[
-                            select([bindparam('bug_id' + str(i), h)
-                                    .label('bug_id')])
+                            select(bindparam('bug_id' + str(i), h)
+                                   .label('bug_id'))
                             for i, h in enumerate(chunk)])
-                        q = select([new_hashes_query.subquery()]) \
+                        q = select(new_hashes_query.subquery()) \
                             .except_(base_hashes)
                         new_hashes.extend([
                             res[0] for res in session.query(q.subquery())])
@@ -1983,10 +1983,10 @@ def getRunResults(self, run_ids, limit, offset, sort_types,
 
             annotation_cols = OrderedDict()
             for col in annotation_keys:
-                annotation_cols[col] = func.max(sqlalchemy.case([(
+                annotation_cols[col] = func.max(sqlalchemy.case((
                     ReportAnnotations.key == col,
                     cast(ReportAnnotations.value,
-                         report_annotation_types[col]["db"]))])) \
+                         report_annotation_types[col]["db"])))) \
                             .label(f"annotation_{col}")
 
             if report_filter.isUnique:
@@ -3960,7 +3960,7 @@ def getMissingContentHashes(self, file_hashes):
         with DBSession(self._Session) as session:
 
             q = session.query(FileContent) \
-                .options(sqlalchemy.orm.load_only('content_hash')) \
+                .options(sqlalchemy.orm.load_only(FileContent.content_hash)) \
                 .filter(FileContent.content_hash.in_(file_hashes))
 
             return list(set(file_hashes) -
@@ -3977,7 +3977,7 @@ def getMissingContentHashesForBlameInfo(self, file_hashes):
         with DBSession(self._Session) as session:
 
             q = session.query(FileContent) \
-                .options(sqlalchemy.orm.load_only('content_hash')) \
+                .options(sqlalchemy.orm.load_only(FileContent.content_hash)) \
                 .filter(FileContent.content_hash.in_(file_hashes)) \
                 .filter(FileContent.blame_info.isnot(None))
 

web/server/codechecker_server/database/config_db_model.py

@@ -14,7 +14,7 @@
 
 from sqlalchemy import Boolean, CHAR, Column, DateTime, Enum, ForeignKey, \
     Integer, MetaData, String, Text, UniqueConstraint
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql.expression import false
 
 from ..permissions import get_permissions

web/server/codechecker_server/database/database.py

@@ -396,15 +396,16 @@ def create_engine(self):
             # FIXME: workaround for locking errors
             # FIXME: why is the connection used by multiple threads
             # is that a problem ??? do we need some extra locking???
-            engine = sqlalchemy.create_engine(self.get_connection_string(),
-                                              encoding='utf8',
-                                              connect_args={'timeout': 600,
-                                              'check_same_thread': False},
-                                              poolclass=NullPool)
+            engine = sqlalchemy.create_engine(
+                self.get_connection_string(),
+                connect_args={'timeout': 600, 'check_same_thread': False},
+                poolclass=NullPool)
         else:
-            engine = sqlalchemy.create_engine(self.get_connection_string(),
-                                              encoding='utf8',
-                                              poolclass=NullPool)
+            engine = sqlalchemy.create_engine(
+                self.get_connection_string(),
+                client_encoding='utf8',
+                #connect_args={'client_encoding': 'UTF8'},
+                poolclass=NullPool)
 
         self._register_engine_hooks(engine)
         return engine
@@ -556,13 +557,14 @@ def _get_connection_string(self, database):
         extra_args = {}
         if driver == "psycopg2":
             extra_args = {'client_encoding': 'utf8'}
-        return str(URL('postgresql+' + driver,
-                       username=self.user,
-                       password=password,
-                       host=self.host,
-                       port=str(self.port),
-                       database=database,
-                       query=extra_args))
+        return URL.create(
+            drivername='postgresql+' + driver,
+            username=self.user,
+            password=password,
+            host=self.host,
+            port=str(self.port),
+            database=database,
+            query=extra_args).render_as_string(hide_password=False)
 
     def connect(self, init=False):
         """
@@ -653,7 +655,9 @@ def connect(self, init=False):
         return self.check_schema()
 
     def get_connection_string(self) -> str:
-        return str(URL('sqlite+pysqlite', None, None, None, None, self.dbpath))
+        return str(URL.create(
+            drivername='sqlite+pysqlite',
+            database=self.dbpath))
 
     def get_db_location(self):
         return self.dbpath

web/server/codechecker_server/database/run_db_model.py

@@ -15,7 +15,7 @@
 
 from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, \
     LargeBinary, MetaData, String, UniqueConstraint, Table, Text
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import declarative_base
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import true, false
 

web/server/codechecker_server/migrations/config/env.py

@@ -43,14 +43,7 @@ def run_migrations_online():
     In this scenario we need to create an Engine and associate a connection
     with the context.
     """
-    connectable = config.attributes.get('connection', None)
-    if connectable is None:
-        connectable = engine_from_config(
-            config.get_section(config.config_ini_section),
-            prefix='sqlalchemy.',
-            poolclass=pool.NullPool)
-
-    with connectable.connect() as connection:
+    def migrate(connection):
         context.configure(
             connection=connection,
             target_metadata=target_metadata
@@ -61,6 +54,18 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+    connection = config.attributes.get('connection', None)
+    if connection:
+        migrate(connection)
+    else:
+        connectable = engine_from_config(
+            config.get_section(config.config_ini_section),
+            prefix='sqlalchemy.',
+            poolclass=pool.NullPool)
+
+        with connectable.connect() as connection:
+            migrate(connection)
+
 
 if context.is_offline_mode():
     raise NotImplementedError(f"Offline '{schema}' migration is not possible!")

web/server/codechecker_server/migrations/report/env.py

@@ -37,14 +37,7 @@ def run_migrations_online():
     In this scenario we need to create an Engine and associate a connection
     with the context.
     """
-    connectable = config.attributes.get('connection', None)
-    if connectable is None:
-        connectable = engine_from_config(
-            config.get_section(config.config_ini_section),
-            prefix='sqlalchemy.',
-            poolclass=pool.NullPool)
-
-    with connectable.connect() as connection:
+    def migrate(connection):
         context.configure(
             connection=connection,
             target_metadata=target_metadata
@@ -55,6 +48,18 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+    connection = config.attributes.get('connection', None)
+    if connection:
+        migrate(connection)
+    else:
+        connectable = engine_from_config(
+            config.get_section(config.config_ini_section),
+            prefix='sqlalchemy.',
+            poolclass=pool.NullPool)
+
+        with connectable.connect() as connection:
+            migrate(connection)
+
 
 if context.is_offline_mode():
     raise NotImplementedError(f"Offline '{schema}' migration is not possible!")