diff --git a/alembic/versions/312c9eb92e40_add_cbs_locations_table.py b/alembic/versions/312c9eb92e40_add_cbs_locations_table.py index e0b02ec62..6c59c9c78 100644 --- a/alembic/versions/312c9eb92e40_add_cbs_locations_table.py +++ b/alembic/versions/312c9eb92e40_add_cbs_locations_table.py @@ -34,7 +34,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) conn = op.get_bind() - conn.execute("""INSERT INTO cbs_locations + conn.execute(sa.text("""INSERT INTO cbs_locations (SELECT ROW_NUMBER() OVER (ORDER BY road1) as id, LOCATIONS.* FROM (SELECT DISTINCT road1, @@ -52,7 +52,7 @@ def upgrade(): WHERE (provider_code=1 OR provider_code=3) AND (longitude is not null - AND latitude is not null)) LOCATIONS)""") + AND latitude is not null)) LOCATIONS)""")) # ### end Alembic commands ### diff --git a/alembic/versions/4c4b79f8c4a_adding_geom_gix_to_markers.py b/alembic/versions/4c4b79f8c4a_adding_geom_gix_to_markers.py index 22690606e..6d2a64159 100644 --- a/alembic/versions/4c4b79f8c4a_adding_geom_gix_to_markers.py +++ b/alembic/versions/4c4b79f8c4a_adding_geom_gix_to_markers.py @@ -12,13 +12,14 @@ depends_on = None from alembic import op +import sqlalchemy as sa def upgrade(): ### commands auto generated by Alembic - please adjust! ### conn = op.get_bind() - conn.execute('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));') - conn.execute('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));') + conn.execute(sa.text('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));')) + conn.execute(sa.text('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));')) ### end Alembic commands ### @@ -26,6 +27,6 @@ def upgrade(): def downgrade(): ### commands auto generated by Alembic - please adjust! ### conn = op.get_bind() - conn.execute('DROP INDEX geom_gix;') - conn.execute('DROP INDEX discussions_gix;') + conn.execute(sa.text('DROP INDEX geom_gix;')) + conn.execute(sa.text('DROP INDEX discussions_gix;')) ### end Alembic commands ### diff --git a/alembic/versions/5a5ffe56bb7_adding_geom_table_to_markers_and_discussions.py b/alembic/versions/5a5ffe56bb7_adding_geom_table_to_markers_and_discussions.py index e87c87532..97b370b0c 100644 --- a/alembic/versions/5a5ffe56bb7_adding_geom_table_to_markers_and_discussions.py +++ b/alembic/versions/5a5ffe56bb7_adding_geom_table_to_markers_and_discussions.py @@ -12,19 +12,20 @@ depends_on = None from alembic import op +import sqlalchemy as sa def upgrade(): ### commands auto generated by Alembic - please adjust! ### conn = op.get_bind() - conn.execute('CREATE EXTENSION IF NOT EXISTS postgis;') - conn.execute('CREATE EXTENSION IF NOT EXISTS postgis_topology;') - conn.execute("SELECT AddGeometryColumn('public','markers','geom',4326,'POINT',2);") - conn.execute('UPDATE markers SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);') - conn.execute('CREATE INDEX idx_markers_geom ON markers USING GIST(geom);') - conn.execute("SELECT AddGeometryColumn('public','discussions','geom',4326,'POINT',2);") - conn.execute('UPDATE discussions SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);') - conn.execute('CREATE INDEX idx_discussions_geom ON discussions USING GIST(geom);') + conn.execute(sa.text('CREATE EXTENSION IF NOT EXISTS postgis;')) + conn.execute(sa.text('CREATE EXTENSION IF NOT EXISTS postgis_topology;')) + conn.execute(sa.text("SELECT AddGeometryColumn('public','markers','geom',4326,'POINT',2);")) + conn.execute(sa.text('UPDATE markers SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);')) + conn.execute(sa.text('CREATE INDEX idx_markers_geom ON markers USING GIST(geom);')) + conn.execute(sa.text("SELECT AddGeometryColumn('public','discussions','geom',4326,'POINT',2);")) + conn.execute(sa.text('UPDATE discussions SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);')) + conn.execute(sa.text('CREATE INDEX idx_discussions_geom ON discussions USING GIST(geom);')) ### end Alembic commands ### @@ -32,11 +33,11 @@ def upgrade(): def downgrade(): ### commands auto generated by Alembic - please adjust! ### conn = op.get_bind() - conn.execute('DROP INDEX idx_markers_geom;') + conn.execute(sa.text('DROP INDEX idx_markers_geom;')) op.drop_column('markers', 'geom') - conn.execute('DROP INDEX idx_discussions_geom;') + conn.execute(sa.text('DROP INDEX idx_discussions_geom;')) op.drop_column('discussions', 'geom') - conn.execute('DROP EXTENSION postgis_topology;') - conn.execute('DROP EXTENSION postgis;') - conn.execute('DROP SCHEMA IF EXISTS topology CASCADE;') + conn.execute(sa.text('DROP EXTENSION postgis_topology;')) + conn.execute(sa.text('DROP EXTENSION postgis;')) + conn.execute(sa.text('DROP SCHEMA IF EXISTS topology CASCADE;')) ### end Alembic commands ### diff --git a/alembic/versions/7574885e1fed_remove_unecessary_table_index.py b/alembic/versions/7574885e1fed_remove_unecessary_table_index.py index 40ec44ba7..768afb1be 100644 --- a/alembic/versions/7574885e1fed_remove_unecessary_table_index.py +++ b/alembic/versions/7574885e1fed_remove_unecessary_table_index.py @@ -34,6 +34,6 @@ def downgrade(): op.create_index('provider_and_id_idx_involved', 'involved', ['provider_and_id'], unique=False) op.create_index('provider_and_id_idx_vehicles', 'vehicles', ['provider_and_id'], unique=False) conn = op.get_bind() - conn.execute('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));') - conn.execute('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));') + conn.execute(sa.text('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));')) + conn.execute(sa.text('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));')) # ### end Alembic commands ### diff --git a/alembic/versions/7f629b4c8891_add_news_flash_fields.py b/alembic/versions/7f629b4c8891_add_news_flash_fields.py index da55c81eb..0a3a16af4 100644 --- a/alembic/versions/7f629b4c8891_add_news_flash_fields.py +++ b/alembic/versions/7f629b4c8891_add_news_flash_fields.py @@ -19,9 +19,9 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### conn = op.get_bind() - conn.execute("ALTER TABLE news_flash ALTER COLUMN id SET DEFAULT nextval('news_flash_id_seq');") - conn.execute("ALTER SEQUENCE news_flash_id_seq OWNED BY news_flash.id;") - conn.execute("SELECT setval('news_flash_id_seq', COALESCE(max(id), 1)) FROM news_flash;") + conn.execute(sa.text("ALTER TABLE news_flash ALTER COLUMN id SET DEFAULT nextval('news_flash_id_seq');")) + conn.execute(sa.text("ALTER SEQUENCE news_flash_id_seq OWNED BY news_flash.id;")) + conn.execute(sa.text("SELECT setval('news_flash_id_seq', COALESCE(max(id), 1)) FROM news_flash;")) op.add_column('news_flash', sa.Column('district_hebrew', sa.Text(), nullable=True)) op.add_column('news_flash', sa.Column('non_urban_intersection_hebrew', sa.Text(), nullable=True)) op.add_column('news_flash', sa.Column('region_hebrew', sa.Text(), nullable=True)) diff --git a/anyway/parsers/infographics_data_cache_updater.py b/anyway/parsers/infographics_data_cache_updater.py index 9e9f96371..d394fe3d6 100755 --- a/anyway/parsers/infographics_data_cache_updater.py +++ b/anyway/parsers/infographics_data_cache_updater.py @@ -2,6 +2,7 @@ from datetime import datetime from sqlalchemy import not_ +import sqlalchemy as sa from anyway.models import ( Base, InfographicsDataCache, @@ -214,13 +215,15 @@ def copy_temp_into_cache(table: Dict[str, Base]): db.session.commit() start = datetime.now() with db.get_engine().begin() as conn: - conn.execute("lock table infographics_data_cache in exclusive mode") + conn.execute(sa.text("lock table infographics_data_cache in exclusive mode")) logging.debug(f"in transaction, after lock") - conn.execute(f"delete from {table[CACHE].__tablename__}") + conn.execute(sa.text(f"delete from {table[CACHE].__tablename__}")) logging.debug(f"in transaction, after delete") conn.execute( - f"insert into {table[CACHE].__tablename__} " - f"SELECT * from {table[TEMP].__tablename__}" + sa.text( + f"insert into {table[CACHE].__tablename__} " + f"SELECT * from {table[TEMP].__tablename__}" + ) ) logging.debug(f"in transaction, after insert into") logging.info(f"cache unavailable time: {str(datetime.now() - start)}") diff --git a/anyway/parsers/registered.py b/anyway/parsers/registered.py index 3764915b5..9ef7aa0f5 100644 --- a/anyway/parsers/registered.py +++ b/anyway/parsers/registered.py @@ -7,7 +7,7 @@ from datetime import datetime from anyway.models import RegisteredVehicle, DeprecatedCity from anyway.utilities import time_delta, CsvReader, ImporterUI, truncate_tables, decode_hebrew -from anyway.app_and_db import db +from anyway.app_and_db import db, app COLUMN_CITY_NAME_ENG = 0 @@ -71,8 +71,8 @@ def import_file(self, inputfile): else: self.header_row(row) row_count += 1 - - db.session.bulk_insert_mappings(RegisteredVehicle, inserts) + with app.app_context(): + db.session.bulk_insert_mappings(RegisteredVehicle, inserts) return total @staticmethod @@ -131,11 +131,11 @@ def main(specific_folder, delete_all, path): started = datetime.now() for fname in dir_files: total += importer.import_file(fname) - - db.session.commit() - db.engine.execute( - "UPDATE {0} SET city_id = (SELECT id FROM {1} WHERE {0}.search_name = {1}.search_heb) WHERE city_id IS NULL".format( - RegisteredVehicle.__tablename__, DeprecatedCity.__tablename__ + with app.app_context(): + db.session.commit() + db.get_engine().execute( + "UPDATE {0} SET city_id = (SELECT id FROM {1} WHERE {0}.search_name = {1}.search_heb) WHERE city_id IS NULL".format( + RegisteredVehicle.__tablename__, DeprecatedCity.__tablename__ + ) ) - ) - logging.info("Total: {0} items in {1}".format(total, time_delta(started))) + logging.info("Total: {0} items in {1}".format(total, time_delta(started))) diff --git a/anyway/utilities.py b/anyway/utilities.py index a3692a39e..f79fb767e 100644 --- a/anyway/utilities.py +++ b/anyway/utilities.py @@ -12,6 +12,7 @@ from urllib.parse import urlparse from sqlalchemy import func, or_ from sqlalchemy.sql import select +import sqlalchemy as sa import phonenumbers from dateutil.relativedelta import relativedelta @@ -196,7 +197,7 @@ def truncate_tables(db, tables): def delete_all_rows_from_table(conn, table): table_name = table.__tablename__ logging.info("Deleting all rows from table " + table_name) - conn.execute("DELETE FROM " + table_name) + conn.execute(sa.text("DELETE FROM " + table_name)) def split_query_to_chunks_by_column(base_select, column_to_chunk_by, chunk_size, conn):