diff --git a/node_normalizer/normalizer.py b/node_normalizer/normalizer.py index ae63b0c..635e5c9 100644 --- a/node_normalizer/normalizer.py +++ b/node_normalizer/normalizer.py @@ -495,7 +495,7 @@ async def get_info_content( return {} # call redis and get the value - info_contents = await app.state.redis_connection4.mget(*canonical_nonan, encoding='utf8') + info_contents = await app.state.info_content_db.mget(*canonical_nonan, encoding='utf8') # get this into a list info_contents = [round(float(ic_ids), 1) if ic_ids is not None else None for ic_ids in info_contents] @@ -515,9 +515,9 @@ async def get_eqids_and_types( batch_size = int(os.environ.get("EQ_BATCH_SIZE", 2500)) eqids = [] for i in range(0, len(canonical_nonan), batch_size): - eqids += await app.state.redis_connection1.mget(*canonical_nonan[i:i+batch_size], encoding='utf-8') + eqids += await app.state.id_to_eqids_db.mget(*canonical_nonan[i:i + batch_size], encoding='utf-8') eqids = [json.loads(value) if value is not None else [None] for value in eqids] - types = await app.state.redis_connection2.mget(*canonical_nonan, encoding='utf-8') + types = await app.state.id_to_type_db.mget(*canonical_nonan, encoding='utf-8') types_with_ancestors = [] for index, typ in enumerate(types): if not typ: @@ -552,7 +552,7 @@ async def get_normalized_nodes( upper_curies = [c.upper() for c in curies] try: - canonical_ids = await app.state.redis_connection0.mget(*upper_curies, encoding='utf-8') + canonical_ids = await app.state.eq_id_to_id_db.mget(*upper_curies, encoding='utf-8') canonical_nonan = [canonical_id for canonical_id in canonical_ids if canonical_id is not None] info_contents = {} @@ -569,12 +569,12 @@ async def get_normalized_nodes( other_ids = [] if conflate_gene_protein: - other_ids.extend(await app.state.redis_connection5.mget(*canonical_nonan, encoding='utf8')) + other_ids.extend(await app.state.gene_protein_db.mget(*canonical_nonan, encoding='utf8')) # logger.error(f"After conflate_gene_protein: {other_ids}") if conflate_chemical_drug: - other_ids.extend(await app.state.redis_connection6.mget(*canonical_nonan, encoding='utf8')) + other_ids.extend(await app.state.chemical_drug_db.mget(*canonical_nonan, encoding='utf8')) # logger.error(f"After conflate_chemical_drug: {other_ids}") @@ -657,7 +657,7 @@ async def get_info_content_attribute(app, canonical_nonan) -> dict: :return: """ # get the information content value - ic_val = await app.state.redis_connection4.get(canonical_nonan, encoding='utf8') + ic_val = await app.state.info_content_db.get(canonical_nonan, encoding='utf8') # did we get a good value if ic_val is not None: @@ -776,7 +776,7 @@ async def get_curie_prefixes( if semantic_types: for item in semantic_types: # get the curies for this type - curies = await app.state.redis_connection3.get(item, encoding='utf-8') + curies = await app.state.curie_to_bl_type_db.get(item, encoding='utf-8') # did we get any data if not curies: @@ -787,11 +787,11 @@ async def get_curie_prefixes( # set the return data ret_val[item] = {'curie_prefix': curies} else: - types = await app.state.redis_connection3.lrange('semantic_types', 0, -1, encoding='utf-8') + types = await app.state.curie_to_bl_type_db.lrange('semantic_types', 0, -1, encoding='utf-8') for item in types: # get the curies for this type - curies = await app.state.redis_connection3.get(item, encoding='utf-8') + curies = await app.state.curie_to_bl_type_db.get(item, encoding='utf-8') # did we get any data if not curies: diff --git a/node_normalizer/server.py b/node_normalizer/server.py index b2d9f1d..c14a14b 100644 --- a/node_normalizer/server.py +++ b/node_normalizer/server.py @@ -58,13 +58,13 @@ async def startup_event(): """ redis_config_file = Path(__file__).parent.parent / "redis_config.yaml" connection_factory = await RedisConnectionFactory.create_connection_pool(redis_config_file) - app.state.redis_connection0 = connection_factory.get_connection(connection_id="eq_id_to_id_db") - app.state.redis_connection1 = connection_factory.get_connection(connection_id="id_to_eqids_db") - app.state.redis_connection2 = connection_factory.get_connection(connection_id="id_to_type_db") - app.state.redis_connection3 = connection_factory.get_connection(connection_id="curie_to_bl_type_db") - app.state.redis_connection4 = connection_factory.get_connection(connection_id="info_content_db") - app.state.redis_connection5 = connection_factory.get_connection(connection_id="gene_protein_db") - app.state.redis_connection6 = connection_factory.get_connection(connection_id="chemical_drug_db") + app.state.eq_id_to_id_db = connection_factory.get_connection(connection_id="eq_id_to_id_db") + app.state.id_to_eqids_db = connection_factory.get_connection(connection_id="id_to_eqids_db") + app.state.id_to_type_db = connection_factory.get_connection(connection_id="id_to_type_db") + app.state.curie_to_bl_type_db = connection_factory.get_connection(connection_id="curie_to_bl_type_db") + app.state.info_content_db = connection_factory.get_connection(connection_id="info_content_db") + app.state.gene_protein_db = connection_factory.get_connection(connection_id="gene_protein_db") + app.state.chemical_drug_db = connection_factory.get_connection(connection_id="chemical_drug_db") app.state.toolkit = Toolkit() app.state.ancestor_map = {} @@ -74,20 +74,20 @@ async def shutdown_event(): """ Shut down Redis connection """ - app.state.redis_connection0.close() - await app.state.redis_connection0.wait_closed() - app.state.redis_connection1.close() - await app.state.redis_connection1.wait_closed() - app.state.redis_connection2.close() - await app.state.redis_connection2.wait_closed() - app.state.redis_connection3.close() - await app.state.redis_connection3.wait_closed() - app.state.redis_connection4.close() - await app.state.redis_connection4.wait_closed() - app.state.redis_connection5.close() - await app.state.redis_connection5.wait_closed() - app.state.redis_connection6.close() - await app.state.redis_connection6.wait_closed() + app.state.eq_id_to_id_db.close() + await app.state.eq_id_to_id_db.wait_closed() + app.state.id_to_eqids_db.close() + await app.state.id_to_eqids_db.wait_closed() + app.state.id_to_type_db.close() + await app.state.id_to_type_db.wait_closed() + app.state.curie_to_bl_type_db.close() + await app.state.curie_to_bl_type_db.wait_closed() + app.state.info_content_db.close() + await app.state.info_content_db.wait_closed() + app.state.gene_protein_db.close() + await app.state.gene_protein_db.wait_closed() + app.state.chemical_drug_db.close() + await app.state.chemical_drug_db.wait_closed() @app.post( @@ -230,7 +230,7 @@ async def get_normalized_node_handler(curies: CurieList): ) async def get_semantic_types_handler() -> SemanticTypes: # look for all biolink semantic types - types = await app.state.redis_connection3.lrange("semantic_types", 0, -1, encoding="utf-8") + types = await app.state.curie_to_bl_type_db.lrange("semantic_types", 0, -1, encoding="utf-8") # did we get any data if not types: diff --git a/requirements.lock b/requirements.lock index 3a585a4..5ca42b7 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,49 +1,71 @@ aioredis==1.3.1 +annotated-types==0.7.0 anyio==3.7.1 -asgiref==3.7.2 +asgiref==3.8.1 async-timeout==4.0.3 -attrs==23.1.0 +attrs==23.2.0 bmt-lite-v3.1.0==2.2.2 -certifi==2023.7.22 -charset-normalizer==2.1.1 +bmt-lite-v3.6.0==2.3.0 +certifi==2024.6.2 +charset-normalizer==3.3.2 click==8.0.4 -coverage==7.3.2 +coverage==7.5.3 deepdiff==5.8.1 +Deprecated==1.2.14 deprecation==2.1.0 -docker==6.1.3 -fastapi==0.83.0 -gunicorn==20.1.0 +docker==7.1.0 +fastapi==0.108.0 +googleapis-common-protos==1.59.1 +grpcio==1.64.1 +gunicorn==22.0.0 h11==0.12.0 -hiredis==2.2.3 +hiredis==2.3.2 httpcore==0.15.0 -httptools==0.5.0 +httptools==0.6.1 httpx==0.23.0 -idna==3.4 +idna==3.7 +importlib-metadata==6.11.0 iniconfig==2.0.0 jsonschema==4.6.2 +opentelemetry-api==1.21.0 +opentelemetry-exporter-jaeger==1.21.0 +opentelemetry-exporter-jaeger-proto-grpc==1.21.0 +opentelemetry-exporter-jaeger-thrift==1.21.0 +opentelemetry-instrumentation==0.42b0 +opentelemetry-instrumentation-asgi==0.42b0 +opentelemetry-instrumentation-fastapi==0.42b0 +opentelemetry-instrumentation-httpx==0.42b0 +opentelemetry-sdk==1.21.0 +opentelemetry-semantic-conventions==0.42b0 +opentelemetry-util-http==0.42b0 ordered-set==4.1.0 -orjson==3.8.10 -packaging==23.2 -pluggy==1.3.0 +orjson==3.9.15 +packaging==24.1 +pluggy==1.5.0 +protobuf==4.25.3 py==1.11.0 -pydantic==1.10.13 -pyrsistent==0.19.3 +pydantic==1.10.16 +pydantic_core==2.18.4 +pyrsistent==0.20.0 pytest==6.2.5 pytest-asyncio==0.18.3 pytest-cov==3.0.0 -PyYAML==6.0 +PyYAML==6.0.1 reasoner-pydantic==4.1.5 redis==3.5.3 redis-py-cluster==2.1.3 requests==2.31.0 rfc3986==1.5.0 -sniffio==1.3.0 -starlette==0.19.1 +setuptools==70.1.0 +six==1.16.0 +sniffio==1.3.1 +starlette==0.32.0.post1 testcontainers==3.6.1 +thrift==0.20.0 toml==0.10.2 -typing_extensions==4.8.0 -urllib3==1.26.17 +typing_extensions==4.12.2 +urllib3==2.2.2 uvicorn==0.17.6 -uvloop==0.17.0 -websocket-client==1.6.4 -wrapt==1.15.0 +uvloop==0.19.0 +wrapt==1.16.0 +zipp==3.19.2 diff --git a/requirements.txt b/requirements.txt index a703fb0..521fb67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,27 @@ -aioredis~=1.3.1 -click==8.0.4 -bmt-lite-v3.1.0==2.2.2 -deepdiff==5.8.1 -fastapi~=0.108.0 -httptools==0.5.0 -jsonschema~=4.6.0 -pytest==6.2.5 -pytest-cov==3.0.0 -pytest-asyncio==0.18.3 -pyyaml==6.0 -reasoner-pydantic==4.1.5 -redis~=3.5.3 -redis-py-cluster==2.1.3 -requests==2.31.0 -testcontainers==3.6.1 -uvicorn==0.17.6 -uvloop==0.17.0 -gunicorn==22.0.0 -orjson==3.9.15 -httpx==0.23.0 +aioredis +click +bmt-lite-v3.6.0 +deepdiff +fastapi +httptools +jsonschema +pytest +pytest-cov +pytest-asyncio +pyyaml +reasoner-pydantic +redis +redis-py-cluster +requests +testcontainers +uvicorn +uvloop +gunicorn +orjson +httpx # To support Open Telemetry -opentelemetry-sdk==1.21.0 -opentelemetry-exporter-jaeger==1.21.0 -opentelemetry-instrumentation-fastapi==0.42b0 -opentelemetry-instrumentation-httpx==0.42b0 +opentelemetry-sdk +opentelemetry-exporter-jaeger +opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-httpx diff --git a/tests/test_norm.py b/tests/test_norm.py index 783cc6b..438ae34 100644 --- a/tests/test_norm.py +++ b/tests/test_norm.py @@ -18,17 +18,17 @@ async def mget(self, *args, **kwargs): # Id -> Canonical -app.state.redis_connection0 = MockRedis( +app.state.eq_id_to_id_db = MockRedis( {"DOID:3812": "MONDO:0005002", "MONDO:0005002": "MONDO:0005002"} ) # Canonical->Equiv -app.state.redis_connection1 = MockRedis( +app.state.id_to_eqids_db = MockRedis( {"MONDO:0005002": json.dumps([{"i": "MONDO:0005002"}, {"i": "DOID:3812"}])} ) -app.state.redis_connection2 = MockRedis({"MONDO:0005002": "biolink:Disease"}) -app.state.redis_connection3 = MockRedis({}) -app.state.redis_connection4 = MockRedis({}) -app.state.redis_connection5 = MockRedis({}) +app.state.id_to_type_db = MockRedis({"MONDO:0005002": "biolink:Disease"}) +app.state.curie_to_bl_type_db = MockRedis({}) +app.state.info_content_db = MockRedis({}) +app.state.gene_protein_db = MockRedis({}) #app.state.ancestor_map = {"biolink:Disease": ["biolink:Disease", "biolink:NamedThing"]} app.state.toolkit = Toolkit() app.state.ancestor_map = {}