Migrating to Python 3.12 #2521

DockerBuild/Dockerfile

@@ -1,10 +1,12 @@
-FROM ubuntu:20.04
+# THIS DOCKER FILE IS FOR TESTING THE TEST DOCKER IMAGE on UBUNTU 24.04 and PYTHON 3.12
+#### IMPORTANT: THIS DOCKER FILE IS FOR TESTING ONLY AND IS NOT USED FOR PRODUCTION ####
+FROM ubuntu:24.04
 
 # suppress debconf error messages:
 ARG DEBIAN_FRONTEND=noninteractive
 
 # build based on branch
-ARG BUILD_BRANCH=master
+ARG BUILD_BRANCH=issue2521
 
 # add user "rt" and give sudo privilege:
 RUN useradd rt -m -s /bin/bash
@@ -30,7 +32,7 @@ RUN chmod -R 700 /home/rt/.ssh
 
 # install network troubleshooting utilities and emacs, in case we need them
 RUN apt-get update && apt-get install -y \
-    netcat \
+    netcat-openbsd \
     dnsutils \
     iputils-ping \
     traceroute \
@@ -42,46 +44,46 @@ RUN apt-get update && apt-get install -y \
     tcpdump \
  && rm -rf /var/lib/apt/lists/*
 
-# clone RTX repo
-RUN mkdir -p /mnt/data/orangeboard/production
-RUN chown -R rt.rt /mnt/data/orangeboard/production
-RUN su rt && cd /mnt/data/orangeboard/production && git clone https://github.com/RTXteam/RTX.git
+# clone RTX repo (FOR TESTING TEST DOCKER IMAGE)
+RUN mkdir -p /mnt/data/orangeboard/test
+RUN chown -R rt.rt /mnt/data/orangeboard/test
+RUN su rt && cd /mnt/data/orangeboard/test && git clone https://github.com/RTXteam/RTX.git
 # 08/12/2022 NCATS change
-RUN su rt && cd /mnt/data/orangeboard/production/RTX && git checkout ${BUILD_BRANCH}
-RUN chown -R rt.rt /mnt/data/orangeboard/production/RTX
+RUN su rt && cd /mnt/data/orangeboard/test/RTX && git checkout ${BUILD_BRANCH}
+RUN chown -R rt.rt /mnt/data/orangeboard/test/RTX
 
 # install apache2 and copy config file
 RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -yq apache2
 COPY 000-default.conf /etc/apache2/sites-available/000-default.conf
-RUN cd /var/www && mv html html-orig && ln -s /mnt/data/orangeboard/production/RTX/code/UI/interactive html
+RUN cd /var/www && mv html html-orig && ln -s /mnt/data/orangeboard/test/RTX/code/UI/interactive html
 RUN cd /var/www && a2enmod proxy && a2enmod proxy_html && a2enmod proxy_http && a2enmod rewrite && a2enmod headers
 RUN a2enmod remoteip
 COPY remoteip.conf /etc/apache2/conf-available/remoteip.conf
 RUN a2enconf remoteip
 RUN cd /var/www && /usr/sbin/apachectl configtest
 
-# install python and packages
+# install python 3.12 and packages
 RUN apt-get update
 RUN apt-get install -y python3 python3-pip
 RUN apt-get install -y software-properties-common
 RUN add-apt-repository -y ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y python3.9 python3.9-dev python3.9-venv
-RUN rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
-RUN su rt && cd /mnt/data/orangeboard/production/RTX && python3.9 -m pip install click==8.1.3 && python3.9 -m pip install -r requirements.txt && python3.9 -m pip install connexion[swagger-ui]
+RUN apt-get update && apt-get install -y python3.12 python3.12-dev python3.12-venv
+RUN apt-get install -y python3-pip
+RUN su rt && cd /mnt/data/orangeboard/test/RTX && python3.12 -m pip install click==8.1.3 --break-system-packages && python3.12 -m pip install -r requirements.txt --break-system-packages
 
 # setup services
-RUN export DEVAREA=production
-RUN cat /mnt/data/orangeboard/production/RTX/code/UI/OpenAPI/python-flask-server/RTX_OpenAPI | sed 's/template/production/g' > /etc/init.d/RTX_OpenAPI_production
-RUN chmod 700 /etc/init.d/RTX_OpenAPI_production
-RUN su rt && cd /mnt/data/orangeboard/production/RTX/code/UI/OpenAPI/python-flask-server && cp -p RTX_OpenAPI RTX_OpenAPI_production && sed -i 's/template/production/g' RTX_OpenAPI_production
-RUN su rt && cd /mnt/data/orangeboard/production/RTX/code/UI/OpenAPI/python-flask-server && cp -p RTX_OpenAPI.start RTX_OpenAPI_production.start && sed -i 's/template/production/g' RTX_OpenAPI_production.start
-RUN update-rc.d RTX_OpenAPI_production defaults
+RUN export DEVAREA=test
+RUN cat /mnt/data/orangeboard/test/RTX/code/UI/OpenAPI/python-flask-server/RTX_OpenAPI | sed 's/template/test/g' > /etc/init.d/RTX_OpenAPI_test
+RUN chmod 700 /etc/init.d/RTX_OpenAPI_test
+RUN su rt && cd /mnt/data/orangeboard/test/RTX/code/UI/OpenAPI/python-flask-server && cp -p RTX_OpenAPI RTX_OpenAPI_test && sed -i 's/template/test/g' RTX_OpenAPI_test
+RUN su rt && cd /mnt/data/orangeboard/test/RTX/code/UI/OpenAPI/python-flask-server && cp -p RTX_OpenAPI.start RTX_OpenAPI_test.start && sed -i 's/template/test/g' RTX_OpenAPI_test.start
+RUN update-rc.d RTX_OpenAPI_test defaults
 
 # autocomplete
-RUN cat /mnt/data/orangeboard/production/RTX/code/autocomplete/RTX_Complete | sed 's/devED/production/g' > /etc/init.d/RTX_Complete
+RUN cat /mnt/data/orangeboard/test/RTX/code/autocomplete/RTX_Complete | sed 's/devED/test/g' > /etc/init.d/RTX_Complete
 RUN chmod 700 /etc/init.d/RTX_Complete
-RUN cd /mnt/data/orangeboard/production/RTX/code/autocomplete && sed -i 's/devED/production/g' RTX_Complete.start
+RUN cd /mnt/data/orangeboard/test/RTX/code/autocomplete && sed -i 's/devED/test/g' RTX_Complete.start
 RUN update-rc.d RTX_Complete defaults
 
-# setup config.domain
-RUN su rt && cd /mnt/data/orangeboard/production/RTX/code/ && echo "arax.ci.transltr.io" > config.domain
+# setup config.domain (Changed from production to test for testing purposes arax.ncats.io/test)
+RUN su rt && cd /mnt/data/orangeboard/test/RTX/code/ && echo "arax.ncats.io/test" > config.domain

DockerBuild/test-instance-scripts/build-test-arax-from-fresh-instance.sh

@@ -42,9 +42,6 @@ ssh -q -oStrictHostKeyChecking=no [email protected] exit
 # copy the config secrets file into the RTX/code directory
 scp [email protected]:config_secrets.json RTX/code
 
-# create an override to point to the local RTX-KG2 API
-echo "http://localhost:5008/api/rtxkg2/v1.2" > RTX/code/kg2_url_override.txt
-
 # download the database files (this step takes a long time)
 python3 RTX/code/ARAX/ARAXQuery/ARAX_database_manager.py --mnt --skip-if-exists
 
@@ -59,10 +56,8 @@ cp RTX/DockerBuild/Merged-Dockerfile .
 # EOF
 # --------------------------------------------------------------------------------------------------
 
-sed -i 's/checkout production/checkout master/g' ./Merged-Dockerfile  # for issue 1740; temporary fix during code freeze until commit `ffbd287` can be merged to `production` branch
-
 # build the Docker image
-sudo docker build --file ./Merged-Dockerfile --no-cache --rm --tag arax:1.0 ./RTX/DockerBuild/
+sudo docker build --file ./Dockerfile --no-cache --rm --tag arax:1.0 ./RTX/DockerBuild/
 
 # create the Docker container
 sudo docker create --name arax --tty --publish "${port_number}":80 \
@@ -74,7 +69,7 @@ sudo docker start arax
 
 # copy the config files into the devareas in the container
 
-for devarea in kg2 production
+for devarea in production
 do
     sudo docker cp RTX/code/config_secrets.json arax:${arax_base}/${devarea}/RTX/code
     sudo docker exec arax chown rt.rt ${arax_base}/${devarea}/RTX/code/config_secrets.json
@@ -84,6 +79,5 @@ done
 
 # start all the services
 sudo docker exec arax service apache2 start
-sudo docker exec arax service RTX_OpenAPI_kg2 start
 sudo docker exec arax service RTX_OpenAPI_production start
-sudo docker exec arax service RTX_Complete start
+sudo docker exec arax service RTX_Complete start  # this is autocomplete; can be commented out for a simple test of ARAX

code/ARAX/ARAXQuery/ARAX_expander.py

@@ -431,9 +431,9 @@ def apply(self, response, input_parameters, mode: str = "ARAX"):
                     kp_answers = loop.run_until_complete(task_group)
                     loop.close()
                 else:
-                    log.error("Expand could not find any KPs to answer "
-                              f"{qedge_key} with.", error_code="NoResults")
-                    return response
+                    kp_answers = []
+                    log.warning("Expand could not find any KPs to answer "
+                                f"{qedge_key} with.")
 
                 # Merge KPs' answers into our overarching KG
                 log.debug("Got answers from all KPs; merging them into one KG")
@@ -694,7 +694,7 @@ def get_inferred_answers(inferred_qedge_keys: list[str],
                         response.info(f"Looking for a previously cached result from {kp_curie}")
                         response_data, response_code, elapsed_time, error = cacher.get_cached_result(kp_curie, infer_input_parameters)
                     else:
-                        response.info(f"KP results caching for xDTD is currently disabled, pending further debugging")
+                        response.info("KP results caching for xDTD is currently disabled, pending further debugging")
                     if enable_caching and response_code != -2: 
                         n_results = cacher._get_n_results(response_data)
                         response.info(f"Found a cached result with response_code={response_code}, n_results={n_results} from the cache in {elapsed_time:.3f} seconds")
@@ -707,12 +707,12 @@ def get_inferred_answers(inferred_qedge_keys: list[str],
                         for edge_key, edge in response_data['message']['knowledge_graph']['edges'].items():
                             response.info(f"Copying qedge_keys for edge {edge_key}") 
                             response.envelope.message.knowledge_graph.edges[edge_key].qedge_keys = edge['qedge_keys']
-                        response.dtd_from_cache = True
+                        response.dtd_from_cache = True   # type: ignore[attr-defined]
 
                     #### Else run the inferer to get the result and then cache it
                     else:
                         inferer = ARAXInfer()
-                        response.info(f"Launching ARAX inferer")
+                        response.info("Launching ARAX inferer")
                         infer_response = inferer.apply(response, infer_input_parameters)
                         elapsed_time = time.time() - start
                         response.info(f"Got result from ARAX inferer after {elapsed_time}. Converting to_dict()")
@@ -722,7 +722,7 @@ def get_inferred_answers(inferred_qedge_keys: list[str],
                             response_object['message']['knowledge_graph']['nodes'][node_key]['qnode_keys'] = node.qnode_keys
                         for edge_key, edge in response.envelope.message.knowledge_graph.edges.items():
                             response_object['message']['knowledge_graph']['edges'][edge_key]['qedge_keys'] = edge.qedge_keys
-                        response.info(f"Storing result in the cache")
+                        response.info("Storing result in the cache")
                         cacher.store_response(
                             kp_curie=kp_curie,
                             query_url=kp_url,
@@ -732,10 +732,10 @@ def get_inferred_answers(inferred_qedge_keys: list[str],
                             elapsed_time=elapsed_time,
                             status="OK"
                         )
-                        response.info(f"Stored result in the cache.")
+                        response.info("Stored result in the cache.")
 
                     # return infer_response
-                    #response = infer_response  # these are already always the same object?
+                    response = infer_response  # these are already always the same object?
                     overarching_kg = eu.convert_standard_kg_to_qg_organized_kg(response.envelope.message.knowledge_graph)
 
                     wait_time = round(time.time() - start, 2)

code/UI/OpenAPI/python-flask-server/RTX_OpenAPI.start

@@ -17,7 +17,8 @@ fi
 
 cd /mnt/data/orangeboard/$DEVAREA/RTX/code/UI/OpenAPI/python-flask-server
 
-export PATH=/mnt/data/python/Python-3.9.18/bin:$PATH
+#export PATH=/mnt/data/python/Python-3.9.18/bin:$PATH # (old)for python 3.9
+export PATH=/usr/bin:$PATH
 
 exec python3 -u -m openapi_server 1>$LOGFILE 2>$ELOGFILE
 

code/autocomplete/RTX_Complete.start

@@ -17,7 +17,8 @@ fi
 
 cd /mnt/data/orangeboard/$DEVAREA/RTX/code/autocomplete
 
-export PATH=/mnt/data/python/Python-3.9.18/bin:$PATH
+#export PATH=/mnt/data/python/Python-3.9.18/bin:$PATH # (old)for python 3.9
+export PATH=/usr/bin:$PATH
 
 exec python3 -u server.py 1>$LOGFILE 2>$ELOGFILE
 

code/config_dbs.json

@@ -10,8 +10,8 @@
       "explainable_dtd_db": "/translator/data/orangeboard/databases/KG2.10.2/ExplainableDTD_v1.0_KG2.10.0_refreshedTo_KG2.10.2.db",
       "cohd_database": "/translator/data/orangeboard/databases/KG2.8.0/COHDdatabase_v1.0_KG2.8.0.db",
       "xcrg_embeddings": "/translator/data/orangeboard/databases/KG2.10.2/chemical_gene_embeddings_v1.0.KG2.10.0_refreshedTo_KG2.10.2.npz",
-      "xcrg_increase_model": "/translator/data/orangeboard/databases/KG2.10.0/xcrg_increase_model_v1.0.KG2.10.0.pt",
-      "xcrg_decrease_model": "/translator/data/orangeboard/databases/KG2.10.0/xcrg_decrease_model_v1.0.KG2.10.0.pt"
+      "xcrg_increase_model": "/translator/data/orangeboard/databases/KG2.10.0/xcrg_increase_model_v1.0.KG2.10.0_new_version.pt",
+      "xcrg_decrease_model": "/translator/data/orangeboard/databases/KG2.10.0/xcrg_decrease_model_v1.0.KG2.10.0_new_version.pt"
    },
    "plover_url_override": null,
    "neo4j": {

requirements.txt

@@ -1,5 +1,7 @@
-pandas==1.5.3
-numpy==1.24.2
+# Python 3.12
+
+pandas==2.3.1  # UPGRADED: pandas 1.5.3 doesn't support Python 3.12
+numpy==1.26.4  # UPGRADED: numpy 1.24.2 doesn't support Python 3.12
 requests==2.32.0
 requests-cache
 matplotlib
@@ -9,10 +11,10 @@ neo4j-driver>=1.7.0,<=1.7.6
 ipython-cypher==0.2.6
 networkx==2.8.8
 graphviz==0.20.1
-lxml==4.9.1
+lxml==4.9.4  # UPGRADED: 4.9.1 doesn't build on Python 3.12, 4.9.4 has pre-built wheels
 nltk==3.7
-scipy==1.10.1
-scikit-learn==1.1.2
+scipy==1.15.3  # UPGRADED: scipy 1.10.1 doesn't support Python 3.12
+scikit-learn==1.7.1 # compatible with Python 3.12
 connexion==2.14.1
 connexion[swagger-ui]
 Flask==2.2.5
@@ -21,24 +23,25 @@ SQLAlchemy==1.4.47
 pymysql==1.1.1
 CacheControl==0.12.11
 lockfile==0.12.2
-simple-crypt==4.1.7
+# simple-crypt==4.1.7 REMOVED - incompatible with Python 3.12 (depends on old pycrypto)
+pycryptodome==3.23.0  # ADDED: Modern replacement for pycrypto, compatible with Python 3.12
 tomark==0.1.4
 md-toc==8.1.4
 pickledb==0.9.2
 pytest==7.2.2
 joblib==1.2.0
-PyYAML==6.0
+PyYAML==6.0.2  # UPGRADED: 6.0 doesn't have pre-built wheels for Python 3.12, 6.0.2 does
 ujson==5.4.0
-asyncio==3.4.3
+# asyncio==3.4.3 # built into Python 3.12
 aiohttp==3.9.4
-boto3==1.24.59
+boto3==1.40.16  # UPGRADED: 1.24.59 doesn't support urllib3>=2.5.0 required by reasoner-validator 5.0.0
 tornado==6.4.1
 MarkupSafe==2.1.2
-reasoner-validator==4.2.3
+reasoner-validator==5.0.0  # UPGRADED to the latest version that supports python3.12 
 pronto==2.5.3
-pygit2==1.10.0
+pygit2==1.18.0  # UPGRADED: 1.10.0 doesn't build on Python 3.12, 1.18.0 has pre-built wheels
 tabulate==0.9.0
-urllib3==1.26.19
+urllib3==2.5.0  # UPGRADED: 1.26.19 doesn't meet reasoner-validator 5.0.0 requirement of urllib3>=2.5.0
 setproctitle==1.3.3
 opentelemetry-api==1.17.0
 opentelemetry-exporter-jaeger==1.17.0
@@ -54,5 +57,5 @@ opentelemetry-proto==1.17.0
 opentelemetry-sdk==1.17.0
 opentelemetry-semantic-conventions==0.38b0
 opentelemetry-util-http==0.38b0
-xgboost
-
+xgboost==1.7.6 # compatible with Python 3.12
+httpx