Using astropy's PyVO to make URL calls.

pyproject.toml

@@ -19,7 +19,7 @@ authors = [
 dependencies = [
   "astropy>5.2.0",
   "sunpy[net]>=5.0",
-  "requests>=2.28.0",
+  "astroquery>=0.4.7"
 ]
 dynamic = ["version"]
 

sunpy_soar/attrs.py

@@ -88,9 +88,9 @@ def apply_and(wlk, and_attr, params):
 
 @walker.add_applier(a.Time)
 def _(wlk, attr, params):  # NOQA: ARG001
-    start = attr.start.strftime("%Y-%m-%d+%H:%M:%S")
-    end = attr.end.strftime("%Y-%m-%d+%H:%M:%S")
-    params.append(f"begin_time>='{start}'+AND+begin_time<='{end}'")
+    start = attr.start.strftime("%Y-%m-%d %H:%M:%S")
+    end = attr.end.strftime("%Y-%m-%d %H:%M:%S")
+    params.append(f"begin_time>='{start}' AND begin_time<='{end}'")
 
 
 @walker.add_applier(a.Level)
@@ -140,4 +140,4 @@ def _(wlk, attr, params):  # NOQA: ARG001
 def _(wlk, attr, params):  # NOQA: ARG001
     wavemin = attr.min.value
     wavemax = attr.max.value
-    params.append(f"Wavemin='{wavemin}'+AND+Wavemax='{wavemax}'")
+    params.append(f"Wavemin='{wavemin}' AND Wavemax='{wavemax}'")

sunpy_soar/client.py

@@ -4,12 +4,11 @@
 
 import astropy.table
 import astropy.units as u
-import requests
 import sunpy.net.attrs as a
+from astroquery.utils.tap.core import TapPlus
 from sunpy import log
 from sunpy.net.attr import and_
 from sunpy.net.base_client import BaseClient, QueryResponseTable
-from sunpy.time import parse_time
 
 from sunpy_soar.attrs import SOOP, Product, walker
 
@@ -31,14 +30,14 @@ def search(self, *query, **kwargs):  # NOQA: ARG002
                 query_parameters.remove("provider='SOAR'")
             results.append(self._do_search(query_parameters))
         table = astropy.table.vstack(results)
+        table["Filesize"] = (table["Filesize"] * u.byte).to(u.Mbyte).round(3)
         qrt = QueryResponseTable(table, client=self)
-        qrt["Filesize"] = (qrt["Filesize"] * u.byte).to(u.Mbyte).round(3)
-        qrt.hide_keys = ["Data item ID", "Filename"]
+        qrt.hide_keys = ["data_item_id", "Filename", "dimension_index"]
         return qrt
 
     def add_join_to_query(query: list[str], data_table: str, instrument_table: str):
         """
-        Construct the WHERE, FROM, and SELECT parts of the ADQL query.
+        Construct the WHERE, FROM, and SELECT parts of the SQL query.
 
         Parameters
         ----------
@@ -69,17 +68,17 @@ def add_join_to_query(query: list[str], data_table: str, instrument_table: str):
                 # To make sure this data is not misleading to the user we do not return any values for PHI AND SPICE.
                 parameter = f"Wavelength='{wavemin_match.group(1)}'"
             elif wavemin_match and wavemax_match:
-                parameter = f"Wavemin='{wavemin_match.group(1)}'+AND+h2.Wavemax='{wavemax_match.group(1)}'"
+                parameter = f"Wavemin='{wavemin_match.group(1)}' AND h2.Wavemax='{wavemax_match.group(1)}'"
             prefix = "h1." if not parameter.startswith("Detector") and not parameter.startswith("Wave") else "h2."
             if parameter.startswith("begin_time"):
-                time_list = parameter.split("+AND+")
-                final_query += f"h1.{time_list[0]}+AND+h1.{time_list[1]}+AND+"
+                time_list = parameter.split(" AND ")
+                final_query += f"h1.{time_list[0]} AND h1.{time_list[1]} AND "
                 # As there are no dimensions in STIX, the dimension index need not be included in the query for STIX.
                 if "stx" not in instrument_table:
                     # To avoid duplicate rows in the output table, the dimension index is set to 1.
-                    final_query += "h2.dimension_index='1'+AND+"
+                    final_query += "h2.dimension_index='1' AND "
             else:
-                final_query += f"{prefix}{parameter}+AND+"
+                final_query += f"{prefix}{parameter} AND "
 
         where_part = final_query[:-5]
         from_part = f"{data_table} AS h1"
@@ -104,8 +103,8 @@ def _construct_payload(query):
 
         Returns
         -------
-        dict
-            Payload dictionary to be sent with the query.
+        str
+            SQL query string.
         """
         # Default data table
         data_table = "v_sc_data_item"
@@ -122,7 +121,7 @@ def _construct_payload(query):
 
         instrument_name = None
         for q in query:
-            if q.startswith("instrument") or q.startswith("descriptor") and not instrument_name:
+            if (q.startswith(("instrument", "descriptor"))) and not instrument_name:
                 instrument_name = q.split("=")[1][1:-1].split("-")[0].upper()
             elif q.startswith("level") and q.split("=")[1][1:3] == "LL":
                 data_table = "v_ll_data_item"
@@ -139,13 +138,11 @@ def _construct_payload(query):
             where_part, from_part, select_part = SOARClient.add_join_to_query(query, data_table, instrument_table)
         else:
             from_part = data_table
-            select_part = "*"
-            where_part = "+AND+".join(query)
-
-        adql_query = {"SELECT": select_part, "FROM": from_part, "WHERE": where_part}
-
-        adql_query_str = "+".join([f"{key}+{value}" for key, value in adql_query.items()])
-        return {"REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "json", "QUERY": adql_query_str}
+            select_part = (
+                "instrument, descriptor, level, begin_time, end_time, data_item_id, filename, filesize, soop_name"
+            )
+            where_part = " AND ".join(query)
+        return f"SELECT {select_part} FROM {from_part} WHERE {where_part}"
 
     @staticmethod
     def _do_search(query):
@@ -159,49 +156,29 @@ def _do_search(query):
 
         Returns
         -------
-        astropy.table.QTable
+        astropy.table.Table
             Query results.
         """
         tap_endpoint = "http://soar.esac.esa.int/soar-sl-tap/tap"
-        payload = SOARClient._construct_payload(query)
-        # Need to force requests to not form-encode the parameters
-        payload = "&".join([f"{key}={val}" for key, val in payload.items()])
-        # Get request info
-        r = requests.get(f"{tap_endpoint}/sync", params=payload)
-        log.debug(f"Sent query: {r.url}")
-        r.raise_for_status()
-
-        # Do some list/dict wrangling
-        names = [m["name"] for m in r.json()["metadata"]]
-        info = {name: [] for name in names}
-
-        for entry in r.json()["data"]:
-            for i, name in enumerate(names):
-                info[name].append(entry[i])
-
-        if len(info["begin_time"]):
-            info["begin_time"] = parse_time(info["begin_time"]).iso
-            info["end_time"] = parse_time(info["end_time"]).iso
-
-        result_table = astropy.table.QTable(
-            {
-                "Instrument": info["instrument"],
-                "Data product": info["descriptor"],
-                "Level": info["level"],
-                "Start time": info["begin_time"],
-                "End time": info["end_time"],
-                "Data item ID": info["data_item_id"],
-                "Filename": info["filename"],
-                "Filesize": info["filesize"],
-                "SOOP Name": info["soop_name"],
-            },
-        )
-        if "detector" in info:
-            result_table["Detector"] = info["detector"]
-        if "wavelength" in info:
-            result_table["Wavelength"] = info["wavelength"]
-        result_table.sort("Start time")
-        return result_table
+        sql_query = SOARClient._construct_payload(query)
+        soar = TapPlus(url=tap_endpoint)
+        job = soar.launch_job_async(sql_query)
+        results = job.results
+        new_colnames = {
+            "instrument": "Instrument",
+            "descriptor": "Data product",
+            "level": "Level",
+            "begin_time": "Start time",
+            "end_time": "End time",
+            "filename": "Filename",
+            "filesize": "Filesize",
+            "soop_name": "SOOP Name",
+        }
+        new_colnames.update({k: k.capitalize() for k in ["wavelength", "detector"] if k in results.colnames})
+        for old_name, new_name in new_colnames.items():
+            results.rename_column(old_name, new_name)
+        results.sort("Start time")
+        return results
 
     def fetch(self, query_results, *, path, downloader, **kwargs):  # NOQA: ARG002
         """
@@ -229,7 +206,7 @@ def fetch(self, query_results, *, path, downloader, **kwargs):  # NOQA: ARG002
                 url += "&product_type=LOW_LATENCY"
             else:
                 url += "&product_type=SCIENCE"
-            data_id = row["Data item ID"]
+            data_id = row["data_item_id"]
             url += f"&data_item_id={data_id}"
             filepath = str(path).format(file=row["Filename"], **row.response_block_map)
             log.debug(f"Queuing URL: {url}")

sunpy_soar/tests/test_sunpy_soar.py

@@ -236,35 +236,35 @@ def test_join_science_query():
     result = SOARClient._construct_payload(  # NOQA: SLF001
         [
             "instrument='EUI'",
-            "begin_time>='2021-02-01+00:00:00'+AND+begin_time<='2021-02-02+00:00:00'",
+            "begin_time>='2021-02-01 00:00:00' AND begin_time<='2021-02-02 00:00:00'",
             "level='L1'",
             "descriptor='eui-fsi174-image'",
         ]
     )
 
-    assert result["QUERY"] == (
-        "SELECT+h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
+    assert result == (
+        "SELECT h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
         "h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, "
-        "h2.dimension_index+FROM+v_sc_data_item AS h1 JOIN v_eui_sc_fits AS h2 USING (data_item_oid)"
-        "+WHERE+h1.instrument='EUI'+AND+h1.begin_time>='2021-02-01+00:00:00'+AND+h1.begin_time<='2021-02-02+00:00:00'"
-        "+AND+h2.dimension_index='1'+AND+h1.level='L1'+AND+h1.descriptor='eui-fsi174-image'"
+        "h2.dimension_index FROM v_sc_data_item AS h1 JOIN v_eui_sc_fits AS h2 USING (data_item_oid)"
+        " WHERE h1.instrument='EUI' AND h1.begin_time>='2021-02-01 00:00:00' AND h1.begin_time<='2021-02-02 00:00:00'"
+        " AND h2.dimension_index='1' AND h1.level='L1' AND h1.descriptor='eui-fsi174-image'"
     )
 
 
 def test_join_low_latency_query():
     result = SOARClient._construct_payload(  # NOQA: SLF001
         [
             "instrument='EUI'",
-            "begin_time>='2021-02-01+00:00:00'+AND+begin_time<='2021-02-02+00:00:00'",
+            "begin_time>='2021-02-01 00:00:00' AND begin_time<='2021-02-02 00:00:00'",
             "level='LL01'",
             "descriptor='eui-fsi174-image'",
         ]
     )
 
-    assert result["QUERY"] == (
-        "SELECT+h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
+    assert result == (
+        "SELECT h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
         "h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, "
-        "h2.dimension_index+FROM+v_ll_data_item AS h1 JOIN v_eui_ll_fits AS h2 USING (data_item_oid)"
-        "+WHERE+h1.instrument='EUI'+AND+h1.begin_time>='2021-02-01+00:00:00'+AND+h1.begin_time<='2021-02-02+00:00:00'"
-        "+AND+h2.dimension_index='1'+AND+h1.level='LL01'+AND+h1.descriptor='eui-fsi174-image'"
+        "h2.dimension_index FROM v_ll_data_item AS h1 JOIN v_eui_ll_fits AS h2 USING (data_item_oid)"
+        " WHERE h1.instrument='EUI' AND h1.begin_time>='2021-02-01 00:00:00' AND h1.begin_time<='2021-02-02 00:00:00'"
+        " AND h2.dimension_index='1' AND h1.level='LL01' AND h1.descriptor='eui-fsi174-image'"
     )

tox.ini

@@ -36,7 +36,7 @@ deps =
     devdeps: astropy>=0.0.dev0
     # Packages without nightly wheels will be built from source like this
     devdeps: git+https://github.com/sunpy/sunpy
-    devdeps: git+https://github.com/psf/requests
+    devdeps: git+https://github.com/astropy/astroquery.git
     oldestdeps: minimum_dependencies
     # old astropy isn't compatible with numpy 2, but numpy isn't a direct dep of sunpy-soar
     oldestdeps: numpy<2