Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Using astropy's PyVO to make URL calls. #129

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ authors = [
dependencies = [
"astropy>5.2.0",
"sunpy[net]>=5.0",
"requests>=2.28.0",
"astroquery>=0.4.7"
]
dynamic = ["version"]

Expand Down
8 changes: 4 additions & 4 deletions sunpy_soar/attrs.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@ def apply_and(wlk, and_attr, params):

@walker.add_applier(a.Time)
def _(wlk, attr, params): # NOQA: ARG001
start = attr.start.strftime("%Y-%m-%d+%H:%M:%S")
end = attr.end.strftime("%Y-%m-%d+%H:%M:%S")
params.append(f"begin_time>='{start}'+AND+begin_time<='{end}'")
start = attr.start.strftime("%Y-%m-%d %H:%M:%S")
end = attr.end.strftime("%Y-%m-%d %H:%M:%S")
params.append(f"begin_time>='{start}' AND begin_time<='{end}'")


@walker.add_applier(a.Level)
Expand Down Expand Up @@ -140,4 +140,4 @@ def _(wlk, attr, params): # NOQA: ARG001
def _(wlk, attr, params): # NOQA: ARG001
wavemin = attr.min.value
wavemax = attr.max.value
params.append(f"Wavemin='{wavemin}'+AND+Wavemax='{wavemax}'")
params.append(f"Wavemin='{wavemin}' AND Wavemax='{wavemax}'")
99 changes: 38 additions & 61 deletions sunpy_soar/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,11 @@

import astropy.table
import astropy.units as u
import requests
import sunpy.net.attrs as a
from astroquery.utils.tap.core import TapPlus
from sunpy import log
from sunpy.net.attr import and_
from sunpy.net.base_client import BaseClient, QueryResponseTable
from sunpy.time import parse_time

from sunpy_soar.attrs import SOOP, Product, walker

Expand All @@ -31,14 +30,14 @@ def search(self, *query, **kwargs): # NOQA: ARG002
query_parameters.remove("provider='SOAR'")
results.append(self._do_search(query_parameters))
table = astropy.table.vstack(results)
table["Filesize"] = (table["Filesize"] * u.byte).to(u.Mbyte).round(3)
qrt = QueryResponseTable(table, client=self)
qrt["Filesize"] = (qrt["Filesize"] * u.byte).to(u.Mbyte).round(3)
qrt.hide_keys = ["Data item ID", "Filename"]
qrt.hide_keys = ["data_item_id", "Filename", "dimension_index"]
return qrt

def add_join_to_query(query: list[str], data_table: str, instrument_table: str):
"""
Construct the WHERE, FROM, and SELECT parts of the ADQL query.
Construct the WHERE, FROM, and SELECT parts of the SQL query.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand, before it was URL-encoded ADQL, now it is just ADQL. Transforming the +s to spaces don't make this SQL.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alrighty, makes sense!


Parameters
----------
Expand Down Expand Up @@ -69,17 +68,17 @@ def add_join_to_query(query: list[str], data_table: str, instrument_table: str):
# To make sure this data is not misleading to the user we do not return any values for PHI AND SPICE.
parameter = f"Wavelength='{wavemin_match.group(1)}'"
elif wavemin_match and wavemax_match:
parameter = f"Wavemin='{wavemin_match.group(1)}'+AND+h2.Wavemax='{wavemax_match.group(1)}'"
parameter = f"Wavemin='{wavemin_match.group(1)}' AND h2.Wavemax='{wavemax_match.group(1)}'"
prefix = "h1." if not parameter.startswith("Detector") and not parameter.startswith("Wave") else "h2."
if parameter.startswith("begin_time"):
time_list = parameter.split("+AND+")
final_query += f"h1.{time_list[0]}+AND+h1.{time_list[1]}+AND+"
time_list = parameter.split(" AND ")
final_query += f"h1.{time_list[0]} AND h1.{time_list[1]} AND "
# As there are no dimensions in STIX, the dimension index need not be included in the query for STIX.
if "stx" not in instrument_table:
# To avoid duplicate rows in the output table, the dimension index is set to 1.
final_query += "h2.dimension_index='1'+AND+"
final_query += "h2.dimension_index='1' AND "
else:
final_query += f"{prefix}{parameter}+AND+"
final_query += f"{prefix}{parameter} AND "

where_part = final_query[:-5]
from_part = f"{data_table} AS h1"
Expand All @@ -104,8 +103,8 @@ def _construct_payload(query):

Returns
-------
dict
Payload dictionary to be sent with the query.
str
SQL query string.
"""
# Default data table
data_table = "v_sc_data_item"
Expand All @@ -122,7 +121,7 @@ def _construct_payload(query):

instrument_name = None
for q in query:
if q.startswith("instrument") or q.startswith("descriptor") and not instrument_name:
if (q.startswith(("instrument", "descriptor"))) and not instrument_name:
instrument_name = q.split("=")[1][1:-1].split("-")[0].upper()
elif q.startswith("level") and q.split("=")[1][1:3] == "LL":
data_table = "v_ll_data_item"
Expand All @@ -139,13 +138,11 @@ def _construct_payload(query):
where_part, from_part, select_part = SOARClient.add_join_to_query(query, data_table, instrument_table)
else:
from_part = data_table
select_part = "*"
where_part = "+AND+".join(query)

adql_query = {"SELECT": select_part, "FROM": from_part, "WHERE": where_part}

adql_query_str = "+".join([f"{key}+{value}" for key, value in adql_query.items()])
return {"REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "json", "QUERY": adql_query_str}
select_part = (
"instrument, descriptor, level, begin_time, end_time, data_item_id, filename, filesize, soop_name"
)
where_part = " AND ".join(query)
return f"SELECT {select_part} FROM {from_part} WHERE {where_part}"

@staticmethod
def _do_search(query):
Expand All @@ -159,49 +156,29 @@ def _do_search(query):

Returns
-------
astropy.table.QTable
astropy.table.Table
Query results.
"""
tap_endpoint = "http://soar.esac.esa.int/soar-sl-tap/tap"
payload = SOARClient._construct_payload(query)
# Need to force requests to not form-encode the parameters
payload = "&".join([f"{key}={val}" for key, val in payload.items()])
# Get request info
r = requests.get(f"{tap_endpoint}/sync", params=payload)
log.debug(f"Sent query: {r.url}")
r.raise_for_status()

# Do some list/dict wrangling
names = [m["name"] for m in r.json()["metadata"]]
info = {name: [] for name in names}

for entry in r.json()["data"]:
for i, name in enumerate(names):
info[name].append(entry[i])

if len(info["begin_time"]):
info["begin_time"] = parse_time(info["begin_time"]).iso
info["end_time"] = parse_time(info["end_time"]).iso

result_table = astropy.table.QTable(
{
"Instrument": info["instrument"],
"Data product": info["descriptor"],
"Level": info["level"],
"Start time": info["begin_time"],
"End time": info["end_time"],
"Data item ID": info["data_item_id"],
"Filename": info["filename"],
"Filesize": info["filesize"],
"SOOP Name": info["soop_name"],
},
)
if "detector" in info:
result_table["Detector"] = info["detector"]
if "wavelength" in info:
result_table["Wavelength"] = info["wavelength"]
result_table.sort("Start time")
return result_table
sql_query = SOARClient._construct_payload(query)
soar = TapPlus(url=tap_endpoint)
job = soar.launch_job_async(sql_query)
results = job.results
new_colnames = {
"instrument": "Instrument",
"descriptor": "Data product",
"level": "Level",
"begin_time": "Start time",
"end_time": "End time",
"filename": "Filename",
"filesize": "Filesize",
"soop_name": "SOOP Name",
}
new_colnames.update({k: k.capitalize() for k in ["wavelength", "detector"] if k in results.colnames})
for old_name, new_name in new_colnames.items():
results.rename_column(old_name, new_name)
results.sort("Start time")
return results

def fetch(self, query_results, *, path, downloader, **kwargs): # NOQA: ARG002
"""
Expand Down Expand Up @@ -229,7 +206,7 @@ def fetch(self, query_results, *, path, downloader, **kwargs): # NOQA: ARG002
url += "&product_type=LOW_LATENCY"
else:
url += "&product_type=SCIENCE"
data_id = row["Data item ID"]
data_id = row["data_item_id"]
url += f"&data_item_id={data_id}"
filepath = str(path).format(file=row["Filename"], **row.response_block_map)
log.debug(f"Queuing URL: {url}")
Expand Down
24 changes: 12 additions & 12 deletions sunpy_soar/tests/test_sunpy_soar.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,35 +236,35 @@ def test_join_science_query():
result = SOARClient._construct_payload( # NOQA: SLF001
[
"instrument='EUI'",
"begin_time>='2021-02-01+00:00:00'+AND+begin_time<='2021-02-02+00:00:00'",
"begin_time>='2021-02-01 00:00:00' AND begin_time<='2021-02-02 00:00:00'",
"level='L1'",
"descriptor='eui-fsi174-image'",
]
)

assert result["QUERY"] == (
"SELECT+h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
assert result == (
"SELECT h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
"h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, "
"h2.dimension_index+FROM+v_sc_data_item AS h1 JOIN v_eui_sc_fits AS h2 USING (data_item_oid)"
"+WHERE+h1.instrument='EUI'+AND+h1.begin_time>='2021-02-01+00:00:00'+AND+h1.begin_time<='2021-02-02+00:00:00'"
"+AND+h2.dimension_index='1'+AND+h1.level='L1'+AND+h1.descriptor='eui-fsi174-image'"
"h2.dimension_index FROM v_sc_data_item AS h1 JOIN v_eui_sc_fits AS h2 USING (data_item_oid)"
" WHERE h1.instrument='EUI' AND h1.begin_time>='2021-02-01 00:00:00' AND h1.begin_time<='2021-02-02 00:00:00'"
" AND h2.dimension_index='1' AND h1.level='L1' AND h1.descriptor='eui-fsi174-image'"
)


def test_join_low_latency_query():
result = SOARClient._construct_payload( # NOQA: SLF001
[
"instrument='EUI'",
"begin_time>='2021-02-01+00:00:00'+AND+begin_time<='2021-02-02+00:00:00'",
"begin_time>='2021-02-01 00:00:00' AND begin_time<='2021-02-02 00:00:00'",
"level='LL01'",
"descriptor='eui-fsi174-image'",
]
)

assert result["QUERY"] == (
"SELECT+h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
assert result == (
"SELECT h1.instrument, h1.descriptor, h1.level, h1.begin_time, h1.end_time, "
"h1.data_item_id, h1.filesize, h1.filename, h1.soop_name, h2.detector, h2.wavelength, "
"h2.dimension_index+FROM+v_ll_data_item AS h1 JOIN v_eui_ll_fits AS h2 USING (data_item_oid)"
"+WHERE+h1.instrument='EUI'+AND+h1.begin_time>='2021-02-01+00:00:00'+AND+h1.begin_time<='2021-02-02+00:00:00'"
"+AND+h2.dimension_index='1'+AND+h1.level='LL01'+AND+h1.descriptor='eui-fsi174-image'"
"h2.dimension_index FROM v_ll_data_item AS h1 JOIN v_eui_ll_fits AS h2 USING (data_item_oid)"
" WHERE h1.instrument='EUI' AND h1.begin_time>='2021-02-01 00:00:00' AND h1.begin_time<='2021-02-02 00:00:00'"
" AND h2.dimension_index='1' AND h1.level='LL01' AND h1.descriptor='eui-fsi174-image'"
)
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ deps =
devdeps: astropy>=0.0.dev0
# Packages without nightly wheels will be built from source like this
devdeps: git+https://github.com/sunpy/sunpy
devdeps: git+https://github.com/psf/requests
devdeps: git+https://github.com/astropy/astroquery.git
oldestdeps: minimum_dependencies
# old astropy isn't compatible with numpy 2, but numpy isn't a direct dep of sunpy-soar
oldestdeps: numpy<2
Expand Down