add cache to function that will eventually query CMR

Author: juliacollins

docs/source/conf.py

@@ -2,34 +2,34 @@
 
 # -- Project information
 
-project = 'MetGenC'
-copyright = '2024, NSIDC'
-author = 'National Snow and Ice Data Center'
+project = "MetGenC"
+copyright = "2024, NSIDC"
+author = "National Snow and Ice Data Center"
 
-release = '0.6'
-version = '0.6.0'
+release = "0.6"
+version = "0.6.0"
 
 # -- General configuration
 
 extensions = [
-    'sphinx.ext.duration',
-    'sphinx.ext.doctest',
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.intersphinx',
+    "sphinx.ext.duration",
+    "sphinx.ext.doctest",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.intersphinx",
 ]
 
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3/', None),
-    'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
+    "python": ("https://docs.python.org/3/", None),
+    "sphinx": ("https://www.sphinx-doc.org/en/master/", None),
 }
-intersphinx_disabled_domains = ['std']
+intersphinx_disabled_domains = ["std"]
 
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # -- Options for HTML output
 
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # -- Options for EPUB output
-epub_show_urls = 'footnote'
+epub_show_urls = "footnote"

src/nsidc/metgen/cli.py

@@ -69,6 +69,7 @@ def validate(config_filename, content_type):
     metgen.init_logging(configuration)
     metgen.validate(configuration, content_type)
 
+
 @cli.command()
 @click.option(
     "-f",
@@ -81,6 +82,7 @@ def assess():
     """Examine a sample data file for metadata completeness"""
     return True
 
+
 @cli.command()
 @click.option(
     "-c",

src/nsidc/metgen/config.py

@@ -4,7 +4,7 @@
 import os.path
 from pathlib import Path
 
-from nsidc.metgen import aws, constants
+from nsidc.metgen import aws, constants, netcdf_reader
 
 
 class ValidationError(Exception):
@@ -32,7 +32,8 @@ class Config:
     dry_run: bool
 
     def __post_init__(self):
-        self.collection = None
+        # data_reader: Callable[[str], dict]
+        self.data_reader = netcdf_reader
 
     def show(self):
         # TODO: add section headings in the right spot

src/nsidc/metgen/metgen.py

@@ -8,6 +8,7 @@
 import sys
 import uuid
 from collections.abc import Callable
+from functools import cache
 from importlib.resources import open_text
 from pathlib import Path
 from string import Template
@@ -18,7 +19,7 @@
 from returns.maybe import Maybe
 from rich.prompt import Confirm, Prompt
 
-from nsidc.metgen import aws, config, constants, netcdf_reader
+from nsidc.metgen import aws, config, constants
 
 # -------------------------------------------------------------------
 CONSOLE_FORMAT = "%(message)s"
@@ -186,7 +187,6 @@ class Collection:
 
     auth_id: str
     version: int
-    data_reader: Callable[[str], dict]
 
 
 @dataclasses.dataclass
@@ -231,15 +231,7 @@ def process(configuration: config.Config) -> None:
     """
     Process all Granules and record the results and summary.
     """
-    # TODO:
-    #   Do any prep actions, like mkdir, etc
-    #   Get real collection information from CMR
-    #   Determine data reader based on actual data file characteristics (e.g. suffix)
-    configuration.collection = Collection(
-            configuration.auth_id,
-            configuration.version,
-            netcdf_reader.extract_metadata
-    )
+    # TODO: Do any prep actions, like mkdir, etc
 
     # Ordered list of operations to perform on each granule
     operations = [
@@ -343,12 +335,21 @@ def null_operation(configuration: config.Config, granule: Granule) -> Granule:
     return granule
 
 
+@cache
+def retrieve_collection(auth_id: str, version: int):
+    # ummc_from_cmr = talk_to_cmr(configuration.auth_id, configuration.version)
+    # pull out fields from UMM-C response and use to create collection object
+    # with more than just auth_id and version number.
+    return Collection(auth_id, version)
+
+
 def granule_collection(configuration: config.Config, granule: Granule) -> Granule:
     """
-    Associate the Collection with the Granule.
+    Associate collection information with the Granule.
     """
     return dataclasses.replace(
-        granule, collection=configuration.collection
+        granule,
+        collection=retrieve_collection(configuration.auth_id, configuration.version),
     )
 
 
@@ -399,7 +400,7 @@ def create_ummg(configuration: config.Config, granule: Granule) -> Granule:
     # }
     metadata_details = {}
     for data_file in granule.data_filenames:
-        metadata_details[data_file] = granule.collection.data_reader(data_file)
+        metadata_details[data_file] = configuration.data_reader(data_file)
 
     # Collapse information about (possibly) multiple files into a granule summary.
     summary = metadata_summary(metadata_details)

src/nsidc/metgen/netcdf_reader.py

@@ -69,11 +69,13 @@ def spatial_values(netcdf):
         for (lon, lat) in perimeter
     ]
 
+
 def pixel_padding(netcdf):
     # Adding padding should give us values that match up to the
     # netcdf.attrs.geospatial_bounds
     return abs(float(netcdf.crs.GeoTransform.split()[1])) / 2
 
+
 def thinned_perimeter(xdata, ydata):
     """
     Extract the thinned perimeter of a grid.

tests/test_config.py

@@ -32,7 +32,7 @@ def expected_keys():
             "checksum_type",
             "number",
             "dry_run",
-            "collection"
+            "data_reader",
         ]
     )
 

tests/test_metgen.py

@@ -102,22 +102,14 @@ def test_returns_datetime_range():
 
 
 def test_s3_object_path_has_no_leading_slash():
-    granule = metgen.Granule(
-        "foo",
-        metgen.Collection("ABCD", 2, "my_reader"),
-        uuid="abcd-1234"
-    )
+    granule = metgen.Granule("foo", metgen.Collection("ABCD", 2), uuid="abcd-1234")
     expected = "external/ABCD/2/abcd-1234/xyzzy.bin"
     assert metgen.s3_object_path(granule, "xyzzy.bin") == expected
 
 
 def test_s3_url_simple_case():
     staging_bucket_name = "xyzzy-bucket"
-    granule = metgen.Granule(
-        "foo",
-        metgen.Collection("ABCD", 2, "my_reader"),
-        uuid="abcd-1234"
-    )
+    granule = metgen.Granule("foo", metgen.Collection("ABCD", 2), uuid="abcd-1234")
     expected = "s3://xyzzy-bucket/external/ABCD/2/abcd-1234/xyzzy.bin"
     assert metgen.s3_url(staging_bucket_name, granule, "xyzzy.bin") == expected