code cleanup

CHANGES.md

@@ -16,6 +16,11 @@
 * Add tests for CLI and implementations when invoked through the CLI
 * Refactored code dealing with requests and authentication to the `STACpopulator/requests.py` file
 * Add `--log-file` command line option to specify a non-default location to write log files to
+* fix incorrect example in README
+* move argument parsing for logging options to the implementation code
+* fix bug where logging options were being set incorrectly
+* rename files to avoid potential naming conflicts with other packages (`logging` and `requests`)
+
 
 ## [0.6.0](https://github.com/crim-ca/stac-populator/tree/0.6.0) (2024-02-22)
 

README.md

@@ -119,7 +119,7 @@ For example, using the [CMIP6_UofT][CMIP6_UofT] implementation, the script can b
 python STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py \
     "http://localhost:8880/stac/" \
     "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html" \
-    "STACpopulator/implementations/CMIP6_UofT/collection_config.yml"
+    --config "STACpopulator/implementations/CMIP6_UofT/collection_config.yml"
 ```
 
 *Note*: <br>

STACpopulator/cli.py

@@ -5,26 +5,20 @@
 import sys
 from types import ModuleType
 import warnings
-from datetime import datetime, timezone
-from typing import Callable
 
 from STACpopulator import __version__, implementations
 from STACpopulator.exceptions import STACPopulatorError
-from STACpopulator.logging import setup_logging
+from STACpopulator.log import setup_logging
 
 
-def add_parser_args(parser: argparse.ArgumentParser) -> dict[str, Callable]:
+def add_parser_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--version",
         "-V",
         action="version",
         version=f"%(prog)s {__version__}",
         help="prints the version of the library and exits",
     )
-    parser.add_argument("--debug", action="store_const", const=logging.DEBUG, help="set logger level to debug")
-    parser.add_argument(
-        "--log-file", help="file to write log output to. By default logs will be written to the current directory."
-    )
     commands_subparser = parser.add_subparsers(
         title="command", dest="command", description="STAC populator command to execute.", required=True
     )
@@ -52,8 +46,7 @@ def implementation_modules() -> dict[str, ModuleType]:
 
 def run(ns: argparse.Namespace) -> int:
     if ns.command == "run":
-        logfile_name = ns.log_file or f"{ns.populator}_log_{datetime.now(timezone.utc).isoformat() + 'Z'}.jsonl"
-        setup_logging(logfile_name, ns.debug or logging.INFO)
+        setup_logging(ns.log_file, ns.debug or logging.INFO)
         return implementation_modules()[ns.populator].runner(ns) or 0
 
 

STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py

@@ -9,7 +9,9 @@
 from pystac.extensions.datacube import DatacubeExtension
 from requests.sessions import Session
 
-from STACpopulator.requests import add_request_options, apply_request_options
+from STACpopulator import cli
+from STACpopulator.log import add_logging_options
+from STACpopulator.request_utils import add_request_options, apply_request_options
 from STACpopulator.extensions.cmip6 import CMIP6Helper, CMIP6Properties
 from STACpopulator.extensions.datacube import DataCubeHelper
 from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper
@@ -31,7 +33,6 @@ def __init__(
         update: Optional[bool] = False,
         session: Optional[Session] = None,
         config_file: Optional[Union[os.PathLike[str], str]] = None,
-        log_debug: Optional[bool] = False,
     ) -> None:
         """Constructor
 
@@ -40,7 +41,7 @@ def __init__(
         :param data_loader: loader to iterate over ingestion data.
         """
         super().__init__(
-            stac_host, data_loader, update=update, session=session, config_file=config_file, log_debug=log_debug
+            stac_host, data_loader, update=update, session=session, config_file=config_file
         )
 
     def create_stac_item(
@@ -106,6 +107,7 @@ def add_parser_args(parser: argparse.ArgumentParser) -> None:
         ),
     )
     add_request_options(parser)
+    add_logging_options(parser)
 
 
 def runner(ns: argparse.Namespace) -> int:
@@ -120,7 +122,7 @@ def runner(ns: argparse.Namespace) -> int:
             data_loader = ErrorLoader()
 
         c = CMIP6populator(
-            ns.stac_host, data_loader, update=ns.update, session=session, config_file=ns.config, log_debug=ns.debug
+            ns.stac_host, data_loader, update=ns.update, session=session, config_file=ns.config
         )
         c.ingest()
     return 0
@@ -130,8 +132,8 @@ def main(*args: str) -> int:
     parser = argparse.ArgumentParser()
     add_parser_args(parser)
     ns = parser.parse_args(args or None)
-    return runner(ns)
-
+    ns.populator = os.path.basename(os.path.dirname(__file__))
+    return cli.run(ns)
 
 if __name__ == "__main__":
     sys.exit(main())

STACpopulator/implementations/DirectoryLoader/crawl_directory.py

@@ -6,7 +6,9 @@
 
 from requests.sessions import Session
 
-from STACpopulator.requests import add_request_options, apply_request_options
+from STACpopulator import cli
+from STACpopulator.log import add_logging_options
+from STACpopulator.request_utils import add_request_options, apply_request_options
 from STACpopulator.input import STACDirectoryLoader
 from STACpopulator.models import GeoJSONPolygon
 from STACpopulator.populator_base import STACpopulatorBase
@@ -51,6 +53,7 @@ def add_parser_args(parser: argparse.ArgumentParser) -> None:
         help="Limit search of STAC Collections only to first top-most matches in the crawled directory structure.",
     )
     add_request_options(parser)
+    add_logging_options(parser)
 
 
 def runner(ns: argparse.Namespace) -> int:
@@ -70,7 +73,8 @@ def main(*args: str) -> int:
     parser = argparse.ArgumentParser()
     add_parser_args(parser)
     ns = parser.parse_args(args or None)
-    return runner(ns)
+    ns.populator = os.path.basename(os.path.dirname(__file__))
+    return cli.run(ns)
 
 
 if __name__ == "__main__":

STACpopulator/logging.py → STACpopulator/log.py

@@ -1,3 +1,4 @@
+import argparse
 import datetime as dt
 import json
 import logging.config
@@ -126,3 +127,14 @@ def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord:
     },
     "loggers": {"root": {"level": "DEBUG", "handlers": ["stderr", "file"]}},
 }
+
+def add_logging_options(parser: argparse.ArgumentParser) -> None:
+    """
+    Adds arguments to a parser to configure logging options.
+    """
+    parser.add_argument("--debug", action="store_const", const=logging.DEBUG, help="set logger level to debug")
+    parser.add_argument(
+        "--log-file", 
+        default=f"stac_populator_log_{dt.datetime.now(dt.timezone.utc).isoformat() + 'Z'}.jsonl",
+        help="file to write log output to. By default logs will be written to the current directory."
+    )

STACpopulator/populator_base.py

@@ -27,10 +27,9 @@ def __init__(
         self,
         stac_host: str,
         data_loader: GenericLoader,
-        update: Optional[bool] = False,
+        update: bool = False,
         session: Optional[Session] = None,
         config_file: Optional[Union[os.PathLike[str], str]] = "collection_config.yml",
-        log_debug: Optional[bool] = False,
     ) -> None:
         """Constructor
 
@@ -41,7 +40,6 @@ def __init__(
         :raises RuntimeError: Raised if one of the required definitions is not found in the collection info filename
         """
 
-        super().__init__()
         self._collection_config_path = config_file
         self._collection_info: MutableMapping[str, Any] = None
         self._session = session