From 0d6cb3058365c35d61c66c90351d63def2710316 Mon Sep 17 00:00:00 2001 From: thevickypedia Date: Wed, 28 Feb 2024 23:26:59 -0600 Subject: [PATCH] Release `v1.0.0` Update README.md and release_notes.rst --- README.md | 15 +++++++++++++++ docs/README.html | 25 +++++++++++++++++++++++++ docs/README.md | 15 +++++++++++++++ docs/_sources/README.md.txt | 15 +++++++++++++++ docs/_sources/index.rst.txt | 10 +++++----- docs/py-modindex.html | 5 +++++ docs/searchindex.js | 2 +- release_notes.rst | 5 +++++ s3/__init__.py | 2 +- 9 files changed, 87 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9c57dbe..436ef99 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,21 @@ if __name__ == '__main__': wrapper.run() ``` +#### Mandatory arg +- **bucket_name** - Name of the s3 bucket. + +#### Optional kwargs +- **prefix** - Prefix to filter the objects based on their path. Defaults to `None` +- **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging. +- **download_dir** - Name/path of the directory where the objects have to be stored. +Defaults to `bucket_name` at current working directory. +

+- **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION` +- **profile_name** - AWS profile name. Defaults to the env var `PROFILE_NAME` +- **aws_access_key_id** - AWS access key ID. Defaults to the env var `AWS_ACCESS_KEY_ID` +- **aws_secret_access_key** - AWS secret access key. Defaults to the env var `AWS_SECRET_ACCESS_KEY` +> AWS values are loaded from env vars or the default config at `~/.aws/config` / `~/.aws/credentials` + ### Coding Standards Docstring format: [`Google`](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings)
Styling conventions: [`PEP 8`](https://www.python.org/dev/peps/pep-0008/)
diff --git a/docs/README.html b/docs/README.html index 8a9bcfe..03f8d36 100644 --- a/docs/README.html +++ b/docs/README.html @@ -90,6 +90,29 @@

Download objects in sequence +

Mandatory arg

+ + +
+

Optional kwargs

+
    +

  • prefix - Prefix to filter the objects based on their path. Defaults to None

    • +

  • logger - Bring your own custom pre-configured logger. Defaults to on-screen logging.

    • +

  • download_dir - Name/path of the directory where the objects have to be stored. +Defaults to bucket_name at current working directory. +

    • +

  • region_name - AWS region name. Defaults to the env var AWS_DEFAULT_REGION

    • +

  • profile_name - AWS profile name. Defaults to the env var PROFILE_NAME

    • +

  • aws_access_key_id - AWS access key ID. Defaults to the env var AWS_ACCESS_KEY_ID

    • +

  • aws_secret_access_key - AWS secret access key. Defaults to the env var AWS_SECRET_ACCESS_KEY

    • +
+
+

AWS values are loaded from env vars or the default config at ~/.aws/config / ~/.aws/credentials

+
+

Coding Standards

@@ -149,6 +172,8 @@

Table of Contents

  • Usage
  • Coding Standards
    • diff --git a/docs/README.md b/docs/README.md index 9c57dbe..436ef99 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,6 +52,21 @@ if __name__ == '__main__': wrapper.run() ``` +#### Mandatory arg +- **bucket_name** - Name of the s3 bucket. + +#### Optional kwargs +- **prefix** - Prefix to filter the objects based on their path. Defaults to `None` +- **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging. +- **download_dir** - Name/path of the directory where the objects have to be stored. +Defaults to `bucket_name` at current working directory. +

    +- **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION` +- **profile_name** - AWS profile name. Defaults to the env var `PROFILE_NAME` +- **aws_access_key_id** - AWS access key ID. Defaults to the env var `AWS_ACCESS_KEY_ID` +- **aws_secret_access_key** - AWS secret access key. Defaults to the env var `AWS_SECRET_ACCESS_KEY` +> AWS values are loaded from env vars or the default config at `~/.aws/config` / `~/.aws/credentials` + ### Coding Standards Docstring format: [`Google`](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings)
    Styling conventions: [`PEP 8`](https://www.python.org/dev/peps/pep-0008/)
    diff --git a/docs/_sources/README.md.txt b/docs/_sources/README.md.txt index 9c57dbe..436ef99 100644 --- a/docs/_sources/README.md.txt +++ b/docs/_sources/README.md.txt @@ -52,6 +52,21 @@ if __name__ == '__main__': wrapper.run() ``` +#### Mandatory arg +- **bucket_name** - Name of the s3 bucket. + +#### Optional kwargs +- **prefix** - Prefix to filter the objects based on their path. Defaults to `None` +- **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging. +- **download_dir** - Name/path of the directory where the objects have to be stored. +Defaults to `bucket_name` at current working directory. +

    +- **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION` +- **profile_name** - AWS profile name. Defaults to the env var `PROFILE_NAME` +- **aws_access_key_id** - AWS access key ID. Defaults to the env var `AWS_ACCESS_KEY_ID` +- **aws_secret_access_key** - AWS secret access key. Defaults to the env var `AWS_SECRET_ACCESS_KEY` +> AWS values are loaded from env vars or the default config at `~/.aws/config` / `~/.aws/credentials` + ### Coding Standards Docstring format: [`Google`](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings)
    Styling conventions: [`PEP 8`](https://www.python.org/dev/peps/pep-0008/)
    diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt index 88a8f8a..e49e230 100644 --- a/docs/_sources/index.rst.txt +++ b/docs/_sources/index.rst.txt @@ -12,10 +12,10 @@ Welcome to S3 Downloader's documentation! README ---------S3 Downloader-------- -============================= -Dumper -====== +S3 Downloader +============= +Main Module +=========== .. automodule:: s3.dumper :members: :undoc-members: @@ -28,7 +28,7 @@ Exceptions Logger ====== -.. autoclass:: s3.logger +.. automodule:: s3.logger :members: :undoc-members: diff --git a/docs/py-modindex.html b/docs/py-modindex.html index 2bf70a7..372f7fa 100644 --- a/docs/py-modindex.html +++ b/docs/py-modindex.html @@ -68,6 +68,11 @@

    Python Module Index

        s3.exceptions + + +     + s3.logger + diff --git a/docs/searchindex.js b/docs/searchindex.js index 526672c..db4261a 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["README", "index"], "filenames": ["README.md", "index.rst"], "titles": ["S3 Download", "Welcome to S3 Downloader\u2019s documentation!"], "terms": {"version": 0, "support": 0, "languag": 0, "stat": 0, "repo": 0, "activ": 0, "build": 0, "statu": 0, "python": 0, "modul": 0, "an": [0, 1], "bucket": [0, 1], "pip": 0, "import": 0, "__name__": 0, "__main__": 0, "wrapper": 0, "bucket_nam": [0, 1], "run_in_parallel": [0, 1], "thread": [0, 1], "10": 0, "default": [0, 1], "5": [0, 1], "run": [0, 1], "docstr": 0, "format": [0, 1], "googl": 0, "style": 0, "convent": 0, "pep": 0, "8": 0, "clean": 0, "pre": 0, "commit": 0, "hook": 0, "flake8": 0, "isort": 0, "requir": 0, "m": 0, "gitvers": 0, "revers": 0, "f": 0, "release_not": 0, "rst": 0, "t": 0, "ensur": 0, "pytest": 0, "gener": 0, "runbook": 0, "valid": 0, "hyperlink": 0, "all": [0, 1], "markdown": 0, "file": [0, 1], "includ": 0, "wiki": 0, "page": [0, 1], "sphinx": 0, "1": 0, "recommonmark": 0, "http": 0, "org": 0, "project": 0, "vignesh": 0, "rao": 0, "under": 0, "mit": 0, "instal": 1, "usag": 1, "code": 1, "standard": 1, "releas": 1, "note": 1, "lint": 1, "licens": 1, "copyright": 1, "class": 1, "dumper": 1, "str": 1, "download_dir": 1, "option": 1, "none": 1, "region_nam": 1, "profile_nam": 1, "aws_access_key_id": 1, "aws_secret_access_kei": 1, "prefix": 1, "initi": 1, "object": 1, "entir": 1, "necessari": 1, "arg": 1, "creat": 1, "boto3": 1, "session": 1, "retri": 1, "logic": 1, "paramet": 1, "name": 1, "directori": 1, "aw": 1, "region": 1, "profil": 1, "access": 1, "kei": 1, "id": 1, "secret": 1, "bring": 1, "your": 1, "own": 1, "specif": 1, "path": 1, "from": 1, "which": 1, "have": 1, "retry_config": 1, "config": 1, "botocor": 1, "init": 1, "instanti": 1, "instanc": 1, "rais": 1, "valueerror": 1, "If": 1, "wa": 1, "pass": 1, "bucketnotfound": 1, "found": 1, "exit": 1, "noreturn": 1, "log": 1, "were": 1, "ani": 1, "failur": 1, "get_object": 1, "list": 1, "get": 1, "target": 1, "invalidprefix": 1, "given": 1, "exist": 1, "noobjectfound": 1, "i": 1, "empti": 1, "return": 1, "type": 1, "exact": 1, "replac": 1, "space": 1, "underscor": 1, "take": 1, "filenam": 1, "argument": 1, "tradit": 1, "loop": 1, "int": 1, "multi": 1, "number": 1, "us": 1, "get_bucket_structur": 1, "form": 1, "hierarch": 1, "folder": 1, "like": 1, "represent": 1, "chosen": 1, "print_bucket_structur": 1, "print": 1, "store": 1, "custom": 1, "formatt": 1, "s3error": 1, "error": 1, "base": 1, "convert_to_folder_structur": 1, "sequenc": 1, "set": 1, "convert": 1, "either": 1, "mutabl": 1, "immut": 1, "string": 1, "architectur": 1, "avail": 1, "invalid": 1, "valu": 1, "inherit": 1, "limit": 1, "format_error_messag": 1, "messag": 1, "load": 1, "streamhandl": 1, "debug": 1, "mode": 1, "default_handl": 1, "assign": 1, "default_format": 1, "datetim": 1, "default_logg": 1, "enabl": 1, "index": 1, "search": 1}, "objects": {"s3": [[1, 0, 0, "-", "dumper"], [1, 0, 0, "-", "exceptions"], [1, 0, 0, "-", "logger"]], "s3.dumper": [[1, 1, 1, "", "Downloader"]], "s3.dumper.Downloader": [[1, 2, 1, "", "RETRY_CONFIG"], [1, 3, 1, "", "downloader"], [1, 3, 1, "", "exit"], [1, 3, 1, "", "get_bucket_structure"], [1, 3, 1, "", "get_objects"], [1, 3, 1, "", "init"], [1, 3, 1, "", "print_bucket_structure"], [1, 3, 1, "", "run"], [1, 3, 1, "", "run_in_parallel"]], "s3.exceptions": [[1, 4, 1, "", "BucketNotFound"], [1, 4, 1, "", "InvalidPrefix"], [1, 4, 1, "", "NoObjectFound"], [1, 4, 1, "", "S3Error"], [1, 5, 1, "", "convert_to_folder_structure"]], "s3.exceptions.InvalidPrefix": [[1, 3, 1, "", "format_error_message"]], "s3.logger": [[1, 5, 1, "", "default_format"], [1, 5, 1, "", "default_handler"], [1, 5, 1, "", "default_logger"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:attribute", "3": "py:method", "4": "py:exception", "5": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "method", "Python method"], "4": ["py", "exception", "Python exception"], "5": ["py", "function", "Python function"]}, "titleterms": {"s3": [0, 1], "download": [0, 1], "instal": 0, "usag": 0, "object": 0, "parallel": 0, "sequenc": 0, "code": 0, "standard": 0, "releas": 0, "note": 0, "lint": 0, "pypi": 0, "packag": 0, "licens": 0, "copyright": 0, "welcom": 1, "": 1, "document": 1, "read": 1, "me": 1, "main": 1, "modul": 1, "except": 1, "logger": 1, "indic": 1, "tabl": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}}) \ No newline at end of file +Search.setIndex({"docnames": ["README", "index"], "filenames": ["README.md", "index.rst"], "titles": ["S3 Download", "Welcome to S3 Downloader\u2019s documentation!"], "terms": {"version": 0, "support": 0, "languag": 0, "stat": 0, "repo": 0, "activ": 0, "build": 0, "statu": 0, "python": 0, "modul": 0, "an": [0, 1], "bucket": [0, 1], "pip": 0, "import": 0, "__name__": 0, "__main__": 0, "wrapper": 0, "bucket_nam": [0, 1], "run_in_parallel": [0, 1], "thread": [0, 1], "10": 0, "default": [0, 1], "5": [0, 1], "run": [0, 1], "_": 0, "name": [0, 1], "prefix": [0, 1], "filter": 0, "base": [0, 1], "path": [0, 1], "none": [0, 1], "logger": 0, "bring": [0, 1], "your": [0, 1], "own": [0, 1], "custom": [0, 1], "pre": 0, "configur": 0, "screen": 0, "log": [0, 1], "dir": 0, "directori": [0, 1], "where": 0, "have": [0, 1], "store": [0, 1], "current": 0, "work": 0, "region": [0, 1], "aw": [0, 1], "env": 0, "var": 0, "aws_default_region": 0, "profil": [0, 1], "profile_nam": [0, 1], "access": [0, 1], "kei": [0, 1], "id": [0, 1], "aws_access_key_id": [0, 1], "secret": [0, 1], "aws_secret_access_kei": [0, 1], "valu": [0, 1], "ar": 0, "load": [0, 1], "from": [0, 1], "config": [0, 1], "credenti": 0, "docstr": 0, "format": [0, 1], "googl": 0, "style": 0, "convent": 0, "pep": 0, "8": 0, "clean": 0, "commit": 0, "hook": 0, "flake8": 0, "isort": 0, "requir": 0, "m": 0, "gitvers": 0, "revers": 0, "f": 0, "release_not": 0, "rst": 0, "t": 0, "ensur": 0, "pytest": 0, "gener": 0, "runbook": 0, "valid": 0, "hyperlink": 0, "all": [0, 1], "markdown": 0, "file": [0, 1], "includ": 0, "wiki": 0, "page": [0, 1], "sphinx": 0, "1": 0, "recommonmark": 0, "http": 0, "org": 0, "project": 0, "vignesh": 0, "rao": 0, "under": 0, "mit": 0, "instal": 1, "usag": 1, "code": 1, "standard": 1, "releas": 1, "note": 1, "lint": 1, "licens": 1, "copyright": 1, "class": 1, "dumper": 1, "str": 1, "download_dir": 1, "option": 1, "region_nam": 1, "initi": 1, "object": 1, "entir": 1, "necessari": 1, "arg": 1, "creat": 1, "boto3": 1, "session": 1, "retri": 1, "logic": 1, "paramet": 1, "specif": 1, "which": 1, "retry_config": 1, "botocor": 1, "init": 1, "instanti": 1, "instanc": 1, "rais": 1, "valueerror": 1, "If": 1, "wa": 1, "pass": 1, "bucketnotfound": 1, "found": 1, "exit": 1, "noreturn": 1, "were": 1, "ani": 1, "failur": 1, "get_object": 1, "list": 1, "get": 1, "target": 1, "invalidprefix": 1, "given": 1, "exist": 1, "noobjectfound": 1, "i": 1, "empti": 1, "return": 1, "type": 1, "exact": 1, "replac": 1, "space": 1, "underscor": 1, "take": 1, "filenam": 1, "argument": 1, "tradit": 1, "loop": 1, "int": 1, "multi": 1, "number": 1, "us": 1, "get_bucket_structur": 1, "form": 1, "hierarch": 1, "folder": 1, "like": 1, "represent": 1, "chosen": 1, "print_bucket_structur": 1, "print": 1, "formatt": 1, "s3error": 1, "error": 1, "convert_to_folder_structur": 1, "sequenc": 1, "set": 1, "convert": 1, "either": 1, "mutabl": 1, "immut": 1, "string": 1, "architectur": 1, "avail": 1, "invalid": 1, "inherit": 1, "limit": 1, "format_error_messag": 1, "messag": 1, "streamhandl": 1, "debug": 1, "mode": 1, "default_handl": 1, "assign": 1, "default_format": 1, "datetim": 1, "default_logg": 1, "enabl": 1, "index": 1, "search": 1}, "objects": {"s3": [[1, 0, 0, "-", "dumper"], [1, 0, 0, "-", "exceptions"], [1, 0, 0, "-", "logger"]], "s3.dumper": [[1, 1, 1, "", "Downloader"]], "s3.dumper.Downloader": [[1, 2, 1, "", "RETRY_CONFIG"], [1, 3, 1, "", "downloader"], [1, 3, 1, "", "exit"], [1, 3, 1, "", "get_bucket_structure"], [1, 3, 1, "", "get_objects"], [1, 3, 1, "", "init"], [1, 3, 1, "", "print_bucket_structure"], [1, 3, 1, "", "run"], [1, 3, 1, "", "run_in_parallel"]], "s3.exceptions": [[1, 4, 1, "", "BucketNotFound"], [1, 4, 1, "", "InvalidPrefix"], [1, 4, 1, "", "NoObjectFound"], [1, 4, 1, "", "S3Error"], [1, 5, 1, "", "convert_to_folder_structure"]], "s3.exceptions.InvalidPrefix": [[1, 3, 1, "", "format_error_message"]], "s3.logger": [[1, 5, 1, "", "default_format"], [1, 5, 1, "", "default_handler"], [1, 5, 1, "", "default_logger"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:attribute", "3": "py:method", "4": "py:exception", "5": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "method", "Python method"], "4": ["py", "exception", "Python exception"], "5": ["py", "function", "Python function"]}, "titleterms": {"s3": [0, 1], "download": [0, 1], "instal": 0, "usag": 0, "object": 0, "parallel": 0, "sequenc": 0, "mandatori": 0, "arg": 0, "option": 0, "kwarg": 0, "code": 0, "standard": 0, "releas": 0, "note": 0, "lint": 0, "pypi": 0, "packag": 0, "licens": 0, "copyright": 0, "welcom": 1, "": 1, "document": 1, "read": 1, "me": 1, "main": 1, "modul": 1, "except": 1, "logger": 1, "indic": 1, "tabl": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}}) \ No newline at end of file diff --git a/release_notes.rst b/release_notes.rst index ba20910..a90754b 100644 --- a/release_notes.rst +++ b/release_notes.rst @@ -1,6 +1,11 @@ Release Notes ============= +v1.0.0 (02/28/2024) +------------------- +- Includes a feature to filter objects using `prefix` +- Minor stability and performance improvements + v0.9.1 (10/04/2023) ------------------- - Handle errors caused by threads and missing filenames diff --git a/s3/__init__.py b/s3/__init__.py index 5d00458..cc2c152 100644 --- a/s3/__init__.py +++ b/s3/__init__.py @@ -1,3 +1,3 @@ from s3.dumper import Downloader # noqa: F401 -version = "0.9.1" +version = "1.0.0"