diff --git a/README.md b/README.md
index 8d4a355d..9f5f8516 100644
--- a/README.md
+++ b/README.md
@@ -115,6 +115,7 @@ Available notebooks provided as examples:
- [DEM](https://eoreader.readthedocs.io/en/latest/notebooks/dem.html)
- [Custom stacks](https://eoreader.readthedocs.io/en/latest/notebooks/custom.html)
- [Methods to clean optical bands](https://eoreader.readthedocs.io/en/latest/notebooks/optical_cleaning_methods.html)
+- [AWS storage](https://eoreader.readthedocs.io/en/latest/notebooks/aws.html)
- [S3 Compatible Storage](https://eoreader.readthedocs.io/en/latest/notebooks/s3_compatible_storage.html)
- [Dask](https://eoreader.readthedocs.io/en/latest/notebooks/dask.html)
- [STAC](https://eoreader.readthedocs.io/en/latest/notebooks/stac.html)
diff --git a/docs/_build/.jupyter_cache/global.db b/docs/_build/.jupyter_cache/global.db
index 7440cbc0..81f6340d 100644
Binary files a/docs/_build/.jupyter_cache/global.db and b/docs/_build/.jupyter_cache/global.db differ
diff --git a/docs/notebooks/advanced.md b/docs/notebooks/advanced.md
index cd474538..28659f0e 100644
--- a/docs/notebooks/advanced.md
+++ b/docs/notebooks/advanced.md
@@ -10,4 +10,6 @@
water_detection
optical_cleaning_methods
windowed_reading
+ aws
+ s3_compatible_storage
```
diff --git a/docs/notebooks/aws.ipynb b/docs/notebooks/aws.ipynb
index dc761b23..2c2302a9 100644
--- a/docs/notebooks/aws.ipynb
+++ b/docs/notebooks/aws.ipynb
@@ -8,15 +8,19 @@
"# AWS\n",
"Let's read Sentinel-2 data stored on AWS.\n",
"\n",
+ "This tutorial is a bit tricky as EOReader uses `rasterio` and `cloudpathlib` libraries and they are currently handling S3 buckets differently.\n",
+ "That's why we'll use the helper context manager `temp_s3` from `sertit.s3`, that will do the right connections for us.\n",
+ "\n",
"
\n",
" \n",
" Note: These products are not stored in the `.SAFE` format.\n",
" \n",
"
\n",
"\n",
- "## Let's read data processed by Element84: Sentinel-2 L2A as COGs\n",
+ "## Cloud data processed by Element84: Sentinel-2 L2A as COGs\n",
"\n",
- "See this [registry](https://registry.opendata.aws/sentinel-2-l2a-cogs) (`arn:aws:s3:::sentinel-cogs`)"
+ "See this [registry](https://registry.opendata.aws/sentinel-2-l2a-cogs) (`arn:aws:s3:::sentinel-cogs`).\n",
+ "This registry is open access, so you don't have to sign-in. "
]
},
{
@@ -72,10 +76,8 @@
"source": [
"with tempenv.TemporaryEnvironment({\n",
" \"AWS_S3_ENDPOINT\": \"s3.us-west-2.amazonaws.com\",\n",
- " \"AWS_SECRET_ACCESS_KEY\": os.getenv(\"AMAZON_AWS_SECRET_ACCESS_KEY\"),\n",
- " \"AWS_ACCESS_KEY_ID\": os.getenv(\"AMAZON_AWS_ACCESS_KEY_ID\"),\n",
"}):\n",
- " with s3.temp_s3():\n",
+ " with s3.temp_s3(no_sign_request=True):\n",
" logs.init_logger(logging.getLogger(\"eoreader\"), logging.DEBUG)\n",
" path = r\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/39/K/ZU/2023/10/S2A_39KZU_20231031_0_L2A\"\n",
" prod = Reader().open(path)\n",
@@ -96,9 +98,10 @@
{
"cell_type": "markdown",
"source": [
- "## Let's read data processed by Sinergise: Sentinel-2 L1C\n",
+ "## Cloudprocessed by Sinergise: Sentinel-2 L1C\n",
"\n",
- "See this [registry](https://registry.opendata.aws/sentinel-2/) (`arn:aws:s3:::sentinel-s2-l1c`)\n",
+ "See this [registry](https://registry.opendata.aws/sentinel-2/) (`arn:aws:s3:::sentinel-s2-l1c`).\n",
+ "This registry needs authentication.\n",
"\n",
"NB: L2A would have been the same (`arn:aws:s3:::sentinel-s2-l2a`)\n",
"\n",
diff --git a/docs/notebooks/experimental.md b/docs/notebooks/experimental.md
index 0e400b6a..d32aa770 100644
--- a/docs/notebooks/experimental.md
+++ b/docs/notebooks/experimental.md
@@ -4,8 +4,6 @@
:maxdepth: 1
:caption: Experimental
- s3_compatible_storage
- aws
dask
stac
```
\ No newline at end of file
diff --git a/docs/notebooks/s3_compatible_storage.ipynb b/docs/notebooks/s3_compatible_storage.ipynb
index 9a373d24..211263ef 100644
--- a/docs/notebooks/s3_compatible_storage.ipynb
+++ b/docs/notebooks/s3_compatible_storage.ipynb
@@ -9,18 +9,11 @@
"Let's use EOReader with products stored in a S3 compatible storage cloud.\n",
"\n",
"This tutorial is a bit tricky as EOReader uses `rasterio` and `cloudpathlib` libraries and they are currently handling S3 buckets differently.\n",
- "\n",
- "\n",
- " \n",
- "Note: This is experimental for now, use it at your own risk !\n",
- " \n",
- "
\n",
+ "That's why we'll use the helper context manager `temp_s3` from `sertit.s3`, that will do the right connections for us.\n",
"\n",
"\n",
" \n",
- "
Warning:\n",
- "
S3 compatible storage is not well handled by rasterio for now, you should use Linux or Docker instead !\n",
- " This tutorial assumes that you have correctly set AWS_ACCESS_KEY_ID
and AWS_SECRET_ACCESS_KEY
as environment variables\n",
+ "Warning: This tutorial assumes that you have correctly set AWS_ACCESS_KEY_ID
and AWS_SECRET_ACCESS_KEY
as environment variables\n",
" \n",
" "
]
@@ -42,37 +35,15 @@
"from cloudpathlib import S3Client, AnyPath\n",
"import matplotlib.pyplot as plt\n",
"\n",
- "import rasterio\n",
+ "from sertit import s3\n",
"\n",
"from eoreader.reader import Reader\n",
"from eoreader.bands import MNDWI, CLOUDS\n",
"\n",
- "# Other\n",
- "AWS_S3_ENDPOINT = os.getenv(\"AWS_S3_ENDPOINT\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "id": "4d2a6657",
- "metadata": {
- "ExecuteTime": {
- "start_time": "2023-05-31T13:47:01.052799Z",
- "end_time": "2023-05-31T13:47:01.405195Z"
- }
- },
- "outputs": [],
- "source": [
- "# Create your S3 compatible storage path with cloudpathlib\n",
- "# See here for more insights about this awesome lib: https://cloudpathlib.drivendata.org/\n",
- "client = S3Client(\n",
- " endpoint_url=f\"https://{AWS_S3_ENDPOINT}\",\n",
- " aws_access_key_id=os.getenv(\"AWS_ACCESS_KEY_ID\"),\n",
- " aws_secret_access_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\"),\n",
- ")\n",
- "client.set_as_default_client()\n",
- "path = AnyPath(\"s3://sertit-eoreader-ci\").joinpath(\n",
- " \"optical/S2B_MSIL2A_20200114T065229_N0213_R020_T40REQ_20200114T094749.SAFE/\")"
+ "# This endpoint points to your S3 compatible bucket\n",
+ "custom_s3_compatible_endpoint = os.getenv(\"AWS_S3_ENDPOINT\")\n",
+ "\n",
+ "# Be sure to have AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables correctly pointing to the credentials of your S3-compatible storage"
]
},
{
@@ -82,13 +53,10 @@
"metadata": {},
"outputs": [],
"source": [
- "# Create a rasterio env to enable S3 compatible storage\n",
- "with rasterio.Env(\n",
- " CPL_CURL_VERBOSE=False,\n",
- " AWS_VIRTUAL_HOSTING=False,\n",
- " AWS_S3_ENDPOINT=AWS_S3_ENDPOINT,\n",
- " GDAL_DISABLE_READDIR_ON_OPEN=False,\n",
- "):\n",
+ "with s3.temp_s3(default_endpoint=custom_s3_compatible_endpoint):\n",
+ " path = AnyPath(\"s3://sertit-eoreader-ci\").joinpath(\n",
+ " \"optical/S2B_MSIL2A_20200114T065229_N0213_R020_T40REQ_20200114T094749.SAFE/\")\n",
+ " \n",
" # Create the reader\n",
" reader = Reader()\n",
"\n",
diff --git a/eoreader/products/custom_product.py b/eoreader/products/custom_product.py
index 69ee7ac9..91332a91 100644
--- a/eoreader/products/custom_product.py
+++ b/eoreader/products/custom_product.py
@@ -90,7 +90,7 @@ def __init__(
"""Custom kwargs"""
# Initialization from the super class
- # (Custom products arte managing constellation on their own)
+ # (Custom products are managing constellation on their own)
super_kwargs = kwargs.copy()
super_kwargs.pop("constellation", None)
super().__init__(
diff --git a/requirements.txt b/requirements.txt
index d89b3573..19a88868 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,7 +40,7 @@ spyndex>=0.3.0
pystac[validation]
# SERTIT libs
-sertit[full]>=1.32.0
+sertit[full]>=1.32.1
# Optimizations
dask[complete]>=2021.10.0