chores: Adds tests for raster clipping (#88)

* chores: Adds tests for raster clipping

Signed-off-by: nagesh bansal <[email protected]>

Author: Nageshbansal

neonwranglerpy/fetcher/fetcher.py

@@ -35,7 +35,7 @@ async def _request(session, url):
         return await response.json()
 
 
-async def _download(session, url, filename, sem, month, size=None):
+async def _download(session, url, filename, sem, size=None):
     """Asynchronous function to download file from url.
 
     Parameters
@@ -58,6 +58,7 @@ async def _download(session, url, filename, sem, month, size=None):
                 size = response.content_length if not size else size
                 block = size
                 copied = 0
+
                 with open(filename, mode='wb') as f:
                     async for chunk in response.content.iter_chunked(block):
                         f.write(chunk)
@@ -66,26 +67,44 @@ async def _download(session, url, filename, sem, month, size=None):
                         # update_progressbar(progress, size)
 
 
-async def _fetcher(data, rate_limit, headers, files_to_stack_path="filesToStack"):
+async def _fetcher(data,
+                   rate_limit,
+                   headers,
+                   files_to_stack_path="filesToStack",
+                   data_type="vst"):
     """Fetcher for downloading files."""
     sem = asyncio.Semaphore(rate_limit)
     data = data['data']
-    dir_name = '.'.join(
-        ['NEON', data['productCode'], data['siteCode'], data['month'], data['release']])
-    zip_dir_path = os.path.join(files_to_stack_path, f'{dir_name}')
-    if not os.path.isdir(zip_dir_path):
-        os.mkdir(zip_dir_path)
 
     d_urls = [f['url'] for f in data["files"]]
     sizes = [f['size'] for f in data["files"]]
     f_names = [f['name'] for f in data["files"]]
-    f_paths = [pjoin(zip_dir_path, name) for name in f_names]
-    month = [data['month']]
+    if data_type == "vst":
+        dir_name = '.'.join([
+            'NEON', data['productCode'], data['siteCode'], data['month'], data['release']
+        ])
+        zip_dir_path = os.path.join(files_to_stack_path, f'{dir_name}')
+        if not os.path.isdir(zip_dir_path):
+            os.mkdir(zip_dir_path)
+        f_paths = [pjoin(zip_dir_path, name) for name in f_names]
+    else:
+        f_paths = []
+        zip_dir_path = os.path.join(files_to_stack_path, data['productCode'])
+        if not os.path.isdir(zip_dir_path):
+            os.mkdir(zip_dir_path)
+        for i in range(len(d_urls)):
+            split_path = d_urls[i].split('/')
+            dir_path = '/'.join(split_path[4:len(split_path) - 1])
+            save_dir_path = pjoin(zip_dir_path, dir_path)
+            if not os.path.exists(save_dir_path):
+                os.makedirs(save_dir_path)
+            f_paths.append(os.path.join(save_dir_path, f_names[i]))
+
     zip_url = zip(d_urls, f_paths, sizes)
     async with aiohttp.ClientSession() as session:
         tasks = []
         for url, name, sz in zip_url:
-            task = asyncio.create_task(_download(session, url, name, sem, month, sz))
+            task = asyncio.create_task(_download(session, url, name, sem, sz))
             tasks.append(task)
 
         await asyncio.gather(*tasks)
@@ -94,7 +113,7 @@ async def _fetcher(data, rate_limit, headers, files_to_stack_path="filesToStack"
 async def vst_fetcher(item, rate_limit, headers, files_to_stack_path="filesToStack"):
     """Vst fetcher gets the urls for the files of vst data."""
     data = requests.get(item).json()
-    await _fetcher(data, rate_limit, headers, files_to_stack_path)
+    await _fetcher(data, rate_limit, headers, files_to_stack_path, "vst")
 
 
 def fetcher(batch, data_type, rate_limit, headers, files_to_stack_path):
@@ -103,7 +122,8 @@ def fetcher(batch, data_type, rate_limit, headers, files_to_stack_path):
         if data_type == 'vst':
             asyncio.run(vst_fetcher(batch, rate_limit, headers, files_to_stack_path))
         elif data_type == 'aop':
-            asyncio.run(_fetcher(batch, rate_limit, headers, files_to_stack_path))
+            asyncio.run(
+                _fetcher(batch, rate_limit, headers, files_to_stack_path, data_type))
 
     except Exception as e:
         print(f"Error processing URLs: {e}")

neonwranglerpy/lib/crop_plot_data.py

@@ -9,8 +9,9 @@
 
 def list_files(path):
     """List all the files in a path of given format."""
-    mask = path + '/**/*.[th][i5]'
-    files = glob(mask, recursive=True)
+    mask_tif = path + "/**/*.tif"
+    mask_h5 = path + "/**/*.h5"
+    files = glob(mask_tif, recursive=True) + glob(mask_h5, recursive=True)
     return files
 
 
@@ -23,12 +24,11 @@ def crop_data_to_plot(plt,
                       parallelized=False,
                       savepath=""):
     """Create shapefiles out of a vegetation structure data with lat/lon coordinates."""
-    dataset_path = os.path.normpath(dataset_path)
+    dataset_path = os.path.abspath(dataset_path)
     full_files = list_files(dataset_path)
     files = [file for file in full_files if dpID in file]
     files = [file for file in files if str(target_year) in file]
 
-    # TODO: add check if files for targeted year and product is not present
     plots = plt[['plotID', 'subplotID', 'siteID', 'utmZone', 'easting', 'northing']]
 
     plots = plots.groupby(['plotID', 'subplotID', 'siteID',

neonwranglerpy/lib/retrieve_aop_data.py

@@ -23,10 +23,10 @@ def retrieve_aop_data(data, year=2019, dpID=['DP3.30006.001'], savepath=""):
     """
     coords_for_tiles = data[['plotID', 'siteID', 'utmZone', 'easting', 'northing']]
     # get tiles dimensions
-    coords_for_tiles['easting'] = (coords_for_tiles[['easting']] /
-                                   1000).astype(int) * 1000
-    coords_for_tiles['northing'] = (coords_for_tiles[['northing']] /
-                                    1000).astype(int) * 1000
+    coords_for_tiles['easting'] = (coords_for_tiles[['easting']] / 1000).astype(
+        int, errors='ignore') * 1000
+    coords_for_tiles['northing'] = (coords_for_tiles[['northing']] / 1000).astype(
+        int, errors='ignore') * 1000
     # if there are more than 1 row, drop duplicates
     if coords_for_tiles.easting.shape[0] > 1:
         # drop duplicates values

neonwranglerpy/utilities/byTileAOP.py

@@ -77,9 +77,9 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None):
     if not len(month_urls):
         print(f"There is no data for site {site} and year {year}")
 
-    if isinstance(easting, (int, float)):
-        easting = [easting]
-        northing = [northing]
+    # if isinstance(easting, (int, float)):
+    #     easting = [easting]
+    #     northing = [northing]
     # convert the easting and northing for BLAN site
     if site == "BLAN":
         if isinstance(easting, (int, list)):
@@ -124,7 +124,7 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None):
     tile_northing = np.floor(northing / 1000).astype(int) * 1000
 
     file_urls = get_tile_urls(month_urls, tile_easting, tile_northing)
-    print(f"Tiles Found for Remote Sensing Data: {len(file_urls)}")
+    # print(f"Tiles Found for Remote Sensing Data: {len(file_urls)}")
     if not savepath:
         savepath = os.path.normpath(os.path.join(os.getcwd(), dpID))
     else:
@@ -133,14 +133,14 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None):
     if not os.path.isdir(savepath):
         os.makedirs(savepath)
 
-    files_to_stack_path = os.path.join(savepath, "filesToStack")
-    if not os.path.isdir(files_to_stack_path):
-        os.mkdir(files_to_stack_path)
+    # files_to_stack_path = os.path.join(savepath, "filesToStack")
+    # if not os.path.isdir(files_to_stack_path):
+    #     os.mkdir(files_to_stack_path)
 
-    if files_to_stack_path:
+    if savepath:
         fetcher.run_threaded_batches(file_urls,
                                      'aop',
                                      rate_limit=2,
                                      headers=None,
-                                     savepath=files_to_stack_path)
+                                     savepath=savepath)
     return savepath

requirements.txt

@@ -1,17 +1,18 @@
+deepforest
 flake8-docstrings
 geopandas
+h5py
+laspy
+laspy[lazrs,laszip]
+lazrs
+numpy
+opencv-python
 pandas
 pytest
 pytest-cov
+rasterio
 requests
+sphinx-press-theme
 toml
 tox
 yapf
-sphinx-press-theme
-laspy
-lazrs
-opencv-python
-numpy
-rasterio
-deepforest
-laspy[lazrs,laszip]

tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/L3/DiscreteLidar/CanopyHeightModelGtif/NEON_D16_ABBY_DP3_559000_5070000_CHM.tif

@@ -0,0 +1,179 @@
+<kml>
+ <Document>
+  <Style id="yellow">
+   <LineStyle>
+   <color>ff00ffff</color>
+   <width>1</width>
+   </LineStyle>
+  </Style>
+  <Style id="red">
+   <LineStyle>
+   <color>ffff0000</color>
+   <width>2</width>
+   </LineStyle>
+  </Style>
+  <Style id="green">
+   <LineStyle>
+   <color>ff00ff00</color>
+   <width>1</width>
+   </LineStyle>
+  </Style>
+  <Placemark>
+  <styleUrl>#yellow</styleUrl>
+   <MultiGeometry>
+   <LineString>
+    <coordinates>
+-122.23238135,45.78998225,472.42
+-122.23251581,45.78998319,502.60
+-122.23293157,45.78998597,495.12
+-122.23323219,45.78998800,528.25
+-122.23336215,45.78998883,533.17
+-122.23352100,45.78998984,529.48
+-122.23391686,45.78999247,536.98
+-122.23418289,45.78999436,550.95
+-122.23446446,45.78999622,557.39
+-122.23488887,45.78999910,579.25
+-122.23518130,45.79000103,596.16
+-122.23558094,45.79000370,596.65
+-122.23583161,45.79000538,607.72
+-122.23637528,45.79000902,614.77
+-122.23667629,45.79001104,630.26
+-122.23691821,45.79001267,627.68
+-122.23746663,45.79001634,638.23
+-122.23787007,45.79001903,647.24
+-122.23829064,45.79002180,649.80
+-122.23889580,45.79002587,642.38
+-122.23941458,45.79002932,647.60
+-122.24003466,45.79003344,620.49
+-122.24060095,45.79003719,602.50
+-122.24072702,45.79003803,616.02
+-122.24081019,45.79003854,616.17
+-122.24090217,45.79003900,630.50
+-122.24090741,45.79003863,605.05
+-122.24090941,45.79003769,619.30
+-122.24090961,45.79003546,624.19
+-122.24091087,45.78994774,632.77
+-122.24091276,45.78981042,627.01
+-122.24091613,45.78956400,629.13
+-122.24091874,45.78937174,622.10
+-122.24092179,45.78914701,649.52
+-122.24092677,45.78878024,663.03
+-122.24092683,45.78877532,662.99
+-122.24092733,45.78873915,662.55
+-122.24093202,45.78839299,670.26
+-122.24093390,45.78825512,658.97
+-122.24093661,45.78805501,677.56
+-122.24093793,45.78795761,671.62
+-122.24094121,45.78771630,697.13
+-122.24094387,45.78751962,696.45
+-122.24094775,45.78723222,691.93
+-122.24095073,45.78701475,716.12
+-122.24095427,45.78675446,739.77
+-122.24095765,45.78650539,741.78
+-122.24095810,45.78647169,740.04
+-122.24095917,45.78639010,728.65
+-122.24096435,45.78600930,755.07
+-122.24096779,45.78575778,774.91
+-122.24096874,45.78568832,774.25
+-122.24096986,45.78560544,753.52
+-122.24097462,45.78525376,772.82
+-122.24097634,45.78512721,795.48
+-122.24098101,45.78478381,801.61
+-122.24098664,45.78436899,819.95
+-122.24098998,45.78411981,819.61
+-122.24099274,45.78391684,831.97
+-122.24099807,45.78352698,840.98
+-122.24099995,45.78338830,840.10
+-122.24100301,45.78316303,845.75
+-122.24100597,45.78294461,851.62
+-122.24101049,45.78261037,830.82
+-122.24101301,45.78242328,831.80
+-122.24101683,45.78214237,827.47
+-122.24102209,45.78175510,845.19
+-122.24102355,45.78164921,840.07
+-122.24102398,45.78161755,838.33
+-122.24102423,45.78159943,845.91
+-122.24102809,45.78131481,832.99
+-122.24102980,45.78118832,825.68
+-122.24103104,45.78109728,840.28
+-122.24103152,45.78105487,831.92
+-122.24103161,45.78104560,830.48
+-122.24103108,45.78104047,834.11
+-122.24103014,45.78103998,837.98
+-122.24101558,45.78103942,835.67
+-122.24100614,45.78103927,832.36
+-122.24084005,45.78103811,844.06
+-122.24046370,45.78103560,842.94
+-122.23990230,45.78103188,820.38
+-122.23967972,45.78103041,832.94
+-122.23910563,45.78102657,823.25
+-122.23871273,45.78102397,823.67
+-122.23841314,45.78102196,806.22
+-122.23814754,45.78102018,811.12
+-122.23750748,45.78101593,804.22
+-122.23735086,45.78101489,806.80
+-122.23683190,45.78101142,782.21
+-122.23654231,45.78100949,775.20
+-122.23599912,45.78100588,769.10
+-122.23558946,45.78100310,761.54
+-122.23505218,45.78099952,753.16
+-122.23487799,45.78099834,760.59
+-122.23446789,45.78099560,748.59
+-122.23445889,45.78099554,761.53
+-122.23389307,45.78099184,760.91
+-122.23373284,45.78099073,745.26
+-122.23346330,45.78098889,744.87
+-122.23342892,45.78098979,743.23
+-122.23342211,45.78099023,731.96
+-122.23339356,45.78099616,732.85
+-122.23337332,45.78101520,731.03
+-122.23337611,45.78113084,732.19
+-122.23337897,45.78121380,733.23
+-122.23338068,45.78159885,735.48
+-122.23337276,45.78171767,734.30
+-122.23336741,45.78183789,733.26
+-122.23335587,45.78205088,731.29
+-122.23334934,45.78225623,729.59
+-122.23334104,45.78236337,728.07
+-122.23326144,45.78280977,709.15
+-122.23314301,45.78324149,682.22
+-122.23308463,45.78343722,666.49
+-122.23308419,45.78346469,666.52
+-122.23312231,45.78388315,676.06
+-122.23312488,45.78390203,676.86
+-122.23317638,45.78427732,688.30
+-122.23320198,45.78454484,693.25
+-122.23321142,45.78467600,695.20
+-122.23322496,45.78479141,697.67
+-122.23324568,45.78503386,702.08
+-122.23324782,45.78505210,702.60
+-122.23325844,45.78545998,703.32
+-122.23325252,45.78553470,701.97
+-122.23321130,45.78598143,690.07
+-122.23319094,45.78613482,684.61
+-122.23318346,45.78620583,682.11
+-122.23315809,45.78642558,675.12
+-122.23313964,45.78656417,669.62
+-122.23312010,45.78675973,663.72
+-122.23306359,45.78712179,648.35
+-122.23298096,45.78754079,625.42
+-122.23290970,45.78781744,606.94
+-122.23288090,45.78791713,598.96
+-122.23284534,45.78803543,589.93
+-122.23278066,45.78826695,573.07
+-122.23277595,45.78828580,571.45
+-122.23272403,45.78848753,558.93
+-122.23265525,45.78876980,542.39
+-122.23260206,45.78897986,530.45
+-122.23258008,45.78908361,525.32
+-122.23250474,45.78943420,506.75
+-122.23245574,45.78963089,494.85
+-122.23242285,45.78975558,487.07
+-122.23238606,45.78992532,475.85
+-122.23238135,45.78998225,472.42
+    </coordinates>
+   </LineString>
+  </MultiGeometry>
+  </Placemark>
+ </Document>
+</kml>

tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/kmls/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.kml

@@ -0,0 +1 @@
+PROJCS["WGS 84 / UTM 10N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-123],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["Meter",1]]