Integrates fetcher with byTileAop

Signed-off-by: nagesh bansal <[email protected]>

Author: Nageshbansal

neonwranglerpy/utilities/byTileAOP.py

@@ -11,7 +11,7 @@
 from neonwranglerpy.utilities.tools import get_api
 from neonwranglerpy.utilities.defaults import NEON_API_BASE_URL
 from neonwranglerpy.utilities.get_tile_urls import get_tile_urls
-
+import neonwranglerpy.fetcher.fetcher as fetcher
 
 def load_shared_flights():
     """Return the dataframe about the table types of Data Products."""
@@ -134,18 +134,10 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None):
     if not os.path.isdir(savepath):
         os.makedirs(savepath)
 
-    for i in range(len(file_urls)):
-        split_path = file_urls[i]['url'].split('/')
-        dir_path = '/'.join(split_path[4:len(split_path) - 1])
-        save_dir_path = os.path.join(savepath, dir_path)
-        save_file_path = os.path.join(save_dir_path, file_urls[i]['name'])
-        if not os.path.exists(save_dir_path):
-            os.makedirs(save_dir_path)
-
-        try:
-            file_path, _ = urlretrieve(file_urls[i]['url'], save_file_path)
-        except HTTPError as e:
-            print("HTTPError :", e)
-            return None
+    files_to_stack_path = os.path.join(savepath, "filesToStack")
+    if not os.path.isdir(files_to_stack_path):
+        os.mkdir(files_to_stack_path)
 
+    if files_to_stack_path:
+        fetcher.run_threaded_batches(file_urls, 'aop', rate_limit=2, headers=None, savepath=files_to_stack_path)
     return savepath

neonwranglerpy/utilities/get_tile_urls.py

@@ -12,22 +12,29 @@ def get_tile_urls(
     """Get tile urls."""
     file_urls = []
     for i in range(len(month_url)):
-        temp = get_api(month_url[i]).json()
+        temp = get_api(month_url[i]).json()['data']
 
         if not len(temp):
             print(f"No files found for site {temp['data']['siteCode']} and "
                   f"year {temp['data']['month']}.")
             continue
 
-        temp_ = temp['data']['files']
-        # temp_ = json.dumps(temp['data']['files'])
-        # df = pd.read_json(temp_)
-        # # get the files for easting and northing
-
-        # if easting is a signle value and northing is a single value
+        temp_ = temp['files']
+        dataSiteMonth = {
+            "data": {
+                "productCode": temp['productCode'],
+                "siteCode": temp['siteCode'],
+                "month": temp['month'],
+                "release": temp['release'],
+                "packages": temp['packages'],
+                "files": []
+            }
+        }
 
         if isinstance(easting.astype(str), str) and isinstance(northing.astype(str), str):
-            file_urls = [x for x in temp_ if f'_{easting}_{northing}' in x['name']]
+            dataSiteMonth['data']['files'] = [x for x in temp_ if f'_{easting}_{northing}' in x['name']]
+            file_urls.append(dataSiteMonth)
+
         elif isinstance(easting, np.ndarray) and isinstance(northing, np.ndarray):
             for j in range(len(easting)):
                 urls = [
@@ -44,6 +51,8 @@ def get_tile_urls(
 
                 if not len(urls):
                     print(f"no tiles found for {easting[j]} and {northing[j]}")
-                file_urls.extend(urls)
+                dataSiteMonth['data']['files'].append(urls)
+            file_urls.append(dataSiteMonth)
+
     print(f'{len(file_urls)} files found')
     return file_urls