Fixed supperted type check in decode_method=data

Author: activesoull

deeplake/enterprise/convert_to_libdeeplake.py

@@ -169,10 +169,17 @@ def dataset_to_libdeeplake(hub2_dataset: Dataset):
     path: str = hub2_dataset.path
 
     token = (
-        hub2_dataset.client.get_token()
-        if (hub2_dataset.token is None or hub2_dataset._token == "")
-        and hub2_dataset.client
-        else hub2_dataset.token
+        hub2_dataset.token
+        if hasattr(hub2_dataset, "token") and hub2_dataset.token is not None
+        else (
+            getattr(hub2_dataset, "_token", None)
+            if hasattr(hub2_dataset, "_token") and hub2_dataset._token != ""
+            else (
+                hub2_dataset.client.get_token()
+                if hasattr(hub2_dataset, "client") and hub2_dataset.client
+                else ""
+            )
+        )
     )
 
     if hub2_dataset.libdeeplake_dataset is not None:
@@ -239,15 +246,18 @@ def dataset_to_libdeeplake(hub2_dataset: Dataset):
         hub2_dataset.libdeeplake_dataset = libdeeplake_dataset
 
     assert libdeeplake_dataset is not None
-    if hasattr(hub2_dataset.storage, "cache_size"):
-        libdeeplake_dataset._max_cache_size = max(
-            hub2_dataset.storage.cache_size, libdeeplake_dataset._max_cache_size
-        )
-    commit_id = hub2_dataset.pending_commit_id
-    libdeeplake_dataset.checkout(commit_id)
-    slice_ = hub2_dataset.index.values[0].value
-    if slice_ != slice(None):
-        if isinstance(slice_, tuple):
-            slice_ = list(slice_)
-        libdeeplake_dataset = libdeeplake_dataset[slice_]
+    try:
+        if hasattr(hub2_dataset.storage, "cache_size"):
+            libdeeplake_dataset._max_cache_size = max(
+                hub2_dataset.storage.cache_size, libdeeplake_dataset._max_cache_size
+            )
+        commit_id = hub2_dataset.pending_commit_id
+        libdeeplake_dataset.checkout(commit_id)
+        slice_ = hub2_dataset.index.values[0].value
+        if slice_ != slice(None):
+            if isinstance(slice_, tuple):
+                slice_ = list(slice_)
+            libdeeplake_dataset = libdeeplake_dataset[slice_]
+    except INDRA_API.api.NoStorageDatasetViewError:  # type: ignore
+        pass
     return libdeeplake_dataset

deeplake/integrations/pytorch/common.py

@@ -102,7 +102,8 @@ def check_tensors(dataset, tensors, verbose=True):
                 f" tensor."
             )
         meta = tensor.meta
-        if meta.sample_compression in supported_image_compressions:
+        comp = meta.sample_compression
+        if comp and meta.sample_compression in supported_image_compressions:
             jpeg_png_compressed_tensors.append(tensor_name)
         elif meta.htype == "json":
             json_tensors.append(tensor_name)
@@ -218,7 +219,11 @@ def get_htype_ndim_tensor_info_dicts(dataset, data_tensors, tensor_info_tensors)
         htype_dict[tensor_name] = tensor.htype
         ndim_dict[tensor_name] = tensor.ndim
         if tensor_name in tensor_info_tensors:
-            tensor_info_dict[tensor_name] = tensor.info._info
+            if "_info" in tensor.info:
+                tensor_info_dict[tensor_name] = tensor.info._info
+            else:
+                tensor_info_dict[tensor_name] = dict(tensor.info)
+
     return htype_dict, ndim_dict, tensor_info_dict
 
 

deeplake/util/dataset.py

@@ -7,6 +7,8 @@ def try_flushing(ds):
         ds.storage.flush()
     except ReadOnlyModeError:
         pass
+    except:
+        pass
 
 
 def map_tensor_keys(dataset, tensor_keys: Optional[Sequence[str]] = None) -> List[str]: