fixed a bug

eth-easl · Jan 10, 2025 · 91a7054 · 91a7054
1 parent 4df0bba
commit 91a7054
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 1,129 deletions.
diff --git a/modyn/models/__init__.py b/modyn/models/__init__.py
@@ -6,7 +6,7 @@
 from .dlrm.dlrm import DLRM  # noqa: F401
 from .dummy.dummy import Dummy  # noqa: F401
 from .fmownet.fmownet import FmowNet  # noqa: F401
-from .gpt2.gpt2 import GPT2  # noqa: F401
+from .gpt2.gpt2 import gpt2  # noqa: F401
 from .resnet18.resnet18 import ResNet18  # noqa: F401
 from .resnet50.resnet50 import ResNet50  # noqa: F401
 from .resnet152.resnet152 import ResNet152  # noqa: F401

diff --git a/modyn/models/gpt2/gpt2.py b/modyn/models/gpt2/gpt2.py
@@ -14,18 +14,18 @@
 # as GPT2_Lora
 
 
-class GPT2:
+class gpt2:
     # pylint: disable-next=unused-argument
     def __init__(self, hparams: Any, device: str, amp: bool) -> None:
-        self.model = GPT2Modyn(hparams)
+        self.model = gpt2Modyn(hparams)
         self.model.to(device)
 
 
 # the following class is adapted from
 # torchvision https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py
 
 
-class GPT2Modyn(CoresetSupportingModule):
+class gpt2Modyn(CoresetSupportingModule):
     def __init__(self, hparams: Any) -> None:
         super().__init__()
 

diff --git a/modyn/storage/include/internal/grpc/storage_service_impl.hpp b/modyn/storage/include/internal/grpc/storage_service_impl.hpp
@@ -380,7 +380,7 @@ class StorageServiceImpl final : public modyn::storage::Storage::Service {
     // return millions of files
     const std::vector<int64_t> file_ids = get_file_ids(session, dataset_id, start_timestamp, end_timestamp);
     session.close();
-
+     
     if (file_ids.empty()) {
       SPDLOG_INFO("No files found for dataset {} with start_timestamp = {} and end_timestamp = {}", dataset_id,
                   start_timestamp, end_timestamp);
@@ -645,10 +645,11 @@ class StorageServiceImpl final : public modyn::storage::Storage::Service {
             response.mutable_labels()->Assign(sample_labels.begin() + static_cast<int64_t>(current_file_start_idx),
                                               sample_labels.begin() + static_cast<int64_t>(sample_idx));
           }
-
+          {
           const std::lock_guard<std::mutex> lock(writer_mutex);
           writer->Write(response);
-
+          }
+          current_file_id = sample_fileid;
           current_file_path = "",
           session << "SELECT path FROM files WHERE file_id = :file_id AND dataset_id = :dataset_id",
           soci::into(current_file_path), soci::use(current_file_id), soci::use(dataset_data.dataset_id);
@@ -671,6 +672,7 @@ class StorageServiceImpl final : public modyn::storage::Storage::Service {
       // Send leftovers
       const std::vector<uint64_t> file_indexes(sample_indices.begin() + static_cast<int64_t>(current_file_start_idx),
                                                sample_indices.end());
+
       const std::vector<std::vector<unsigned char>> data =
           file_wrapper->get_samples_from_indices(file_indexes, include_labels);
 
@@ -691,9 +693,9 @@ class StorageServiceImpl final : public modyn::storage::Storage::Service {
         response.mutable_labels()->Assign(sample_labels.begin() + static_cast<int64_t>(current_file_start_idx),
                                           sample_labels.end());
       }
-
+     {
       const std::lock_guard<std::mutex> lock(writer_mutex);
-      writer->Write(response);
+      writer->Write(response);}
     } catch (const std::exception& e) {
       SPDLOG_ERROR("Error in send_sample_data_for_keys_and_file: {}", e.what());
       SPDLOG_ERROR("Propagating error up the call chain to handle gRPC calls.");

diff --git a/modyn/storage/src/internal/database/grpc2/storage_pb2.py b/modyn/storage/src/internal/database/grpc2/storage_pb2.py