using Kfold cross-Validation on Experiences needs a seperate benchmark ?

[AymenTlili131]

I'm trying to work on a smaller portion of one of 2 SplitMNIST experiences .
for this end I wanted to use stratified Kfold splits to get a hold of a test set with the desired smaller number of instances and train/fine tune a model on it. but to my understanding CL_Strategy.train method takes an experience as arguement so I'd have to create a custom benchmark for this use case ?
Here's a code sample , I feel i'm so close yet so far to get the default stream to work as intended , any tips or guidance on which direction I should pursue ?

class CNN(nn.Module):
    def __init__(
        self,
        channels_in,
        nlin="leakyrelu",
        dropout=0.0,
        init_type="uniform",
    ):
        super().__init__()
        # init module list
        self.module_list = nn.ModuleList()
        ### ASSUMES 28x28 image size
        ## compose layer 1
        self.module_list.append(nn.Conv2d(channels_in, 8, 5))
        self.module_list.append(nn.MaxPool2d(2, 2))
        self.module_list.append(self.get_nonlin(nlin))
        # apply dropout
        if dropout > 0:
            self.module_list.append(nn.Dropout(dropout))
        ## compose layer 2
        self.module_list.append(nn.Conv2d(8, 6, 5))
        self.module_list.append(nn.MaxPool2d(2, 2))
        self.module_list.append(self.get_nonlin(nlin))
        ## add dropout
        if dropout > 0:
            self.module_list.append(nn.Dropout(dropout))
        ## compose layer 3
        self.module_list.append(nn.Conv2d(6, 4, 2))
        self.module_list.append(self.get_nonlin(nlin))
        ## add flatten layer
        self.module_list.append(nn.Flatten())
        ## add linear layer 1
        self.module_list.append(nn.Linear(3 * 3 * 4, 20))
        self.module_list.append(self.get_nonlin(nlin))
        ## add dropout
        if dropout > 0:
            self.module_list.append(nn.Dropout(dropout))
        ## add linear layer 1
        self.module_list.append(nn.Linear(20, 10))

        ### initialize weights with se methods
        self.initialize_weights(init_type)

    def initialize_weights(self, init_type):
        # print("initialze model")
        for m in self.module_list:
            if type(m) == nn.Linear or type(m) == nn.Conv2d:
                if init_type == "xavier_uniform":
                    torch.nn.init.xavier_uniform_(m.weight)
                if init_type == "xavier_normal":
                    torch.nn.init.xavier_normal_(m.weight)
                if init_type == "uniform":
                    torch.nn.init.uniform_(m.weight)
                if init_type == "normal":
                    torch.nn.init.normal_(m.weight)
                if init_type == "kaiming_normal":
                    torch.nn.init.kaiming_normal_(m.weight)
                if init_type == "kaiming_uniform":
                    torch.nn.init.kaiming_uniform_(m.weight)
                # set bias to some small non-zero value
                m.bias.data.fill_(0.01)

    def get_nonlin(self, nlin):
        # apply nonlinearity
        if nlin == "leakyrelu":
            return nn.LeakyReLU()
        if nlin == "relu":
            return nn.ReLU()
        if nlin == "tanh":
            return nn.Tanh()
        if nlin == "sigmoid":
            return nn.Sigmoid()
        if nlin == "silu":
            return nn.SiLU()
        if nlin == "gelu":
            return nn.GELU()
        if nlin == "ORU":
            return ORU()
        if nlin == "ERU":
            return ERU()

    def forward(self, x):
        # forward prop through module_list
        for layer in self.module_list:
            x = layer(x)
        return x

    def forward_activations(self, x):
        # forward prop through module_list
        activations = []
        for layer in self.module_list:
            x = layer(x)
            if (
                isinstance(layer, nn.Tanh)
                or isinstance(layer, nn.Sigmoid)
                or isinstance(layer, nn.ReLU)
                or isinstance(layer, nn.LeakyReLU)
                or isinstance(layer, nn.SiLU)
                or isinstance(layer, nn.GELU)
                or isinstance(layer, ORU)
                or isinstance(layer, ERU)
            ):
                activations.append(x)
        return x, activations

model = CNN(1)
model=model.to(device)

optimizer = Adam(model.parameters(), lr=0.05)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-3, max_lr=0.1, step_size_up=1, mode="triangular2", cycle_momentum=False)


checkpoint_pluginCum = CheckpointPlugin(
    FileSystemCheckpointStorage(
        directory='./checkpoints/Cumulative/{}/Leakyrelu/uniform'.format(ListExperiences[0][:5]),
    ),
    map_location=device
)

criterion = CrossEntropyLoss()
cl_strategy = Cumulative(
    model, optimizer, criterion,
    train_mb_size=150, train_epochs=4, eval_mb_size=150, evaluator=eval_plugin,eval_every=1,device=device,plugins=[checkpoint_pluginCum]
)

  skf = StratifiedKFold(n_splits=20,random_state=1234,shuffle=True)
  split_mnist = SplitMNIST(n_experiences=2, fixed_class_order=[0,1,2,3,4],shuffle=False)
  print(len(split_mnist.train_stream[0].dataset))
  print('Starting experiment...')
  skf.get_n_splits(split_mnist.train_stream)
  print(skf)
  c=0
  for j, (train_index, test_index) in enumerate(skf.split(split_mnist.train_stream[0].dataset,split_mnist.train_stream[0].dataset.targets)):
      print(f"Fold {j}:")
      print(f"  Train: index={len(train_index)}")
      print(f"  Test:  index={len(test_index)}")
      cl_strategy.train(split_mnist.train_stream[0].dataset[test_index])
            
      print('Training completed')

      res.append(cl_strategy.eval(split_mnist.test_stream))
      c=+1
      
  print(c)

The error raised when passing test_index to split_mnist.train_stream[0].dataset is TypeError: only integer scalar arrays can be converted to a scalar index .
Thank avalanche team

[AymenTlili131]

Using a TensorDataset with the indices indicated by the stratified Kfold should resolve this issue .

[AymenTlili131]

going back to the creation of the TensorDataset , I tried creating the lists of tensors as advised by the api by running

  skf = StratifiedKFold(n_splits=100,random_state=1234,shuffle=True)
  split_mnist = SplitMNIST(n_experiences=2, fixed_class_order=ListExperiences[class_order_idx],shuffle=False)
  print(len(split_mnist.train_stream[0].dataset))
  print('Starting experiment...')
  skf.get_n_splits(split_mnist.train_stream)
  print(skf)
  c=0
  L_images1=torch.empty((1,28,28))
  L_labels1=torch.empty((1))
  L_images2=torch.empty((1,28,28))
  L_labels2=torch.empty((1))
  res=[]
  for j, (train_index, test_index) in enumerate(skf.split(split_mnist.train_stream[0].dataset,split_mnist.train_stream[0].dataset.targets)):
      print(f"Fold {j}:")
      print(f"  Train: index={len(train_index)}")
      print(f"  Test:  index={len(test_index)}")
      if j==0 :
        for img_idx in test_index:
          L_images1=torch.vstack((L_images1,split_mnist.train_stream[0].dataset[img_idx][0]))
          L_labels1=torch.vstack((L_labels1,torch.tensor(split_mnist.train_stream[0].dataset[img_idx][1])))
      if j==1 :
        for img_idx in test_index:
          L_images2=torch.vstack((L_images2,split_mnist.train_stream[0].dataset[img_idx][0]))
          L_labels2=torch.vstack((L_labels2,torch.tensor(split_mnist.train_stream[0].dataset[img_idx][1])))

      TN_Dataset=tensors_benchmark(((L_images1[1:],L_labels1[1:]),(L_images2[1:],L_labels2[1:])),((L_images1[1:],L_labels1[1:]),(L_images2[1:],L_labels2[1:])),task_labels=(0,0))

for creating the dataset works but when calling the stream for training

for experience in TN_Dataset.train_stream:
     
      cl_strategy.train(experience)
      
      print('Training completed')

      # assume you have the following dictionary
      print('Computing accuracy on the whole test set')
      res.append(cl_strategy.eval(split_mnist.test_stream))

I get a type error

----> 1 for experience in TN_Dataset.train_stream:
     2 
     3       cl_strategy.train(experience)
     4 
     5       print('Training completed')

6 frames
[/usr/lib/python3.10/_collections_abc.py](https://localhost:8080/#) in __iter__(self)
  1041         try:
  1042             while True:
-> 1043                 v = self[i]
  1044                 yield v
  1045                 i += 1

[/usr/local/lib/python3.10/dist-packages/avalanche/benchmarks/scenarios/classification_scenario.py](https://localhost:8080/#) in __getitem__(self, exp_idx)
   662             if exp_idx < len(self):
   663                 if self.slice_ids is None:
--> 664                     return self.benchmark.experience_factory(self, exp_idx)
   665                 else:
   666                     return self.benchmark.experience_factory(

[/usr/local/lib/python3.10/dist-packages/avalanche/benchmarks/scenarios/classification_scenario.py](https://localhost:8080/#) in __init__(self, origin_stream, current_experience)
   945             classes_seen_so_far,
   946             future_classes,
--> 947         ) = origin_stream.benchmark.get_classes_timeline(
   948             current_experience, stream=origin_stream.name
   949         )

[/usr/local/lib/python3.10/dist-packages/avalanche/benchmarks/scenarios/classification_scenario.py](https://localhost:8080/#) in get_classes_timeline(self, current_experience, stream)
   333         """
   334 
--> 335         class_set_current_exp = self.classes_in_experience[stream][
   336             current_experience
   337         ]

[/usr/local/lib/python3.10/dist-packages/avalanche/benchmarks/scenarios/classification_scenario.py](https://localhost:8080/#) in __getitem__(self, exp_id)
   764 
   765     def __getitem__(self, exp_id) -> Set[int]:
--> 766         return manage_advanced_indexing(
   767             exp_id,
   768             self._get_single_exp_classes,

[/usr/local/lib/python3.10/dist-packages/avalanche/benchmarks/utils/dataset_utils.py](https://localhost:8080/#) in manage_advanced_indexing(idx, single_element_getter, max_length, collate_fn)
   357     elements = []
   358     for single_idx in indexes_iterator:
--> 359         single_element = single_element_getter(int(single_idx))
   360         elements.append(single_element)
   361 

[/usr/local/lib/python3.10/dist-packages/avalanche/benchmarks/scenarios/classification_scenario.py](https://localhost:8080/#) in _get_single_exp_classes(self, exp_id)
   783         if targets is None:
   784             return None
--> 785         return set(targets)
   786 
   787     @staticmethod

TypeError: unhashable type: 'list'

[AymenTlili131]

Hi again , you guys are probably busy with the 0.4 roadmap but would be great if I got an answer .
The tensorBenchmark wasn't successful so I tried to create a path benchmark using the images indixed by the stratified kfold algorithm.
This saves the images in a structure fold number/class/index in orifinal splitMnist

L_images=[]
L_labels=[]
L_images1=[]
L_labels1=[]
res=[]

for j, (train_index, test_index) in enumerate(skf.split(split_mnist.train_stream[0].dataset,split_mnist.train_stream[0].dataset.targets)):
    if not(os.path.isdir(".//data//SplitMnist//fold {}".format(j))):
        os.mkdir(".//data//SplitMnist//fold {}".format(j))
    for k in range(10):
        if not(os.path.isdir(".//data//SplitMnist//fold {}//{}".format(j,k))):
            os.mkdir(".//data//SplitMnist//fold {}//{}".format(j,k))
    print(f"Fold {j}:")
    print(f"  Train: index={len(train_index)}")
    print(f"  Test:  index={len(test_index)}")
    for img_idx in tqdm(range(len(test_index))):
        L_images1.append(split_mnist.train_stream[0].dataset[test_index[img_idx]][0])
        L_labels1.append(split_mnist.train_stream[0].dataset[test_index[img_idx]][1])
    for img_idx in tqdm(range(len(test_index))):
        save_image(L_images1[img_idx], ".//data//SplitMnist//fold {}//{}//{}.png".format(j,L_labels1[img_idx],test_index[img_idx]))
        #print("fold {}//{}//{}.png saved".format(j,L_labels1[img_idx],test_index[img_idx]))
    L_images.append(L_images1)
    L_labels.append(L_labels1) 

followed by this code which creates the list of instance tuples (path,label) pertaining to each fold as a seperate experience then combines all folds to be inside L_experiences .

dirpath=".[replace this with absolute path]/data/SplitMnist"
L_experiences=[]
for fold in os.listdir( dirpath ) :
    if fold[0]=='f':
        L_paths=[]
        L_labels=[]
        experience=[]
        for classe in os.listdir( dirpath+"//"+fold ) :
            if len(os.listdir( dirpath+"//"+fold+"//"+classe))>0 :
                for image in os.listdir( dirpath+"//"+fold+"//"+classe) :
                    #print(dirpath+"//"+fold+"//"+classe+"//"+image)
                    L_paths.append(dirpath+"//"+fold+"//"+classe+"//"+image)
                    L_paths.append(int(classe))
                    instance_tuple= dirpath+"//"+fold+"//"+classe+"//"+image, classe
                    experience.append(instance_tuple)
        #print(experience)
        print(fold ,len(experience))
        L_experiences.append(experience)
    #print(len(L_paths))
    #print(len(L_labels))
len(L_experiences)

following the 0 to hero guide , this is my generic scenerio

generic_scenario = paths_benchmark(
   L_experiences,
   [L_experiences[0]], 
   task_labels= list(range(20)), #20 is the number of test folds I created
   train_transform=ToTensor(),
   train_target_transform=ToTensor(),
   eval_transform=ToTensor()
)

I go back and try to run this as on a cumulative strategy

skf = StratifiedKFold(n_splits=20,random_state=1234,shuffle=True)
split_mnist = SplitMNIST(n_experiences=2, fixed_class_order=[0,1,2,3,4,5,6,7,8,9],shuffle=False)
print(len(split_mnist.train_stream[0].dataset))
print('Starting experiment...')
skf.get_n_splits(split_mnist.train_stream)


model = CNN(1,nlin=list(i)[0],init_type=list(i)[1])
model=model.to(device)

optimizer = Adam(model.parameters(), lr=0.05)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-3, max_lr=0.1, step_size_up=1, mode="triangular2", cycle_momentum=False)


checkpoint_pluginCum = CheckpointPlugin(
    FileSystemCheckpointStorage(
        directory='./fientuning/Cumulative/{}/Leakyrelu/uniform'.format(ListExperiences[0][:5]),
    ),
    map_location=device
)

criterion = CrossEntropyLoss()
cl_strategy = Cumulative(
    model, optimizer, criterion,
    train_mb_size=150, train_epochs=4, eval_mb_size=150, evaluator=eval_plugin,eval_every=1,device=device,plugins=[checkpoint_pluginCum]
)
#for class_order_idx in range(len(ListExperiences)):
#    skf = StratifiedKFold(n_splits=100,random_state=1234,shuffle=True)
#    split_mnist = SplitMNIST(n_experiences=2, fixed_class_order=ListExperiences[class_order_idx],shuffle=False)
#    print(len(split_mnist.train_stream[0].dataset))
#    print('Starting experiment...')
#    skf.get_n_splits(split_mnist.train_stream)
print(skf)
c=0
res=[]

for experience in generic_scenario.train_stream:

    cl_strategy.train(experience)

    print('Training completed')

    # assume you have the following dictionary
    print('Computing accuracy on the whole test set')
    res.append(cl_strategy.eval(generic_scenario))

    print('Training completed')

but fail with typeError List object has no attribute ‘to’ in

~/anaconda3/lib/python3.9/site-packages/avalanche/training/templates/base_sgd.py in _unpack_minibatch(self)
                 self.mbatch = list(self.mbatch)
 384         for i in range(len(self.mbatch)):
 385             self.mbatch[i] = self.mbatch[i].to(self.device)

I tried moving everything to cpu including the device but I receive Runtime errors about the input shape not matching of the CNN's .

any help replicating my code or suggesting a turnaround would be great while some us still have to work with the soon to be deprecated generic benchmarks .
In the meantime i'll create a custom dataloader seperate from avalanche's and try to fine tune there