Warning for Evaluation Metric

[nina-wiegers]

Hello everyone,

in the training process i get the following warning:

UserWarning: Evaluation stream is not equal to the complete test stream. This may result in inconsistent metrics. Use at your own risk.

My training loop looks like this:

from avalanche.training.storage_policy import ParametricBuffer, RandomExemplarsSelectionStrategy, ClassBalancedBuffer
from avalanche.training.plugins import ReplayPlugin

# freigeben von Speicherplatz von der GPU
torch.cuda.empty_cache()

# scenario
pvc_benchmark = nc_benchmark(ava_train_pcv, ava_test_pcv, n_experiences=2, one_dataset_per_exp=False, shuffle=False, \
                        task_labels=True, per_exp_classes={0: 2})

# DEFINE THE EVALUATION PLUGIN and LOGGERS
# The evaluation plugin manages the metrics computation.
# It takes as argument a list of metrics, collectes their results and returns
# them to the strategy it is attached to.

# Device config
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# model definition
ic_model = IncrementalCNN(576)

# log to Tensorboard
tb_logger = TensorboardLogger()

# log to text file
text_logger = TextLogger(open('log.txt', 'a'))

# print to stdout
interactive_logger = InteractiveLogger()

print("Classes in this Scenario: ", pvc_benchmark.n_classes)

eval_plugin = EvaluationPlugin(
    accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True),
    loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
    timing_metrics(epoch=True, epoch_running=True, experience=True),
    confusion_matrix_metrics(num_classes=pvc_benchmark.n_classes, save_image=False, stream=True),
    forgetting_metrics(experience=True, stream=True),
    loggers=[interactive_logger, text_logger, tb_logger],
    benchmark=pvc_benchmark
)

# epoch list for individual epochs for experience
EpochList = [60, 5]

# set optimizer and scheduler
optimizer = torch.optim.Adam(ic_model.parameters(), lr=0.001)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1/8, gamma=0.9)

# CREATE THE STRATEGY INSTANCE (NAIVE)
"""
cl_strategy = Naive(
    ic_model, optimizer,
    CrossEntropyLoss(), train_mb_size=30, train_epochs=60, eval_mb_size=30, device=device,
    evaluator=eval_plugin, eval_every=0, plugins=[LRSchedulerPlugin(scheduler), TaskSpecificEpochNumberPlugin(EpochList)])
"""

"""
cl_strategy = Cumulative(ic_model, optimizer,
    CrossEntropyLoss(), train_mb_size=30, train_epochs=60, eval_mb_size=30, #device=device,
    evaluator=eval_plugin, eval_every=0, plugins=[LRSchedulerPlugin(scheduler), TaskSpecificEpochNumberPlugin(EpochList)])
"""


#"""
# choose buffer strategy for replay
storage_p = ParametricBuffer(
    max_size=50,
    groupby='class',
    selection_strategy=RandomExemplarsSelectionStrategy()
)

storage_c = ClassBalancedBuffer(max_size=400, adaptive_size = True)

replay = ReplayPlugin(mem_size = 400, storage_policy = storage_c)

# CREATE REPLAY STRATEGY INSTANCE
cl_strategy = Replay(ic_model, optimizer, CrossEntropyLoss(), 
    mem_size=400, train_mb_size=30, train_epochs=60, eval_mb_size=30, device=device,
    evaluator=eval_plugin, eval_every=0, plugins=[LRSchedulerPlugin(scheduler), replay,
    TaskSpecificEpochNumberPlugin(EpochList)])
#"""

"""
cl_strategy = GDumb(ic_model, optimizer,
    CrossEntropyLoss(), train_mb_size=30, train_epochs=60, eval_mb_size=30, #device=device,
    evaluator=eval_plugin, eval_every=0, plugins=[LRSchedulerPlugin(scheduler), TaskSpecificEpochNumberPlugin(EpochList),
                                                 GDumbPlugin(mem_size=40)])
"""

"""
cl_strategy = EWC(ic_model, optimizer, CrossEntropyLoss(), ewc_lambda=0.5, mode='separate', 
    keep_importance_data=True, train_mb_size=30, train_epochs=60, eval_mb_size=30, device=device,
    evaluator=eval_plugin, eval_every=20, plugins=[LRSchedulerPlugin(scheduler), TaskSpecificEpochNumberPlugin(EpochList)])
"""

# TRAINING LOOP
print('Starting experiment...')
results = []

# access current test stream images
test_stream = scenario.test_stream

# save prediction and ground truth for task 0
prediction_0 = []
ground_truth_0 = []
proba_0 = []

# save prediction and ground truth for task 1
prediction_1 = []
ground_truth_1 = []
proba_1 = []

for experience in pvc_benchmark.train_stream:
      print("Start of experience: ", experience.current_experience)
      print("Current Classes: ", experience.classes_in_this_experience)
      
      #ic_model.adaptation(experience.dataset)
      
      print("Size of old Train Dataset:", len(experience.dataset))
      
      # use own oversampling function
      experience.dataset = oversampling(experience.dataset)
      
      t = experience.task_label
      exp_id = experience.current_experience
      print('Task {} batch {} -> train'.format(t, exp_id))
      print('This batch contains', len(experience.dataset), 'patterns')
      
      print("Size of new Train Dataset:", len(experience.dataset))
  
      # start the train loop 
      cl_strategy.train(experience)
      print('Training completed')
      
      # get current test set and current trained net
      current_test_set = test_stream[experience.current_experience].dataset
      
      # save prediction and ground truth for every task
      if(current_test_set[0][2] == 0):
          y_gt, y_pred, proba = eval(ic_model, current_test_set)
          prediction_0 = y_pred
          ground_truth_0 = y_gt
          proba_0 = proba
      
      elif(current_test_set[0][2] == 1):
          y_gt, y_pred, proba = eval(ic_model, current_test_set)
          prediction_1 = y_pred
          ground_truth_1 = y_gt
          proba_1 = proba
          
  
      print('Computing accuracy on the whole test set')
      results.append(cl_strategy.eval(pvc_benchmark.test_stream))