Merge branch 'main' into robinholzi/feat/thesis-plotting

.github/workflows/workflow.yaml

@@ -56,7 +56,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup mamba
         uses: ./.github/actions/mamba
@@ -72,7 +72,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup mamba
         uses: ./.github/actions/mamba
@@ -220,7 +220,7 @@ jobs:
 
       - name: Upload HTML coverage report
         if: ${{ matrix.compiler.coverage && matrix.build-type == 'Debug' }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-results
           path: ${{github.workspace}}/build/modyn/tests/coverage
@@ -236,7 +236,7 @@ jobs:
       line-coverage: ${{steps.run_main_test_with_coverage.outputs.LINE_COVERAGE}}
       branch-coverage: ${{steps.run_main_test_with_coverage.outputs.BRANCH_COVERAGE}}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           ref: main
 
@@ -315,7 +315,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup base container
         uses: ./.github/actions/base
@@ -335,7 +335,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Debug
@@ -351,7 +351,7 @@ jobs:
       - cpp_build_and_test
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Asan
@@ -368,7 +368,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Tsan
@@ -385,7 +385,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Start docker compose and exit when tests run through
         run: bash scripts/run_integrationtests.sh Release

docs/pipeline/triggers/COST_TRIGGER.md

@@ -35,6 +35,11 @@ classDiagram
         -bool evaluate_batch(batch, trigger_index)*
     }
 
+    class WarmupTrigger {
+        -int warmup_intervals
+        +delegate_inform(batch)
+    }
+
     class PerformanceTriggerMixin {
         +DataDensityTracker data_density
         +PerformanceTracker performance_tracker
@@ -78,6 +83,10 @@ classDiagram
 
     Trigger <|-- BatchedTrigger
     BatchedTrigger <|-- CostTrigger
+
+    BatchedTrigger o-- "0..1" WarmupTrigger
+    Trigger "1" --* WarmupTrigger
+
     PerformanceTriggerMixin <|-- CostTrigger
 
     CostTrigger *-- "1" CostTracker

docs/pipeline/triggers/DRIFT_TRIGGER.md

@@ -23,6 +23,12 @@ classDiagram
         -bool evaluate_batch(batch, trigger_index)*
     }
 
+    class WarmupTrigger {
+        -int warmup_intervals
+        +delegate_inform(batch)
+    }
+
+
     class DataDriftTrigger {
         +DataDriftTriggerConfig config
         +DetectionWindows windows
@@ -49,6 +55,9 @@ classDiagram
     Trigger <|-- BatchedTrigger
     BatchedTrigger <|-- DataDriftTrigger
 
+    Trigger "1" --* WarmupTrigger
+    BatchedTrigger o-- "0..1" WarmupTrigger
+
     DataDriftTrigger "warmup_trigger" *-- "1" Trigger
     DataDriftTrigger *-- "1" DataDriftTriggerConfig
     DataDriftTrigger *-- "1" DetectionWindows
@@ -188,8 +197,8 @@ classDiagram
         int window_size = 10
     }
 
-    class DynamicPercentileThresholdCriterion {
-        float percentile = 0.05
+    class DynamicQuantileThresholdCriterion {
+        float quantile = 0.05
     }
 
     class DynamicRollingAverageThresholdCriterion {
@@ -200,7 +209,7 @@ classDiagram
     DriftDecisionCriterion <|-- ThresholdDecisionCriterion
     DriftDecisionCriterion <|-- DynamicThresholdCriterion
 
-    DynamicThresholdCriterion <|-- DynamicPercentileThresholdCriterion
+    DynamicThresholdCriterion <|-- DynamicQuantileThresholdCriterion
     DynamicThresholdCriterion <|-- DynamicRollingAverageThresholdCriterion
 ```
 
@@ -243,8 +252,8 @@ classDiagram
         +Deque~float~ score_observations
     }
 
-    class DynamicPercentileThresholdPolicy {
-        +DynamicPercentileThresholdCriterion config
+    class DynamicQuantileThresholdPolicy {
+        +DynamicQuantileThresholdCriterion config
         +bool evaluate_decision(float distance)
     }
 
@@ -260,13 +269,10 @@ classDiagram
 
     DriftDecisionPolicy <|-- ThresholdDecisionPolicy
     DriftDecisionPolicy <|-- DynamicDecisionPolicy
-    DynamicDecisionPolicy <|-- DynamicPercentileThresholdPolicy
+    DynamicDecisionPolicy <|-- DynamicQuantileThresholdPolicy
     DynamicDecisionPolicy <|-- DynamicRollingAverageThresholdPolicy
     DriftDecisionPolicy <|-- HypothesisTestDecisionPolicy
 
-    DynamicDecisionPolicy <|-- DynamicPercentileThresholdPolicy
-    DynamicDecisionPolicy <|-- DynamicRollingAverageThresholdPolicy
-
     style HypothesisTestDecisionPolicy fill:#DDDDDD,stroke:#A9A9A9,stroke-width:2px
 ```
 

docs/pipeline/triggers/PERFORMANCE_TRIGGER.md

@@ -36,6 +36,11 @@ classDiagram
         -bool evaluate_batch(batch, trigger_index)*
     }
 
+    class WarmupTrigger {
+        -int warmup_intervals
+        +delegate_inform(batch)
+    }
+
     class PerformanceTriggerMixin {
         +DataDensityTracker data_density
         +PerformanceTracker performance_tracker
@@ -67,6 +72,10 @@ classDiagram
 
     Trigger <|-- BatchedTrigger
     BatchedTrigger <|-- PerformanceTrigger
+
+    BatchedTrigger o-- "0..1" WarmupTrigger
+    Trigger "1" --* WarmupTrigger
+
     PerformanceTriggerMixin <|-- PerformanceTrigger
     PerformanceTrigger *-- "1" PerformanceTriggerConfig
     PerformanceTrigger *-- "1" DataDensityTracker

experiments/yearbook/compare_trigger_policies/run.py

@@ -31,16 +31,21 @@
     AlibiDetectMmdDriftMetric,
 )
 from modyn.config.schema.pipeline.trigger.drift.criterion import (
-    DynamicPercentileThresholdCriterion,
+    DynamicQuantileThresholdCriterion,
     DynamicRollingAverageThresholdCriterion,
     ThresholdDecisionCriterion,
 )
-from modyn.config.schema.pipeline.trigger.drift.detection_window.time_ import TimeWindowingStrategy
-from modyn.config.schema.pipeline.trigger.ensemble import AtLeastNEnsembleStrategy, EnsembleTriggerConfig
+from modyn.config.schema.pipeline.trigger.drift.detection_window.time_ import (
+    TimeWindowingStrategy,
+)
+from modyn.config.schema.pipeline.trigger.ensemble import (
+    AtLeastNEnsembleStrategy,
+    EnsembleTriggerConfig,
+)
 from modyn.config.schema.pipeline.trigger.performance.criterion import (
-    DynamicPerformanceThresholdCriterion,
     StaticNumberAvoidableMisclassificationCriterion,
     StaticPerformanceThresholdCriterion,
+    _DynamicPerformanceThresholdCriterion,
 )
 from modyn.config.schema.pipeline.trigger.performance.performance import (
     PerformanceTriggerConfig,
@@ -49,7 +54,10 @@
 from modyn.utils.utils import SECONDS_PER_UNIT
 from modynclient.config.schema.client_config import ModynClientConfig, Supervisor
 
-from .pipeline_config import yb_bytes_parser_function, yb_evaluation_transformer_function
+from .pipeline_config import (
+    yb_bytes_parser_function,
+    yb_evaluation_transformer_function,
+)
 
 _FIRST_TIMESTAMP = 0
 _LAST_TIMESTAMP = SECONDS_PER_UNIT["d"] * (2014 - 1930)  # 2014: dummy
@@ -126,7 +134,10 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # time baselines
     1: Experiment(
         name="yb-baseline-time",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         time_triggers={
             f"{schedule}y": TimeTriggerConfig(every=f"{schedule}d", start_timestamp=_FIRST_TIMESTAMP)
             for schedule in [1, 2, 3, 5, 15, 25, 40]
@@ -136,18 +147,34 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # data amount baselines
     2: Experiment(
         name="yb-baseline-dataamount",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         data_amount_triggers={
             f"{num_samples}": DataAmountTriggerConfig(num_samples=num_samples)
-            for num_samples in [100, 200, 500, 1_000, 2_500, 5_000, 10_000, 15_000, 30_000]
+            for num_samples in [
+                100,
+                200,
+                500,
+                1_000,
+                2_500,
+                5_000,
+                10_000,
+                15_000,
+                30_000,
+            ]
         },
         gpu_device="cuda:1",
     ),
     # -------------------------------- Drift triggers -------------------------------- #
     # Static threshold drift
     3: Experiment(
         name="yb-baseline-datadrift-static",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         drift_detection_triggers={
             f"{criterion_name}_int{interval}_win{window_size}": DataDriftTriggerConfig(
                 evaluation_interval_data_points=interval,
@@ -176,7 +203,10 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # Dynamic threshold drift
     4: Experiment(
         name="yb-baseline-datadrift-dynamic",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         drift_detection_triggers={
             f"{criterion_name}_int{interval}_win{window_size}": DataDriftTriggerConfig(
                 evaluation_interval_data_points=interval,
@@ -197,16 +227,16 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
             for window_size in [5]
             for criterion_name, criterion in (
                 {
-                    f"mmd-perc-{percentile}-{window_size}": DynamicPercentileThresholdCriterion(
-                        window_size=window_size, percentile=percentile
+                    f"mmd-perc-{quantile}-{window_size}": DynamicQuantileThresholdCriterion(
+                        window_size=window_size, quantile=quantile
                     )
-                    for percentile in [0.05, 0.1, 0.2, 0.3]
+                    for quantile in [0.05, 0.1, 0.2, 0.3]
                     for window_size in [15]  # TODO [10, 20, 30]
                 }
                 | {
                     f"mmd-rollavg-{deviation}-{window_size}": DynamicRollingAverageThresholdCriterion(
                         window_size=window_size, deviation=deviation, absolute=False
-                    )  # TODO: avg / percentile
+                    )  # TODO: avg / quantile
                     for deviation in [0.025, 0.05, 0.1, 0.2, 0.3]
                     for window_size in [15]  # TODO [10, 20, 30]
                 }
@@ -217,7 +247,10 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # ----------------------------- Performance triggers ----------------------------- #
     5: Experiment(
         name="yb-performancetrigger",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         performance_triggers={
             f"{criterion_name}-int{interval}y": PerformanceTriggerConfig(
                 evaluation_interval_data_points=interval,
@@ -248,7 +281,7 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
                     for perf_threshold in [0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
                 }
                 | {
-                    f"dynamic-{deviation}": DynamicPerformanceThresholdCriterion(
+                    f"dynamic-{deviation}": _DynamicPerformanceThresholdCriterion(
                         metric="Accuracy",
                         deviation=deviation,
                         absolute=False,
@@ -260,7 +293,7 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
                         expected_accuracy=0.9,  # TODO: variable
                         allow_reduction=allow_reduction,
                         avoidable_misclassification_threshold=num_misclassifications,
-                    )  # TODO: avg / percentile
+                    )  # TODO: avg / quantile
                     for num_misclassifications in [100, 200, 500, 1000, 2000, 5000]
                     for allow_reduction in [True, False]
                 }
@@ -273,7 +306,10 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # Data integration latency trigger
     10: Experiment(
         name="yb-costtrigger-dataincorporation",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         cost_triggers={
             f"int{interval}_exch{exchange_rate}": DataIncorporationLatencyCostTriggerConfig(
                 evaluation_interval_data_points=interval,
@@ -288,7 +324,10 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # avoidable misclassfication integration trigger
     11: Experiment(
         name="yb-costtrigger-avoidablemisclassification",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         cost_triggers={
             f"int{interval}_exch{exchange_rate}_red{allow_reduction}": AvoidableMisclassificationCostTriggerConfig(
                 # cost trigger params
@@ -324,7 +363,10 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
     # with best working previous triggers
     20: Experiment(
         name="yb-ensemble",
-        eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
+        eval_handlers=[
+            construct_slicing_eval_handler(),
+            construct_between_trigger_eval_handler(),
+        ],
         ensemble_triggers={
             "ensemble1": EnsembleTriggerConfig(
                 subtriggers={