Implements incremental analysis experiment

tests/data/test_report.py

@@ -19,7 +19,7 @@ class TestFileStatusExtension(unittest.TestCase):
     """Test basic FileStatusExtension functionality."""
 
     def test_status_extension(self):
-        """"""
+        """Tests if we convert a FSE to it's string representation."""
         self.assertEqual(
             FileStatusExtension.SUCCESS.get_status_extension(), "success"
         )
@@ -62,6 +62,11 @@ def test_wrong_status_lookup(self):
 class TestReportFilename(unittest.TestCase):
     """Test basic TestReportFilename functionality."""
 
+    correct_UUID: str
+    raw_filename: str
+    report_filename: ReportFilename
+    broken_report_filename: ReportFilename
+
     @classmethod
     def setUpClass(cls):
         """Setup file and CommitReport."""
@@ -151,6 +156,10 @@ def test_get_uuid(self):
 class TestBaseReport(unittest.TestCase):
     """Test basic BaseReport functionality."""
 
+    success_filename_cr: str
+    success_filename: str
+    fail_filename: str
+
     @classmethod
     def setUpClass(cls):
         """Setup report file paths."""

tests/report/test_tef_report.py

@@ -0,0 +1,235 @@
+"""Test TEFReport."""
+
+import json
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from varats.report.tef_report import TEFReport, TraceEvent, TraceEventType
+
+TRACE_EVENT_FORMAT_OUTPUT = """{
+    "traceEvents": [{
+        "name": "Base",
+        "cat": "Feature",
+        "ph": "B",
+        "ts": 1637675320727304236,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Foo",
+        "cat": "Feature",
+        "ph": "B",
+        "ts": 1637675320727316656,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Foo",
+        "cat": "Feature",
+        "ph": "E",
+        "ts": 1637675325727410375,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Bar",
+        "cat": "Feature",
+        "ph": "B",
+        "ts": 1637675328727504858,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Bar",
+        "cat": "Feature",
+        "ph": "E",
+        "ts": 1637675331727788401,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Foo_2",
+        "cat": "Feature",
+        "ph": "B",
+        "ts": 1637675335727890982,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Foo_2",
+        "cat": "Feature",
+        "ph": "E",
+        "ts": 1637675341728002649,
+        "pid": 91098,
+        "tid": 91098
+    }, {
+        "name": "Base",
+        "cat": "Feature",
+        "ph": "E",
+        "ts": 1637675341728008439,
+        "pid": 91098,
+        "tid": 91098
+    } ],
+    "displayTimeUnit": "ns",
+    "stackFrames": {}
+}
+"""
+
+
+class TestTraceEventType(unittest.TestCase):
+    """Test if we can correclty parse TraceEventTypes."""
+
+    def test_parse_duration_events(self) -> None:
+        """Test if we correctly parse duration event types."""
+        self.assertEqual(
+            TraceEventType.parse_event_type("B"),
+            TraceEventType.DURATION_EVENT_BEGIN
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("E"),
+            TraceEventType.DURATION_EVENT_END
+        )
+
+    def test_parse_async_events(self) -> None:
+        """Test if we correctly parse async event types."""
+        self.assertEqual(
+            TraceEventType.parse_event_type("b"),
+            TraceEventType.ASYNC_EVENT_START
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("n"),
+            TraceEventType.ASYNC_EVENT_INSTANT
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("e"), TraceEventType.ASYNC_EVENT_END
+        )
+
+    def test_parse_flow_events(self) -> None:
+        """Test if we correctly parse flow event types."""
+        self.assertEqual(
+            TraceEventType.parse_event_type("s"),
+            TraceEventType.FLOW_EVENT_START
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("t"), TraceEventType.FLOW_EVENT_STEP
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("f"), TraceEventType.FLOW_EVENT_END
+        )
+
+    def test_parse_other_events(self) -> None:
+        """Test if we correctly parse other event types."""
+        self.assertEqual(
+            TraceEventType.parse_event_type("X"), TraceEventType.COMPLETE_EVENT
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("i"), TraceEventType.INSTANT_EVENT
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("C"), TraceEventType.COUNTER_EVENT
+        )
+        self.assertEqual(
+            TraceEventType.parse_event_type("P"), TraceEventType.SAMPLE_EVENT
+        )
+
+    def test_fail_at_wrong_event_string(self) -> None:
+        """Test if we fail should an event type not match."""
+        self.assertRaises(LookupError, TraceEventType.parse_event_type, "42")
+        self.assertRaises(LookupError, TraceEventType.parse_event_type, "I")
+        self.assertRaises(LookupError, TraceEventType.parse_event_type, "D")
+        self.assertRaises(LookupError, TraceEventType.parse_event_type, "d")
+
+
+SINGLE_TRACE_EVENT = """{
+    "name": "Base",
+    "cat": "Feature",
+    "ph": "E",
+    "ts": 1637675341728008439,
+    "pid": 91098,
+    "tid": 91099
+}
+"""
+
+
+class TestTraceEvent(unittest.TestCase):
+    """Test if we can correctly load trace events and parse values."""
+
+    trace_event: TraceEvent
+
+    @classmethod
+    def setUpClass(cls):
+        """Load trace event."""
+        cls.trace_event = TraceEvent(json.loads(SINGLE_TRACE_EVENT))
+
+    def test_name_parsing(self):
+        """Test if we can correctly parse event names."""
+        self.assertEqual(self.trace_event.name, "Base")
+        self.assertNotEqual(self.trace_event.name, "Foo")
+
+    def test_category_parsing(self):
+        """Test if we can correctly parse event categories."""
+        self.assertEqual(self.trace_event.category, "Feature")
+        self.assertNotEqual(self.trace_event.name, "Foo")
+
+    def test_event_type_parsing(self):
+        """Test if we can correctly parse event type."""
+        self.assertEqual(
+            self.trace_event.event_type, TraceEventType.DURATION_EVENT_END
+        )
+        self.assertIsInstance(self.trace_event.event_type, TraceEventType)
+        self.assertNotEqual(
+            self.trace_event.event_type, TraceEventType.DURATION_EVENT_BEGIN
+        )
+
+    def test_timestamp_parsing(self):
+        """Test if we can correctly parse event timestamps."""
+        self.assertEqual(self.trace_event.timestamp, 1637675341728008439)
+        self.assertIsInstance(self.trace_event.timestamp, int)
+
+        self.assertNotEqual(self.trace_event.name, 1637675341728008438)
+        self.assertNotEqual(self.trace_event.name, 0)
+
+    def test_pid_parsing(self):
+        """Test if we can correctly parse event pid."""
+        self.assertEqual(self.trace_event.pid, 91098)
+        self.assertIsInstance(self.trace_event.pid, int)
+
+        self.assertNotEqual(self.trace_event.name, 91099)
+        self.assertNotEqual(self.trace_event.name, 91097)
+        self.assertNotEqual(self.trace_event.name, 0)
+
+    def test_tid_parsing(self):
+        """Test if we can correctly parse event tid."""
+        self.assertEqual(self.trace_event.tid, 91099)
+        self.assertIsInstance(self.trace_event.tid, int)
+
+        self.assertNotEqual(self.trace_event.name, 91100)
+        self.assertNotEqual(self.trace_event.name, 91098)
+        self.assertNotEqual(self.trace_event.name, 0)
+
+
+class TestTEFReportParser(unittest.TestCase):
+    """Tests if the trace-event-format report can be parsed correctly."""
+
+    report: TEFReport
+
+    @classmethod
+    def setUpClass(cls):
+        """Load and prepare TEF report."""
+        with mock.patch(
+            'builtins.open',
+            new=mock.mock_open(read_data=TRACE_EVENT_FORMAT_OUTPUT)
+        ):
+            cls.report = TEFReport(Path("fake_file_path"))
+
+    def test_parse_time_unit(self) -> None:
+        """Test if the time unit field is correclty parsed."""
+        self.assertEqual(self.report.display_time_unit, "ns")
+        self.assertNotEqual(self.report.display_time_unit, "ms")
+
+    def test_parse_trace_events(self) -> None:
+        """Test if we correctly parse the listed trace events."""
+        self.assertEqual(len(self.report.trace_events), 8)
+
+        self.assertEqual(self.report.trace_events[0].name, "Base")
+
+    def test_parse_stack_frames(self) -> None:
+        """Test if we correctly parse stack frames."""
+        # Currently, not implemented so we should get an exception.
+        with self.assertRaises(NotImplementedError):
+            _ = self.report.stack_frames

tests/tools/test_driver_casestudy.py

@@ -103,6 +103,36 @@ def test_vara_cs_gen_sample_start_end_timestamp(self):
             case_study.revisions
         )
 
+    @run_in_test_environment()
+    def test_vara_cs_gen_sample_start_before_initial_commit(self):
+        """Check if vara-cs gen select_sample with start timestamp before the
+        initial commit selects the right revisiosn."""
+        runner = CliRunner()
+        Path(vara_cfg()["paper_config"]["folder"].value + "/" +
+             "test_gen").mkdir()
+        vara_cfg()["paper_config"]["current_config"] = "test_gen"
+        result = runner.invoke(
+            driver_casestudy.main, [
+                'gen', '-p', 'brotli', 'select_sample', '--num-rev', '6',
+                '--start', '1991-01-01', '--end', '2013-10-20',
+                'UniformSamplingMethod'
+            ]
+        )
+
+        self.assertEqual(0, result.exit_code, result.exception)
+        case_study_path = Path(
+            vara_cfg()["paper_config"]["folder"].value +
+            "/test_gen/brotli_0.case_study"
+        )
+        self.assertTrue(case_study_path.exists())
+
+        case_study = load_case_study_from_file(case_study_path)
+        self.assertEqual(len(case_study.revisions), 1)
+        self.assertTrue(
+            FullCommitHash('e0346c826249368f0f4a68a2b95f4ab5cf1e235b') in
+            case_study.revisions
+        )
+
     @run_in_test_environment()
     def test_vara_cs_gen_latest(self):
         """Test for vara-cs gen select_latest."""

tests/utils/test_bb_config.py

@@ -76,7 +76,8 @@ def test_if_experiments_were_added(self):
             "varats.experiments.discover_experiments",
             "varats.experiments.vara.region_instrumentation",
             "varats.experiments.vara.commit_annotation_report",
-            "varats.experiments.vara.blame_experiment"
+            "varats.experiments.vara.blame_experiment",
+            "varats.experiments.phasar.incremental_analysis"
         ]
 
         loaded_plugins = bb_cfg()["plugins"]["experiments"].value

tests/utils/test_git_util.py

@@ -90,9 +90,9 @@ def test_get_all_revisions_between_full(self):
             self.assertSetEqual(
                 set(revs), {
                     FullCommitHash("5692e422da6af1e991f9182345d58df87866bc5e"),
-                    FullCommitHash("2f9277ff2f2d0b4113b1ffd9753cc0f6973d354a"),
+                    FullCommitHash("2a51a85aa86abb4c294c65fab57f3d9c69f10080"),
                     FullCommitHash("63be8a99401992075c23e99f7c84de1c653e39e2"),
-                    FullCommitHash("2a51a85aa86abb4c294c65fab57f3d9c69f10080")
+                    FullCommitHash("2f9277ff2f2d0b4113b1ffd9753cc0f6973d354a")
                 }
             )
 
@@ -108,9 +108,9 @@ def test_get_all_revisions_between_short(self):
             self.assertSetEqual(
                 set(revs), {
                     ShortCommitHash("5692e422da"),
-                    ShortCommitHash("2f9277ff2f"),
+                    ShortCommitHash("2a51a85aa8"),
                     ShortCommitHash("63be8a9940"),
-                    ShortCommitHash("2a51a85aa8")
+                    ShortCommitHash("2f9277ff2f")
                 }
             )
 

varats-core/setup.py

@@ -10,7 +10,7 @@
     setup_requires=["pytest-runner", "setuptools_scm"],
     tests_require=["pytest", "pytest-cov"],
     install_requires=[
-        "benchbuild>=6.3.0",
+        "benchbuild>=6.3.1",
         "plumbum>=1.6.6",
         "PyGithub>=1.47",
         "PyDriller>=2.0",

varats-core/varats/experiment/experiment_util.py

@@ -1,11 +1,13 @@
 """Utility module for BenchBuild experiments."""
 
 import os
+import shutil
 import random
 import traceback
 import typing as tp
 from abc import abstractmethod
 from pathlib import Path
+import tempfile
 
 from benchbuild import source
 from benchbuild.experiment import Experiment
@@ -472,3 +474,28 @@ def experiment_filter(file_path: Path) -> bool:
     return get_tagged_revisions(
         project_cls, result_file_type, tag_blocked, experiment_filter
     )
+
+
+class ZippedReportFolder(tempfile.TemporaryDirectory):
+    """
+    Context manager for creating a folder report, i.e., a report file which is
+    actually a folder containing multiple files and other folders.
+
+    Example usage: An experiment step can, with this context manager, simply
+        create a folder into which all kinds of data is dropped into. After the
+        completion of the step (leaving the context manager), all files dropped
+        into the folder will be compressed and stored as a single report.
+    """
+
+    def __init__(self, result_report_path: Path) -> None:
+        super().__init__()
+        self.__result_report_name: Path = result_report_path.with_suffix('')
+
+    def __enter__(self) -> Path:
+        return Path(super().__enter__())
+
+    def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
+        shutil.make_archive(
+            str(self.__result_report_name), "zip", Path(self.name)
+        )
+        super().__exit__(exc_type, exc_value, exc_traceback)