Bachlorthesis Friedel

varats-core/varats/experiment/experiment_util.py

@@ -366,7 +366,7 @@ def report_spec(cls) -> ReportSpecification:
         return cls.REPORT_SPEC
 
     @classmethod
-    def file_belongs_to_experiment(cls, file_name: str) -> bool:
+    def file_belongs_to_experiment(cls, file_name: ReportFilename) -> bool:
         """
         Checks if the file belongs to this experiment.
 
@@ -377,7 +377,7 @@ def file_belongs_to_experiment(cls, file_name: str) -> bool:
             True, if the file belongs to this experiment type
         """
         try:
-            other_short_hand = ReportFilename(file_name).experiment_shorthand
+            other_short_hand = file_name.experiment_shorthand
             return cls.shorthand() == other_short_hand
         except ValueError:
             return False

varats-core/varats/utils/git_util.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 from types import TracebackType
 
+import numpy
 import pygit2
 from benchbuild.utils.cmd import git, grep
 from plumbum import local, TF, RETCODE
@@ -37,7 +38,7 @@ class CommitHash(abc.ABC):
 
     def __init__(self, short_commit_hash: str):
         if not len(short_commit_hash) >= self.hash_length():
-            raise ValueError("Commit hash too short")
+            raise ValueError(f"Commit hash too short {short_commit_hash}")
         self.__commit_hash = short_commit_hash[:self.hash_length()]
 
     @property
@@ -474,7 +475,7 @@ class Language(Enum):
         value: tp.Set[str]  # pylint: disable=invalid-name
 
         C = {"h", "c"}
-        CPP = {"h", "hxx", "hpp", "cxx", "cpp"}
+        CPP = {"h", "hxx", "hpp", "cxx", "cpp", "cc"}
 
     def __init__(self) -> None:
         self.__enabled_languages: tp.List[ChurnConfig.Language] = []
@@ -1133,15 +1134,17 @@ def branch_has_upstream(
     return tp.cast(bool, exit_code == 0)
 
 
-class RepositoryAtCommit():
+class RepositoryAtCommit:
     """Context manager to work with a repository at a specific revision, without
     duplicating the repository."""
 
     def __init__(self, project_name: str, revision: ShortCommitHash) -> None:
         self.__repo = pygit2.Repository(
             get_local_project_git_path(project_name)
         )
-        self.__initial_head = self.__repo.head
+
+        self.__initial_head: pygit2.Reference = self.__repo.head
+        print(self.__initial_head.name)
         self.__revision = self.__repo.get(revision.hash)
 
     def __enter__(self) -> Path:
@@ -1153,4 +1156,46 @@ def __exit__(
         exc_value: tp.Optional[BaseException],
         exc_traceback: tp.Optional[TracebackType]
     ) -> None:
-        self.__repo.checkout(self.__initial_head)
+        self.__repo.checkout(
+            self.__initial_head, strategy=pygit2.GIT_CHECKOUT_FORCE
+        )
+
+
+def calc_surviving_lines(project_name: str, revision: CommitHash) -> \
+tp.Dict[str, int]:
+    """
+
+    Args:
+        project_name: project to analyze
+        revision: revision to analyze at
+
+    returns: number of lines per prior commit
+    """
+    churn_config = ChurnConfig.create_c_style_languages_config()
+    file_pattern = re.compile(
+        "|".join(churn_config.get_extensions_repr(r"^.*\.", r"$"))
+    )
+    revision = revision.to_short_commit_hash()
+    lines_per_revision: dict = {}
+    print(revision)
+    with RepositoryAtCommit(project_name, revision) as project_path:
+        print(project_path)
+        with local.cwd(project_path):
+            files = git("ls-tree", "-r", "--name-only",
+                        revision.hash).splitlines()
+
+            for file in files:
+                if file_pattern.match(file):
+                    lines = git("blame", "--root", "-l", f"{file}").splitlines()
+                    for line in lines:
+                        if line:
+                            last_change = line[:FullCommitHash.hash_length()]
+                            last_change = FullCommitHash(last_change)
+                            if lines_per_revision.keys(
+                            ).__contains__(last_change):
+                                lines_per_revision[
+                                    last_change
+                                ] = lines_per_revision[last_change] + 1
+                            else:
+                                lines_per_revision[last_change] = 1
+    return lines_per_revision

varats/varats/data/databases/commit_interaction_aggregate_database.py

@@ -0,0 +1,101 @@
+import typing as tp
+
+import pandas as pd
+
+from varats.data.cache_helper import build_cached_report_table
+from varats.data.databases.evaluationdatabase import EvaluationDatabase
+from varats.data.reports.blame_report import (
+    gen_base_to_inter_commit_repo_pair_mapping,
+)
+from varats.experiments.vara.blame_report_experiment import (
+    BlameReportExperiment,
+)
+from varats.jupyterhelper.file import load_blame_report
+from varats.mapping.commit_map import CommitMap
+from varats.paper.case_study import CaseStudy
+from varats.paper_mgmt.case_study import get_case_study_file_name_filter
+from varats.report.report import ReportFilepath
+from varats.revision.revisions import (
+    get_processed_revisions_files,
+    get_failed_revisions_files,
+)
+from varats.utils.git_util import FullCommitHash
+
+
+class SurvivingInteractionsDatabase(
+    EvaluationDatabase,
+    cache_id="survivng_interactions_data",
+    column_types={
+        "base_hash": 'str',
+        "interactions": 'int32',
+    }
+):
+    """Provides access to total interactions of commits."""
+
+    @classmethod
+    def _load_dataframe(
+        cls, project_name: str, commit_map: CommitMap,
+        case_study: tp.Optional[CaseStudy], **kwargs: tp.Dict[str, tp.Any]
+    ) -> pd.DataFrame:
+
+        def create_dataframe_layout() -> pd.DataFrame:
+            df_layout = pd.DataFrame(columns=cls.COLUMNS)
+            df_layout = df_layout.astype(cls.COLUMN_TYPES)
+            return df_layout
+
+        def create_data_frame_for_report(
+            report_path: ReportFilepath
+        ) -> tp.Tuple[pd.DataFrame, str, str]:
+            report = load_blame_report(report_path)
+            base_inter_c_repo_pair_mapping = \
+                gen_base_to_inter_commit_repo_pair_mapping(report)
+            revision = report.head_commit
+
+            def build_dataframe_row(chash: FullCommitHash,
+                                    interactions: int) -> tp.Dict[str, tp.Any]:
+
+                data_dict: tp.Dict[str, tp.Any] = {
+                    'revision': revision.hash,
+                    'time_id': commit_map.short_time_id(revision),
+                    'base_hash': chash.hash,
+                    'interactions': interactions
+                }
+                return data_dict
+
+            result_data_dicts: tp.List[tp.Dict[str, tp.Any]] = []
+
+            for base_pair in base_inter_c_repo_pair_mapping:
+                inter_pair_amount_dict = base_inter_c_repo_pair_mapping[
+                    base_pair]
+                interactions_amount = sum(inter_pair_amount_dict.values())
+                result_data_dicts.append(
+                    build_dataframe_row(
+                        chash=base_pair.commit.commit_hash,
+                        interactions=interactions_amount
+                    )
+                )
+            return pd.DataFrame(result_data_dicts
+                               ), report.head_commit.hash, str(
+                                   report_path.stat().st_mtime_ns
+                               )
+
+        report_files = get_processed_revisions_files(
+            project_name,
+            BlameReportExperiment,
+            file_name_filter=get_case_study_file_name_filter(case_study)
+        )
+
+        failed_report_files = get_failed_revisions_files(
+            project_name,
+            BlameReportExperiment,
+            file_name_filter=get_case_study_file_name_filter(case_study)
+        )
+
+        data_frame = build_cached_report_table(
+            cls.CACHE_ID, project_name, report_files, failed_report_files,
+            create_dataframe_layout, create_data_frame_for_report,
+            lambda path: path.report_filename.commit_hash.hash,
+            lambda path: str(path.stat().st_mtime_ns),
+            lambda a, b: int(a) > int(b)
+        )
+        return data_frame

varats/varats/data/databases/survivng_lines_database.py

@@ -0,0 +1,72 @@
+import typing as tp
+
+import pandas as pd
+from pygit2._pygit2 import GIT_SORT_TOPOLOGICAL
+
+from varats.data.cache_helper import load_cached_df_or_none, cache_dataframe
+from varats.data.databases.evaluationdatabase import EvaluationDatabase
+from varats.mapping.commit_map import CommitMap
+from varats.paper.case_study import CaseStudy
+from varats.project.project_util import get_local_project_git
+from varats.utils.git_util import (
+    calc_surviving_lines,
+    FullCommitHash,
+    ShortCommitHash,
+)
+
+
+class SurvivingLinesDatabase(
+    EvaluationDatabase,
+    cache_id="survivng_lines_data",
+    column_types={
+        "commit_hash": 'str',
+        "lines": 'int32'
+    }
+):
+
+    @classmethod
+    def _load_dataframe(
+        cls, project_name: str, commit_map: CommitMap,
+        case_study: tp.Optional[CaseStudy], **kwargs: tp.Dict[str, tp.Any]
+    ) -> pd.DataFrame:
+        data_frame = load_cached_df_or_none(
+            cls.CACHE_ID, project_name, cls.COLUMN_TYPES
+        )
+        project_repo = get_local_project_git(case_study.project_name)
+        revisions = case_study.revisions if case_study else [
+            FullCommitHash.from_pygit_commit(commit) for commit in
+            project_repo.walk(project_repo.head.target, GIT_SORT_TOPOLOGICAL)
+        ]
+        data_dicts: tp.List[tp.Dict[str, tp.Any]] = []
+        cached_revisions = data_frame.groupby("revision").groups.keys(
+        ) if data_frame is not None else set()
+        revisions_to_compute: tp.Set[str] = set(
+            map(lambda r: r.hash, revisions)
+        ) - cached_revisions
+
+        for revision in revisions_to_compute:
+            lines_per_commit = calc_surviving_lines(
+                case_study.project_name, ShortCommitHash(revision)
+            )
+
+            def build_dataframe_row(chash: FullCommitHash,
+                                    lines: int) -> tp.Dict[str, tp.Any]:
+                data_dict: tp.Dict[str, tp.Any] = {
+                    'revision': revision,
+                    'time_id': commit_map.time_id(FullCommitHash(revision)),
+                    'commit_hash': chash.hash,
+                    'lines': lines
+                }
+                return data_dict
+
+            for entry in lines_per_commit.items():
+                data_dicts.append(build_dataframe_row(entry[0], entry[1]))
+        if data_frame is None:
+            data_frame = pd.DataFrame(data_dicts)
+        else:
+            data_frame = pd.concat([data_frame,
+                                    pd.DataFrame(data_dicts)],
+                                   ignore_index=True,
+                                   copy=False)
+        cache_dataframe(cls.CACHE_ID, project_name, data_frame)
+        return data_frame

varats/varats/data/reports/blame_interaction_graph.py

@@ -29,6 +29,7 @@
     ChurnConfig,
     UNCOMMITTED_COMMIT_HASH,
     FullCommitHash,
+    CommitHash,
     get_submodule_head,
 )
 
@@ -85,6 +86,17 @@ class CAIGEdgeAttrs(TypedDict):
     amount: int
 
 
+class FIGNodeAttrs(TypedDict):
+    """Funition interaction graph node attributes."""
+    function: tp.Optional[str]
+    num_commits: int
+
+
+class FIGEdgeAttrs(TypedDict):
+    """Function interaction graph edge attributes."""
+    amount: int
+
+
 class InteractionGraph(abc.ABC):
     """Graph/Network built from interaction data."""
 
@@ -299,6 +311,64 @@ def commit_author_interaction_graph(
                               ]["amount"] += data["amount"]
         return caig
 
+    def function_interaction_graph(self):
+        """
+        Return a digraph with functions as nodes and interactions as edges.
+
+        Nodes can be referenced via their function name.
+        The graph has the following attributes:
+        Nodes:
+          - function: name of the function
+          - num_commits: number of commits aggregated in this node
+        Edges:
+          - amount: how often an interaction between two functions was found
+
+        Returns:
+            the author interaction graph
+        """
+        interaction_graph = self._interaction_graph()
+
+        def partition(node_u: BIGNodeTy, node_v: BIGNodeTy):
+            return node_u.function_name == node_v.function_name
+
+        def edge_data(
+            partition_a: tp.Set[BIGNodeTy], partition_b: tp.Set[BIGNodeTy]
+        ) -> FIGEdgeAttrs:
+            amount = 0
+            interactions: tp.List[tp.Tuple[CommitRepoPair, CommitRepoPair]] = []
+            for source in partition_a:
+                for sink in partition_b:
+                    if interaction_graph.has_edge(source, sink):
+                        amount += int(interaction_graph[source][sink]["amount"])
+                        interactions.append((source.commit, sink.commit))
+
+            return {"amount": amount}
+
+        def node_data(nodes: tp.Set[BIGNodeTy]) -> FIGNodeAttrs:
+            functions = {
+                node.function_name if node.function_name else "Unknown"
+                for node in nodes
+            }
+            assert len(functions) == 1, "Some node has more then one function."
+            return {
+                "function": next(iter(functions)),
+                "num_commits": len(nodes)
+            }
+
+        fig = nx.quotient_graph(
+            interaction_graph,
+            partition=partition,
+            edge_data=edge_data,
+            node_data=node_data,
+            create_using=nx.DiGraph
+        )
+        relabel_dict: tp.Dict[tp.FrozenSet[BIGNodeTy], str] = {}
+        for node in fig.nodes:
+            relabel_dict[node] = tp.cast(AIGNodeAttrs,
+                                         fig.nodes[node])["function"]
+        nx.relabel_nodes(fig, relabel_dict, copy=False)
+        return fig
+
 
 class BlameInteractionGraph(InteractionGraph):
     """Graph/Network built from blame interaction data."""

varats/varats/experiments/vara/blame_report_experiment.py

@@ -27,7 +27,7 @@
 from varats.experiment.wllvm import get_cached_bc_file_path, BCFileExtensions
 from varats.project.project_util import get_local_project_git_paths
 from varats.project.varats_project import VProject
-from varats.report.report import ReportSpecification
+from varats.report.report import ReportSpecification, ReportFilename
 
 
 class BlameReportGeneration(actions.ProjectStep):  # type: ignore
@@ -147,6 +147,10 @@ def actions_for_project(
 
         return analysis_actions
 
+    @classmethod
+    def file_belongs_to_experiment(cls, file_name: ReportFilename) -> bool:
+        return file_name.experiment_shorthand in ["BRE", "BRER", "BRECIF"]
+
 
 class BlameReportExperimentRegion(BlameReportExperiment, shorthand="BRER"):
     """Generates a blame report with region scoped taints."""