Merge pull request #604 from cdolfi/repo_info_page

Repo Info Page

8Knot/cache_manager/db_init.py

@@ -261,6 +261,72 @@ def _create_application_tables() -> None:
         )
         logging.warning("CREATED repo_files_query TABLE")
 
+        cur.execute(
+            """
+            CREATE UNLOGGED TABLE IF NOT EXISTS repo_languages_query(
+                id int,
+                programming_language text,
+                code_lines int,
+                files int
+            )
+            """
+        )
+        logging.warning("CREATED repo_languages_query TABLE")
+
+        cur.execute(
+            """
+            CREATE UNLOGGED TABLE IF NOT EXISTS package_version_query(
+                id int,
+                name text,
+                current_release_date text,
+                latest_release_date text,
+                libyear float4
+            )
+            """
+        )
+        logging.warning("CREATED package_version_query TABLE")
+
+        cur.execute(
+            """
+            CREATE UNLOGGED TABLE IF NOT EXISTS repo_releases_query(
+                id int,
+                release_name text,
+                release_created_at text,
+                release_published_at text,
+                release_updated_at text
+            )
+            """
+        )
+        logging.warning("CREATED repo_releases_query TABLE")
+
+        cur.execute(
+            """
+            CREATE UNLOGGED TABLE IF NOT EXISTS ossf_score_query(
+                id int,
+                name text,
+                score float4
+            )
+            """
+        )
+        logging.warning("CREATED ossf_score_query TABLE")
+
+        cur.execute(
+            """
+            CREATE UNLOGGED TABLE IF NOT EXISTS repo_info_query(
+                id int,
+                issues_enabled text,
+                fork_count int,
+                watchers_count int,
+                license text,
+                stars_count int,
+                code_of_conduct_file text,
+                security_issue_file text,
+                security_audit_file text
+            )
+            """
+        )
+        logging.warning("CREATED repo_info_query TABLE")
+
         cur.execute(
             """
             CREATE UNLOGGED TABLE IF NOT EXISTS pr_response_query(

8Knot/pages/index/index_callbacks.py

@@ -25,12 +25,18 @@
 from queries.cntrb_per_file_query import cntrb_per_file_query as cpfq
 from queries.repo_files_query import repo_files_query as rfq
 from queries.pr_files_query import pr_file_query as prfq
+from queries.repo_languages_query import repo_languages_query as rlq
+from queries.package_version_query import package_version_query as pvq
+from queries.repo_releases_query import repo_releases_query as rrq
+from queries.ossf_score_query import ossf_score_query as osq
+from queries.repo_info_query import repo_info_query as riq
 import redis
 import flask
 
 
 # list of queries to be run
-QUERIES = [iq, cq, cnq, prq, aq, iaq, praq, prr, cpfq, rfq, prfq]
+QUERIES = [iq, cq, cnq, prq, cmq, iaq, praq, prr, cpfq, rfq, prfq, rlq, pvq, rrq, osq, riq]
+
 
 # check if login has been enabled in config
 login_enabled = os.getenv("AUGUR_LOGIN_ENABLED", "False") == "True"

8Knot/pages/index/index_layout.py

@@ -90,6 +90,7 @@
                             dbc.Nav(
                                 [
                                     dbc.NavLink("Welcome", href="/", active="exact"),
+                                    dbc.NavLink("Repo Overview", href="/repo_overview", active="exact"),
                                     dbc.NavLink(
                                         "Contributions",
                                         href="/contributions",

8Knot/pages/repo_overview/repo_overview.py

@@ -0,0 +1,82 @@
+from dash import html, dcc, callback
+from dash.dependencies import Input, Output, State
+import dash
+import dash_bootstrap_components as dbc
+import warnings
+import dash_mantine_components as dmc
+from app import augur
+
+# import visualization cards
+from .visualizations.code_languages import gc_code_language
+from .visualizations.package_version import gc_package_version
+from .visualizations.ossf_scorecard import gc_ossf_scorecard
+from .visualizations.repo_general_info import gc_repo_general_info
+
+warnings.filterwarnings("ignore")
+
+dash.register_page(__name__, path="/repo_overview")
+
+layout = dbc.Container(
+    [
+        html.H1("Search Bar Populated Analysis", style={"text-align": "center", "marginBottom": "1%"}),
+        dbc.Row(
+            [
+                dbc.Col(gc_code_language, width=5),
+                dbc.Col(gc_package_version, width=5),
+            ],
+            align="center",
+            justify="evenly",
+            style={"marginBottom": "1%"},
+        ),
+        dbc.Row(
+            [
+                dbc.Col(
+                    [
+                        html.H1("Per Repo Analysis:"),
+                    ],
+                    width=2,
+                ),
+                dbc.Col(
+                    [
+                        dmc.Select(
+                            id="repo-info-selection",
+                            placeholder="Repo for info section",
+                            classNames={"values": "dmc-multiselect-custom"},
+                            searchable=True,
+                            clearable=True,
+                        ),
+                    ],
+                    width=3,
+                ),
+            ],
+            justify="center",
+            align="center",
+            style={"marginBottom": "1%"},
+        ),
+        dbc.Row(
+            [
+                dbc.Col(gc_ossf_scorecard, width=6),
+                dbc.Col(gc_repo_general_info, width=6),
+            ],
+            align="center",
+            style={"marginBottom": ".5%"},
+        ),
+    ],
+    fluid=True,
+)
+
+# callback for populating repo drop down
+@callback(
+    [
+        Output("repo-info-selection", "data"),
+        Output("repo-info-selection", "value"),
+    ],
+    [Input("repo-choices", "data")],
+)
+def repo_dropdown(repo_ids):
+    # array to hold repo_id and git url pairing for dropdown
+    data_array = []
+    for repo_id in repo_ids:
+        entry = {"value": repo_id, "label": augur.repo_id_to_git(repo_id)}
+        data_array.append(entry)
+    return data_array, repo_ids[0]

8Knot/pages/repo_overview/visualizations/code_languages.py

@@ -0,0 +1,198 @@
+from dash import html, dcc, callback
+import dash
+import dash_bootstrap_components as dbc
+from dash.dependencies import Input, Output, State
+import plotly.graph_objects as go
+import pandas as pd
+import logging
+from dateutil.relativedelta import *  # type: ignore
+import plotly.express as px
+from pages.utils.graph_utils import color_seq
+from queries.repo_languages_query import repo_languages_query as rlq
+from pages.utils.job_utils import nodata_graph
+import time
+import datetime as dt
+import cache_manager.cache_facade as cf
+
+PAGE = "repo_info"
+VIZ_ID = "code-languages"
+
+gc_code_language = dbc.Card(
+    [
+        dbc.CardBody(
+            [
+                html.H3(
+                    id=f"graph-title-{PAGE}-{VIZ_ID}",
+                    className="card-title",
+                    style={"textAlign": "center"},
+                ),
+                dbc.Popover(
+                    [
+                        dbc.PopoverHeader("Graph Info:"),
+                        dbc.PopoverBody(
+                            """
+                            Visualizes the percent of files or lines of code by language.
+                            """
+                        ),
+                    ],
+                    id=f"popover-{PAGE}-{VIZ_ID}",
+                    target=f"popover-target-{PAGE}-{VIZ_ID}",
+                    placement="top",
+                    is_open=False,
+                ),
+                dcc.Loading(
+                    dcc.Graph(id=f"{PAGE}-{VIZ_ID}"),
+                ),
+                dbc.Form(
+                    [
+                        dbc.Row(
+                            [
+                                dbc.Label(
+                                    "Graph View:",
+                                    html_for=f"graph-view-{PAGE}-{VIZ_ID}",
+                                    width="auto",
+                                ),
+                                dbc.Col(
+                                    dbc.RadioItems(
+                                        id=f"graph-view-{PAGE}-{VIZ_ID}",
+                                        options=[
+                                            {
+                                                "label": "Files",
+                                                "value": "file",
+                                            },
+                                            {
+                                                "label": "Lines of Code",
+                                                "value": "line",
+                                            },
+                                        ],
+                                        value="file",
+                                        inline=True,
+                                    ),
+                                    className="me-2",
+                                ),
+                                dbc.Col(
+                                    dbc.Button(
+                                        "About Graph",
+                                        id=f"popover-target-{PAGE}-{VIZ_ID}",
+                                        color="secondary",
+                                        size="sm",
+                                    ),
+                                    width="auto",
+                                    style={"paddingTop": ".5em"},
+                                ),
+                            ],
+                            align="center",
+                        ),
+                    ]
+                ),
+            ]
+        )
+    ],
+)
+
+
+# callback for graph info popover
+@callback(
+    Output(f"popover-{PAGE}-{VIZ_ID}", "is_open"),
+    [Input(f"popover-target-{PAGE}-{VIZ_ID}", "n_clicks")],
+    [State(f"popover-{PAGE}-{VIZ_ID}", "is_open")],
+)
+def toggle_popover(n, is_open):
+    if n:
+        return not is_open
+    return is_open
+
+
+# callback for dynamically changing the graph title
+@callback(
+    Output(f"graph-title-{PAGE}-{VIZ_ID}", "children"),
+    Input(f"graph-view-{PAGE}-{VIZ_ID}", "value"),
+)
+def graph_title(view):
+    title = ""
+    if view == "file":
+        title = "File Lanugage by File"
+    else:
+        title = "File Lanugage by Line"
+    return title
+
+
+# callback for code languages graph
+@callback(
+    Output(f"{PAGE}-{VIZ_ID}", "figure"),
+    [
+        Input("repo-choices", "data"),
+        Input(f"graph-view-{PAGE}-{VIZ_ID}", "value"),
+    ],
+    background=True,
+)
+def code_languages_graph(repolist, view):
+    # wait for data to asynchronously download and become available.
+    while not_cached := cf.get_uncached(func_name=rlq.__name__, repolist=repolist):
+        logging.warning(f"{VIZ_ID}- WAITING ON DATA TO BECOME AVAILABLE")
+        time.sleep(0.5)
+
+    start = time.perf_counter()
+    logging.warning(f"{VIZ_ID}- START")
+
+    # GET ALL DATA FROM POSTGRES CACHE
+    df = cf.retrieve_from_cache(
+        tablename=rlq.__name__,
+        repolist=repolist,
+    )
+
+    # test if there is data
+    if df.empty:
+        logging.warning(f"{VIZ_ID} - NO DATA AVAILABLE")
+        return nodata_graph
+
+    # function for all data pre processing
+    df = process_data(df)
+
+    fig = create_figure(df, view)
+
+    logging.warning(f"{VIZ_ID} - END - {time.perf_counter() - start}")
+    return fig
+
+
+def process_data(df: pd.DataFrame):
+
+    # group files by their programing language and sum code lines and files
+    df_lang = df[["programming_language", "code_lines", "files"]].groupby("programming_language").sum().reset_index()
+
+    # require a language to have atleast .1 % of total files to be shown, if not grouped into other
+    min_files = df_lang["files"].sum() / 1000
+    df_lang.loc[df_lang.files <= min_files, "programming_language"] = "Other"
+    df_lang = (
+        df_lang[["programming_language", "code_lines", "files"]].groupby("programming_language").sum().reset_index()
+    )
+
+    # order by descending file number and reset format
+    df_lang = df_lang.sort_values(by="files", axis=0, ascending=False).reset_index()
+    df_lang.drop("index", axis=1, inplace=True)
+
+    # calculate percentages
+    df_lang["Code %"] = (df_lang["code_lines"] / df_lang["code_lines"].sum()) * 100
+    df_lang["Files %"] = (df_lang["files"] / df_lang["files"].sum()) * 100
+
+    return df_lang
+
+
+def create_figure(df: pd.DataFrame, view):
+
+    value = "files"
+    if view == "line":
+        value = "code_lines"
+
+    # graph generation
+    fig = px.pie(df, names="programming_language", values=value, color_discrete_sequence=color_seq)
+    fig.update_traces(
+        textposition="inside",
+        textinfo="percent+label",
+        hovertemplate="%{label} <br>Amount: %{value}<br><extra></extra>",
+    )
+
+    # add legend title
+    fig.update_layout(legend_title_text="Languages")
+
+    return fig