Skip to content

Commit

Permalink
repo info page
Browse files Browse the repository at this point in the history
  • Loading branch information
cdolfi committed Jan 12, 2024
1 parent 9dc9193 commit cbe0f54
Show file tree
Hide file tree
Showing 16 changed files with 1,151 additions and 1 deletion.
66 changes: 66 additions & 0 deletions 8Knot/cache_manager/db_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,72 @@ def _create_application_tables() -> None:
)
logging.warning("CREATED repo_files_query TABLE")

cur.execute(
"""
CREATE UNLOGGED TABLE IF NOT EXISTS repo_languages_query(
id int,
programming_language text,
code_lines int,
files int
)
"""
)
logging.warning("CREATED repo_languages_query TABLE")

cur.execute(
"""
CREATE UNLOGGED TABLE IF NOT EXISTS package_version_query(
id int,
name text,
current_release_date text,
latest_release_date text,
libyear float4
)
"""
)
logging.warning("CREATED package_version_query TABLE")

cur.execute(
"""
CREATE UNLOGGED TABLE IF NOT EXISTS repo_releases_query(
id int,
release_name text,
release_created_at text,
release_published_at text,
release_updated_at text
)
"""
)
logging.warning("CREATED repo_releases_query TABLE")

cur.execute(
"""
CREATE UNLOGGED TABLE IF NOT EXISTS ossf_score_query(
id int,
name text,
score float4
)
"""
)
logging.warning("CREATED ossf_score_query TABLE")

cur.execute(
"""
CREATE UNLOGGED TABLE IF NOT EXISTS repo_info_query(
id int,
issues_enabled text,
fork_count int,
watchers_count int,
license text,
stars_count int,
code_of_conduct_file text,
security_issue_file text,
security_audit_file text
)
"""
)
logging.warning("CREATED repo_info_query TABLE")

cur.execute(
"""
CREATE UNLOGGED TABLE IF NOT EXISTS pr_response_query(
Expand Down
7 changes: 6 additions & 1 deletion 8Knot/pages/index/index_callbacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@
from queries.cntrb_per_file_query import cntrb_per_file_query as cpfq
from queries.repo_files_query import repo_files_query as rfq
from queries.pr_files_query import pr_file_query as prfq
from queries.repo_languages_query import repo_languages_query as rlq
from queries.package_version_query import package_version_query as pvq
from queries.repo_releases_query import repo_releases_query as rrq
from queries.ossf_score_query import ossf_score_query as osq
from queries.repo_info_query import repo_info_query as riq
import redis
import flask


# list of queries to be run
QUERIES = [iq, cq, cnq, prq, cmq, iaq, praq, prr, cpfq, rfq, prfq]
QUERIES = [iq, cq, cnq, prq, cmq, iaq, praq, prr, cpfq, rfq, prfq, rlq, pvq, rrq, osq, riq]

# check if login has been enabled in config
login_enabled = os.getenv("AUGUR_LOGIN_ENABLED", "False") == "True"
Expand Down
1 change: 1 addition & 0 deletions 8Knot/pages/index/index_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
dbc.Nav(
[
dbc.NavLink("Welcome", href="/", active="exact"),
dbc.NavLink("Repo Overview", href="/repo_overview", active="exact"),
dbc.NavLink(
"Contributions",
href="/contributions",
Expand Down
Empty file.
82 changes: 82 additions & 0 deletions 8Knot/pages/repo_overview/repo_overview.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
from dash import html, dcc, callback
from dash.dependencies import Input, Output, State
import dash
import dash_bootstrap_components as dbc
import warnings
import dash_mantine_components as dmc
from app import augur

# import visualization cards
from .visualizations.code_languages import gc_code_language
from .visualizations.package_version import gc_package_version
from .visualizations.ossf_scorecard import gc_ossf_scorecard
from .visualizations.repo_general_info import gc_repo_general_info

warnings.filterwarnings("ignore")

dash.register_page(__name__, path="/repo_overview")

layout = dbc.Container(
[
html.H1("Search Bar Populated Analysis", style={"text-align": "center", "marginBottom": "1%"}),
dbc.Row(
[
dbc.Col(gc_code_language, width=5),
dbc.Col(gc_package_version, width=5),
],
align="center",
justify="evenly",
style={"marginBottom": "1%"},
),
dbc.Row(
[
dbc.Col(
[
html.H1("Per Repo Analysis:"),
],
width=2,
),
dbc.Col(
[
dmc.Select(
id="repo-info-selection",
placeholder="Repo for info section",
classNames={"values": "dmc-multiselect-custom"},
searchable=True,
clearable=True,
),
],
width=3,
),
],
justify="center",
align="center",
style={"marginBottom": "1%"},
),
dbc.Row(
[
dbc.Col(gc_ossf_scorecard, width=6),
dbc.Col(gc_repo_general_info, width=6),
],
align="center",
style={"marginBottom": ".5%"},
),
],
fluid=True,
)

# callback for populating repo drop down
@callback(
[
Output("repo-info-selection", "data"),
Output("repo-info-selection", "value"),
],
[Input("repo-choices", "data")],
)
def repo_dropdown(repo_ids):
# array to hold repo_id and git url pairing for dropdown
data_array = []
for repo_id in repo_ids:
entry = {"value": repo_id, "label": augur.repo_id_to_git(repo_id)}
data_array.append(entry)
return data_array, repo_ids[0]
Empty file.
198 changes: 198 additions & 0 deletions 8Knot/pages/repo_overview/visualizations/code_languages.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
from dash import html, dcc, callback
import dash
import dash_bootstrap_components as dbc
from dash.dependencies import Input, Output, State
import plotly.graph_objects as go
import pandas as pd
import logging
from dateutil.relativedelta import * # type: ignore
import plotly.express as px
from pages.utils.graph_utils import color_seq
from queries.repo_languages_query import repo_languages_query as rlq
from pages.utils.job_utils import nodata_graph
import time
import datetime as dt
import cache_manager.cache_facade as cf

PAGE = "repo_info"
VIZ_ID = "code-languages"

gc_code_language = dbc.Card(
[
dbc.CardBody(
[
html.H3(
id=f"graph-title-{PAGE}-{VIZ_ID}",
className="card-title",
style={"textAlign": "center"},
),
dbc.Popover(
[
dbc.PopoverHeader("Graph Info:"),
dbc.PopoverBody(
"""
Visualizes the percent of files or lines of code by language.
"""
),
],
id=f"popover-{PAGE}-{VIZ_ID}",
target=f"popover-target-{PAGE}-{VIZ_ID}",
placement="top",
is_open=False,
),
dcc.Loading(
dcc.Graph(id=f"{PAGE}-{VIZ_ID}"),
),
dbc.Form(
[
dbc.Row(
[
dbc.Label(
"Graph View:",
html_for=f"graph-view-{PAGE}-{VIZ_ID}",
width="auto",
),
dbc.Col(
dbc.RadioItems(
id=f"graph-view-{PAGE}-{VIZ_ID}",
options=[
{
"label": "Files",
"value": "file",
},
{
"label": "Lines of Code",
"value": "line",
},
],
value="file",
inline=True,
),
className="me-2",
),
dbc.Col(
dbc.Button(
"About Graph",
id=f"popover-target-{PAGE}-{VIZ_ID}",
color="secondary",
size="sm",
),
width="auto",
style={"paddingTop": ".5em"},
),
],
align="center",
),
]
),
]
)
],
)


# callback for graph info popover
@callback(
Output(f"popover-{PAGE}-{VIZ_ID}", "is_open"),
[Input(f"popover-target-{PAGE}-{VIZ_ID}", "n_clicks")],
[State(f"popover-{PAGE}-{VIZ_ID}", "is_open")],
)
def toggle_popover(n, is_open):
if n:
return not is_open
return is_open


# callback for dynamically changing the graph title
@callback(
Output(f"graph-title-{PAGE}-{VIZ_ID}", "children"),
Input(f"graph-view-{PAGE}-{VIZ_ID}", "value"),
)
def graph_title(view):
title = ""
if view == "file":
title = "File Lanugage by File"
else:
title = "File Lanugage by Line"
return title


# callback for code languages graph
@callback(
Output(f"{PAGE}-{VIZ_ID}", "figure"),
[
Input("repo-choices", "data"),
Input(f"graph-view-{PAGE}-{VIZ_ID}", "value"),
],
background=True,
)
def code_languages_graph(repolist, view):
# wait for data to asynchronously download and become available.
while not_cached := cf.get_uncached(func_name=rlq.__name__, repolist=repolist):
logging.warning(f"{VIZ_ID}- WAITING ON DATA TO BECOME AVAILABLE")
time.sleep(0.5)

start = time.perf_counter()
logging.warning(f"{VIZ_ID}- START")

# GET ALL DATA FROM POSTGRES CACHE
df = cf.retrieve_from_cache(
tablename=rlq.__name__,
repolist=repolist,
)

# test if there is data
if df.empty:
logging.warning(f"{VIZ_ID} - NO DATA AVAILABLE")
return nodata_graph

# function for all data pre processing
df = process_data(df)

fig = create_figure(df, view)

logging.warning(f"{VIZ_ID} - END - {time.perf_counter() - start}")
return fig


def process_data(df: pd.DataFrame):

# group files by their programing language and sum code lines and files
df_lang = df[["programming_language", "code_lines", "files"]].groupby("programming_language").sum().reset_index()

# require a language to have atleast .1 % of total files to be shown, if not grouped into other
min_files = df_lang["files"].sum() / 1000
df_lang.loc[df_lang.files <= min_files, "programming_language"] = "Other"
df_lang = (
df_lang[["programming_language", "code_lines", "files"]].groupby("programming_language").sum().reset_index()
)

# order by descending file number and reset format
df_lang = df_lang.sort_values(by="files", axis=0, ascending=False).reset_index()
df_lang.drop("index", axis=1, inplace=True)

# calculate percentages
df_lang["Code %"] = (df_lang["code_lines"] / df_lang["code_lines"].sum()) * 100
df_lang["Files %"] = (df_lang["files"] / df_lang["files"].sum()) * 100

return df_lang


def create_figure(df: pd.DataFrame, view):

value = "files"
if view == "line":
value = "code_lines"

# graph generation
fig = px.pie(df, names="programming_language", values=value, color_discrete_sequence=color_seq)
fig.update_traces(
textposition="inside",
textinfo="percent+label",
hovertemplate="%{label} <br>Amount: %{value}<br><extra></extra>",
)

# add legend title
fig.update_layout(legend_title_text="Languages")

return fig
Loading

0 comments on commit cbe0f54

Please sign in to comment.