Refactor integration dashboard display for TA1 (#7)

* Start on graph creation

* Render graph

* Render to near parity

* Render text correctly

* Render scenario names on feature tables

* Render as boxes

* Round tables

* Update README usage

* Remove TODOs

* Update screenshot

Makefile

@@ -10,7 +10,7 @@ dev-init:.env
 PHONY:up
 up: .env
  docker build -f docker/Dockerfile -t integration-dashboard .
- docker run --name dashboard -p8501:8501 -e USE_LOCAL='TRUE' -d integration-dashboard
+ docker run --env-file ./.env --name dashboard -p8501:8501 -d integration-dashboard
 
 
 PHONY:down

README.md

@@ -9,33 +9,28 @@ Specifically, the dashboard indicates
 Currently, the dashboard is testing `knowledge-middleware` integration
 but other services and TAs might be checked in the future.
 
-![TA1 Dashboard Screenshot](https://github.com/DARPA-ASKEM/integration-dashboard/assets/14170067/da57d762-6e22-4130-ad34-ff790ef590e2)
+![TA1 Dashboard Screenshot](https://github.com/DARPA-ASKEM/integration-dashboard/assets/14170067/90e84a0b-ea0e-4d51-81b5-0c9b714a9713)
+
 
 
 ## Usage
 
 To view the current status, start the [Streamlit](https://streamlit.io/) app
 by running:
 ```
+cp sample.env .env
+# Change `.env` as needed
 make up
 ```
-Upon execution, you can pass the following environment variables (with `docker run` do `-e ENV_NAME='ENV_VAL'` for each variable).
 
+### Options
 - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`: Standard credentials for reading and writing to S3
 - `BUCKET`: The bucket you'd like to read and write to.
+- `USE_LOCAL`: Read from the gitignore files in `output/ta*` instead of S3.
 
-
+## Developing
 To set up the project for development, run
 ```
 make dev-init 
 ```
 
-To add a new report, run from [`knowledge-middleware`](https://github.com/DARPA-ASKEM/knowledge-middleware) (NOT THIS REPO)
-```
-# REMINDER: RUN THIS IN `knowledge-middleware`
-poetry run poe report
-```
-This uploads a `report_{datetime}.json` to S3 which the dashboard reads
-off of directly.
-
-

dashboard/sections.py

@@ -0,0 +1,89 @@
+from pyvis.network import Network
+import networkx as nx
+import pandas as pd
+import streamlit as st
+import streamlit.components.v1 as components
+
+from dashboard.formatting import custom_title
+
+
+def render_scenario_viewer(scenarios):
+ st.write(f"### Scenario Viewer")
+ scenario_name = st.selectbox("Select a pipeline:", list(scenarios.keys()))
+ scenario = scenarios[scenario_name]
+
+ graph = nx.DiGraph()
+ steps = scenario["steps"]
+ total_time = 0
+ for step_name, step_details in steps.items():
+ total_time += step_details["time"]
+ status_color = 'green' if step_details['success'] else 'red'
+ graph.add_node(step_name, color=status_color)
+ shape = scenario["shape"]
+ for link in shape:
+ graph.add_edge(link["from"], link["to"])
+ nx.set_node_attributes(graph, "box", "shape")
+ pipeline = Network(notebook=False, directed=True)
+ pipeline.from_nx(graph)
+ pipeline.options = {
+ 'layout': {
+ 'hierarchical': {
+ 'enabled': True,
+ 'direction': 'LR',
+ 'sortMethod': 'directed'
+ },
+ },
+ }
+ display = pipeline.generate_html()
+
+
+ st.write(f"### {scenario_name}")
+ st.text(scenario["description"])
+ st.metric("Total Time", round(total_time,2))
+ components.html(display, height=800, width=800)
+
+
+def render_section_scenario_status(scenarios):
+ st.write(f"### Scenario Status")
+ status_data = {
+ "Scenario": list(scenarios.keys()),
+ "Success": [scenario["success"] for scenario in scenarios.values()], 
+ "Total Time": [
+ round(sum([step["time"] for step in scenario["steps"].values()]),2) 
+ for scenario in scenarios.values()
+ ]
+ }
+ df = pd.DataFrame(status_data)
+ df.replace({False: "❌", True: "✅"}, inplace=True)
+ st.dataframe(df, hide_index=True)
+
+
+def get_feature_table(scenarios, feature):
+ results = {}
+ step_names = set()
+ for scenario_name, scenario in scenarios.items():
+ for step_name, step in scenario["steps"].items():
+ step_names.add(step_name)
+ results[(scenario_name, step_name)] = step[feature]
+ df = pd.DataFrame(index=list(scenarios.keys()), columns=list(step_names))
+ for (scenario_name, step_name), result in results.items():
+ df.at[scenario_name, step_name] = result
+ return df
+
+def render_section_integration_status(scenarios):
+ st.write(f"### Integration Status")
+ df = get_feature_table(scenarios, "success")
+ df.replace({False: "❌", True: "✅", None: ""}, inplace=True)
+ st.dataframe(df)
+
+def render_section_time(scenarios):
+ st.write(f"### Execution Time")
+ df = get_feature_table(scenarios, "time")
+ df = df.applymap(lambda t: round(t,2), ) # `df.round(2)` is ineffective
+ st.dataframe(df)
+
+
+def render_section_accuracy(scenarios):
+ st.write(f"### Accuracy")
+ df = get_feature_table(scenarios, "accuracy")
+ st.dataframe(df)

dashboard/utils/storage.py → dashboard/storage.py

@@ -7,6 +7,8 @@
 
 import streamlit as st
 
+from dashboard.formatting import custom_title
+
 BUCKET = os.environ.get("BUCKET")
 USE_LOCAL = os.environ.get("USE_LOCAL", "FALSE").lower() == "true" 
 s3 = boto3.client("s3")
@@ -54,4 +56,10 @@ def select_report(ta):
  st.stop()
  timestamp_to_filename = {format_timestamp_from_filename(f): f for f in report_files}
  selected_timestamp = st.selectbox("Select a report", sorted(timestamp_to_filename.keys(), reverse=True))
- return report_files[timestamp_to_filename[selected_timestamp]]
+ report = report_files[timestamp_to_filename[selected_timestamp]] 
+ # TODO: Remove conditional once TA1 and TA3 match
+ if ta == "ta1":
+ for scenario in report["scenarios"].values(): 
+ scenario["steps"] = { custom_title(name): step for name, step in scenario["steps"].items()}
+ scenario["shape"] = [ {"from": custom_title(edge["from"]), "to": custom_title(edge["to"])} for edge in scenario["shape"]]
+ return report 

dashboard/ui/pages/1_TA1.py

@@ -7,88 +7,43 @@
 import streamlit as st
 import pandas as pd
 
-from dashboard.utils.storage import select_report
-from dashboard.utils.formatting import custom_title
+from dashboard.storage import select_report
+from dashboard.formatting import custom_title
+from dashboard.sections import (
+ render_scenario_viewer,
+ render_section_scenario_status, 
+ render_section_integration_status, 
+ render_section_time, 
+ render_section_accuracy
+)
 
-# Let the user select a report based on formatted timestamps
 st.title("TA1 Integration Dashboard")
 
 
-# Let the user select a report based on formatted timestamps
 report = select_report("ta1")
 
-if "scenarios" not in report: # OLD FORMAT
- report_scenarios = report
- services = None
-else: # NEW FORMAT
- report_scenarios = report["scenarios"]
- services = report["services"]
-
-
-test_results = defaultdict(lambda: defaultdict())
-
-for scenario, content in report_scenarios.items():
- for operation, tests in content["operations"].items():
- for name, result in tests.items():
- test_results[name][(content["name"], operation)] = result
-
-scenarios = [report_scenarios[scenario]["name"] for scenario in report_scenarios.keys()]
-operations = list(reduce(lambda left, right: left.union(right), [set(content["operations"].keys()) for content in report_scenarios.values()], set()))
-tests = sorted([i for i in test_results.keys() if i != "Logs"], reverse=True)
-tests.append("Logs")
-
-
-dataframes = {name: pd.DataFrame(index=scenarios, columns=operations) for name in tests}
-
 st.sidebar.markdown("""
 # TA1
 
 TA1 integration status and quality metrics.
- 
-The current metrics are:
-- Status of `knowledge-middleware` integration
-- F-score for conversion of code/equations to AMR
-- Execution time
-- Application logs
 """)
 
-"""
-### Tests Overview
-
-* `Equations to AMR`: tests the ability to send a set of equations to `SKEMA` and to receive a valid AMR as response. Currently only LaTeX is tested.
-* `Code to AMR`: tests the ability to send a code snippet (model core dynamics) to `SKEMA` and to receive a valid AMR as response. Currently only Python is tested.
-* `PDF Extraction`: tests the ability to send a PDF’s text content to `SKEMA` and to receive metadata extractions and groundings in response.
-* `Profile Dataset`: tests the ability to send a CSV dataset and corresponding documentation to `MIT` and to receive a “data card” in response.
-* `PDF to Text`: tests the ability to send a PDF to `Cosmos` and to receive extracted text in response. 
-* `Profile Model`: tests the ability to send a valid AMR and corresponding documentation to `MIT` and to receive a “model card” in response.
-
-Currently tests are run against SKEMA, MIT and Cosmos public instances.
-"""
-
-if services is not None:
- st.write("### Service Info")
- service_names = list(services.keys())
- service_data = {
- "Service": service_names,
- "Source": [services[name]["source"] for name in service_names],
- "Version": [services[name]["version"] for name in service_names],
- }
- st.dataframe(pd.DataFrame(service_data), hide_index=True)
-
-
-st.write("### Scenario Overview")
-scenarios_overview = ""
-for kk, vv in sorted(report_scenarios.items(), key=lambda item: item[1]['name']):
- scenarios_overview += f"- **{vv['name']}**: {vv['description']}\n"
-st.write(scenarios_overview)
-
-for test in tests:
- df = dataframes[test]
- results = test_results[test]
- for (scenario_name, operation), result in results.items():
- df.at[scenario_name, operation] = result
- st.write(f"### {test}")
- df.replace({False: "❌", True: "✅", None: ""}, inplace=True)
- df.columns = [custom_title(col) for col in df.columns]
- df = df.sort_index()
- df
+services = report["services"]
+st.write("## Service Info")
+service_names = list(services.keys())
+service_data = {
+ "Service": service_names,
+ "Source": [services[name]["source"] for name in service_names],
+ "Version": [services[name]["version"] for name in service_names],
+}
+st.dataframe(pd.DataFrame(service_data), hide_index=True)
+
+
+scenarios = report["scenarios"]
+
+st.write("## Testing")
+render_section_scenario_status(scenarios)
+render_section_integration_status(scenarios)
+render_section_time(scenarios)
+render_section_accuracy(scenarios)
+render_scenario_viewer(scenarios)

dashboard/ui/pages/2_TA3.py

@@ -8,8 +8,8 @@
 import streamlit as st
 import pandas as pd
 
-from dashboard.utils.storage import select_report
-from dashboard.utils.formatting import custom_title
+from dashboard.storage import select_report
+from dashboard.formatting import custom_title
 
 
 st.title("TA3 Integration Dashboard")

docker/Dockerfile

@@ -6,9 +6,10 @@ RUN poetry config virtualenvs.create false
 COPY pyproject.toml pyproject.toml
 COPY poetry.lock poetry.lock
 COPY README.md README.md
-COPY dashboard dashboard
+RUN poetry install --no-root
 
-RUN poetry install
+COPY dashboard dashboard
+RUN poetry install --only-root
 
 COPY .streamlit .streamlit
 COPY outputs outputs