From 224423334cb050d74cd327f49b44d3867fe5fb18 Mon Sep 17 00:00:00 2001 From: Spyros Date: Wed, 6 Sep 2023 20:35:30 +0100 Subject: [PATCH 01/30] add action to deploy staging --- .github/workflows/deploy-staging.yml | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/deploy-staging.yml diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 000000000..0d1f15704 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,43 @@ +--- +name: "Deploy to tmp-cre" +on: + workflow_dispatch: # allow manual triggering + workflow_run: + workflows: ["Test-e2e","Lint Code Base","Test"] + types: + - completed + branches: + - staging +jobs: + deploy: + environment: tmp-cre + name: deploy + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: staging + - name: Deploy backend to heroku + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + HEROKU_APP_NAME: tmp-cre + run: | + git config user.name spyros + git config user.email "foo.bar.baz@example.com" # git just needs AN email, it doesn't matter which + # git fetch --all --unshallow --tags + + highest_tag="$(git tag -l --sort=-v:refname | grep ^v | sed 1q)" + echo "highest tag is ${highest_tag}" + echo "autodeployment only supports minor tags" + git remote add heroku https://heroku:${{ secrets.HEROKU_API_KEY }}@git.heroku.com/tmp-cre.git + commit_sha=$(git rev-parse origin/main) + major=$(echo "${highest_tag}" | cut -f1 -d. | tr -dc '0-9') + minor=$(echo "${highest_tag}" | cut -f2 -d. | tr -dc '0-9') + patch=$(echo "${highest_tag}" | cut -f3 -d. | tr -dc '0-9') + patch=$((patch+1)) + new_tag="v${major}.${minor}.${patch}" + echo "new tag is ${new_tag}" + git tag --annotate "${new_tag}" --message "${new_tag}" "${commit_sha}" + git push --force heroku main --follow-tags From 6d6d93d5a57abf7d7e68f096e1bd31a143e0cd91 Mon Sep 17 00:00:00 2001 From: Spyros Date: Wed, 6 Sep 2023 20:40:42 +0100 Subject: [PATCH 02/30] trigger first staging build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e46c683a8..6ccf6a916 100644 --- a/README.md +++ b/README.md @@ -84,4 +84,4 @@ Please see [Contributing](CONTRIBUTING.md) for contributing instructions Roadmap --- -For a roadmap of what we would like to be done please see the [issues](https://github.com/OWASP/common-requirement-enumeration/issues). +For a roadmap of what we would like to be done please see the [issues](https://github.com/OWASP/common-requirement-enumeration/issues). \ No newline at end of file From ccaefc39d2c8aa0b3d096e1f50510312351464dd Mon Sep 17 00:00:00 2001 From: john681611 Date: Wed, 2 Aug 2023 16:11:15 +0100 Subject: [PATCH 03/30] Inital Hack of NEO4j DB creation --- application/database/db.py | 47 +++++++++++++++++++++++ requirements.txt | 76 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/application/database/db.py b/application/database/db.py index cf5c38da8..acd82ae5a 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -1,3 +1,4 @@ +from neo4j import GraphDatabase from sqlalchemy.orm import aliased import os import logging @@ -179,6 +180,13 @@ def add_node(self, *args, **kwargs): @classmethod def add_cre(cls, dbcre: CRE, graph: nx.DiGraph) -> nx.DiGraph: if dbcre: + Neo4j_driver.execute_query( + "MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})", + nid=dbcre.id, + name=dbcre.name, + description=dbcre.description, + external_id=dbcre.external_id, + database_="neo4j") graph.add_node( f"CRE: {dbcre.id}", internal_id=dbcre.id, external_id=dbcre.external_id ) @@ -189,6 +197,21 @@ def add_cre(cls, dbcre: CRE, graph: nx.DiGraph) -> nx.DiGraph: @classmethod def add_dbnode(cls, dbnode: Node, graph: nx.DiGraph) -> nx.DiGraph: if dbnode: + Neo4j_driver.execute_query( + "MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})", + nid=dbnode.id, + name=dbnode.name, + section=dbnode.section, + section_id=dbnode.section_id, + subsection=dbnode.subsection or "", + tags=dbnode.tags, + version=dbnode.version or "", + description=dbnode.description, + ntype=dbnode.ntype, + database_="neo4j") + + # coma separated tags + graph.add_node( "Node: " + str(dbnode.id), internal_id=dbnode.id, @@ -215,6 +238,16 @@ def load_cre_graph(cls, session) -> nx.Graph: graph = cls.add_cre(dbcre=cre, graph=graph) graph.add_edge(f"CRE: {il.group}", f"CRE: {il.cre}", ltype=il.type) + Neo4j_driver.execute_query( + "MATCH (a:CRE), (b:CRE) " + "WHERE a.id = $aID AND b.id = $bID " + "CALL apoc.create.relationship(a,$relType, {},b) " + "YIELD rel " + "RETURN rel", + aID=il.group, + bID=il.cre, + relType=str.upper(il.type).replace(' ', '_'), + database_="neo4j") for lnk in session.query(Links).all(): node = session.query(Node).filter(Node.id == lnk.node).first() @@ -226,6 +259,16 @@ def load_cre_graph(cls, session) -> nx.Graph: graph = cls.add_cre(dbcre=cre, graph=graph) graph.add_edge(f"CRE: {lnk.cre}", f"Node: {str(lnk.node)}", ltype=lnk.type) + Neo4j_driver.execute_query( + "MATCH (a:CRE), (b:Node) " + "WHERE a.id = $aID AND b.id = $bID " + "CALL apoc.create.relationship(a,$relType, {},b) " + "YIELD rel " + "RETURN rel", + aID=lnk.cre, + bID=lnk.node, + relType=str.upper(lnk.type).replace(' ', '_'), + database_="neo4j") return graph @@ -1475,3 +1518,7 @@ def dbCREfromCRE(cre: cre_defs.CRE) -> CRE: external_id=cre.id, tags=",".join(tags), ) + +URI = "neo4j://localhost:7687" +AUTH = ("neo4j", "password") +Neo4j_driver = GraphDatabase.driver(URI, auth=AUTH) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7eb93e3db..025dbc184 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,5 +29,81 @@ semver setuptools==66.1.1 simplify_docx==0.1.2 SQLAlchemy==2.0.20 +compliance-trestle +nose==1.3.7 +numpy==1.23.0 +neo4j==5.11.0 +openapi-schema-validator==0.3.4 +openapi-spec-validator==0.5.1 +openpyxl==3.1.0 +orderedmultidict==1.0.1 +orjson==3.8.5 +packaging +paramiko==3.0.0 +pathable==0.4.3 +pathspec==0.9.0 +pbr==5.8.0 +pep517==0.8.2 +Pillow==9.1.1 +pip-autoremove==0.9.1 +platformdirs==2.2.0 +playwright==1.33.0 +pluggy==1.0.0 +prance +prompt-toolkit==3.0.19 +proto-plus==1.22.2 +protobuf==4.23.1 +psycopg2==2.9.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.7.0 +pycparser==2.21 +pydantic==1.10.4 +pyee==9.0.4 +pyflakes==2.3.1 +PyGithub==1.53 +PyJWT==1.7.1 +PyNaCl==1.5.0 +pyparsing==2.4.6 +pyrsistent==0.17.3 +PySnooper==1.1.1 +pytest==7.3.1 +pytest-base-url==2.0.0 +pytest-playwright==0.3.3 +python-dateutil==2.8.1 +python-docx==0.8.11 +python-dotenv==0.21.1 +python-frontmatter==1.0.0 +python-markdown-maker==1.0 +python-slugify==8.0.1 +PyYAML==5.3.1 +regex==2021.11.10 +requests==2.27.1 +requests-oauthlib==1.3.1 +rfc3986==1.5.0 +rsa==4.7 +ruamel.yaml==0.17.21 +ruamel.yaml.clib==0.2.7 +scikit-learn==1.2.2 +Shapely==1.8.5.post1 +simplify-docx==0.1.2 +six==1.15.0 +smmap==3.0.4 +sniffio==1.3.0 +soupsieve==2.4.1 +SQLAlchemy==1.3.23 +sqlalchemy-stubs==0.4 +testresources==2.0.1 +text-unidecode==1.3 +threadpoolctl==3.1.0 +toml==0.10.2 +tomli==1.2.2 +tqdm==4.65.0 +typed-ast==1.5.4 +types-PyYAML==5.4.8 +typing-inspect==0.7.1 +typing_extensions==4.4.0 +untangle==1.1.1 +urllib3==1.26.8 vertexai==0.0.1 xmltodict==0.13.0 From 1499be4e0605e95037089f43664251239ac8c6d4 Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 10 Aug 2023 13:50:10 +0100 Subject: [PATCH 04/30] Added: Neo4j docker run --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 1e9f86ac7..ef43d81e7 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ docker: docker-run: docker run -it -p 5000:5000 opencre:$(shell git rev-parse HEAD) +docker-neo4j: + docker run --env NEO4J_PLUGINS='["apoc"]' --volume=/Users/johnharvey/neo4j/data:/data --volume=/data --volume=/logs --workdir=/var/lib/neo4j -p 7474:7474 -p 7687:7687 -d neo4j + lint: [ -d "./venv" ] && . ./venv/bin/activate && black . && yarn lint From 560e1566c5f43cac0e9c889e39db710d26578026 Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 10 Aug 2023 13:50:20 +0100 Subject: [PATCH 05/30] Added NEO_DB Class --- application/database/db.py | 144 +++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 46 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index acd82ae5a..764e18938 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -1,4 +1,6 @@ + from neo4j import GraphDatabase +import neo4j from sqlalchemy.orm import aliased import os import logging @@ -157,14 +159,102 @@ class Embeddings(BaseModel): # type: ignore ) + +class NEO_DB: + __instance = None + + driver = None + connected = False + @classmethod + def instance(self): + if self.__instance is None: + self.__instance = self.__new__(self) + + URI = os.getenv('NEO4J_URI') or "neo4j://localhost:7687" + AUTH = (os.getenv('NEO4J_USR') or "neo4j", os.getenv('NEO4J_PASS') or "password") + self.driver = GraphDatabase.driver(URI, auth=AUTH) + + try: + self.driver.verify_connectivity() + self.connected = True + except neo4j.exceptions.ServiceUnavailable: + logger.error("NEO4J ServiceUnavailable error - disabling neo4j related features") + + return self.__instance + + def __init__(sel): + raise ValueError("NEO_DB is a singleton, please call instance() instead") + + @classmethod + def add_cre(self, dbcre: CRE): + if not self.connected: + return + self.driver.execute_query( + "MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})", + nid=dbcre.id, + name=dbcre.name, + description=dbcre.description, + external_id=dbcre.external_id, + database_="neo4j") + + @classmethod + def add_dbnode(self, dbnode: Node): + if not self.connected: + return + self.driver.execute_query( + "MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})", + nid=dbnode.id, + name=dbnode.name, + section=dbnode.section, + section_id=dbnode.section_id, + subsection=dbnode.subsection or "", + tags=dbnode.tags, + version=dbnode.version or "", + description=dbnode.description, + ntype=dbnode.ntype, + database_="neo4j") + + @classmethod + def link_CRE_to_CRE(self, id1, id2, link_type): + if not self.connected: + return + self.driver.execute_query( + "MATCH (a:CRE), (b:CRE) " + "WHERE a.id = $aID AND b.id = $bID " + "CALL apoc.create.relationship(a,$relType, {},b) " + "YIELD rel " + "RETURN rel", + aID=id1, + bID=id2, + relType=str.upper(link_type).replace(' ', '_'), + database_="neo4j") + + @classmethod + def link_CRE_to_Node(self, CRE_id, node_id, link_type): + if not self.connected: + return + self.driver.execute_query( + "MATCH (a:CRE), (b:Node) " + "WHERE a.id = $aID AND b.id = $bID " + "CALL apoc.create.relationship(a,$relType, {},b) " + "YIELD rel " + "RETURN rel", + aID=CRE_id, + bID=node_id, + relType=str.upper(link_type).replace(' ', '_'), + database_="neo4j") + + class CRE_Graph: graph: nx.Graph = None + neo_db: NEO_DB = None __instance = None @classmethod - def instance(cls, session): + def instance(cls, session, neo_db: NEO_DB): if cls.__instance is None: cls.__instance = cls.__new__(cls) + cls.neo_db = neo_db cls.graph = cls.load_cre_graph(session) return cls.__instance @@ -180,13 +270,7 @@ def add_node(self, *args, **kwargs): @classmethod def add_cre(cls, dbcre: CRE, graph: nx.DiGraph) -> nx.DiGraph: if dbcre: - Neo4j_driver.execute_query( - "MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})", - nid=dbcre.id, - name=dbcre.name, - description=dbcre.description, - external_id=dbcre.external_id, - database_="neo4j") + cls.neo_db.add_cre(dbcre) graph.add_node( f"CRE: {dbcre.id}", internal_id=dbcre.id, external_id=dbcre.external_id ) @@ -197,19 +281,7 @@ def add_cre(cls, dbcre: CRE, graph: nx.DiGraph) -> nx.DiGraph: @classmethod def add_dbnode(cls, dbnode: Node, graph: nx.DiGraph) -> nx.DiGraph: if dbnode: - Neo4j_driver.execute_query( - "MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})", - nid=dbnode.id, - name=dbnode.name, - section=dbnode.section, - section_id=dbnode.section_id, - subsection=dbnode.subsection or "", - tags=dbnode.tags, - version=dbnode.version or "", - description=dbnode.description, - ntype=dbnode.ntype, - database_="neo4j") - + cls.neo_db.add_dbnode(dbnode) # coma separated tags graph.add_node( @@ -238,16 +310,7 @@ def load_cre_graph(cls, session) -> nx.Graph: graph = cls.add_cre(dbcre=cre, graph=graph) graph.add_edge(f"CRE: {il.group}", f"CRE: {il.cre}", ltype=il.type) - Neo4j_driver.execute_query( - "MATCH (a:CRE), (b:CRE) " - "WHERE a.id = $aID AND b.id = $bID " - "CALL apoc.create.relationship(a,$relType, {},b) " - "YIELD rel " - "RETURN rel", - aID=il.group, - bID=il.cre, - relType=str.upper(il.type).replace(' ', '_'), - database_="neo4j") + cls.neo_db.link_CRE_to_CRE(il.group, il.cre, il.type) for lnk in session.query(Links).all(): node = session.query(Node).filter(Node.id == lnk.node).first() @@ -259,26 +322,19 @@ def load_cre_graph(cls, session) -> nx.Graph: graph = cls.add_cre(dbcre=cre, graph=graph) graph.add_edge(f"CRE: {lnk.cre}", f"Node: {str(lnk.node)}", ltype=lnk.type) - Neo4j_driver.execute_query( - "MATCH (a:CRE), (b:Node) " - "WHERE a.id = $aID AND b.id = $bID " - "CALL apoc.create.relationship(a,$relType, {},b) " - "YIELD rel " - "RETURN rel", - aID=lnk.cre, - bID=lnk.node, - relType=str.upper(lnk.type).replace(' ', '_'), - database_="neo4j") + cls.neo_db.link_CRE_to_Node(lnk.cre, lnk.node, lnk.type) return graph class Node_collection: graph: nx.Graph = None + neo_db: NEO_DB = None session = sqla.session def __init__(self) -> None: if not os.environ.get("NO_LOAD_GRAPH"): - self.graph = CRE_Graph.instance(sqla.session) + self.neo_db = NEO_DB.instance() + self.graph = CRE_Graph.instance(sqla.session, self.neo_db) self.session = sqla.session def __get_external_links(self) -> List[Tuple[CRE, Node, str]]: @@ -1518,7 +1574,3 @@ def dbCREfromCRE(cre: cre_defs.CRE) -> CRE: external_id=cre.id, tags=",".join(tags), ) - -URI = "neo4j://localhost:7687" -AUTH = ("neo4j", "password") -Neo4j_driver = GraphDatabase.driver(URI, auth=AUTH) \ No newline at end of file From a5b56b2c0b73ce1807b1a32c249176fe06557aa9 Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 10 Aug 2023 15:26:16 +0100 Subject: [PATCH 06/30] Create Inital Path API response --- application/database/db.py | 99 +++++++++++++++++++++++++++++-------- application/web/web_main.py | 14 ++++-- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index 764e18938..db8e6aa2a 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -243,7 +243,64 @@ def link_CRE_to_Node(self, CRE_id, node_id, link_type): bID=node_id, relType=str.upper(link_type).replace(' ', '_'), database_="neo4j") + @classmethod + def gap_analysis(self, name_1, name_2): + if not self.connected: + return + records, _, _ = self.driver.execute_query( + "MATCH" + "(BaseStandard:Node {name: $name1}), " + "(CompareStandard:Node {name: $name2}), " + "p = shortestPath((BaseStandard)-[*]-(CompareStandard)) " + "WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n = BaseStandard or n = CompareStandard) " + "RETURN p ", + name1=name_1, + name2=name_2, + database_="neo4j" + ) + def format_segment(seg): + return { + "start": { + "name": seg.start_node["name"], + "sectionID": seg.start_node["section_id"], + "section": seg.start_node["section"], + "subsection": seg.start_node["subsection"], + "description": seg.start_node["description"], + "id": seg.start_node["id"] + }, + "end": { + "name": seg.end_node["name"], + "sectionID": seg.end_node["section_id"], + "section": seg.end_node["section"], + "subsection": seg.end_node["subsection"], + "description": seg.end_node["description"], + "id": seg.end_node["id"] + }, + "relationship": seg.type + } + + def format_record(rec): + return { + "start": { + "name": rec.start_node["name"], + "sectionID": rec.start_node["section_id"], + "section": rec.start_node["section"], + "subsection": rec.start_node["subsection"], + "description": rec.start_node["description"], + "id": rec.start_node["id"] + }, + "end": { + "name": rec.end_node["name"], + "sectionID": rec.end_node["section_id"], + "section": rec.end_node["section"], + "subsection": rec.end_node["subsection"], + "description": rec.end_node["description"], + "id": rec.end_node["id"] + }, + "path": [format_segment(seg) for seg in rec.relationships] + } + return [format_record(rec['p']) for rec in records] class CRE_Graph: graph: nx.Graph = None @@ -255,7 +312,7 @@ def instance(cls, session, neo_db: NEO_DB): if cls.__instance is None: cls.__instance = cls.__new__(cls) cls.neo_db = neo_db - cls.graph = cls.load_cre_graph(session) + # cls.graph = cls.load_cre_graph(session) return cls.__instance def __init__(sel): @@ -1158,30 +1215,30 @@ def find_path_between_nodes( return res - def gap_analysis(self, node_names: List[str]) -> List[cre_defs.Node]: + def gap_analysis(self, node_names: List[str]): """Since the CRE structure is a tree-like graph with leaves being nodes we can find the paths between nodes find_path_between_nodes() is a graph-path-finding method """ - processed_nodes = [] - dbnodes: List[Node] = [] - for name in node_names: - dbnodes.extend(self.session.query(Node).filter(Node.name == name).all()) - - for node in dbnodes: - working_node = nodeFromDB(node) - for other_node in dbnodes: - if node.id == other_node.id: - continue - if self.find_path_between_nodes(node.id, other_node.id): - working_node.add_link( - cre_defs.Link( - ltype=cre_defs.LinkTypes.LinkedTo, - document=nodeFromDB(other_node), - ) - ) - processed_nodes.append(working_node) - return processed_nodes + # processed_nodes = [] + # dbnodes: List[Node] = [] + # for name in node_names: + # dbnodes.extend(self.session.query(Node).filter(Node.name == name).all()) + + # for node in dbnodes: + # working_node = nodeFromDB(node) + # for other_node in dbnodes: + # if node.id == other_node.id: + # continue + # if self.find_path_between_nodes(node.id, other_node.id): + # working_node.add_link( + # cre_defs.Link( + # ltype=cre_defs.LinkTypes.LinkedTo, + # document=nodeFromDB(other_node), + # ) + # ) + # processed_nodes.append(working_node) + return self.neo_db.gap_analysis(node_names[0], node_names[1]) def text_search(self, text: str) -> List[Optional[cre_defs.Document]]: """Given a piece of text, tries to find the best match diff --git a/application/web/web_main.py b/application/web/web_main.py index 50955eed9..989b6462d 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -208,10 +208,16 @@ def find_document_by_tag() -> Any: def gap_analysis() -> Any: # TODO (spyros): add export result to spreadsheet database = db.Node_collection() standards = request.args.getlist("standard") - documents = database.gap_analysis(standards=standards) - if documents: - res = [doc.todict() for doc in documents] - return jsonify(res) + paths = database.gap_analysis(standards) + grouped_paths = {} + for path in paths: + key = path['start']['id'] + if key not in grouped_paths: + grouped_paths[key] = {"start": path['start'], "paths": []} + del path['start'] + grouped_paths[key]['paths'].append(path) + + return jsonify(grouped_paths) @app.route("/rest/v1/text_search", methods=["GET"]) From 2f1c62e3c5cfefd28a5125281dde97625dd8c72b Mon Sep 17 00:00:00 2001 From: john681611 Date: Fri, 11 Aug 2023 17:22:06 +0100 Subject: [PATCH 07/30] Build basic UI for testing --- application/database/db.py | 129 +++++++++-------- application/frontend/src/const.ts | 1 + .../src/pages/GapAnalysis/GapAnalysis.tsx | 133 ++++++++++++++++++ application/frontend/src/routes.tsx | 20 ++- application/web/web_main.py | 10 +- 5 files changed, 228 insertions(+), 65 deletions(-) create mode 100644 application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx diff --git a/application/database/db.py b/application/database/db.py index db8e6aa2a..4987a1bfb 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -1,4 +1,3 @@ - from neo4j import GraphDatabase import neo4j from sqlalchemy.orm import aliased @@ -159,90 +158,100 @@ class Embeddings(BaseModel): # type: ignore ) - class NEO_DB: __instance = None - + driver = None connected = False + @classmethod def instance(self): if self.__instance is None: self.__instance = self.__new__(self) - URI = os.getenv('NEO4J_URI') or "neo4j://localhost:7687" - AUTH = (os.getenv('NEO4J_USR') or "neo4j", os.getenv('NEO4J_PASS') or "password") + URI = os.getenv("NEO4J_URI") or "neo4j://localhost:7687" + AUTH = ( + os.getenv("NEO4J_USR") or "neo4j", + os.getenv("NEO4J_PASS") or "password", + ) self.driver = GraphDatabase.driver(URI, auth=AUTH) try: self.driver.verify_connectivity() self.connected = True - except neo4j.exceptions.ServiceUnavailable: - logger.error("NEO4J ServiceUnavailable error - disabling neo4j related features") - + except neo4j.exceptions.ServiceUnavailable: + logger.error( + "NEO4J ServiceUnavailable error - disabling neo4j related features" + ) + return self.__instance def __init__(sel): raise ValueError("NEO_DB is a singleton, please call instance() instead") - + @classmethod def add_cre(self, dbcre: CRE): - if not self.connected: + if not self.connected: return - self.driver.execute_query( - "MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})", - nid=dbcre.id, - name=dbcre.name, - description=dbcre.description, - external_id=dbcre.external_id, - database_="neo4j") - + self.driver.execute_query( + "MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})", + nid=dbcre.id, + name=dbcre.name, + description=dbcre.description, + external_id=dbcre.external_id, + database_="neo4j", + ) + @classmethod def add_dbnode(self, dbnode: Node): if not self.connected: return self.driver.execute_query( - "MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})", - nid=dbnode.id, - name=dbnode.name, - section=dbnode.section, - section_id=dbnode.section_id, - subsection=dbnode.subsection or "", - tags=dbnode.tags, - version=dbnode.version or "", - description=dbnode.description, - ntype=dbnode.ntype, - database_="neo4j") - + "MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})", + nid=dbnode.id, + name=dbnode.name, + section=dbnode.section, + section_id=dbnode.section_id, + subsection=dbnode.subsection or "", + tags=dbnode.tags, + version=dbnode.version or "", + description=dbnode.description, + ntype=dbnode.ntype, + database_="neo4j", + ) + @classmethod def link_CRE_to_CRE(self, id1, id2, link_type): if not self.connected: return self.driver.execute_query( - "MATCH (a:CRE), (b:CRE) " - "WHERE a.id = $aID AND b.id = $bID " - "CALL apoc.create.relationship(a,$relType, {},b) " - "YIELD rel " - "RETURN rel", - aID=id1, - bID=id2, - relType=str.upper(link_type).replace(' ', '_'), - database_="neo4j") - + "MATCH (a:CRE), (b:CRE) " + "WHERE a.id = $aID AND b.id = $bID " + "CALL apoc.create.relationship(a,$relType, {},b) " + "YIELD rel " + "RETURN rel", + aID=id1, + bID=id2, + relType=str.upper(link_type).replace(" ", "_"), + database_="neo4j", + ) + @classmethod def link_CRE_to_Node(self, CRE_id, node_id, link_type): if not self.connected: return self.driver.execute_query( - "MATCH (a:CRE), (b:Node) " - "WHERE a.id = $aID AND b.id = $bID " - "CALL apoc.create.relationship(a,$relType, {},b) " - "YIELD rel " - "RETURN rel", - aID=CRE_id, - bID=node_id, - relType=str.upper(link_type).replace(' ', '_'), - database_="neo4j") + "MATCH (a:CRE), (b:Node) " + "WHERE a.id = $aID AND b.id = $bID " + "CALL apoc.create.relationship(a,$relType, {},b) " + "YIELD rel " + "RETURN rel", + aID=CRE_id, + bID=node_id, + relType=str.upper(link_type).replace(" ", "_"), + database_="neo4j", + ) + @classmethod def gap_analysis(self, name_1, name_2): if not self.connected: @@ -256,18 +265,18 @@ def gap_analysis(self, name_1, name_2): "RETURN p ", name1=name_1, name2=name_2, - database_="neo4j" + database_="neo4j", ) def format_segment(seg): - return { + return { "start": { "name": seg.start_node["name"], "sectionID": seg.start_node["section_id"], "section": seg.start_node["section"], "subsection": seg.start_node["subsection"], "description": seg.start_node["description"], - "id": seg.start_node["id"] + "id": seg.start_node["id"], }, "end": { "name": seg.end_node["name"], @@ -275,9 +284,9 @@ def format_segment(seg): "section": seg.end_node["section"], "subsection": seg.end_node["subsection"], "description": seg.end_node["description"], - "id": seg.end_node["id"] + "id": seg.end_node["id"], }, - "relationship": seg.type + "relationship": seg.type, } def format_record(rec): @@ -288,7 +297,7 @@ def format_record(rec): "section": rec.start_node["section"], "subsection": rec.start_node["subsection"], "description": rec.start_node["description"], - "id": rec.start_node["id"] + "id": rec.start_node["id"], }, "end": { "name": rec.end_node["name"], @@ -296,11 +305,13 @@ def format_record(rec): "section": rec.end_node["section"], "subsection": rec.end_node["subsection"], "description": rec.end_node["description"], - "id": rec.end_node["id"] + "id": rec.end_node["id"], }, - "path": [format_segment(seg) for seg in rec.relationships] + "path": [format_segment(seg) for seg in rec.relationships], } - return [format_record(rec['p']) for rec in records] + + return [format_record(rec["p"]) for rec in records] + class CRE_Graph: graph: nx.Graph = None @@ -339,7 +350,7 @@ def add_cre(cls, dbcre: CRE, graph: nx.DiGraph) -> nx.DiGraph: def add_dbnode(cls, dbnode: Node, graph: nx.DiGraph) -> nx.DiGraph: if dbnode: cls.neo_db.add_dbnode(dbnode) - # coma separated tags + # coma separated tags graph.add_node( "Node: " + str(dbnode.id), diff --git a/application/frontend/src/const.ts b/application/frontend/src/const.ts index 231f78447..cc2afdfc8 100644 --- a/application/frontend/src/const.ts +++ b/application/frontend/src/const.ts @@ -36,3 +36,4 @@ export const CRE = '/cre'; export const GRAPH = '/graph'; export const DEEPLINK = '/deeplink'; export const BROWSEROOT = '/root_cres'; +export const GAP_ANALYSIS = '/gap_analysis'; diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx new file mode 100644 index 000000000..99aeff39c --- /dev/null +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -0,0 +1,133 @@ +import React, { useEffect, useState } from 'react'; +import { Dropdown, Label, Popup, Segment, Table } from 'semantic-ui-react'; + +import { useEnvironment } from '../../hooks'; + +const GetSegmentText = (segment, segmentID) => { + let textPart = segment.end; + let nextID = segment.end.id; + let arrow = '->'; + if (segmentID !== segment.start.id) { + textPart = segment.start; + nextID = segment.start.id; + arrow = '<-'; + } + const text = `${arrow} ${segment.relationship} ${arrow} ${textPart.name} ${textPart.sectionID} ${textPart.section} ${textPart.subsection} ${textPart.description}`; + return { text, nextID }; +}; + +export const GapAnalysis = () => { + const standardOptions = [ + { key: '', text: '', value: undefined }, + { key: 'OWASP Top 10 2021', text: 'OWASP Top 10 2021', value: 'OWASP Top 10 2021' }, + { key: 'NIST 800-53 v5', text: 'NIST 800-53 v5', value: 'NIST 800-53 v5' }, + { key: 'ISO 27001', text: 'ISO 27001', value: 'ISO 27001' }, + { key: 'Cloud Controls Matrix', text: 'Cloud Controls Matrix', value: 'Cloud Controls Matrix' }, + { key: 'ASVS', text: 'ASVS', value: 'ASVS' }, + { key: 'OWASP Proactive Controls', text: 'OWASP Proactive Controls', value: 'OWASP Proactive Controls' }, + { key: 'SAMM', text: 'SAMM', value: 'SAMM' }, + { key: 'CWE', text: 'CWE', value: 'CWE' }, + { key: 'OWASP Cheat Sheets', text: 'OWASP Cheat Sheets', value: 'OWASP Cheat Sheets' }, + { + key: 'OWASP Web Security Testing Guide (WSTG)', + text: 'OWASP Web Security Testing Guide (WSTG)', + value: 'OWASP Web Security Testing Guide (WSTG)', + }, + { key: 'NIST 800-63', text: 'NIST 800-63', value: 'NIST 800-63' }, + { key: 'Cheat_sheets', text: 'Cheat_sheets', value: 'Cheat_sheets' }, + { key: 'CAPEC', text: 'CAPEC', value: 'CAPEC' }, + { key: 'ZAP Rule', text: 'ZAP Rule', value: 'ZAP Rule' }, + { key: 'OWASP', text: 'OWASP', value: 'OWASP' }, + { + key: 'OWASP Secure Headers Project', + text: 'OWASP Secure Headers Project', + value: 'OWASP Secure Headers Project', + }, + { key: 'PCI DSS', text: 'PCI DSS', value: 'PCI DSS' }, + { key: 'OWASP Juice Shop', text: 'OWASP Juice Shop', value: 'OWASP Juice Shop' }, + ]; + const [BaseStandard, setBaseStandard] = useState(); + const [CompareStandard, setCompareStandard] = useState(); + const [gapAnalysis, setGapAnalysis] = useState(); + const { apiUrl } = useEnvironment(); + useEffect(() => { + const fetchData = async () => { + const result = await fetch( + `${apiUrl}/gap_analysis?standard=${BaseStandard}&standard=${CompareStandard}` + ); + const resultObj = await result.json(); + setGapAnalysis(resultObj); + }; + + if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; + fetchData().catch(console.error); + }, [BaseStandard, CompareStandard, setGapAnalysis]); + + return ( +
+ setBaseStandard(value?.toString())} + /> + setCompareStandard(value?.toString())} + /> + {gapAnalysis && ( + + + + {BaseStandard} + {CompareStandard} + + + + + {Object.keys(gapAnalysis).map((key) => ( + + + + + + {gapAnalysis[key].paths.map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} + {path.end.description},{' '} + + } + /> + ); + })} +
({gapAnalysis[key].paths.length}) + + + ))} + +
+ )} +
+ ); +}; diff --git a/application/frontend/src/routes.tsx b/application/frontend/src/routes.tsx index 876462503..548c2d7a3 100644 --- a/application/frontend/src/routes.tsx +++ b/application/frontend/src/routes.tsx @@ -1,10 +1,22 @@ import { ReactNode } from 'react'; -import { BROWSEROOT, CRE, DEEPLINK, GRAPH, INDEX, SEARCH, SECTION, SECTION_ID, STANDARD } from './const'; +import { + BROWSEROOT, + CRE, + DEEPLINK, + GAP_ANALYSIS, + GRAPH, + INDEX, + SEARCH, + SECTION, + SECTION_ID, + STANDARD, +} from './const'; import { CommonRequirementEnumeration, Graph, Search, Standard } from './pages'; import { BrowseRootCres } from './pages/BrowseRootCres/browseRootCres'; import { Chatbot } from './pages/chatbot/chatbot'; import { Deeplink } from './pages/Deeplink/Deeplink'; +import { GapAnalysis } from './pages/GapAnalysis/GapAnalysis'; import { MembershipRequired } from './pages/MembershipRequired/MembershipRequired'; import { SearchName } from './pages/Search/SearchName'; import { StandardSection } from './pages/Standard/StandardSection'; @@ -23,6 +35,12 @@ export const ROUTES: IRoute[] = [ showFilter: false, showHeader: false, }, + { + path: GAP_ANALYSIS, + component: GapAnalysis, + showHeader: true, + showFilter: false, + }, { path: `/node${STANDARD}/:id${SECTION}/:section`, component: StandardSection, diff --git a/application/web/web_main.py b/application/web/web_main.py index 989b6462d..546fb6d00 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -211,12 +211,12 @@ def gap_analysis() -> Any: # TODO (spyros): add export result to spreadsheet paths = database.gap_analysis(standards) grouped_paths = {} for path in paths: - key = path['start']['id'] + key = path["start"]["id"] if key not in grouped_paths: - grouped_paths[key] = {"start": path['start'], "paths": []} - del path['start'] - grouped_paths[key]['paths'].append(path) - + grouped_paths[key] = {"start": path["start"], "paths": []} + del path["start"] + grouped_paths[key]["paths"].append(path) + return jsonify(grouped_paths) From eeb853352ac7f579ebff7104b1414cecb96c2adb Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 21 Aug 2023 17:23:41 +0100 Subject: [PATCH 08/30] Mock score and WIP UI --- application/database/db.py | 12 +++ .../src/pages/GapAnalysis/GapAnalysis.tsx | 94 ++++++++++++++----- application/utils/gap_analysis.py | 5 + application/web/web_main.py | 2 + 4 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 application/utils/gap_analysis.py diff --git a/application/database/db.py b/application/database/db.py index 4987a1bfb..42dd1a361 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -268,6 +268,18 @@ def gap_analysis(self, name_1, name_2): database_="neo4j", ) + # records_no_related, _, _ = self.driver.execute_query( + # "MATCH" + # "(BaseStandard:Node {name: $name1}), " + # "(CompareStandard:Node {name: $name2}), " + # "p = shortestPath((BaseStandard)-[*]-(CompareStandard)) " + # "WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n = BaseStandard or n = CompareStandard) AND ALL(r IN relationships(p) WHERE NOT r:RELATED) " + # "RETURN p ", + # name1=name_1, + # name2=name_2, + # database_="neo4j", + # ) + def format_segment(seg): return { "start": { diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 99aeff39c..2b952ce82 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react'; -import { Dropdown, Label, Popup, Segment, Table } from 'semantic-ui-react'; +import { Accordion, Dropdown, Icon, Label, Popup, Segment, Table } from 'semantic-ui-react'; import { useEnvironment } from '../../hooks'; @@ -49,6 +49,7 @@ export const GapAnalysis = () => { const [BaseStandard, setBaseStandard] = useState(); const [CompareStandard, setCompareStandard] = useState(); const [gapAnalysis, setGapAnalysis] = useState(); + const [activeIndex, SetActiveIndex] = useState(); const { apiUrl } = useEnvironment(); useEffect(() => { const fetchData = async () => { @@ -63,6 +64,12 @@ export const GapAnalysis = () => { fetchData().catch(console.error); }, [BaseStandard, CompareStandard, setGapAnalysis]); + const handleAccordionClick = (e, titleProps) => { + const { index } = titleProps + const newIndex = activeIndex === index ? -1 : index + SetActiveIndex(newIndex) + } + return (
{ - {gapAnalysis[key].paths.map((path) => { - let segmentID = gapAnalysis[key].start.id; - return ( - { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} - trigger={ - - {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} - {path.end.description},{' '} - - } - /> - ); - })} -
({gapAnalysis[key].paths.length}) + + + + {gapAnalysis[key].paths.sort((a, b) => a.score - b.score).slice(0, 3).map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + <> + { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} + {path.end.description}{' '}({path.score}) + + } + /> +
+ + ); + })} + (Total Links: {gapAnalysis[key].paths.length}) + + + {gapAnalysis[key].paths.sort((a, b) => a.score - b.score).slice(2, gapAnalysis[key].paths.length).map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + <> + { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} + {path.end.description}{' '}({path.score}) + + } + /> +
+ + ); + })} + + ))} diff --git a/application/utils/gap_analysis.py b/application/utils/gap_analysis.py new file mode 100644 index 000000000..fe3be0e39 --- /dev/null +++ b/application/utils/gap_analysis.py @@ -0,0 +1,5 @@ +import random + + +def get_path_score(path): + return random.randint(10, 100) \ No newline at end of file diff --git a/application/web/web_main.py b/application/web/web_main.py index 546fb6d00..f4cf126ec 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -15,6 +15,7 @@ from application.defs import osib_defs as odefs from application.utils import spreadsheet as sheet_utils from application.utils import mdutils, redirectors +from application.utils.gap_analysis import get_path_score from application.prompt_client import prompt_client as prompt_client from enum import Enum from flask import ( @@ -214,6 +215,7 @@ def gap_analysis() -> Any: # TODO (spyros): add export result to spreadsheet key = path["start"]["id"] if key not in grouped_paths: grouped_paths[key] = {"start": path["start"], "paths": []} + path['score'] = get_path_score(path) del path["start"] grouped_paths[key]["paths"].append(path) From 7cb9ef2f808e0a2f8cab8803b3309b5c49d3fc5d Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 24 Aug 2023 09:57:58 +0100 Subject: [PATCH 09/30] implement scoring and basic tests --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 122 +++++++++--------- application/tests/gap_analysis_test.py | 103 +++++++++++++++ application/utils/gap_analysis.py | 27 +++- application/web/web_main.py | 2 +- 4 files changed, 190 insertions(+), 64 deletions(-) create mode 100644 application/tests/gap_analysis_test.py diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 2b952ce82..be6041207 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -65,10 +65,10 @@ export const GapAnalysis = () => { }, [BaseStandard, CompareStandard, setGapAnalysis]); const handleAccordionClick = (e, titleProps) => { - const { index } = titleProps - const newIndex = activeIndex === index ? -1 : index - SetActiveIndex(newIndex) - } + const { index } = titleProps; + const newIndex = activeIndex === index ? -1 : index; + SetActiveIndex(newIndex); + }; return (
@@ -107,65 +107,67 @@ export const GapAnalysis = () => { - - - {gapAnalysis[key].paths.sort((a, b) => a.score - b.score).slice(0, 3).map((path) => { - let segmentID = gapAnalysis[key].start.id; - return ( - <> - { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} - trigger={ - - {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} - {path.end.description}{' '}({path.score}) - - } - /> -
- - ); - })} + + + {gapAnalysis[key].paths + .sort((a, b) => a.score - b.score) + .slice(0, 3) + .map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + <> + { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section}{' '} + {path.end.subsection} {path.end.description} ({path.score}) + + } + /> +
+ + ); + })} (Total Links: {gapAnalysis[key].paths.length}) - {gapAnalysis[key].paths.sort((a, b) => a.score - b.score).slice(2, gapAnalysis[key].paths.length).map((path) => { - let segmentID = gapAnalysis[key].start.id; - return ( - <> - { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} - trigger={ - - {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} - {path.end.description}{' '}({path.score}) - - } - /> -
- - ); - })} + {gapAnalysis[key].paths + .sort((a, b) => a.score - b.score) + .slice(2, gapAnalysis[key].paths.length) + .map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + <> + { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section}{' '} + {path.end.subsection} {path.end.description} ({path.score}) + + } + /> +
+ + ); + })} diff --git a/application/tests/gap_analysis_test.py b/application/tests/gap_analysis_test.py new file mode 100644 index 000000000..a1414c6d9 --- /dev/null +++ b/application/tests/gap_analysis_test.py @@ -0,0 +1,103 @@ +import unittest + +from application.utils.gap_analysis import ( + get_path_score, + get_relation_direction, + get_next_id, + PENALTIES +) + + +class TestGapAnalysis(unittest.TestCase): + def tearDown(self) -> None: + return None + + def setUp(self) -> None: + return None + + def test_get_relation_direction_UP(self): + step = {"start": {"id": "123"}, "end": {"id": "234"}} + self.assertEqual(get_relation_direction(step, "123"), "UP") + + def test_get_relation_direction_DOWN(self): + step = {"start": {"id": "123"}, "end": {"id": "234"}} + self.assertEqual(get_relation_direction(step, "234"), "DOWN") + + def test_get_next_id_start(self): + step = {"start": {"id": "123"}, "end": {"id": "234"}} + self.assertEqual(get_next_id(step, "234"), "123") + + def test_get_next_id_end(self): + step = {"start": {"id": "123"}, "end": {"id": "234"}} + self.assertEqual(get_next_id(step, "123"), "234") + + def test_get_path_score_direct_siblings_zero(self): + path = { + "start": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "path": [ + { + "end": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "relationship": "LINKED_TO", + "start": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + }, + { + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "relationship": "LINKED_TO", + "start": { + "id": "e2ac59b2-c1d8-4525-a6b3-155d480aecc9", + }, + }, + ], + } + self.assertEqual(get_path_score(path), 0) + + def test_get_path_score_one_up_zero(self): + path = { + "start": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "path": [ + { + "end": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "relationship": "LINKED_TO", + "start": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + }, + { + "end": { + "id": "123", + }, + "relationship": "CONTAINS", + "start": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + }, + { + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "relationship": "LINKED_TO", + "start": { + "id": "123", + }, + }, + ], + } + self.assertEqual(get_path_score(path), PENALTIES['CONTAINS_UP']) diff --git a/application/utils/gap_analysis.py b/application/utils/gap_analysis.py index fe3be0e39..308714592 100644 --- a/application/utils/gap_analysis.py +++ b/application/utils/gap_analysis.py @@ -1,5 +1,26 @@ -import random +PENALTIES = {"RELATED": 20, "CONTAINS_UP": 2, "CONTAINS_DOWN": 1, "LINKED_TO": 0} -def get_path_score(path): - return random.randint(10, 100) \ No newline at end of file +def get_path_score(path, start_id): + score = 0 + previous_id = start_id + for step in path["path"]: + penalty_type = step["relationship"] + + if step["relationship"] == "CONTAINS": + penalty_type = f"CONTAINS_{get_relation_direction(step, previous_id)}" + score += PENALTIES[penalty_type] + previous_id = get_next_id(step, previous_id) + return score + + +def get_relation_direction(step, previous_id): + if step["start"]["id"] == previous_id: + return "UP" + return "DOWN" + + +def get_next_id(step, previous_id): + if step["start"]["id"] == previous_id: + return step["end"]["id"] + return step["start"]["id"] diff --git a/application/web/web_main.py b/application/web/web_main.py index f4cf126ec..8547097ca 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -215,7 +215,7 @@ def gap_analysis() -> Any: # TODO (spyros): add export result to spreadsheet key = path["start"]["id"] if key not in grouped_paths: grouped_paths[key] = {"start": path["start"], "paths": []} - path['score'] = get_path_score(path) + path["score"] = get_path_score(path) del path["start"] grouped_paths[key]["paths"].append(path) From 1be0235437fd1275e23542672bc921f7d98e624a Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 24 Aug 2023 10:13:09 +0100 Subject: [PATCH 10/30] scoring passing tests --- application/tests/gap_analysis_test.py | 151 ++++++++++++++++++++++++- application/utils/gap_analysis.py | 4 +- 2 files changed, 149 insertions(+), 6 deletions(-) diff --git a/application/tests/gap_analysis_test.py b/application/tests/gap_analysis_test.py index a1414c6d9..396da8ee4 100644 --- a/application/tests/gap_analysis_test.py +++ b/application/tests/gap_analysis_test.py @@ -4,7 +4,7 @@ get_path_score, get_relation_direction, get_next_id, - PENALTIES + PENALTIES, ) @@ -31,7 +31,7 @@ def test_get_next_id_end(self): step = {"start": {"id": "123"}, "end": {"id": "234"}} self.assertEqual(get_next_id(step, "123"), "234") - def test_get_path_score_direct_siblings_zero(self): + def test_get_path_score_direct_siblings_returns_zero(self): path = { "start": { "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", @@ -62,7 +62,7 @@ def test_get_path_score_direct_siblings_zero(self): } self.assertEqual(get_path_score(path), 0) - def test_get_path_score_one_up_zero(self): + def test_get_path_score_one_up_returns_one_up_penaltiy(self): path = { "start": { "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", @@ -100,4 +100,147 @@ def test_get_path_score_one_up_zero(self): }, ], } - self.assertEqual(get_path_score(path), PENALTIES['CONTAINS_UP']) + self.assertEqual(get_path_score(path), PENALTIES["CONTAINS_UP"]) + + def test_get_path_score_one_down_one_returns_one_down_penaltiy(self): + path = { + "start": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "path": [ + { + "end": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "relationship": "LINKED_TO", + "start": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + }, + { + "end": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + "relationship": "CONTAINS", + "start": { + "id": "123", + }, + }, + { + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "relationship": "LINKED_TO", + "start": { + "id": "123", + }, + }, + ], + } + self.assertEqual(get_path_score(path), PENALTIES["CONTAINS_DOWN"]) + + def test_get_path_score_related_returns_related_penalty(self): + path = { + "start": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "path": [ + { + "end": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "relationship": "LINKED_TO", + "start": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + }, + { + "end": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + "relationship": "RELATED", + "start": { + "id": "123", + }, + }, + { + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "relationship": "LINKED_TO", + "start": { + "id": "123", + }, + }, + ], + } + self.assertEqual(get_path_score(path), PENALTIES["RELATED"]) + + def test_get_path_score_one_of_each_returns_penalty(self): + path = { + "start": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "path": [ + { + "end": { + "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", + }, + "relationship": "LINKED_TO", + "start": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + }, + { + "end": { + "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", + }, + "relationship": "CONTAINS", + "start": { + "id": "123", + }, + }, + { + "end": { + "id": "456", + }, + "relationship": "RELATED", + "start": { + "id": "123", + }, + }, + { + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "relationship": "CONTAINS", + "start": { + "id": "456", + }, + }, + { + "end": { + "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", + }, + "relationship": "LINKED_TO", + "start": { + "id": "456", + }, + }, + ], + } + self.assertEqual( + get_path_score(path), + PENALTIES["RELATED"] + + PENALTIES["CONTAINS_UP"] + + PENALTIES["CONTAINS_DOWN"], + ) diff --git a/application/utils/gap_analysis.py b/application/utils/gap_analysis.py index 308714592..47f97e830 100644 --- a/application/utils/gap_analysis.py +++ b/application/utils/gap_analysis.py @@ -1,9 +1,9 @@ PENALTIES = {"RELATED": 20, "CONTAINS_UP": 2, "CONTAINS_DOWN": 1, "LINKED_TO": 0} -def get_path_score(path, start_id): +def get_path_score(path): score = 0 - previous_id = start_id + previous_id = path["start"]["id"] for step in path["path"]: penalty_type = step["relationship"] From f320d63216f6b1a745804e0373651e6667a34e75 Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 24 Aug 2023 11:21:05 +0100 Subject: [PATCH 11/30] Update the UI --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 158 +++++++++++------- 1 file changed, 95 insertions(+), 63 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index be6041207..1c5383f37 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -1,5 +1,7 @@ import React, { useEffect, useState } from 'react'; -import { Accordion, Dropdown, Icon, Label, Popup, Segment, Table } from 'semantic-ui-react'; +import { Accordion, Button, Dropdown, Grid, Popup, Table } from 'semantic-ui-react'; +import { useLocation } from "react-router-dom"; +import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; import { useEnvironment } from '../../hooks'; @@ -12,12 +14,18 @@ const GetSegmentText = (segment, segmentID) => { nextID = segment.start.id; arrow = '<-'; } - const text = `${arrow} ${segment.relationship} ${arrow} ${textPart.name} ${textPart.sectionID} ${textPart.section} ${textPart.subsection} ${textPart.description}`; + const text = `${arrow} ${segment.relationship} ${arrow} ${textPart.name} ${textPart.sectionID ?? ""} ${textPart.section ?? ""} ${textPart.subsection ?? ''} ${textPart.description ?? ''}`; return { text, nextID }; }; +function useQuery() { + const { search } = useLocation(); + + return React.useMemo(() => new URLSearchParams(search), [search]); +} + export const GapAnalysis = () => { - const standardOptions = [ + const standardOptions = [ // TODO: Automate this list { key: '', text: '', value: undefined }, { key: 'OWASP Top 10 2021', text: 'OWASP Top 10 2021', value: 'OWASP Top 10 2021' }, { key: 'NIST 800-53 v5', text: 'NIST 800-53 v5', value: 'NIST 800-53 v5' }, @@ -46,23 +54,34 @@ export const GapAnalysis = () => { { key: 'PCI DSS', text: 'PCI DSS', value: 'PCI DSS' }, { key: 'OWASP Juice Shop', text: 'OWASP Juice Shop', value: 'OWASP Juice Shop' }, ]; - const [BaseStandard, setBaseStandard] = useState(); - const [CompareStandard, setCompareStandard] = useState(); + const searchParams = useQuery(); + const [BaseStandard, setBaseStandard] = useState(searchParams.get('base') ?? ""); + const [CompareStandard, setCompareStandard] = useState(searchParams.get('compare') ?? ""); const [gapAnalysis, setGapAnalysis] = useState(); const [activeIndex, SetActiveIndex] = useState(); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); const { apiUrl } = useEnvironment(); + + const GetStrength = (score) => { + if(score < 5) return 'Strong' + if(score > 20) return 'Weak' + return 'Average' + } useEffect(() => { const fetchData = async () => { const result = await fetch( `${apiUrl}/gap_analysis?standard=${BaseStandard}&standard=${CompareStandard}` ); const resultObj = await result.json(); + setLoading(false); setGapAnalysis(resultObj); }; if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; - fetchData().catch(console.error); - }, [BaseStandard, CompareStandard, setGapAnalysis]); + setLoading(true); + fetchData().catch(e => setError(e)); + }, [BaseStandard, CompareStandard, setGapAnalysis, setLoading, setError]); const handleAccordionClick = (e, titleProps) => { const { index } = titleProps; @@ -72,22 +91,33 @@ export const GapAnalysis = () => { return (
- setBaseStandard(value?.toString())} - /> - setCompareStandard(value?.toString())} - /> + + + + setBaseStandard(value?.toString())} + value={BaseStandard} + /> + + + setCompareStandard(value?.toString())} + value={CompareStandard} + /> + + + + {gapAnalysis && ( - +
{BaseStandard} @@ -97,58 +127,60 @@ export const GapAnalysis = () => { {Object.keys(gapAnalysis).map((key) => ( - - - + + +

+ {gapAnalysis[key].start.name} {gapAnalysis[key].start.section} {gapAnalysis[key].start.subsection}
+ {gapAnalysis[key].start.sectionID} + {gapAnalysis[key].start.description} +

- + + {gapAnalysis[key].paths + .sort((a, b) => a.score - b.score) + .slice(0, 3) + .map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + + { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section}{' '} + {path.end.subsection} {path.end.description} ({GetStrength(path.score)}:{path.score}) + + } + /> +
+ + ); + })} + - - {gapAnalysis[key].paths - .sort((a, b) => a.score - b.score) - .slice(0, 3) - .map((path) => { - let segmentID = gapAnalysis[key].start.id; - return ( - <> - { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} - trigger={ - - {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} ({path.score}) - - } - /> -
- - ); - })} - (Total Links: {gapAnalysis[key].paths.length}) + + Weaker Links:
{gapAnalysis[key].paths .sort((a, b) => a.score - b.score) .slice(2, gapAnalysis[key].paths.length) .map((path) => { let segmentID = gapAnalysis[key].start.id; return ( - <> + { @@ -160,12 +192,12 @@ export const GapAnalysis = () => { trigger={ {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} ({path.score}) + {path.end.subsection} {path.end.description} {GetStrength(path.score)}:{path.score}) } />
- + ); })} From e299f1ffd7650b284c758c470a81712780226c7b Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 13:25:55 +0100 Subject: [PATCH 12/30] Updated: Dropdowns are now dynamic --- application/database/db.py | 49 +++++------------ .../src/pages/GapAnalysis/GapAnalysis.tsx | 54 ++++++++----------- application/web/web_main.py | 17 +++++- 3 files changed, 52 insertions(+), 68 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index 42dd1a361..a4e564747 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -268,18 +268,6 @@ def gap_analysis(self, name_1, name_2): database_="neo4j", ) - # records_no_related, _, _ = self.driver.execute_query( - # "MATCH" - # "(BaseStandard:Node {name: $name1}), " - # "(CompareStandard:Node {name: $name2}), " - # "p = shortestPath((BaseStandard)-[*]-(CompareStandard)) " - # "WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n = BaseStandard or n = CompareStandard) AND ALL(r IN relationships(p) WHERE NOT r:RELATED) " - # "RETURN p ", - # name1=name_1, - # name2=name_2, - # database_="neo4j", - # ) - def format_segment(seg): return { "start": { @@ -323,7 +311,17 @@ def format_record(rec): } return [format_record(rec["p"]) for rec in records] - + + @classmethod + def standards(self): + if not self.connected: + return + records, _, _ = self.driver.execute_query( + 'MATCH (n:Node {ntype: "Standard"}) ' + "RETURN collect(distinct n.name)", + database_="neo4j", + ) + return records[0][0] class CRE_Graph: graph: nx.Graph = None @@ -1239,30 +1237,11 @@ def find_path_between_nodes( return res def gap_analysis(self, node_names: List[str]): - """Since the CRE structure is a tree-like graph with - leaves being nodes we can find the paths between nodes - find_path_between_nodes() is a graph-path-finding method - """ - # processed_nodes = [] - # dbnodes: List[Node] = [] - # for name in node_names: - # dbnodes.extend(self.session.query(Node).filter(Node.name == name).all()) - - # for node in dbnodes: - # working_node = nodeFromDB(node) - # for other_node in dbnodes: - # if node.id == other_node.id: - # continue - # if self.find_path_between_nodes(node.id, other_node.id): - # working_node.add_link( - # cre_defs.Link( - # ltype=cre_defs.LinkTypes.LinkedTo, - # document=nodeFromDB(other_node), - # ) - # ) - # processed_nodes.append(working_node) return self.neo_db.gap_analysis(node_names[0], node_names[1]) + def standards(self): + return self.neo_db.standards() + def text_search(self, text: str) -> List[Optional[cre_defs.Document]]: """Given a piece of text, tries to find the best match for the text in the database. diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 1c5383f37..32356baaa 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -1,9 +1,10 @@ import React, { useEffect, useState } from 'react'; -import { Accordion, Button, Dropdown, Grid, Popup, Table } from 'semantic-ui-react'; +import { Accordion, Button, Dropdown, DropdownItemProps, Grid, Popup, Table } from 'semantic-ui-react'; import { useLocation } from "react-router-dom"; import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; import { useEnvironment } from '../../hooks'; +import axios from 'axios'; const GetSegmentText = (segment, segmentID) => { let textPart = segment.end; @@ -25,36 +26,12 @@ function useQuery() { } export const GapAnalysis = () => { - const standardOptions = [ // TODO: Automate this list + const standardOptionsDefault = [ { key: '', text: '', value: undefined }, - { key: 'OWASP Top 10 2021', text: 'OWASP Top 10 2021', value: 'OWASP Top 10 2021' }, - { key: 'NIST 800-53 v5', text: 'NIST 800-53 v5', value: 'NIST 800-53 v5' }, - { key: 'ISO 27001', text: 'ISO 27001', value: 'ISO 27001' }, - { key: 'Cloud Controls Matrix', text: 'Cloud Controls Matrix', value: 'Cloud Controls Matrix' }, - { key: 'ASVS', text: 'ASVS', value: 'ASVS' }, - { key: 'OWASP Proactive Controls', text: 'OWASP Proactive Controls', value: 'OWASP Proactive Controls' }, - { key: 'SAMM', text: 'SAMM', value: 'SAMM' }, - { key: 'CWE', text: 'CWE', value: 'CWE' }, - { key: 'OWASP Cheat Sheets', text: 'OWASP Cheat Sheets', value: 'OWASP Cheat Sheets' }, - { - key: 'OWASP Web Security Testing Guide (WSTG)', - text: 'OWASP Web Security Testing Guide (WSTG)', - value: 'OWASP Web Security Testing Guide (WSTG)', - }, - { key: 'NIST 800-63', text: 'NIST 800-63', value: 'NIST 800-63' }, - { key: 'Cheat_sheets', text: 'Cheat_sheets', value: 'Cheat_sheets' }, - { key: 'CAPEC', text: 'CAPEC', value: 'CAPEC' }, - { key: 'ZAP Rule', text: 'ZAP Rule', value: 'ZAP Rule' }, - { key: 'OWASP', text: 'OWASP', value: 'OWASP' }, - { - key: 'OWASP Secure Headers Project', - text: 'OWASP Secure Headers Project', - value: 'OWASP Secure Headers Project', - }, - { key: 'PCI DSS', text: 'PCI DSS', value: 'PCI DSS' }, - { key: 'OWASP Juice Shop', text: 'OWASP Juice Shop', value: 'OWASP Juice Shop' }, + ]; const searchParams = useQuery(); + const [standardOptions, setStandardOptions] = useState(standardOptionsDefault); const [BaseStandard, setBaseStandard] = useState(searchParams.get('base') ?? ""); const [CompareStandard, setCompareStandard] = useState(searchParams.get('compare') ?? ""); const [gapAnalysis, setGapAnalysis] = useState(); @@ -68,19 +45,32 @@ export const GapAnalysis = () => { if(score > 20) return 'Weak' return 'Average' } + + useEffect(() => { + const fetchData = async () => { + const result = await axios.get( + `${apiUrl}/standards` + ); + setLoading(false); + setStandardOptions(standardOptionsDefault.concat(result.data.map(x => ({ key: x, text: x, value: x })))); + }; + + setLoading(true); + fetchData().catch(e => {setLoading(false); setError(e.response.data.message ?? e.message)}); + }, [setStandardOptions, setLoading, setError]); + useEffect(() => { const fetchData = async () => { - const result = await fetch( + const result = await axios.get( `${apiUrl}/gap_analysis?standard=${BaseStandard}&standard=${CompareStandard}` ); - const resultObj = await result.json(); setLoading(false); - setGapAnalysis(resultObj); + setGapAnalysis(result.data); }; if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; setLoading(true); - fetchData().catch(e => setError(e)); + fetchData().catch(e => {setLoading(false); setError(e.response.data.message ?? e.message)}); }, [BaseStandard, CompareStandard, setGapAnalysis, setLoading, setError]); const handleAccordionClick = (e, titleProps) => { diff --git a/application/web/web_main.py b/application/web/web_main.py index 8547097ca..95883befa 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -66,6 +66,10 @@ def extend_cre_with_tag_links( return cre +def neo4j_not_running_rejection(): + logger.info("Neo4j is disabled") + return jsonify({"message": "Backend services connected to this feature are not running at the moment."}), 500 + @app.route("/rest/v1/id/", methods=["GET"]) @app.route("/rest/v1/name/", methods=["GET"]) @cache.cached(timeout=50) @@ -206,10 +210,12 @@ def find_document_by_tag() -> Any: @app.route("/rest/v1/gap_analysis", methods=["GET"]) @cache.cached(timeout=50) -def gap_analysis() -> Any: # TODO (spyros): add export result to spreadsheet +def gap_analysis() -> Any: database = db.Node_collection() standards = request.args.getlist("standard") paths = database.gap_analysis(standards) + if paths is None: + return neo4j_not_running_rejection() grouped_paths = {} for path in paths: key = path["start"]["id"] @@ -221,6 +227,15 @@ def gap_analysis() -> Any: # TODO (spyros): add export result to spreadsheet return jsonify(grouped_paths) +@app.route("/rest/v1/standards", methods=["GET"]) +@cache.cached(timeout=50) +def standards() -> Any: + database = db.Node_collection() + standards = database.standards() + if standards is None: + neo4j_not_running_rejection() + return standards + @app.route("/rest/v1/text_search", methods=["GET"]) # @cache.cached(timeout=50) From 79c6009140a035c27a5ccffd18478dc99095849e Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 15:20:11 +0100 Subject: [PATCH 13/30] Localise neo4j --- .gitignore | 5 ++++- Makefile | 2 +- application/database/db.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index cbf1dd8c1..d6db6dd2b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,7 @@ yarn-error.log coverage/ ### Dev db -standards_cache.sqlite \ No newline at end of file +standards_cache.sqlite + +### Neo4j +neo4j/ \ No newline at end of file diff --git a/Makefile b/Makefile index ef43d81e7..5da2b61ea 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ docker-run: docker run -it -p 5000:5000 opencre:$(shell git rev-parse HEAD) docker-neo4j: - docker run --env NEO4J_PLUGINS='["apoc"]' --volume=/Users/johnharvey/neo4j/data:/data --volume=/data --volume=/logs --workdir=/var/lib/neo4j -p 7474:7474 -p 7687:7687 -d neo4j + docker run --env NEO4J_PLUGINS='["apoc"]' --volume=./neo4j/data:/data --volume=/data --volume=/logs --workdir=/var/lib/neo4j -p 7474:7474 -p 7687:7687 -d neo4j lint: [ -d "./venv" ] && . ./venv/bin/activate && black . && yarn lint diff --git a/application/database/db.py b/application/database/db.py index a4e564747..4b6518f80 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -333,7 +333,7 @@ def instance(cls, session, neo_db: NEO_DB): if cls.__instance is None: cls.__instance = cls.__new__(cls) cls.neo_db = neo_db - # cls.graph = cls.load_cre_graph(session) + cls.graph = cls.load_cre_graph(session) return cls.__instance def __init__(sel): From 4f973e0fa79bbd63209fabf24cd58fcf72f667d6 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 15:20:53 +0100 Subject: [PATCH 14/30] Added Navigation method --- application/frontend/src/scaffolding/Header/Header.tsx | 4 ++++ application/frontend/src/scaffolding/Header/header.scss | 1 + 2 files changed, 5 insertions(+) diff --git a/application/frontend/src/scaffolding/Header/Header.tsx b/application/frontend/src/scaffolding/Header/Header.tsx index aa872fb43..c2652d80e 100644 --- a/application/frontend/src/scaffolding/Header/Header.tsx +++ b/application/frontend/src/scaffolding/Header/Header.tsx @@ -13,6 +13,10 @@ const getLinks = (): { to: string; name: string }[] => [ to: `/`, name: 'Open CRE', }, + { + to: `/gap_analysis`, + name: 'Gap Analysis', + }, ]; export const Header = () => { diff --git a/application/frontend/src/scaffolding/Header/header.scss b/application/frontend/src/scaffolding/Header/header.scss index e01e85568..faec51d53 100644 --- a/application/frontend/src/scaffolding/Header/header.scss +++ b/application/frontend/src/scaffolding/Header/header.scss @@ -20,6 +20,7 @@ padding-top: 10px; padding-bottom: 10px; text-align: center; + margin: 0 2px; .item { color: white !important; From 03d7de3aafcd5fd2d674a8a3178308f0a194dabc Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 15:21:07 +0100 Subject: [PATCH 15/30] Add share and nav links --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 32356baaa..89c563235 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react'; -import { Accordion, Button, Dropdown, DropdownItemProps, Grid, Popup, Table } from 'semantic-ui-react'; +import { Accordion, Button, Dropdown, DropdownItemProps, Grid, Icon, Popup, Table } from 'semantic-ui-react'; import { useLocation } from "react-router-dom"; import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; @@ -104,10 +104,17 @@ export const GapAnalysis = () => { /> + {gapAnalysis && ( + + + + )} {gapAnalysis && ( -
+
{BaseStandard} @@ -120,7 +127,11 @@ export const GapAnalysis = () => {

- {gapAnalysis[key].start.name} {gapAnalysis[key].start.section} {gapAnalysis[key].start.subsection}
+ {gapAnalysis[key].start.name} {gapAnalysis[key].start.section} {gapAnalysis[key].start.subsection} + + + +
{gapAnalysis[key].start.sectionID} {gapAnalysis[key].start.description}

@@ -146,7 +157,10 @@ export const GapAnalysis = () => { trigger={ {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} ({GetStrength(path.score)}:{path.score}) + {path.end.subsection} {path.end.description} ({GetStrength(path.score)}:{path.score}){' '} + + + } /> @@ -182,7 +196,10 @@ export const GapAnalysis = () => { trigger={ {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} {GetStrength(path.score)}:{path.score}) + {path.end.subsection} {path.end.description} {GetStrength(path.score)}:{path.score}){' '} + + + } /> From e7459970668446b45cf01de36f02c69b8bd2c441 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 15:40:58 +0100 Subject: [PATCH 16/30] readme improvement --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e46c683a8..44899a8f1 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,13 @@ To run the web application for development you can run Alternatively, you can use the dockerfile with 
make docker && make docker-run
+Some features like Gap Analysis require a neo4j DB running you can start this with +
make docker-neo4j
+enviroment varaibles for app to connect to neo4jDB (default): +- NEO4J_URI (localhost) +- NEO4J_USR (neo4j) +- NEO4J_PASS (password) + To run the web application for production you need gunicorn and you can run from within the cre_sync dir 
make prod-run
From 433755bca8bd3395a2146fa88a2d4da47f81fda5 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 16:19:54 +0100 Subject: [PATCH 17/30] Hide table on new search --- application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 89c563235..246cc9d29 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -69,6 +69,7 @@ export const GapAnalysis = () => { }; if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; + setGapAnalysis(undefined); setLoading(true); fetchData().catch(e => {setLoading(false); setError(e.response.data.message ?? e.message)}); }, [BaseStandard, CompareStandard, setGapAnalysis, setLoading, setError]); From 6dbd7f7437b210744db1aa198bdd22795abe3a70 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 16:20:12 +0100 Subject: [PATCH 18/30] Optermise query to remove relates to --- application/database/db.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index 4b6518f80..a84f83c4b 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -257,12 +257,14 @@ def gap_analysis(self, name_1, name_2): if not self.connected: return records, _, _ = self.driver.execute_query( - "MATCH" - "(BaseStandard:Node {name: $name1}), " - "(CompareStandard:Node {name: $name2}), " - "p = shortestPath((BaseStandard)-[*]-(CompareStandard)) " - "WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n = BaseStandard or n = CompareStandard) " - "RETURN p ", + """ + OPTIONAL MATCH (BaseStandard:Node {name: $name1}) + OPTIONAL MATCH (CompareStandard:Node {name: $name2}) + OPTIONAL MATCH p = shortestPath((BaseStandard)-[:(LINKED_TO|CONTAINS)*..20]-(CompareStandard)) + WITH p + WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n.name = $name1 or n.name = $name2) + RETURN p + """, name1=name_1, name2=name_2, database_="neo4j", From 665c9b5525918920877ff6fd05a74026e08b4cc7 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 16:53:41 +0100 Subject: [PATCH 19/30] Get duel running method working and show empty values --- application/database/db.py | 41 ++++++- .../src/pages/GapAnalysis/GapAnalysis.tsx | 112 +++++++++--------- application/web/web_main.py | 16 ++- 3 files changed, 106 insertions(+), 63 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index a84f83c4b..04f450037 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -255,8 +255,30 @@ def link_CRE_to_Node(self, CRE_id, node_id, link_type): @classmethod def gap_analysis(self, name_1, name_2): if not self.connected: - return - records, _, _ = self.driver.execute_query( + return None, None + base_standard, _, _ = self.driver.execute_query( + """ + MATCH (BaseStandard:Node {name: $name1}) + RETURN BaseStandard + """, + name1=name_1, + database_="neo4j", + ) + + path_records_all, _, _ = self.driver.execute_query( + """ + OPTIONAL MATCH (BaseStandard:Node {name: $name1}) + OPTIONAL MATCH (CompareStandard:Node {name: $name2}) + OPTIONAL MATCH p = shortestPath((BaseStandard)-[*..20]-(CompareStandard)) + WITH p + WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n.name = $name1 or n.name = $name2) + RETURN p + """, + name1=name_1, + name2=name_2, + database_="neo4j", + ) + path_records, _, _ = self.driver.execute_query( """ OPTIONAL MATCH (BaseStandard:Node {name: $name1}) OPTIONAL MATCH (CompareStandard:Node {name: $name2}) @@ -291,7 +313,7 @@ def format_segment(seg): "relationship": seg.type, } - def format_record(rec): + def format_path_record(rec): return { "start": { "name": rec.start_node["name"], @@ -311,8 +333,19 @@ def format_record(rec): }, "path": [format_segment(seg) for seg in rec.relationships], } + + def format_record(rec): + return { + "name": rec["name"], + "sectionID": rec["section_id"], + "section": rec["section"], + "subsection": rec["subsection"], + "description": rec["description"], + "id": rec["id"], + } + - return [format_record(rec["p"]) for rec in records] + return [format_record(rec["BaseStandard"]) for rec in base_standard], [format_path_record(rec["p"]) for rec in (path_records + path_records_all)] @classmethod def standards(self): diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 246cc9d29..19b7f6b7a 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -28,7 +28,7 @@ function useQuery() { export const GapAnalysis = () => { const standardOptionsDefault = [ { key: '', text: '', value: undefined }, - + ]; const searchParams = useQuery(); const [standardOptions, setStandardOptions] = useState(standardOptionsDefault); @@ -41,10 +41,10 @@ export const GapAnalysis = () => { const { apiUrl } = useEnvironment(); const GetStrength = (score) => { - if(score < 5) return 'Strong' - if(score > 20) return 'Weak' + if (score < 5) return 'Strong' + if (score > 20) return 'Weak' return 'Average' - } + } useEffect(() => { const fetchData = async () => { @@ -56,7 +56,7 @@ export const GapAnalysis = () => { }; setLoading(true); - fetchData().catch(e => {setLoading(false); setError(e.response.data.message ?? e.message)}); + fetchData().catch(e => { setLoading(false); setError(e.response.data.message ?? e.message) }); }, [setStandardOptions, setLoading, setError]); useEffect(() => { @@ -71,7 +71,7 @@ export const GapAnalysis = () => { if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; setGapAnalysis(undefined); setLoading(true); - fetchData().catch(e => {setLoading(false); setError(e.response.data.message ?? e.message)}); + fetchData().catch(e => { setLoading(false); setError(e.response.data.message ?? e.message) }); }, [BaseStandard, CompareStandard, setGapAnalysis, setLoading, setError]); const handleAccordionClick = (e, titleProps) => { @@ -106,16 +106,16 @@ export const GapAnalysis = () => { {gapAnalysis && ( - - - + + + )} {gapAnalysis && ( -
+
{BaseStandard} @@ -132,19 +132,19 @@ export const GapAnalysis = () => { -
+
{gapAnalysis[key].start.sectionID} {gapAnalysis[key].start.description}

- {gapAnalysis[key].paths + {Object.values(gapAnalysis[key].paths) .sort((a, b) => a.score - b.score) .slice(0, 3) .map((path) => { let segmentID = gapAnalysis[key].start.id; return ( - + { ); })} + {Object.keys(gapAnalysis[key].paths).length > 3 && ( + + + + + + Weaker Links:
+ {Object.values(gapAnalysis[key].paths) + .sort((a, b) => a.score - b.score) + .slice(3, gapAnalysis[key].paths.length) + .map((path) => { + let segmentID = gapAnalysis[key].start.id; + return ( + + - - - - - Weaker Links:
- {gapAnalysis[key].paths - .sort((a, b) => a.score - b.score) - .slice(2, gapAnalysis[key].paths.length) - .map((path) => { - let segmentID = gapAnalysis[key].start.id; - return ( - - { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} - trigger={ - - {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} {GetStrength(path.score)}:{path.score}){' '} - - - - - } - /> -
- - ); - })} - - + hoverable + content={path.path + .map((segment) => { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + }) + .join('')} + trigger={ + + {path.end.name} {path.end.sectionID} {path.end.section}{' '} + {path.end.subsection} {path.end.description} {GetStrength(path.score)}:{path.score}){' '} + + + + + } + /> +
+ + ); + })} + + + )} + {Object.keys(gapAnalysis[key].paths).length === 0 && (No links Found)} ))} diff --git a/application/web/web_main.py b/application/web/web_main.py index 95883befa..cfca6d394 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -213,17 +213,25 @@ def find_document_by_tag() -> Any: def gap_analysis() -> Any: database = db.Node_collection() standards = request.args.getlist("standard") - paths = database.gap_analysis(standards) + base_standard, paths = database.gap_analysis(standards) if paths is None: return neo4j_not_running_rejection() grouped_paths = {} + for node in base_standard: + key = node["id"] + if key not in grouped_paths: + grouped_paths[key] = {"start": node, "paths": {}} + for path in paths: key = path["start"]["id"] - if key not in grouped_paths: - grouped_paths[key] = {"start": path["start"], "paths": []} + end_key = path["end"]["id"] path["score"] = get_path_score(path) del path["start"] - grouped_paths[key]["paths"].append(path) + if end_key in grouped_paths[key]["paths"]: + if grouped_paths[key]["paths"][end_key]['score'] > path["score"]: + grouped_paths[key]["paths"][end_key] = path + else: + grouped_paths[key]["paths"][end_key] = path return jsonify(grouped_paths) From 10f26b5ae30eca26b2543fe97d04ec3c0cfb886d Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 16:58:05 +0100 Subject: [PATCH 20/30] Refactor grouping & scoring code locations --- application/database/db.py | 25 ++++++++++++++++++++++++- application/web/web_main.py | 24 +++--------------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index 04f450037..ef3fd0a32 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -17,6 +17,8 @@ from sqlalchemy.sql.expression import desc # type: ignore import uuid +from application.utils.gap_analysis import get_path_score + from .. import sqla # type: ignore logging.basicConfig() @@ -1272,7 +1274,28 @@ def find_path_between_nodes( return res def gap_analysis(self, node_names: List[str]): - return self.neo_db.gap_analysis(node_names[0], node_names[1]) + if not self.neo_db.connected: + return None + base_standard, paths = self.neo_db.gap_analysis(node_names[0], node_names[1]) + if base_standard is None: + return None + grouped_paths = {} + for node in base_standard: + key = node["id"] + if key not in grouped_paths: + grouped_paths[key] = {"start": node, "paths": {}} + + for path in paths: + key = path["start"]["id"] + end_key = path["end"]["id"] + path["score"] = get_path_score(path) + del path["start"] + if end_key in grouped_paths[key]["paths"]: + if grouped_paths[key]["paths"][end_key]['score'] > path["score"]: + grouped_paths[key]["paths"][end_key] = path + else: + grouped_paths[key]["paths"][end_key] = path + return grouped_paths def standards(self): return self.neo_db.standards() diff --git a/application/web/web_main.py b/application/web/web_main.py index cfca6d394..ae73a4fc5 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -15,7 +15,6 @@ from application.defs import osib_defs as odefs from application.utils import spreadsheet as sheet_utils from application.utils import mdutils, redirectors -from application.utils.gap_analysis import get_path_score from application.prompt_client import prompt_client as prompt_client from enum import Enum from flask import ( @@ -213,27 +212,10 @@ def find_document_by_tag() -> Any: def gap_analysis() -> Any: database = db.Node_collection() standards = request.args.getlist("standard") - base_standard, paths = database.gap_analysis(standards) - if paths is None: + gap_analysis = database.gap_analysis(standards) + if gap_analysis is None: return neo4j_not_running_rejection() - grouped_paths = {} - for node in base_standard: - key = node["id"] - if key not in grouped_paths: - grouped_paths[key] = {"start": node, "paths": {}} - - for path in paths: - key = path["start"]["id"] - end_key = path["end"]["id"] - path["score"] = get_path_score(path) - del path["start"] - if end_key in grouped_paths[key]["paths"]: - if grouped_paths[key]["paths"][end_key]['score'] > path["score"]: - grouped_paths[key]["paths"][end_key] = path - else: - grouped_paths[key]["paths"][end_key] = path - - return jsonify(grouped_paths) + return jsonify(gap_analysis) @app.route("/rest/v1/standards", methods=["GET"]) @cache.cached(timeout=50) From 6e116cd99337c5f6149f5b1d2de827999662fb46 Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 4 Sep 2023 17:01:25 +0100 Subject: [PATCH 21/30] Add colour to strength raiting --- application/database/db.py | 29 ++--- .../src/pages/GapAnalysis/GapAnalysis.tsx | 113 ++++++++++++------ application/web/web_main.py | 17 ++- 3 files changed, 105 insertions(+), 54 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index ef3fd0a32..ed9eeec11 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -335,31 +335,32 @@ def format_path_record(rec): }, "path": [format_segment(seg) for seg in rec.relationships], } - + def format_record(rec): return { - "name": rec["name"], - "sectionID": rec["section_id"], - "section": rec["section"], - "subsection": rec["subsection"], - "description": rec["description"], - "id": rec["id"], - } + "name": rec["name"], + "sectionID": rec["section_id"], + "section": rec["section"], + "subsection": rec["subsection"], + "description": rec["description"], + "id": rec["id"], + } + return [format_record(rec["BaseStandard"]) for rec in base_standard], [ + format_path_record(rec["p"]) for rec in (path_records + path_records_all) + ] - return [format_record(rec["BaseStandard"]) for rec in base_standard], [format_path_record(rec["p"]) for rec in (path_records + path_records_all)] - @classmethod def standards(self): if not self.connected: return records, _, _ = self.driver.execute_query( - 'MATCH (n:Node {ntype: "Standard"}) ' - "RETURN collect(distinct n.name)", + 'MATCH (n:Node {ntype: "Standard"}) ' "RETURN collect(distinct n.name)", database_="neo4j", ) return records[0][0] + class CRE_Graph: graph: nx.Graph = None neo_db: NEO_DB = None @@ -1276,7 +1277,7 @@ def find_path_between_nodes( def gap_analysis(self, node_names: List[str]): if not self.neo_db.connected: return None - base_standard, paths = self.neo_db.gap_analysis(node_names[0], node_names[1]) + base_standard, paths = self.neo_db.gap_analysis(node_names[0], node_names[1]) if base_standard is None: return None grouped_paths = {} @@ -1291,7 +1292,7 @@ def gap_analysis(self, node_names: List[str]): path["score"] = get_path_score(path) del path["start"] if end_key in grouped_paths[key]["paths"]: - if grouped_paths[key]["paths"][end_key]['score'] > path["score"]: + if grouped_paths[key]["paths"][end_key]["score"] > path["score"]: grouped_paths[key]["paths"][end_key] = path else: grouped_paths[key]["paths"][end_key] = path diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 19b7f6b7a..8a538e4d4 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -1,10 +1,10 @@ +import axios from 'axios'; import React, { useEffect, useState } from 'react'; +import { useLocation } from 'react-router-dom'; import { Accordion, Button, Dropdown, DropdownItemProps, Grid, Icon, Popup, Table } from 'semantic-ui-react'; -import { useLocation } from "react-router-dom"; -import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; +import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; import { useEnvironment } from '../../hooks'; -import axios from 'axios'; const GetSegmentText = (segment, segmentID) => { let textPart = segment.end; @@ -15,7 +15,9 @@ const GetSegmentText = (segment, segmentID) => { nextID = segment.start.id; arrow = '<-'; } - const text = `${arrow} ${segment.relationship} ${arrow} ${textPart.name} ${textPart.sectionID ?? ""} ${textPart.section ?? ""} ${textPart.subsection ?? ''} ${textPart.description ?? ''}`; + const text = `${arrow} ${segment.relationship} ${arrow} ${textPart.name} ${textPart.sectionID ?? ''} ${ + textPart.section ?? '' + } ${textPart.subsection ?? ''} ${textPart.description ?? ''}`; return { text, nextID }; }; @@ -26,14 +28,15 @@ function useQuery() { } export const GapAnalysis = () => { - const standardOptionsDefault = [ - { key: '', text: '', value: undefined }, - - ]; + const standardOptionsDefault = [{ key: '', text: '', value: undefined }]; const searchParams = useQuery(); - const [standardOptions, setStandardOptions] = useState(standardOptionsDefault); - const [BaseStandard, setBaseStandard] = useState(searchParams.get('base') ?? ""); - const [CompareStandard, setCompareStandard] = useState(searchParams.get('compare') ?? ""); + const [standardOptions, setStandardOptions] = useState( + standardOptionsDefault + ); + const [BaseStandard, setBaseStandard] = useState(searchParams.get('base') ?? ''); + const [CompareStandard, setCompareStandard] = useState( + searchParams.get('compare') ?? '' + ); const [gapAnalysis, setGapAnalysis] = useState(); const [activeIndex, SetActiveIndex] = useState(); const [loading, setLoading] = useState(false); @@ -41,22 +44,31 @@ export const GapAnalysis = () => { const { apiUrl } = useEnvironment(); const GetStrength = (score) => { - if (score < 5) return 'Strong' - if (score > 20) return 'Weak' - return 'Average' - } + if (score < 5) return 'Strong'; + if (score > 20) return 'Weak'; + return 'Average'; + }; + + const GetStrengthColor = (score) => { + if (score < 5) return 'Green'; + if (score > 20) return 'Red'; + return 'Orange'; + }; useEffect(() => { const fetchData = async () => { - const result = await axios.get( - `${apiUrl}/standards` - ); + const result = await axios.get(`${apiUrl}/standards`); setLoading(false); - setStandardOptions(standardOptionsDefault.concat(result.data.map(x => ({ key: x, text: x, value: x })))); + setStandardOptions( + standardOptionsDefault.concat(result.data.map((x) => ({ key: x, text: x, value: x }))) + ); }; setLoading(true); - fetchData().catch(e => { setLoading(false); setError(e.response.data.message ?? e.message) }); + fetchData().catch((e) => { + setLoading(false); + setError(e.response.data.message ?? e.message); + }); }, [setStandardOptions, setLoading, setError]); useEffect(() => { @@ -71,7 +83,10 @@ export const GapAnalysis = () => { if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; setGapAnalysis(undefined); setLoading(true); - fetchData().catch(e => { setLoading(false); setError(e.response.data.message ?? e.message) }); + fetchData().catch((e) => { + setLoading(false); + setError(e.response.data.message ?? e.message); + }); }, [BaseStandard, CompareStandard, setGapAnalysis, setLoading, setError]); const handleAccordionClick = (e, titleProps) => { @@ -107,7 +122,14 @@ export const GapAnalysis = () => { {gapAnalysis && ( - @@ -115,7 +137,7 @@ export const GapAnalysis = () => { {gapAnalysis && ( -
+
{BaseStandard} @@ -126,10 +148,16 @@ export const GapAnalysis = () => { {Object.keys(gapAnalysis).map((key) => ( - +

- {gapAnalysis[key].start.name} {gapAnalysis[key].start.section} {gapAnalysis[key].start.subsection} - + + {gapAnalysis[key].start.name} {gapAnalysis[key].start.section}{' '} + {gapAnalysis[key].start.subsection} + +
@@ -157,9 +185,16 @@ export const GapAnalysis = () => { .join('')} trigger={ - {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} ({GetStrength(path.score)}:{path.score}){' '} - + {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} + {path.end.description} ( + + {GetStrength(path.score)}:{path.score} + + ){' '} + @@ -171,11 +206,14 @@ export const GapAnalysis = () => { })} {Object.keys(gapAnalysis[key].paths).length > 3 && ( - - + + - Weaker Links:
{Object.values(gapAnalysis[key].paths) .sort((a, b) => a.score - b.score) .slice(3, gapAnalysis[key].paths.length) @@ -185,7 +223,6 @@ export const GapAnalysis = () => { { @@ -197,8 +234,12 @@ export const GapAnalysis = () => { trigger={ {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} {GetStrength(path.score)}:{path.score}){' '} - + {path.end.subsection} {path.end.description} {GetStrength(path.score)}: + {path.score}){' '} + @@ -211,7 +252,7 @@ export const GapAnalysis = () => { )} - {Object.keys(gapAnalysis[key].paths).length === 0 && (No links Found)} + {Object.keys(gapAnalysis[key].paths).length === 0 && No links Found} ))} diff --git a/application/web/web_main.py b/application/web/web_main.py index ae73a4fc5..c6fd97907 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -67,7 +67,15 @@ def extend_cre_with_tag_links( def neo4j_not_running_rejection(): logger.info("Neo4j is disabled") - return jsonify({"message": "Backend services connected to this feature are not running at the moment."}), 500 + return ( + jsonify( + { + "message": "Backend services connected to this feature are not running at the moment." + } + ), + 500, + ) + @app.route("/rest/v1/id/", methods=["GET"]) @app.route("/rest/v1/name/", methods=["GET"]) @@ -213,17 +221,18 @@ def gap_analysis() -> Any: database = db.Node_collection() standards = request.args.getlist("standard") gap_analysis = database.gap_analysis(standards) - if gap_analysis is None: + if gap_analysis is None: return neo4j_not_running_rejection() return jsonify(gap_analysis) + @app.route("/rest/v1/standards", methods=["GET"]) @cache.cached(timeout=50) def standards() -> Any: database = db.Node_collection() standards = database.standards() - if standards is None: - neo4j_not_running_rejection() + if standards is None: + neo4j_not_running_rejection() return standards From 030c044b0e763f02c6f8e8780edb5da1d6ae60de Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 7 Sep 2023 12:49:08 +0100 Subject: [PATCH 22/30] Add gap analysis tests --- application/tests/db_test.py | 304 ++++++++++++++++++----------------- 1 file changed, 157 insertions(+), 147 deletions(-) diff --git a/application/tests/db_test.py b/application/tests/db_test.py index 84de848ec..b2f6a57b2 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -1,6 +1,7 @@ import os import tempfile import unittest +from unittest.mock import patch import uuid from copy import copy, deepcopy from pprint import pprint @@ -761,153 +762,6 @@ def test_get_nodes_with_pagination(self) -> None: (None, None, None), ) - def test_gap_analysis(self) -> None: - """Given - the following standards SA1, SA2, SA3 SAA1 , SB1, SD1, SDD1, SW1, SX1 - the following CREs CA, CB, CC, CD, CDD , CW, CX - the following links - CC -> CA, CB,CD - CD -> CDD - CA-> SA1, SAA1 - CB -> SB1 - CD -> SD1 - CDD -> SDD1 - CW -> SW1 - CX -> SA3, SX1 - NoCRE -> SA2 - - Then: - gap_analysis(SA) returns SA1, SA2, SA3 - gap_analysis(SA,SAA) returns SA1 <-> SAA1, SA2, SA3 - gap_analysis(SA,SDD) returns SA1 <-> SDD1, SA2, SA3 - gap_analysis(SA, SW) returns SA1,SA2,SA3, SW1 # no connection - gap_analysis(SA, SB, SD, SW) returns SA1 <->(SB1,SD1), SA2 , SW1, SA3 - gap_analysis(SA, SX) returns SA1, SA2, SA3->SX1 - - give me a single standard - give me two standards connected by same cre - give me two standards connected by cres who are children of the same cre - give me two standards connected by completely different cres - give me two standards with sections on different trees. - - give me two standards without connections - give me 3 or more standards - - """ - - collection = db.Node_collection() - collection.graph.graph = db.CRE_Graph.load_cre_graph(sqla.session) - - cres = { - "dbca": collection.add_cre(defs.CRE(id="1", description="CA", name="CA")), - "dbcb": collection.add_cre(defs.CRE(id="2", description="CB", name="CB")), - "dbcc": collection.add_cre(defs.CRE(id="3", description="CC", name="CC")), - "dbcd": collection.add_cre(defs.CRE(id="4", description="CD", name="CD")), - "dbcdd": collection.add_cre( - defs.CRE(id="5", description="CDD", name="CDD") - ), - "dbcw": collection.add_cre(defs.CRE(id="6", description="CW", name="CW")), - "dbcx": collection.add_cre(defs.CRE(id="7", description="CX", name="CX")), - } - def_standards = { - "sa1": defs.Standard(name="SA", section="SA1"), - "sa2": defs.Standard(name="SA", section="SA2"), - "sa3": defs.Standard(name="SA", section="SA3"), - "saa1": defs.Standard(name="SAA", section="SAA1"), - "sb1": defs.Standard(name="SB", section="SB1"), - "sd1": defs.Standard(name="SD", section="SD1"), - "sdd1": defs.Standard(name="SDD", section="SDD1"), - "sw1": defs.Standard(name="SW", section="SW1"), - "sx1": defs.Standard(name="SX", section="SX1"), - } - standards = {} - for k, s in def_standards.items(): - standards["db" + k] = collection.add_node(s) - ltype = defs.LinkTypes.LinkedTo - collection.add_link(cre=cres["dbca"], node=standards["dbsa1"]) - collection.add_link(cre=cres["dbca"], node=standards["dbsaa1"]) - collection.add_link(cre=cres["dbcb"], node=standards["dbsb1"]) - collection.add_link(cre=cres["dbcd"], node=standards["dbsd1"]) - collection.add_link(cre=cres["dbcdd"], node=standards["dbsdd1"]) - collection.add_link(cre=cres["dbcw"], node=standards["dbsw1"]) - collection.add_link(cre=cres["dbcx"], node=standards["dbsa3"]) - collection.add_link(cre=cres["dbcx"], node=standards["dbsx1"]) - - collection.add_internal_link(group=cres["dbcc"], cre=cres["dbca"]) - collection.add_internal_link(group=cres["dbcc"], cre=cres["dbcb"]) - collection.add_internal_link(group=cres["dbcc"], cre=cres["dbcd"]) - collection.add_internal_link(group=cres["dbcd"], cre=cres["dbcdd"]) - - expected = { - "SA": [def_standards["sa1"], def_standards["sa2"], def_standards["sa3"]], - "SA,SAA": [ - copy(def_standards["sa1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["saa1"]) - ), - copy(def_standards["saa1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["sa1"]) - ), - def_standards["sa2"], - def_standards["sa3"], - ], - "SAA,SA": [ - copy(def_standards["sa1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["saa1"]) - ), - copy(def_standards["saa1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["sa1"]) - ), - def_standards["sa2"], - def_standards["sa3"], - ], - "SA,SDD": [ - copy(def_standards["sa1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["sdd1"]) - ), - copy(def_standards["sdd1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["sa1"]) - ), - def_standards["sa2"], - def_standards["sa3"], - ], - "SA,SW": [ - def_standards["sa1"], - def_standards["sa2"], - def_standards["sa3"], - def_standards["sw1"], - ], - "SA,SB,SD,SW": [ - copy(def_standards["sa1"]) - .add_link(defs.Link(ltype=ltype, document=def_standards["sb1"])) - .add_link(defs.Link(ltype=ltype, document=def_standards["sd1"])), - copy(def_standards["sb1"]) - .add_link(defs.Link(ltype=ltype, document=def_standards["sa1"])) - .add_link(defs.Link(ltype=ltype, document=def_standards["sd1"])), - copy(def_standards["sd1"]) - .add_link(defs.Link(ltype=ltype, document=def_standards["sa1"])) - .add_link(defs.Link(ltype=ltype, document=def_standards["sb1"])), - def_standards["sa2"], - def_standards["sa3"], - def_standards["sw1"], - ], - "SA,SX": [ - def_standards["sa1"], - def_standards["sa2"], - copy(def_standards["sa3"]).add_link( - defs.Link(ltype=ltype, document=def_standards["sx1"]) - ), - copy(def_standards["sx1"]).add_link( - defs.Link(ltype=ltype, document=def_standards["sa3"]) - ), - ], - } - - self.maxDiff = None - for args, expected_vals in expected.items(): - stands = args.split(",") - res = collection.gap_analysis(stands) - self.assertCountEqual(res, expected_vals) - def test_add_internal_link(self) -> None: """test that internal links are added successfully, edge cases: @@ -1283,6 +1137,162 @@ def test_get_root_cres(self): self.maxDiff = None self.assertEqual(root_cres, [cres[0], cres[1], cres[7]]) + def test_gap_analysis_disconnected(self): + collection = db.Node_collection() + collection.neo_db.connected = False + self.assertEqual(collection.gap_analysis(["a", "b"]), None) + + @patch.object(db.NEO_DB, 'gap_analysis') + def test_gap_analysis_no_nodes(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + + gap_mock.return_value = ([], []) + self.assertEqual(collection.gap_analysis(["a", "b"]), {}) + + @patch.object(db.NEO_DB, 'gap_analysis') + def test_gap_analysis_no_links(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + + gap_mock.return_value = ([{'id': 1}], []) + self.assertEqual(collection.gap_analysis(["a", "b"]), {1: {'start': {'id': 1}, 'paths': {}}} ) + + @patch.object(db.NEO_DB, 'gap_analysis') + def test_gap_analysis_one_link(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": { + "id": 1, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a", + }, + }, + { + "end": { + "id": 2, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a" + }, + }, + ] + gap_mock.return_value = ([{'id': 1}], [{'start':{'id': 1}, 'end': {'id': 2}, 'path': path}]) + expected = {1: {'start': {'id': 1}, 'paths': { + 2: {'end': {'id': 2}, + 'path': path, + 'score': 0}} + }} + self.assertEqual(collection.gap_analysis(["a", "b"]), expected) + + @patch.object(db.NEO_DB, 'gap_analysis') + def test_gap_analysis_duplicate_link_path_existing_lower(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": { + "id": 1, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a", + }, + }, + { + "end": { + "id": 2, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a" + }, + }, + ] + path2 = [ + { + "end": { + "id": 1, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a", + }, + }, + { + "end": { + "id": 2, + }, + "relationship": "RELATED", + "start": { + "id": "a" + }, + }, + ] + gap_mock.return_value = ([{'id': 1}], [{'start':{'id': 1}, 'end': {'id': 2}, 'path': path}, {'start':{'id': 1}, 'end': {'id': 2}, 'path': path2}]) + expected = {1: {'start': {'id': 1}, 'paths': { + 2: {'end': {'id': 2}, + 'path': path, + 'score': 0}} + }} + self.assertEqual(collection.gap_analysis(["a", "b"]), expected) + + @patch.object(db.NEO_DB, 'gap_analysis') + def test_gap_analysis_duplicate_link_path_existing_higher(self, gap_mock): + collection = db.Node_collection() + collection.neo_db.connected = True + path = [ + { + "end": { + "id": 1, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a", + }, + }, + { + "end": { + "id": 2, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a" + }, + }, + ] + path2 = [ + { + "end": { + "id": 1, + }, + "relationship": "LINKED_TO", + "start": { + "id": "a", + }, + }, + { + "end": { + "id": 2, + }, + "relationship": "RELATED", + "start": { + "id": "a" + }, + }, + ] + gap_mock.return_value = ([{'id': 1}], [{'start':{'id': 1}, 'end': {'id': 2}, 'path': path2}, {'start':{'id': 1}, 'end': {'id': 2}, 'path': path}]) + expected = {1: {'start': {'id': 1}, 'paths': { + 2: {'end': {'id': 2}, + 'path': path, + 'score': 0}} + }} + self.assertEqual(collection.gap_analysis(["a", "b"]), expected) if __name__ == "__main__": unittest.main() From 121ce3579f160a7115b369833dc5a327998d2aed Mon Sep 17 00:00:00 2001 From: john681611 Date: Thu, 7 Sep 2023 13:05:11 +0100 Subject: [PATCH 23/30] Short drop down list --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 12 +- application/tests/db_test.py | 245 +++++++++--------- 2 files changed, 136 insertions(+), 121 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 8a538e4d4..139387693 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -60,7 +60,7 @@ export const GapAnalysis = () => { const result = await axios.get(`${apiUrl}/standards`); setLoading(false); setStandardOptions( - standardOptionsDefault.concat(result.data.map((x) => ({ key: x, text: x, value: x }))) + standardOptionsDefault.concat(result.data.sort().map((x) => ({ key: x, text: x, value: x }))) ); }; @@ -159,7 +159,8 @@ export const GapAnalysis = () => { target="_blank" > - + {' '} + {gapAnalysis[key].start.id}
{gapAnalysis[key].start.sectionID} {gapAnalysis[key].start.description} @@ -234,8 +235,11 @@ export const GapAnalysis = () => { trigger={ {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description} {GetStrength(path.score)}: - {path.score}){' '} + {path.end.subsection} {path.end.description}( + + {GetStrength(path.score)}:{path.score} + + ){' '} Date: Thu, 7 Sep 2023 13:25:03 +0100 Subject: [PATCH 24/30] Styling improvements and legends --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 93 ++++++++++++------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 139387693..833407f23 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -1,7 +1,18 @@ import axios from 'axios'; import React, { useEffect, useState } from 'react'; import { useLocation } from 'react-router-dom'; -import { Accordion, Button, Dropdown, DropdownItemProps, Grid, Icon, Popup, Table } from 'semantic-ui-react'; +import { + Accordion, + Button, + Container, + Dropdown, + DropdownItemProps, + Grid, + Icon, + Label, + Popup, + Table, +} from 'semantic-ui-react'; import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; import { useEnvironment } from '../../hooks'; @@ -96,43 +107,64 @@ export const GapAnalysis = () => { }; return ( -

+
- setBaseStandard(value?.toString())} - value={BaseStandard} - /> + - setCompareStandard(value?.toString())} - value={CompareStandard} - /> + {gapAnalysis && ( - - - + <> + + Generally: lower is better +
+ {GetStrength(0)}: Closely connected likely to have + majority overlap +
+ {GetStrength(6)}: Connected likely to have partial + overlap +
+ {GetStrength(22)}: Weakly connected likely to + have small or no overlap +
+ + + + + )} @@ -160,7 +192,6 @@ export const GapAnalysis = () => { > {' '} - {gapAnalysis[key].start.id}
{gapAnalysis[key].start.sectionID} {gapAnalysis[key].start.description} From 79473fbb4d7c0a32e2c94905ab28f2ef8701c372 Mon Sep 17 00:00:00 2001 From: Spyros Date: Wed, 6 Sep 2023 20:43:40 +0100 Subject: [PATCH 25/30] Staging heroku (#353) * add action to deploy staging * fix push --- .github/workflows/deploy-staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0d1f15704..6268d2d3f 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -40,4 +40,4 @@ jobs: new_tag="v${major}.${minor}.${patch}" echo "new tag is ${new_tag}" git tag --annotate "${new_tag}" --message "${new_tag}" "${commit_sha}" - git push --force heroku main --follow-tags + git push --force heroku staging:main --follow-tags From a243aa5b9cb9938b819a33c5b1c2279daa5c47a7 Mon Sep 17 00:00:00 2001 From: Spyros Date: Fri, 8 Sep 2023 09:41:55 +0100 Subject: [PATCH 26/30] wip rate limiting --- application/web/web_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/web/web_main.py b/application/web/web_main.py index 50955eed9..05925d7e1 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -379,9 +379,9 @@ def login_r(*args, **kwargs): return login_r - @app.route("/rest/v1/completion", methods=["POST"]) @login_required +@limiter.limit("10 per minute", key_func = lambda : logged_in_user) def chat_cre() -> Any: message = request.get_json(force=True) database = db.Node_collection() From ccae86fcf4f6dd4837cc061062a8934266c79f32 Mon Sep 17 00:00:00 2001 From: Spyros Date: Fri, 8 Sep 2023 09:45:02 +0100 Subject: [PATCH 27/30] lint --- application/web/web_main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/application/web/web_main.py b/application/web/web_main.py index 05925d7e1..7ad5912f3 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -42,6 +42,8 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +RATE_LIMIT = os.environ.get("OPENCRE_CHAT_RATE_LIMIT") or "10 per minute" + class SupportedFormats(Enum): Markdown = "md" @@ -379,9 +381,10 @@ def login_r(*args, **kwargs): return login_r + @app.route("/rest/v1/completion", methods=["POST"]) @login_required -@limiter.limit("10 per minute", key_func = lambda : logged_in_user) +@limiter.limit(RATE_LIMIT, key_func=lambda: logged_in_user) def chat_cre() -> Any: message = request.get_json(force=True) database = db.Node_collection() From a7ff11ba8e838c3c9920f20be84eca76f35dc2cf Mon Sep 17 00:00:00 2001 From: Spyros Date: Sun, 10 Sep 2023 21:14:34 +0100 Subject: [PATCH 28/30] rm scikit version pin --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 025dbc184..f2daf8de2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ playwright psycopg2-binary pygithub python_markdown_maker==1.0 -scikit_learn==1.3.0 +scikit_learn scipy==1.11.2 semver setuptools==66.1.1 From 96c7cc027990884b22168bdefd65227c8cbdce1e Mon Sep 17 00:00:00 2001 From: Spyros Date: Sun, 10 Sep 2023 21:21:52 +0100 Subject: [PATCH 29/30] loosen requirement in sqlalchemy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f2daf8de2..811bd393e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ scipy==1.11.2 semver setuptools==66.1.1 simplify_docx==0.1.2 -SQLAlchemy==2.0.20 +SQLAlchemy compliance-trestle nose==1.3.7 numpy==1.23.0 From 713b4c0fe87740f935bcaa0bd2d6057e501af433 Mon Sep 17 00:00:00 2001 From: Spyros Date: Sun, 10 Sep 2023 22:07:02 +0100 Subject: [PATCH 30/30] more deps changes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 811bd393e..b0c02b9ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask_Caching==2.0.2 flask_compress==1.13 Flask_Cors==4.0.0 Flask_Migrate==4.0.4 -Flask-SQLAlchemy==3.0.5 +Flask-SQLAlchemy gitpython google-api-core google_auth_oauthlib