Improve coverage (#199)

* coverage improvements

* add zap parser tests, start on adding alerts tags parsing

* add cheatsheet parser tests

* made zap parser also recognise top10 tags

application/utils/external_project_parsers/cheatsheets_parser.py

@@ -18,13 +18,18 @@ def cheatsheet(section: str, hyperlink: str, tags: List[str]) -> defs.Standard:
 
 def parse_cheatsheets(cache: db.Node_collection):
     c_repo = "https://github.com/OWASP/CheatSheetSeries.git"
-    cheasheets_path = "cheatsheets/"
+    cheatsheets_path = "cheatsheets/"
+    repo = git.clone(c_repo)
+    register_cheatsheets(repo=repo, cache=cache, cheatsheets_path=cheatsheets_path)
+
+
+def register_cheatsheets(cache: db.Node_collection, repo, cheatsheets_path, repo_path):
+
     title_regexp = r"# (?P<title>.+)"
     cre_link = r"(https://www\.)?opencre.org/cre/(?P<cre>\d+-\d+)"
-    repo = git.clone(c_repo)
-    files = os.listdir(os.path.join(repo.working_dir, cheasheets_path))
+    files = os.listdir(os.path.join(repo.working_dir, cheatsheets_path))
     for mdfile in files:
-        pth = os.path.join(repo.working_dir, cheasheets_path, mdfile)
+        pth = os.path.join(repo.working_dir, cheatsheets_path, mdfile)
         name = None
         tag = None
         section = None
@@ -39,9 +44,7 @@ def parse_cheatsheets(cache: db.Node_collection):
                 name = title.group("title")
                 cre_id = cre.group("cre")
                 cres = cache.get_CREs(external_id=cre_id)
-                hyperlink = (
-                    f"{c_repo.replace('.git','')}/tree/master/{cheasheets_path}{mdfile}"
-                )
+                hyperlink = f"{repo_path.replace('.git','')}/tree/master/{cheatsheets_path}{mdfile}"
                 for dbcre in cres:
                     cs = cheatsheet(
                         section=name,

application/utils/external_project_parsers/zap_alerts_parser.py

@@ -17,10 +17,10 @@
 def zap_alert(
     name: str, id: str, description: str, tags: List[str], code: str
 ) -> defs.Tool:
+    tags.append(id)
     return defs.Tool(
         tooltype=defs.ToolTypes.Offensive,
         name=f"ZAP Rule: {name}",
-        id=id,
         description=description,
         tags=tags,
         hyperlink=code,
@@ -30,14 +30,19 @@ def zap_alert(
 def parse_zap_alerts(cache: db.Node_collection):
     zaproxy_website = "https://github.com/zaproxy/zaproxy-website.git"
     alerts_path = "site/content/docs/alerts/"
+    repo = git.clone(zaproxy_website)
+    register_alerts(repo=repo, cache=cache, alerts_path=alerts_path)
+
+
+def register_alerts(cache: db.Node_collection, repo: git.git, alerts_path: str):
     zap_md_cwe_regexp = r"cwe: ?(?P<cweId>\d+)"
     zap_md_title_regexp = r"title: ?(?P<title>\".+\")"
     zap_md_alert_id_regexp = r"alertid: ?(?P<id>\d+)"
     zap_md_alert_type_regexp = r"alerttype: ?(?P<type>\".+\")"
     zap_md_solution_regexp = r"solution: ?(?P<solution>\".+\")"
     zap_md_code_regexp = r"code: ?(?P<code>.+)"
+    zap_md_top10_regexp = r"OWASP_(?P<year>\d\d\d\d)_A(?P<num>\d\d?)"
 
-    repo = git.clone(zaproxy_website)
     for mdfile in os.listdir(os.path.join(repo.working_dir, alerts_path)):
         pth = os.path.join(repo.working_dir, alerts_path, mdfile)
         name = None
@@ -72,20 +77,48 @@ def parse_zap_alerts(cache: db.Node_collection):
                 )
                 continue
             cwe = re.search(zap_md_cwe_regexp, mdtext)
+            alert = zap_alert(
+                name=name,
+                id=externalId,
+                description=description,
+                tags=[tag],
+                code=code,
+            )
+            dbnode = cache.add_node(alert)
+
+            top10 = re.finditer(zap_md_top10_regexp, mdtext)
+            if top10:
+                for match in top10:
+                    year = match.group("year")
+                    num = match.group("num")
+                    entries = cache.get_nodes(name=f"Top10 {year}", ntype="Standard")
+                    entry = [e for e in entries if str(int(num)) in e.section]
+                    if entry:
+                        logger.info(
+                            f"Found zap alert {name} linking to {entry[0].name}{entry[0].section}"
+                        )
+                        for cre in [
+                            nl
+                            for nl in entry[0].links
+                            if nl.document.doctype == defs.Credoctypes.CRE
+                        ]:
+                            cache.add_link(
+                                cre=db.dbCREfromCRE(cre.document), node=dbnode
+                            )
+                    else:
+                        logger.error(
+                            f"Zap Alert {name} links to OWASP top 10 {year}:{num} but CRE doesn't know about it, incomplete data?"
+                        )
             if cwe:
                 cweId = cwe.group("cweId")
+                logger.info(f"Found zap alert {name} linking to CWE {cweId}")
                 cwe_nodes = cache.get_nodes(name="CWE", section=cweId)
                 for node in cwe_nodes:
                     for link in node.links:
                         if link.document.doctype == defs.Credoctypes.CRE:
-                            alert = zap_alert(
-                                name=name,
-                                id=externalId,
-                                description=description,
-                                tags=[tag],
-                                code=code,
-                            )
-                            dbnode = cache.add_node(alert)
+
                             cache.add_link(
                                 cre=db.dbCREfromCRE(link.document), node=dbnode
                             )
+            else:
+                logger.info(f"CWE id not found in alert {externalId}, skipping linking")