Skip to content

Commit

Permalink
Merge pull request ivre#1596 from p-l-/enh-scan2db-tags
Browse files Browse the repository at this point in the history 
CLI/scan2db: add --tags option
  • Loading branch information
@p-l-
p-l- authored Oct 5, 2023
2 parents 4223d0a + 4a66a65 commit 06ed8a9
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 19 deletions.
55 changes: 55 additions & 0 deletions ivre/db/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2336,6 +2336,7 @@ def store_scan_json_ivre(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -2354,6 +2355,8 @@ def store_scan_json_ivre(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand All @@ -2366,6 +2369,8 @@ def store_scan_json_ivre(
del host["_id"]
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
# Update schema if/as needed.
Expand Down Expand Up @@ -2393,6 +2398,7 @@ def store_scan_json_zgrab(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
zgrab_port=None,
**_,
Expand All @@ -2412,6 +2418,8 @@ def store_scan_json_zgrab(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
if zgrab_port is not None:
zgrab_port = int(zgrab_port)
Expand Down Expand Up @@ -2450,6 +2458,8 @@ def store_scan_json_zgrab(
pass
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
for key, value in rec.pop("data", {}).items():
Expand Down Expand Up @@ -2507,6 +2517,7 @@ def store_scan_json_zdns_ptr(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -2525,6 +2536,8 @@ def store_scan_json_zdns_ptr(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand Down Expand Up @@ -2560,6 +2573,8 @@ def store_scan_json_zdns_ptr(
}
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -2575,6 +2590,7 @@ def store_scan_json_zdns_a(
needports=False,
needopenports=False,
categories=None,
tags=None,
source=None,
callback=None,
**_,
Expand All @@ -2594,6 +2610,8 @@ def store_scan_json_zdns_a(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand Down Expand Up @@ -2626,6 +2644,8 @@ def store_scan_json_zdns_a(
}
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -2642,6 +2662,7 @@ def store_scan_json_zdns_recursion(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
masscan_probes=None,
**_,
Expand All @@ -2661,6 +2682,8 @@ def store_scan_json_zdns_recursion(
"""
if categories is None:
categories = []
if tags is None:
tags = []
answers = set()
for probe in masscan_probes or []:
if probe.startswith("ZDNS:"):
Expand Down Expand Up @@ -2746,6 +2769,8 @@ def store_scan_json_zdns_recursion(
port["scripts"] = [script]
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand Down Expand Up @@ -2796,6 +2821,7 @@ def store_scan_json_dnsx(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -2814,6 +2840,8 @@ def store_scan_json_dnsx(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand All @@ -2840,6 +2868,8 @@ def store_scan_json_dnsx(
for host in self._gen_records_json_dnsx(rec, name, timestamp):
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -2862,6 +2892,7 @@ def store_scan_json_nuclei(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -2880,6 +2911,8 @@ def store_scan_json_nuclei(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand Down Expand Up @@ -3068,6 +3101,8 @@ def store_scan_json_nuclei(
)
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -3084,6 +3119,7 @@ def store_scan_json_httpx(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -3102,6 +3138,8 @@ def store_scan_json_httpx(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand Down Expand Up @@ -3167,6 +3205,8 @@ def store_scan_json_httpx(
# status-code response-time failed
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -3183,6 +3223,7 @@ def store_scan_json_tlsx(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -3201,6 +3242,8 @@ def store_scan_json_tlsx(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand Down Expand Up @@ -3297,6 +3340,8 @@ def store_scan_json_tlsx(
# remaining fields (TODO): jarm_hash tls_connection cipher tls_version
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -3313,6 +3358,7 @@ def store_scan_json_shodan(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -3331,6 +3377,8 @@ def store_scan_json_shodan(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
for line in fdesc:
Expand Down Expand Up @@ -3439,6 +3487,8 @@ def store_scan_json_shodan(
# tags (["cloud"]) / cloud
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand All @@ -3455,6 +3505,7 @@ def store_scan_json_dismap(
needopenports=False,
categories=None,
source=None,
tags=None,
callback=None,
**_,
):
Expand All @@ -3473,6 +3524,8 @@ def store_scan_json_dismap(
"""
if categories is None:
categories = []
if tags is None:
tags = []
self.start_store_hosts()
with utils.open_file(fname) as fdesc:
try:
Expand Down Expand Up @@ -3623,6 +3676,8 @@ def store_scan_json_dismap(
# banner.string note path uri
if categories:
host["categories"] = categories
if tags:
add_tags(host, tags)
if source is not None:
host["source"] = source
host = self.json2dbrec(host)
Expand Down
22 changes: 22 additions & 0 deletions ivre/tools/scan2db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@ def main() -> None:
parser.add_argument(
"-t", "--test", action="store_true", help="Test mode (JSON output)."
)
parser.add_argument(
"--tags",
metavar="TAG:LEVEL:INFO[,TAG:LEVEL:INFO]",
help="Add tags to the results; e.g. "
'--tags=CDN:info:"My CDN",Honeypot:warning:"My Masscanned Honeypot"',
)
parser.add_argument(
"--test-normal", action="store_true", help='Test mode ("normal" Nmap output).'
)
Expand Down Expand Up @@ -107,6 +113,21 @@ def main() -> None:
args = parser.parse_args()
database = ivre.db.db.nmap
categories = args.categories.split(",") if args.categories else []
tags = [
{
"value": value,
"type": type_,
"info": [info],
}
if info
else {
"value": value,
"type": type_,
}
for value, type_, info in (
tag.split(":", 3) for tag in (args.tags.split(",") if args.tags else [])
)
]
if args.test:
args.update_view = False
args.no_update_view = True
Expand Down Expand Up @@ -152,6 +173,7 @@ def callback(x: Record) -> None:
scan,
categories=categories,
source=args.source,
tags=tags,
needports=args.ports,
needopenports=args.open_ports,
masscan_probes=args.masscan_probes,
Expand Down
Loading

0 comments on commit 06ed8a9

Please sign in to comment.