diff --git a/changelog.d/2945.added.md b/changelog.d/2945.added.md new file mode 100644 index 0000000000..9045c3694c --- /dev/null +++ b/changelog.d/2945.added.md @@ -0,0 +1 @@ +Add script for populating database with OUIs and corresponding vendors diff --git a/pyproject.toml b/pyproject.toml index a4e403b140..17a6f7a8c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ sortedstats_cacher = "nav.bin.sortedstats_cacher:main" start_arnold = "nav.bin.start_arnold:main" t1000 = "nav.bin.t1000:main" thresholdmon = "nav.bin.thresholdmon:main" +navoui = "nav.bin.update_ouis:main" [tool.setuptools] include-package-data = true diff --git a/python/nav/bin/update_ouis.py b/python/nav/bin/update_ouis.py new file mode 100755 index 0000000000..fbfad9f141 --- /dev/null +++ b/python/nav/bin/update_ouis.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# -*- testargs: -h -*- +# +# Copyright (C) 2024 Sikt +# +# This file is part of Network Administration Visualized (NAV). +# +# NAV is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License version 3 as published by +# the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. You should have received a copy of the GNU General Public License +# along with NAV. If not, see . +# + +import argparse +import sys +import logging + +from typing import Iterable, Generator, Tuple + +import requests +from requests.exceptions import RequestException + +from nav.bootstrap import bootstrap_django + +bootstrap_django(__file__) + +import django.db + +from nav.macaddress import MacPrefix +from nav.logs import init_stderr_logging + +_logger = logging.getLogger(__name__) + +FILE_URL = "https://standards-oui.ieee.org/oui/oui.txt" + +MAX_ERRORS = 100 + + +def main(): + init_stderr_logging() + argparse.ArgumentParser( + description="Updates the database with OUIs and their related organizations" + ).parse_args() + run() + + +def run(): + _logger.debug(f"Downloading OUIs from {FILE_URL}") + try: + data = _download_oui_file(FILE_URL) + except RequestException as error: + _logger.error("Error while downloading OUIs: %s", error) + sys.exit(1) + ouis = _parse_ouis(data) + _update_database(ouis) + _logger.debug("Completed updating OUI records") + + +def _download_oui_file(url: str) -> str: + response = requests.get(url) + response.raise_for_status() + return response.text + + +def _parse_ouis(oui_data: str) -> Generator[Tuple[str, str], None, None]: + """Returns lists of tuples containing OUI and vendor name for + each vendor + """ + error_count = 0 + for line in oui_data.split('\n'): + if "(hex)" not in line: + continue + try: + yield _parse_line(line) + except ValueError: + error_count += 1 + if error_count >= MAX_ERRORS: + _logger.error("Reached max amount of errors (%d), exiting", MAX_ERRORS) + sys.exit(1) + + +def _parse_line(line: str) -> Tuple[str, str]: + prefix, _, vendor = line.strip().split(None, 2) + oui = str(MacPrefix(prefix)[0]) + return oui, vendor + + +@django.db.transaction.atomic +def _update_database(ouis: Iterable[Tuple[str, str]]): + # Begin by dumping everything into a PostgreSQL temporary table + _logger.debug("Updating database") + cursor = django.db.connection.cursor() + cursor.execute( + """ + CREATE TEMPORARY TABLE new_oui ( + oui macaddr PRIMARY KEY, + vendor varchar, + CHECK (oui = trunc(oui))) + """ + ) + cursor.executemany( + """ + INSERT INTO new_oui (oui, vendor) + VALUES (%s, %s) + ON CONFLICT (oui) + DO NOTHING + """, + ouis, + ) + # Then make the necessary updates to the live OUI table, letting PostgreSQL do the + # heavy lifting of resolving conflicts and changes + cursor.execute( + """ + INSERT INTO oui (oui, vendor) + SELECT + oui, + vendor + FROM + new_oui + ON CONFLICT (oui) + DO UPDATE SET + vendor = EXCLUDED.vendor + WHERE + oui.vendor IS DISTINCT FROM EXCLUDED.vendor + """ + ) + cursor.execute("DELETE FROM oui WHERE oui NOT IN (SELECT oui FROM new_oui)") + + +if __name__ == '__main__': + main() diff --git a/requirements/base.txt b/requirements/base.txt index e95e419e3d..088e566cde 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -47,3 +47,5 @@ git+https://github.com/Uninett/drf-oidc-auth@v4.0#egg=drf-oidc-auth PyOpenSSL==23.3.0 # service-identity is required to make TLS communication libraries shut up about potential MITM attacks service-identity==21.1.0 + +requests diff --git a/tests/integration/update_ouis_test.py b/tests/integration/update_ouis_test.py new file mode 100644 index 0000000000..d60eb7301c --- /dev/null +++ b/tests/integration/update_ouis_test.py @@ -0,0 +1,133 @@ +"""Tests nav/bin/update_ouis.py script for updating OUI records""" + +from unittest.mock import Mock +import pytest + +from nav.models.oui import OUI +from nav.bin.update_ouis import run as update_ouis + + +def test_vendor_name_should_be_registered_correctly(db, mock_oui_file): + update_ouis() + ingram = OUI.objects.get(oui="10-E9-92-00-00-00") + assert ingram.vendor == "INGRAM MICRO SERVICES" + + +def test_oui_should_be_registered_as_mac_address_with_last_3_octets_as_zeros( + db, mock_oui_file +): + update_ouis() + assert OUI.objects.filter(oui="10-E9-92-00-00-00").exists() + + +def test_all_unique_ouis_should_be_registered(db, mock_oui_file): + update_ouis() + assert OUI.objects.count() == 5 + + +def test_duplicate_ouis_should_not_be_registered(db, mock_duplicate_oui_file): + """The OUI file we use is known to have duplicate OUIs, but only one should be registered""" + update_ouis() + assert OUI.objects.count() == 1 + + +def test_old_ouis_should_be_deleted_if_they_dont_exist_in_new_oui_file( + db, mock_oui_file +): + old_vendor_oui = "AA-AA-AA-00-00-00" + OUI.objects.create(oui=old_vendor_oui, vendor="Old vendor") + update_ouis() + assert OUI.objects.count() == 5 + assert not OUI.objects.filter(oui=old_vendor_oui).exists() + + +def test_invalid_oui_should_not_be_registered(db, mock_invalid_oui_file): + update_ouis() + assert OUI.objects.count() == 0 + + +@pytest.fixture() +def mock_oui_file(monkeypatch): + mocked_oui_data = """ +OUI/MA-L Organization +company_id Organization + Address + +10-E9-92 (hex) INGRAM MICRO SERVICES +10E992 (base 16) INGRAM MICRO SERVICES + 100 CHEMIN DE BAILLOT + MONTAUBAN 82000 + FR + +78-F2-76 (hex) Cyklop Fastjet Technologies (Shanghai) Inc. +78F276 (base 16) Cyklop Fastjet Technologies (Shanghai) Inc. + No 18?Lane 699, Zhang Wengmiao Rd, Fengxian district, Shanghai China + Shanghai 201401 + CN + +28-6F-B9 (hex) Nokia Shanghai Bell Co., Ltd. +286FB9 (base 16) Nokia Shanghai Bell Co., Ltd. + No.388 Ning Qiao Road,Jin Qiao Pudong Shanghai + Shanghai 201206 + CN + +E0-A1-29 (hex) Extreme Networks Headquarters +E0A129 (base 16) Extreme Networks Headquarters + 2121 RDU Center Drive + Morrisville NC 27560 + US + +A8-C6-47 (hex) Extreme Networks Headquarters +A8C647 (base 16) Extreme Networks Headquarters + 2121 RDU Center Drive + Morrisville NC 27560 + US + """ + download_file_mock = Mock(return_value=mocked_oui_data) + monkeypatch.setattr("nav.bin.update_ouis._download_oui_file", download_file_mock) + + +@pytest.fixture() +def mock_duplicate_oui_file(monkeypatch): + mocked_oui_data = """ +OUI/MA-L Organization +company_id Organization + Address + +08-00-30 (hex) NETWORK RESEARCH CORPORATION +080030 (base 16) NETWORK RESEARCH CORPORATION + 2380 N. ROSE AVENUE + OXNARD CA 93010 + US + +08-00-30 (hex) ROYAL MELBOURNE INST OF TECH +080030 (base 16) ROYAL MELBOURNE INST OF TECH + GPO BOX 2476V + MELBOURNE VIC 3001 + AU + +08-00-30 (hex) CERN +080030 (base 16) CERN + CH-1211 + GENEVE SUISSE/SWITZ 023 + CH + """ + download_file_mock = Mock(return_value=mocked_oui_data) + monkeypatch.setattr("nav.bin.update_ouis._download_oui_file", download_file_mock) + + +@pytest.fixture() +def mock_invalid_oui_file(monkeypatch): + mocked_oui_data = """ +OUI/MA-L Organization +company_id Organization + Address + +invalidhex (hex) INGRAM MICRO SERVICES +10E992 (base 16) INGRAM MICRO SERVICES + 100 CHEMIN DE BAILLOT + MONTAUBAN 82000 + FR + """ + download_file_mock = Mock(return_value=mocked_oui_data) + monkeypatch.setattr("nav.bin.update_ouis._download_oui_file", download_file_mock)