Add script for updating OUI records

changelog.d/2945.added.md

@@ -0,0 +1 @@
+Add script for populating database with OUIs and corresponding vendors

pyproject.toml

@@ -56,6 +56,7 @@ sortedstats_cacher = "nav.bin.sortedstats_cacher:main"
 start_arnold = "nav.bin.start_arnold:main"
 t1000 = "nav.bin.t1000:main"
 thresholdmon = "nav.bin.thresholdmon:main"
+navoui = "nav.bin.update_ouis:main"
 
 [tool.setuptools]
 include-package-data = true

python/nav/bin/update_ouis.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# -*- testargs: -h -*-
+#
+# Copyright (C) 2024 Sikt
+#
+# This file is part of Network Administration Visualized (NAV).
+#
+# NAV is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License version 3 as published by
+# the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.  You should have received a copy of the GNU General Public License
+# along with NAV. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import argparse
+import sys
+import logging
+
+from typing import Iterable, Generator, Tuple
+
+import requests
+from requests.exceptions import RequestException
+
+from nav.bootstrap import bootstrap_django
+
+bootstrap_django(__file__)
+
+import django.db
+
+from nav.macaddress import MacPrefix
+from nav.logs import init_stderr_logging
+
+_logger = logging.getLogger(__name__)
+
+FILE_URL = "https://standards-oui.ieee.org/oui/oui.txt"
+
+MAX_ERRORS = 100
+
+
+def main():
+    init_stderr_logging()
+    argparse.ArgumentParser(
+        description="Updates the database with OUIs and their related organizations"
+    ).parse_args()
+    run()
+
+
+def run():
+    _logger.debug(f"Downloading OUIs from {FILE_URL}")
+    try:
+        data = _download_oui_file(FILE_URL)
+    except RequestException as error:
+        _logger.error("Error while downloading OUIs: %s", error)
+        sys.exit(1)
+    ouis = _parse_ouis(data)
+    _update_database(ouis)
+    _logger.debug("Completed updating OUI records")
+
+
+def _download_oui_file(url: str) -> str:
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.text
+
+
+def _parse_ouis(oui_data: str) -> Generator[Tuple[str, str], None, None]:
+    """Returns lists of tuples containing OUI and vendor name for
+    each vendor
+    """
+    error_count = 0
+    for line in oui_data.split('\n'):
+        if "(hex)" not in line:
+            continue
+        try:
+            yield _parse_line(line)
+        except ValueError:
+            error_count += 1
+            if error_count >= MAX_ERRORS:
+                _logger.error("Reached max amount of errors (%d), exiting", MAX_ERRORS)
+                sys.exit(1)
+
+
+def _parse_line(line: str) -> Tuple[str, str]:
+    prefix, _, vendor = line.strip().split(None, 2)
+    oui = str(MacPrefix(prefix)[0])
+    return oui, vendor
+
+
+@django.db.transaction.atomic
+def _update_database(ouis: Iterable[Tuple[str, str]]):
+    # Begin by dumping everything into a PostgreSQL temporary table
+    _logger.debug("Updating database")
+    cursor = django.db.connection.cursor()
+    cursor.execute(
+        """
+        CREATE TEMPORARY TABLE new_oui (
+            oui macaddr PRIMARY KEY,
+            vendor varchar,
+            CHECK (oui = trunc(oui)))
+        """
+    )
+    cursor.executemany(
+        """
+        INSERT INTO new_oui (oui, vendor)
+            VALUES (%s, %s)
+        ON CONFLICT (oui)
+            DO NOTHING
+        """,
+        ouis,
+    )
+    # Then make the necessary updates to the live OUI table, letting PostgreSQL do the
+    # heavy lifting of resolving conflicts and changes
+    cursor.execute(
+        """
+        INSERT INTO oui (oui, vendor)
+        SELECT
+            oui,
+            vendor
+        FROM
+            new_oui
+        ON CONFLICT (oui)
+            DO UPDATE SET
+                vendor = EXCLUDED.vendor
+            WHERE
+                oui.vendor IS DISTINCT FROM EXCLUDED.vendor
+        """
+    )
+    cursor.execute("DELETE FROM oui WHERE oui NOT IN (SELECT oui FROM new_oui)")
+
+
+if __name__ == '__main__':
+    main()

requirements/base.txt

@@ -47,3 +47,5 @@ git+https://github.com/Uninett/[email protected]#egg=drf-oidc-auth
 PyOpenSSL==23.3.0
 # service-identity is required to make TLS communication libraries shut up about potential MITM attacks
 service-identity==21.1.0
+
+requests

tests/integration/update_ouis_test.py

@@ -0,0 +1,133 @@
+"""Tests nav/bin/update_ouis.py script for updating OUI records"""
+
+from unittest.mock import Mock
+import pytest
+
+from nav.models.oui import OUI
+from nav.bin.update_ouis import run as update_ouis
+
+
+def test_vendor_name_should_be_registered_correctly(db, mock_oui_file):
+    update_ouis()
+    ingram = OUI.objects.get(oui="10-E9-92-00-00-00")
+    assert ingram.vendor == "INGRAM MICRO SERVICES"
+
+
+def test_oui_should_be_registered_as_mac_address_with_last_3_octets_as_zeros(
+    db, mock_oui_file
+):
+    update_ouis()
+    assert OUI.objects.filter(oui="10-E9-92-00-00-00").exists()
+
+
+def test_all_unique_ouis_should_be_registered(db, mock_oui_file):
+    update_ouis()
+    assert OUI.objects.count() == 5
+
+
+def test_duplicate_ouis_should_not_be_registered(db, mock_duplicate_oui_file):
+    """The OUI file we use is known to have duplicate OUIs, but only one should be registered"""
+    update_ouis()
+    assert OUI.objects.count() == 1
+
+
+def test_old_ouis_should_be_deleted_if_they_dont_exist_in_new_oui_file(
+    db, mock_oui_file
+):
+    old_vendor_oui = "AA-AA-AA-00-00-00"
+    OUI.objects.create(oui=old_vendor_oui, vendor="Old vendor")
+    update_ouis()
+    assert OUI.objects.count() == 5
+    assert not OUI.objects.filter(oui=old_vendor_oui).exists()
+
+
+def test_invalid_oui_should_not_be_registered(db, mock_invalid_oui_file):
+    update_ouis()
+    assert OUI.objects.count() == 0
+
+
+@pytest.fixture()
+def mock_oui_file(monkeypatch):
+    mocked_oui_data = """
+OUI/MA-L                                                    Organization
+company_id                                                  Organization
+                                                            Address
+
+10-E9-92   (hex)		    INGRAM MICRO SERVICES
+10E992     (base 16)		INGRAM MICRO SERVICES
+                            100 CHEMIN DE BAILLOT
+                            MONTAUBAN    82000
+                            FR
+
+78-F2-76   (hex)		    Cyklop Fastjet Technologies (Shanghai) Inc.
+78F276     (base 16)		Cyklop Fastjet Technologies (Shanghai) Inc.
+                            No 18?Lane 699, Zhang Wengmiao Rd,  Fengxian district, Shanghai China
+                            Shanghai    201401
+                            CN
+
+28-6F-B9   (hex)		    Nokia Shanghai Bell Co., Ltd.
+286FB9     (base 16)		Nokia Shanghai Bell Co., Ltd.
+                            No.388 Ning Qiao Road,Jin Qiao Pudong Shanghai
+                            Shanghai     201206
+                            CN
+
+E0-A1-29   (hex)		    Extreme Networks Headquarters
+E0A129     (base 16)		Extreme Networks Headquarters
+                            2121 RDU Center Drive
+                            Morrisville  NC  27560
+                            US
+
+A8-C6-47   (hex)		    Extreme Networks Headquarters
+A8C647     (base 16)		Extreme Networks Headquarters
+                            2121 RDU Center Drive
+                            Morrisville  NC  27560
+                            US
+    """
+    download_file_mock = Mock(return_value=mocked_oui_data)
+    monkeypatch.setattr("nav.bin.update_ouis._download_oui_file", download_file_mock)
+
+
+@pytest.fixture()
+def mock_duplicate_oui_file(monkeypatch):
+    mocked_oui_data = """
+OUI/MA-L                                                    Organization
+company_id                                                  Organization
+                                                            Address
+
+08-00-30   (hex)		    NETWORK RESEARCH CORPORATION
+080030     (base 16)		NETWORK RESEARCH CORPORATION
+                            2380 N. ROSE AVENUE
+                            OXNARD  CA  93010
+                            US
+
+08-00-30   (hex)		    ROYAL MELBOURNE INST OF TECH
+080030     (base 16)		ROYAL MELBOURNE INST OF TECH
+                            GPO BOX 2476V
+                            MELBOURNE  VIC  3001
+                            AU
+
+08-00-30   (hex)		    CERN
+080030     (base 16)		CERN
+                            CH-1211
+                            GENEVE  SUISSE/SWITZ  023
+                            CH
+    """
+    download_file_mock = Mock(return_value=mocked_oui_data)
+    monkeypatch.setattr("nav.bin.update_ouis._download_oui_file", download_file_mock)
+
+
+@pytest.fixture()
+def mock_invalid_oui_file(monkeypatch):
+    mocked_oui_data = """
+OUI/MA-L                                                    Organization
+company_id                                                  Organization
+                                                            Address
+
+invalidhex   (hex)		    INGRAM MICRO SERVICES
+10E992     (base 16)		INGRAM MICRO SERVICES
+                            100 CHEMIN DE BAILLOT
+                            MONTAUBAN    82000
+                            FR
+    """
+    download_file_mock = Mock(return_value=mocked_oui_data)
+    monkeypatch.setattr("nav.bin.update_ouis._download_oui_file", download_file_mock)