From 4b8d740cc2b7cc0af7de5112c874c2002d57bfda Mon Sep 17 00:00:00 2001 From: James Kachel Date: Wed, 25 Oct 2023 11:31:01 -0500 Subject: [PATCH] Finishing up initial release of this code - Adds the remaining prereqs for an ol-django app - Adds the migration for storing the MaxMind data - Moves the tests to where they should go - Added a new management command to create GeoIP2 assignments for private IPv4 nets --- src/BUILD | 1 + src/mitol/geoip/BUILD | 26 ++++ src/mitol/geoip/CHANGELOG.md | 5 + src/mitol/geoip/README.md | 38 +++-- src/mitol/geoip/admin.py | 2 +- src/mitol/geoip/api.py | 3 +- src/mitol/geoip/apps.py | 19 ++- .../20231025_162713_jkachel_geoip_app.rst | 37 +++++ src/mitol/geoip/factories.py | 3 +- .../commands/create_private_maxmind_data.py | 75 +++++++++ .../commands/import_maxmind_data.py | 5 +- .../geoip/migrations/0001_add_geoip_tables.py | 147 ++++++++++++++++++ src/mitol/geoip/py.typed | 0 tests/mitol/geoip/__init__.py | 0 .../mitol/geoip/test_api.py | 11 +- tests/testapp/settings/shared.py | 1 + 16 files changed, 339 insertions(+), 34 deletions(-) create mode 100644 src/mitol/geoip/BUILD create mode 100644 src/mitol/geoip/CHANGELOG.md create mode 100644 src/mitol/geoip/changelog.d/20231025_162713_jkachel_geoip_app.rst create mode 100644 src/mitol/geoip/management/commands/create_private_maxmind_data.py create mode 100644 src/mitol/geoip/migrations/0001_add_geoip_tables.py create mode 100644 src/mitol/geoip/py.typed create mode 100644 tests/mitol/geoip/__init__.py rename src/mitol/geoip/api_test.py => tests/mitol/geoip/test_api.py (79%) diff --git a/src/BUILD b/src/BUILD index 8868040b..c7013b6e 100644 --- a/src/BUILD +++ b/src/BUILD @@ -14,6 +14,7 @@ target( "oauth_toolkit_extensions", "openedx", "payment_gateway", + "geoip", ] ], ) diff --git a/src/mitol/geoip/BUILD b/src/mitol/geoip/BUILD new file mode 100644 index 00000000..7734fc8a --- /dev/null +++ b/src/mitol/geoip/BUILD @@ -0,0 +1,26 @@ +resources( + name="files", + sources=[ + "CHANGELOG.md", + "README.md", + "py.typed", + ], +) + +python_sources( + name="geoip", + sources=["**/*.py"], + dependencies=[ + ":files", + "//:reqs#setuptools", + ], +) + +python_distribution( + name="mitol-django-geoip", + dependencies=[":geoip"], + provides=setup_py( + name="mitol-django-geoip", + description="Django application to handle IP-based geolocation", + ), +) diff --git a/src/mitol/geoip/CHANGELOG.md b/src/mitol/geoip/CHANGELOG.md new file mode 100644 index 00000000..c2793fe2 --- /dev/null +++ b/src/mitol/geoip/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project uses date-based versioning. diff --git a/src/mitol/geoip/README.md b/src/mitol/geoip/README.md index 0f434b17..4a3d5a3e 100644 --- a/src/mitol/geoip/README.md +++ b/src/mitol/geoip/README.md @@ -6,9 +6,9 @@ client's IP address to it, and it will send you back a good estimation of what country the client belongs to. Specifically, this wraps lookup around using a _local_ copy of the MaxMind -GeoIP2 data. This choice was made to avoid having to hit an API during checkout -operations, where doing so may end up blocking the client for an unknown amount -of time. +GeoIP2/GeoLite2 data. This choice was made to avoid having to hit an API during +checkout operations, where doing so may end up blocking the client for an +unknown amount of time. The `geoip` app provides lookup for a user's country code only so, while you can use datasets that contain more granular location data in them, you'll still only @@ -16,17 +16,20 @@ get back the country ISO code from the API in this app. ### Setup -You will need a copy of the MaxMind GeoIP2 data in CSV format. This is provided -in several ways: +The MaxMind geolocation databases are provided in two forms: -* For most purposes, the **GeoLite2** dataset is the recommended option. This option provides less features but the dataset is available for free. -* The paid **GeoIP2** dataset can also be used with this library. +* The **GeoLite2** database is free. +* The **GeoIP2** database is not free - a subscription fee applies. -In both cases, you will need the CSV format versions of the databases. The Lite -version is available in ASN, City and Country versions; choose between the City -and Country versions. (The ASN database is not used nor supported.) If the -dataset is only to be used with this app, the Country version is sufficient and -also the smallest. +Which one you need will depend on your use case - however, if your intent is +only to use the dataset with this app, the **GeoLite2** database is sufficient. + +In both cases, you will need the CSV format versions of the databases. You have +the option of downloading the ASN, City, or Country versions of the database. +For the purposes of this app, the Country version is sufficient; if you plan on +using the MaxMind data elsewhere in your project and need more granular location +data, you can use the City version instead. The ASN database is not supported +and can be skipped unless you have a specific need for it. The downloaded datasets include the network block information in two CSV files - one each for IPv4 and IPv6 network blocks - and the location data in a variety @@ -35,9 +38,10 @@ import any others that you require. After installing the app into your Django project in the normal way, you should have a management command called `import_maxmind_data` available to you. Run -this command to import the data from the CSV file you downloaded. You _have_ to +this command to import the data from the CSV file you downloaded. You _must_ import the IPv4 netblock file and the English langauge location file but it is -recommended to also import the IPv6 netblock file. +recommended to import the IPv6 netblock file as well, especially if your app +will be capable of serving IPv6 clients. #### Working Locally @@ -46,10 +50,10 @@ datasets don't include the private network blocks in them as they don't really map to anything, so there's no match for any of the private network IPs. If you'd like these IPs to match something, you can add that data by running the -`create_private_maxmind_data` command. This will create a fake location record -and netblock records for the 3 private network blocks available: +`create_private_maxmind_data` command. This will create a set of fake netblock +records for the 3 private network blocks available: * 10.0.0.0/24 -* 172.16.0.0/20 _(172.16.0.0-172.31.255.255)_ +* 172.16.0.0/12 _(172.16.0.0-172.31.255.255)_ * 192.168.0.0/16 You can specify what ISO code you wish to assign to these as well. Running the diff --git a/src/mitol/geoip/admin.py b/src/mitol/geoip/admin.py index db77b5a4..970ef282 100644 --- a/src/mitol/geoip/admin.py +++ b/src/mitol/geoip/admin.py @@ -2,7 +2,7 @@ from django.contrib import admin -from geoip import models +from mitol.geoip import models class NetBlockAdmin(admin.ModelAdmin): diff --git a/src/mitol/geoip/api.py b/src/mitol/geoip/api.py index 4abc8f82..0e51ed2f 100644 --- a/src/mitol/geoip/api.py +++ b/src/mitol/geoip/api.py @@ -7,8 +7,7 @@ from django.db import transaction from django.db.models import Q -from geoip import models - +from mitol.geoip import models MAXMIND_CSV_COUNTRY_LOCATIONS_LITE = "geolite2-country-locations" MAXMIND_CSV_COUNTRY_BLOCKS_IPV4_LITE = "geolite2-country-ipv4" diff --git a/src/mitol/geoip/apps.py b/src/mitol/geoip/apps.py index 74adfcee..703b8635 100644 --- a/src/mitol/geoip/apps.py +++ b/src/mitol/geoip/apps.py @@ -1,6 +1,17 @@ -from django.apps import AppConfig +"""GeoIP app AppConfigs""" +import os +from mitol.common.apps import BaseApp -class GeoipConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "geoip" + +class GeoIPApp(BaseApp): + """Default configuration for the GeoIP app""" + + name = "mitol.geoip" + label = "geoip" + verbose_name = "GeoIP" + + required_settings = [] + + # necessary because this is a namespaced app + path = os.path.dirname(os.path.abspath(__file__)) diff --git a/src/mitol/geoip/changelog.d/20231025_162713_jkachel_geoip_app.rst b/src/mitol/geoip/changelog.d/20231025_162713_jkachel_geoip_app.rst new file mode 100644 index 00000000..938f1723 --- /dev/null +++ b/src/mitol/geoip/changelog.d/20231025_162713_jkachel_geoip_app.rst @@ -0,0 +1,37 @@ +.. A new scriv changelog fragment. +.. +.. Uncomment the header that is right (remove the leading dots). +.. +.. Removed +.. ------- +.. +.. - A bullet item for the Removed category. +.. + +Added +----- + +- Migrated the MaxMind GeoIP code from xPRO into a ol-django app. +- Added a new management command (`create_private_maxmind_data`) to create GeoIP data for local networks. + +.. +.. Changed +.. ------- +.. +.. - A bullet item for the Changed category. +.. +.. Deprecated +.. ---------- +.. +.. - A bullet item for the Deprecated category. +.. +.. Fixed +.. ----- +.. +.. - A bullet item for the Fixed category. +.. +.. Security +.. -------- +.. +.. - A bullet item for the Security category. +.. diff --git a/src/mitol/geoip/factories.py b/src/mitol/geoip/factories.py index c13bbccf..c2632b6d 100644 --- a/src/mitol/geoip/factories.py +++ b/src/mitol/geoip/factories.py @@ -7,8 +7,7 @@ from factory import LazyAttribute, LazyFunction, fuzzy from factory.django import DjangoModelFactory -from geoip import models - +from mitol.geoip import models fake = faker.Faker() diff --git a/src/mitol/geoip/management/commands/create_private_maxmind_data.py b/src/mitol/geoip/management/commands/create_private_maxmind_data.py new file mode 100644 index 00000000..2839a1f0 --- /dev/null +++ b/src/mitol/geoip/management/commands/create_private_maxmind_data.py @@ -0,0 +1,75 @@ +""" +Creates a set of netblocks for the private IPv4 netblocks, which aren't in the +MaxMind dataset. + +The ISO code must match one that we've imported from MaxMind, so if you haven't +imported the geonames data yet, this will not work (since we have to map the +created netblock to a geoname anyway). +""" + +import ipaddress +from decimal import Decimal + +from django.core.management import BaseCommand, CommandError + +from mitol.geoip.models import Geoname, NetBlock + + +class Command(BaseCommand): + """ + Creates MaxMind assignments for the private IPv4 netblocks. + """ + + help = "Creates MaxMind assignments for the private IPv4 netblocks." + MAX_BIGINTEGERFIELD = 9223372036854775807 + + def add_arguments(self, parser) -> None: + parser.add_argument( + "iso", + type=str, + help="The ISO 3166 alpha2 code to assign these to.", + ) + + parser.add_argument( + "--remove", + action="store_true", + help="Remove the local address netblocks rather than create them.", + ) + + def handle(self, *args, **kwargs): + try: + geoname = Geoname.objects.filter(country_iso_code=kwargs["iso"]).first() + except Geoname.DoesNotExist: + raise CommandError( + f"Could not find a Geoname record for {kwargs['iso']} - have you imported the MaxMind databases?" + ) + + netblocks = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + + if "remove" in kwargs and kwargs["remove"]: + removed_count = NetBlock.objects.filter(network__in=netblocks).delete() + + self.stdout.write( + self.style.SUCCESS(f"{removed_count[0]} netblocks removed!") + ) + else: + for netblock in netblocks: + network = ipaddress.ip_network(netblock) + + (_, created) = NetBlock.objects.update_or_create( + network=netblock, + defaults={ + "is_ipv6": False, + "decimal_ip_start": Decimal(int(network[0])), + "decimal_ip_end": Decimal(int(network[-1])), + "ip_start": network[0], + "ip_end": network[-1], + "geoname_id": geoname.geoname_id, + }, + ) + + self.stdout.write( + self.style.SUCCESS( + f"{'Created' if created else 'Updated'} record for {netblock} for ISO {kwargs['iso']}" + ) + ) diff --git a/src/mitol/geoip/management/commands/import_maxmind_data.py b/src/mitol/geoip/management/commands/import_maxmind_data.py index fbe3ebd8..eba18546 100644 --- a/src/mitol/geoip/management/commands/import_maxmind_data.py +++ b/src/mitol/geoip/management/commands/import_maxmind_data.py @@ -3,10 +3,11 @@ API call that does.) """ -from django.core.management import BaseCommand, CommandError from os import path -from maxmind import api +from django.core.management import BaseCommand, CommandError + +from mitol.geoip import api class Command(BaseCommand): diff --git a/src/mitol/geoip/migrations/0001_add_geoip_tables.py b/src/mitol/geoip/migrations/0001_add_geoip_tables.py new file mode 100644 index 00000000..f260de01 --- /dev/null +++ b/src/mitol/geoip/migrations/0001_add_geoip_tables.py @@ -0,0 +1,147 @@ +# Generated by Django 3.2.21 on 2023-09-22 21:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Geoname", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("geoname_id", models.IntegerField()), + ("locale_code", models.TextField()), + ("continent_code", models.CharField(max_length=2)), + ("continent_name", models.TextField()), + ("country_iso_code", models.CharField(max_length=2)), + ("country_name", models.TextField()), + ( + "subdivision_1_iso_code", + models.CharField(blank=True, max_length=3, null=True), + ), + ("subdivision_1_name", models.TextField(blank=True, null=True)), + ( + "subdivision_2_iso_code", + models.CharField(blank=True, max_length=3, null=True), + ), + ("subdivision_2_name", models.TextField(blank=True, null=True)), + ("city_name", models.TextField(blank=True, null=True)), + ("metro_code", models.IntegerField(blank=True, null=True)), + ("time_zone", models.TextField(blank=True, null=True)), + ( + "is_in_european_union", + models.BooleanField(blank=True, default=False, null=True), + ), + ], + ), + migrations.CreateModel( + name="NetBlock", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("is_ipv6", models.BooleanField(blank=True, default=False)), + ( + "decimal_ip_start", + models.DecimalField( + blank=True, decimal_places=0, max_digits=39, null=True + ), + ), + ( + "decimal_ip_end", + models.DecimalField( + blank=True, decimal_places=0, max_digits=39, null=True + ), + ), + ("ip_start", models.TextField(blank=True)), + ("ip_end", models.TextField(blank=True)), + ("network", models.TextField()), + ("geoname_id", models.BigIntegerField(blank=True, null=True)), + ( + "registered_country_geoname_id", + models.BigIntegerField(blank=True, null=True), + ), + ( + "represented_country_geoname_id", + models.BigIntegerField(blank=True, null=True), + ), + ( + "is_anonymous_proxy", + models.BooleanField(blank=True, default=False, null=True), + ), + ( + "is_satellite_provider", + models.BooleanField(blank=True, default=False, null=True), + ), + ("postal_code", models.CharField(blank=True, max_length=10, null=True)), + ( + "latitude", + models.DecimalField( + blank=True, decimal_places=6, max_digits=16, null=True + ), + ), + ( + "longitude", + models.DecimalField( + blank=True, decimal_places=6, max_digits=16, null=True + ), + ), + ("accuracy_radius", models.IntegerField(blank=True, null=True)), + ], + ), + migrations.AddConstraint( + model_name="netblock", + constraint=models.CheckConstraint( + check=models.Q( + ("geoname_id__isnull", False), + ("registered_country_geoname_id__isnull", False), + ("represented_country_geoname_id__isnull", False), + _connector="OR", + ), + name="at_least_one_geoname_id", + ), + ), + migrations.AddConstraint( + model_name="geoname", + constraint=models.UniqueConstraint( + fields=("geoname_id", "locale_code"), name="unique_geoname_id_locale" + ), + ), + migrations.AddIndex( + model_name="netblock", + index=models.Index( + fields=["decimal_ip_start"], name="maxmind_net_decimal_dfdb1a_idx" + ), + ), + migrations.AddIndex( + model_name="netblock", + index=models.Index( + fields=["decimal_ip_end"], name="maxmind_net_decimal_6166c4_idx" + ), + ), + migrations.AddIndex( + model_name="netblock", + index=models.Index( + fields=["decimal_ip_start", "decimal_ip_end"], + name="maxmind_net_decimal_ab7249_idx", + ), + ), + ] diff --git a/src/mitol/geoip/py.typed b/src/mitol/geoip/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/tests/mitol/geoip/__init__.py b/tests/mitol/geoip/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/mitol/geoip/api_test.py b/tests/mitol/geoip/test_api.py similarity index 79% rename from src/mitol/geoip/api_test.py rename to tests/mitol/geoip/test_api.py index faf2a1d3..647fbbd9 100644 --- a/src/mitol/geoip/api_test.py +++ b/tests/mitol/geoip/test_api.py @@ -7,9 +7,8 @@ import faker import pytest -from geoip.api import ip_to_country_code -from geoip.factories import NetBlockIPv4Factory, NetBlockIPv6Factory - +from mitol.geoip.api import ip_to_country_code +from mitol.geoip.factories import NetBlockIPv4Factory, NetBlockIPv6Factory fake = faker.Factory.create() @@ -43,8 +42,8 @@ def test_ipv4_lookup(v4, in_block): if ( in_block and ( - int(test_address) > int(netblock[0]) - and int(test_address) < int(netblock[-1]) + int(test_address) >= int(netblock[0]) + and int(test_address) <= int(netblock[-1]) ) ) or ( int(test_address) < int(netblock[0]) @@ -54,4 +53,4 @@ def test_ipv4_lookup(v4, in_block): result = ip_to_country_code(str(test_address)) - return result is None and not in_block + assert (result is not None and in_block) or result is None diff --git a/tests/testapp/settings/shared.py b/tests/testapp/settings/shared.py index fa50d88a..d5367409 100644 --- a/tests/testapp/settings/shared.py +++ b/tests/testapp/settings/shared.py @@ -69,6 +69,7 @@ "mitol.oauth_toolkit_extensions.apps.OAuthToolkitExtensionsApp", "mitol.openedx.apps.OpenedxApp", "mitol.payment_gateway.apps.PaymentGatewayApp", + "mitol.geoip.apps.GeoIPApp", # test app, integrates the reusable apps "testapp", ]