store waf headers (#24087)

* store waf headers

* set_request_metadata to consistently use `None` to clear

* rename select_request_fingerprint_headers to select_request_metadata

* lint fix

Author: eviljeff

src/olympia/access/middleware.py

@@ -17,12 +17,15 @@ def process_request(self, request):
         # lazy to avoid early database queries.
         core.set_user(request.user)
         core.set_remote_addr(request.META.get('REMOTE_ADDR'))
+        core.set_request_metadata(core.select_request_metadata(request.headers))
 
     def process_response(self, request, response):
         core.set_user(None)
         core.set_remote_addr(None)
+        core.set_request_metadata(None)
         return response
 
     def process_exception(self, request, exception):
         core.set_user(None)
         core.set_remote_addr(None)
+        core.set_request_metadata(None)

src/olympia/access/tests.py

@@ -1,11 +1,12 @@
 from django.contrib.auth.models import AnonymousUser
+from django.http import HttpResponse
 
 import pytest
 
-from olympia import amo
+from olympia import amo, core
 from olympia.access.models import Group, GroupUser
 from olympia.addons.models import Addon, AddonUser
-from olympia.amo.tests import TestCase, addon_factory, user_factory
+from olympia.amo.tests import RequestFactory, TestCase, addon_factory, user_factory
 from olympia.users.models import UserProfile
 
 from .acl import (
@@ -20,6 +21,7 @@
     match_rules,
     reserved_guid_addon_submission_allowed,
 )
+from .middleware import UserAndAddrMiddleware
 
 
 pytestmark = pytest.mark.django_db
@@ -321,3 +323,39 @@ def test_reserved_guid_addon_submission_allowed_not_mozilla_not_allowed(guid):
     user = user_factory()
     data = {'guid': guid}
     assert not reserved_guid_addon_submission_allowed(user, data)
+
+
+def test_user_and_addr_middleware():
+    middleware = UserAndAddrMiddleware(lambda x: response)
+    wanted_headers = {
+        'Client-JA4': 'd1234-5678-0000',
+        'X-SigSci-Tags': 'TAG,ANOTHERTAG',
+    }
+    request = RequestFactory(
+        REMOTE_ADDR='123.45.67.89',
+        headers={**wanted_headers, 'another_HEADER': 'Ignored'},
+    ).get('/')
+    assert request.headers
+    response = HttpResponse()
+    user = user_factory()
+
+    request.user = user
+
+    assert core.get_user() is None
+    assert core.get_remote_addr() is None
+    assert core.get_request_metadata() == {}
+
+    middleware.process_request(request)
+    assert core.get_user() == user
+    assert core.get_remote_addr() == '123.45.67.89'
+    assert core.get_request_metadata() == wanted_headers, request.headers
+
+    middleware.process_response(request, response)
+    assert core.get_user() is None
+    assert core.get_remote_addr() is None
+    assert core.get_request_metadata() == {}
+
+    middleware.process_exception(request, Exception())
+    assert core.get_user() is None
+    assert core.get_remote_addr() is None
+    assert core.get_request_metadata() == {}

src/olympia/activity/migrations/0030_requestfingerprintlog.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.25 on 2025-10-29 16:38
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import olympia.amo.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('activity', '0029_alter_activitylog_action'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RequestFingerprintLog',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(blank=True, default=django.utils.timezone.now, editable=False)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('ja4', models.CharField(db_index=True, max_length=36)),
+                ('signals', models.JSONField(default=list)),
+                ('activity_log', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='activity.activitylog')),
+            ],
+            options={
+                'db_table': 'log_activity_request_fingerprint',
+                'ordering': ('-created',),
+            },
+            bases=(olympia.amo.models.SaveUpdateMixin, models.Model),
+        ),
+    ]

src/olympia/activity/models.py

@@ -284,6 +284,23 @@ def save(self, *args, **kwargs):
         return super().save(*args, **kwargs)
 
 
+class RequestFingerprintLog(ModelBase):
+    """
+    This table is for indexing the activity log by some request fingerprints, (only for
+    specific actions).
+    """
+
+    activity_log = models.OneToOneField('ActivityLog', on_delete=models.CASCADE)
+    # https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md
+    # e.g. t13d1516h2_8daaf6152771_b186095e22b6
+    ja4 = models.CharField(max_length=36, db_index=True)
+    signals = models.JSONField(default=list)
+
+    class Meta:
+        db_table = 'log_activity_request_fingerprint'
+        ordering = ('-created',)
+
+
 class RatingLog(ModelBase):
     """
     This table is for indexing the activity log by Ratings (user reviews).
@@ -515,6 +532,21 @@ def create(self, *args, **kw):
                 created=kw.get('created', timezone.now()),
             )
 
+            # Also, if we're storing the ip address, store the request fingerprints too
+            ja4 = core.get_request_metadata().get('Client-JA4') or ''
+            if ja4:
+                ja4 = ja4[: RequestFingerprintLog._meta.get_field('ja4').max_length]
+            # it should be a comma-seperated string
+            signals = (core.get_request_metadata().get('X-SigSci-Tags') or '').split(
+                ','
+            )
+            RequestFingerprintLog.objects.create(
+                ja4=ja4,
+                signals=signals,
+                activity_log=al,
+                created=kw.get('created', timezone.now()),
+            )
+
         return al
 
 

src/olympia/activity/tests/test_admin.py

@@ -53,16 +53,16 @@ def test_search_for_single_ip(self):
         user2 = user_factory()
         user3 = user_factory()
         addon = addon_factory(users=[user3])
-        with core.override_remote_addr('127.0.0.2'):
+        with core.override_remote_addr_or_metadata(ip_address='127.0.0.2'):
             user2.update(email='[email protected]')
             # That will make user2 match our query.
             ActivityLog.objects.create(amo.LOG.LOG_IN, user=user2)
-        with core.override_remote_addr('127.0.0.2'):
+        with core.override_remote_addr_or_metadata(ip_address='127.0.0.2'):
             # That will make user3 match our query.
             ActivityLog.objects.create(
                 amo.LOG.ADD_VERSION, addon.current_version, addon, user=user3
             )
-        with core.override_remote_addr('127.0.0.1'):
+        with core.override_remote_addr_or_metadata(ip_address='127.0.0.1'):
             extra_user = user_factory()  # Extra user that shouldn't match
             ActivityLog.objects.create(amo.LOG.LOG_IN, user=extra_user)
         with self.assertNumQueries(11):

src/olympia/activity/tests/test_models.py

@@ -17,6 +17,7 @@
     DraftComment,
     GenericMozillaUser,
     IPLog,
+    RequestFingerprintLog,
     ReviewActionReasonLog,
     attachment_upload_path,
 )
@@ -374,7 +375,7 @@ def test_output(self):
     def test_to_string_num_queries_model_depending_on_addon(self):
         addon = Addon.objects.get()
         addon2 = addon_factory()
-        with core.override_remote_addr('1.1.1.1'):
+        with core.override_remote_addr_or_metadata(ip_address='1.1.1.1'):
             ActivityLog.objects.create(
                 amo.LOG.ADD_VERSION,
                 addon,
@@ -410,7 +411,7 @@ def test_ip_log(self):
         # create an IPLog.
         action = amo.LOG.REJECT_VERSION
         assert not getattr(action, 'store_ip', False)
-        with core.override_remote_addr('127.0.4.8'):
+        with core.override_remote_addr_or_metadata(ip_address='127.0.4.8'):
             activity = ActivityLog.objects.create(
                 action,
                 addon,
@@ -422,7 +423,7 @@ def test_ip_log(self):
         # create an IPLog.
         action = amo.LOG.ADD_VERSION
         assert getattr(action, 'store_ip', False)
-        with core.override_remote_addr('15.16.23.42'):
+        with core.override_remote_addr_or_metadata(ip_address='15.16.23.42'):
             activity = ActivityLog.objects.create(
                 action,
                 addon,
@@ -435,6 +436,48 @@ def test_ip_log(self):
         assert ip_log._ip_address == '15.16.23.42'
         assert ip_log.ip_address_binary == IPv4Address('15.16.23.42')
 
+    def test_request_fingerprint_log(self):
+        addon = Addon.objects.get()
+        assert RequestFingerprintLog.objects.count() == 0
+        # 37 charactors to test truncation to 36 characters.
+        metadata = {
+            'Client-JA4': 'a' * 37,
+            'X-SigSci-Tags': 'TAG1,TAG2',
+            'other': 'data',
+        }
+        # Creating an activity log for an action without store_ip=True doesn't
+        # create an RequestFingerprintLog.
+        action = amo.LOG.REJECT_VERSION
+        assert not getattr(action, 'store_ip', False)
+        with core.override_remote_addr_or_metadata(
+            ip_address='127.0.4.8', metadata=metadata
+        ):
+            activity = ActivityLog.objects.create(
+                action,
+                addon,
+                addon.current_version,
+                user=self.request.user,
+            )
+        assert RequestFingerprintLog.objects.count() == 0
+        # Creating an activity log for an action *with* store_ip=True *does*
+        # create an RequestFingerprintLog.
+        action = amo.LOG.ADD_VERSION
+        assert getattr(action, 'store_ip', False)
+        with core.override_remote_addr_or_metadata(
+            ip_address='15.16.23.42', metadata=metadata
+        ):
+            activity = ActivityLog.objects.create(
+                action,
+                addon,
+                addon.current_version,
+                user=self.request.user,
+            )
+        assert RequestFingerprintLog.objects.count() == 1
+        fingerprint_log = RequestFingerprintLog.objects.get()
+        assert fingerprint_log.activity_log == activity
+        assert fingerprint_log.ja4 == 'a' * 36  # Truncated to 36 characters.
+        assert fingerprint_log.signals == ['TAG1', 'TAG2']
+
     def test_review_action_reason_log(self):
         addon = Addon.objects.get()
         assert ReviewActionReasonLog.objects.count() == 0

src/olympia/addons/tests/test_admin.py

@@ -267,25 +267,25 @@ def test_search_by_ip(self):
         self.client.force_login(user)
 
         addon = addon_factory(guid='@foo')
-        with core.override_remote_addr('4.8.15.16'):
+        with core.override_remote_addr_or_metadata(ip_address='4.8.15.16'):
             ActivityLog.objects.create(
                 amo.LOG.ADD_VERSION, addon.current_version, addon, user=user
             )
         version_factory(addon=addon)
-        with core.override_remote_addr('4.8.15.16'):
+        with core.override_remote_addr_or_metadata(ip_address='4.8.15.16'):
             ActivityLog.objects.create(
                 amo.LOG.ADD_VERSION, addon.current_version, addon, user=user
             )
         second_addon = addon_factory(guid='@bar')
-        with core.override_remote_addr('4.8.15.16'):
+        with core.override_remote_addr_or_metadata(ip_address='4.8.15.16'):
             ActivityLog.objects.create(
                 amo.LOG.ADD_VERSION,
                 second_addon.current_version,
                 second_addon,
                 user=user,
             )
         third_addon = addon_factory(guid='@xyz')
-        with core.override_remote_addr('127.0.0.1'):
+        with core.override_remote_addr_or_metadata(ip_address='127.0.0.1'):
             ActivityLog.objects.create(
                 amo.LOG.ADD_VERSION,
                 third_addon.current_version,

src/olympia/core/__init__.py

@@ -1,10 +1,12 @@
 import contextlib
 import threading
+from collections.abc import Mapping
 
 
 _locals = threading.local()
 _locals.user = None
 _locals.remote_addr = None
+_locals.request_metadata = None
 
 
 def get_user():
@@ -28,10 +30,40 @@ def set_remote_addr(remote_addr):
     _locals.remote_addr = remote_addr
 
 
+def get_request_metadata():
+    return getattr(_locals, 'request_metadata', None) or {}
+
+
+def set_request_metadata(data):
+    if data and isinstance(data, Mapping):
+        _locals.request_metadata = {key: value for key, value in data.items() if value}
+    else:
+        _locals.request_metadata = None
+
+
+def select_request_metadata(headers):
+    """Get the two headers from from request.headers that we currently care about,
+    if present.
+    Note this function also normalizes the header names if it's called with
+    request.headers (HttpHeaders is case-insensitive)."""
+    return {
+        key: val
+        for key in ('Client-JA4', 'X-SigSci-Tags')
+        if (val := headers.get(key)) is not None
+    }
+
+
 @contextlib.contextmanager
-def override_remote_addr(remote_addr_override):
+def override_remote_addr_or_metadata(*, ip_address=None, metadata=None):
     """Override value returned by get_remote_addr() for a specific context."""
-    original = get_remote_addr()
-    set_remote_addr(remote_addr_override)
+    original_ip = get_remote_addr()
+    original_metadata = get_request_metadata()
+    if ip_address is not None:
+        set_remote_addr(ip_address)
+    if metadata is not None:
+        set_request_metadata(metadata)
     yield
-    set_remote_addr(original)
+    if ip_address is not None:
+        set_remote_addr(original_ip)
+    if metadata is not None:
+        set_request_metadata(original_metadata)

src/olympia/core/tests/test_init.py

@@ -1,13 +1,14 @@
 from django.contrib.auth.models import AnonymousUser
+from django.http.request import HttpHeaders
 
 from olympia import core
 from olympia.users.models import UserProfile
 
 
-def test_override_remote_addr():
+def test_override_remote_addr_or_metadata():
     original = core.get_remote_addr()
 
-    with core.override_remote_addr('some other value'):
+    with core.override_remote_addr_or_metadata(ip_address='some other value'):
         assert core.get_remote_addr() == 'some other value'
 
     assert core.get_remote_addr() == original
@@ -23,3 +24,21 @@ def test_set_get_user_anonymous():
 
     core.set_user(None)
     assert core.get_user() is None
+
+
+def test_get_request_metadata_and_set_request_metadata():
+    assert core.get_request_metadata() == {}
+
+    core.set_request_metadata(data=None)
+    assert core.get_request_metadata() == {}
+
+    core.set_request_metadata({'a': 'b', 'c': None})
+    assert core.get_request_metadata() == {'a': 'b'}
+
+
+def test_select_request_metadata():
+    assert core.select_request_metadata(HttpHeaders({})) == {}
+
+    assert core.select_request_metadata(
+        HttpHeaders({'HTTP_Client-JA4': None, 'HTTP_X_SigSci-TAGS': 'SOME,tags'})
+    ) == {'X-SigSci-Tags': 'SOME,tags'}

src/olympia/devhub/tests/test_feeds.py

@@ -38,7 +38,7 @@ def log_updates(self, num, version_string='1'):
         version = Version.objects.create(version=version_string, addon=self.addon)
 
         for _i in range(num):
-            with core.override_remote_addr('127.0.0.1'):
+            with core.override_remote_addr_or_metadata(ip_address='127.0.0.1'):
                 ActivityLog.objects.create(amo.LOG.ADD_VERSION, self.addon, version)
 
     def log_status(self, num):