From 624b44de6d3a171dd4550d4650e61107f7c5b0cd Mon Sep 17 00:00:00 2001 From: Thomas Da Costa Date: Wed, 9 Oct 2024 12:08:09 +0200 Subject: [PATCH 1/2] Add parameter to DateTime-based range fields Prior to this change, values returned from DB were converted in local timezone. This change adds an optional parameter allowing to choose whether or not to convert values into local timezone. --- docs/xocto/model_fields.md | 6 ++++-- tests/fields/postgres/test_ranges.py | 30 ++++++++++++++++++++++----- tests/models/models.py | 8 +++++++ xocto/fields/postgres/ranges.py | 31 +++++++++++++++++++--------- 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/docs/xocto/model_fields.md b/docs/xocto/model_fields.md index f175bc3..266cc0a 100644 --- a/docs/xocto/model_fields.md +++ b/docs/xocto/model_fields.md @@ -92,7 +92,8 @@ Type: [xocto.ranges.FiniteDatetimeRange](xocto.ranges.FiniteDatetimeRange) A field that represents an inclusive-exclusive `[)` ranges of timezone-aware datetimes. Both the start and end of the range must not be `None`. -The values returned from the database will always be converted to the local timezone +The values returned from the database will be converted to `timezone` if +provided. If not provided, the values will be converted to the local timezone as per the `TIME_ZONE` setting in `settings.py`. ```python @@ -132,7 +133,8 @@ Type: [xocto.ranges.HalfFiniteDatetimeRange](xocto.ranges.HalfFiniteRange) A field that represents an inclusive-exclusive `[)` ranges of timezone-aware datetimes. The end of the range may be open-ended, represented by `None`. -The values returned from the database will always be converted to the local timezone +The values returned from the database will be converted to `timezone` if +provided. If not provided, the values will be converted to the local timezone as per the `TIME_ZONE` setting in `settings.py`. ```python diff --git a/tests/fields/postgres/test_ranges.py b/tests/fields/postgres/test_ranges.py index 53b29ee..5babf8d 100644 --- a/tests/fields/postgres/test_ranges.py +++ b/tests/fields/postgres/test_ranges.py @@ -5,7 +5,6 @@ from dateutil import relativedelta from django.conf import settings from django.core import serializers -from django.utils import timezone from tests.models import models from xocto import localtime, ranges @@ -247,6 +246,7 @@ def test_timezone_conversions(self): """ Timezones are converted correctly when round tripping. """ + TZ_UTC = zoneinfo.ZoneInfo("UTC") TZ_MELB = zoneinfo.ZoneInfo("Australia/Melbourne") TZ_DEFAULT = zoneinfo.ZoneInfo(settings.TIME_ZONE) @@ -256,19 +256,28 @@ def test_timezone_conversions(self): ) obj = models.FiniteDateTimeRangeModel.objects.create( finite_datetime_range=finite_datetime_range_melb, + finite_datetime_range_utc=finite_datetime_range_melb, ) finite_datetime_range_london = ranges.FiniteDatetimeRange( - start=timezone.localtime(finite_datetime_range_melb.start), - end=timezone.localtime(finite_datetime_range_melb.end), + start=localtime.as_localtime(finite_datetime_range_melb.start), + end=localtime.as_localtime(finite_datetime_range_melb.end), + ) + finite_datetime_range_utc = ranges.FiniteDatetimeRange( + start=localtime.as_utc(finite_datetime_range_melb.start), + end=localtime.as_utc(finite_datetime_range_melb.end), ) obj.refresh_from_db() assert ( obj.finite_datetime_range + == obj.finite_datetime_range_utc == finite_datetime_range_london == finite_datetime_range_melb + == finite_datetime_range_utc ) assert obj.finite_datetime_range.start.tzinfo == TZ_DEFAULT assert obj.finite_datetime_range.start.tzinfo != TZ_MELB + assert obj.finite_datetime_range_utc.start.tzinfo == TZ_UTC + assert obj.finite_datetime_range.start.tzinfo != TZ_MELB class TestHalfFiniteDateTimeRangeField: @@ -428,24 +437,35 @@ def test_timezone_conversions(self): """ Timezones are converted correctly when round tripping. """ + TZ_UTC = zoneinfo.ZoneInfo("UTC") TZ_MELB = zoneinfo.ZoneInfo("Australia/Melbourne") TZ_DEFAULT = zoneinfo.ZoneInfo(settings.TIME_ZONE) + half_finite_datetime_range_melb = ranges.HalfFiniteDatetimeRange( start=datetime.datetime(2024, 1, 10, tzinfo=TZ_MELB), end=None, ) obj = models.HalfFiniteDateTimeRangeModel.objects.create( half_finite_datetime_range=half_finite_datetime_range_melb, + half_finite_datetime_range_utc=half_finite_datetime_range_melb, + ) + half_finite_datetime_range_london = ranges.FiniteDatetimeRange( + start=localtime.as_localtime(half_finite_datetime_range_melb.start), + end=None, ) - half_finite_datetime_range_london = ranges.HalfFiniteDatetimeRange( - start=timezone.localtime(half_finite_datetime_range_melb.start), + half_finite_datetime_range_utc = ranges.FiniteDatetimeRange( + start=localtime.as_utc(half_finite_datetime_range_melb.start), end=None, ) obj.refresh_from_db() assert ( obj.half_finite_datetime_range + == obj.half_finite_datetime_range_utc == half_finite_datetime_range_london == half_finite_datetime_range_melb + == half_finite_datetime_range_utc ) assert obj.half_finite_datetime_range.start.tzinfo == TZ_DEFAULT assert obj.half_finite_datetime_range.start.tzinfo != TZ_MELB + assert obj.half_finite_datetime_range_utc.start.tzinfo == TZ_UTC + assert obj.half_finite_datetime_range_utc.start.tzinfo != TZ_MELB diff --git a/tests/models/models.py b/tests/models/models.py index 50d3733..1ba35ec 100644 --- a/tests/models/models.py +++ b/tests/models/models.py @@ -1,3 +1,5 @@ +import zoneinfo + from django.db import models from xocto.fields.postgres import ranges as range_fields @@ -11,6 +13,9 @@ class FiniteDateRangeModel(models.Model): class FiniteDateTimeRangeModel(models.Model): finite_datetime_range = range_fields.FiniteDateTimeRangeField() finite_datetime_range_nullable = range_fields.FiniteDateTimeRangeField(null=True) + finite_datetime_range_utc = range_fields.FiniteDateTimeRangeField( + timezone=zoneinfo.ZoneInfo("UTC"), null=True + ) class HalfFiniteDateTimeRangeModel(models.Model): @@ -18,3 +23,6 @@ class HalfFiniteDateTimeRangeModel(models.Model): half_finite_datetime_range_nullable = range_fields.HalfFiniteDateTimeRangeField( null=True ) + half_finite_datetime_range_utc = range_fields.HalfFiniteDateTimeRangeField( + timezone=zoneinfo.ZoneInfo("UTC"), null=True + ) diff --git a/xocto/fields/postgres/ranges.py b/xocto/fields/postgres/ranges.py index 2e3fdca..21bc684 100644 --- a/xocto/fields/postgres/ranges.py +++ b/xocto/fields/postgres/ranges.py @@ -93,7 +93,18 @@ def _upper_to_inclusive(self, value: pg_ranges.DateRange) -> datetime.date: return value.upper - datetime.timedelta(days=1) -class FiniteDateTimeRangeField(pg_fields.DateTimeRangeField): +class _LocaliserMixin: + def __init__( + self, *args: Any, timezone: Optional[datetime.tzinfo] = None, **kwargs: Any + ): + super().__init__(*args, **kwargs) + self._timezone = timezone + + def localise(self, value: datetime.datetime) -> datetime.datetime: + return localtime.as_localtime(value, self._timezone) + + +class FiniteDateTimeRangeField(_LocaliserMixin, pg_fields.DateTimeRangeField): """ A DateTimeRangeField with Inclusive-Exclusive [) bounds that aren't infinite. @@ -125,8 +136,8 @@ def from_db_value( if value is None: return None return ranges.FiniteDatetimeRange( - start=localtime.as_localtime(value.lower), - end=localtime.as_localtime(value.upper), + start=self.localise(value.lower), + end=self.localise(value.upper), ) def to_python(self, value: Optional[str]) -> Optional[ranges.FiniteDatetimeRange]: @@ -134,8 +145,8 @@ def to_python(self, value: Optional[str]) -> Optional[ranges.FiniteDatetimeRange return None obj = json.loads(value) return ranges.FiniteDatetimeRange( - start=localtime.as_localtime(self.base_field.to_python(obj["start"])), - end=localtime.as_localtime(self.base_field.to_python(obj["end"])), + start=self.localise(self.base_field.to_python(obj["start"])), + end=self.localise(self.base_field.to_python(obj["end"])), ) def value_to_string(self, obj: models.Model) -> Optional[str]: @@ -157,7 +168,7 @@ def value_to_string(self, obj: models.Model) -> Optional[str]: ) -class HalfFiniteDateTimeRangeField(pg_fields.DateTimeRangeField): +class HalfFiniteDateTimeRangeField(_LocaliserMixin, pg_fields.DateTimeRangeField): """ A DateTimeRangeField with Inclusive-Exclusive [) bounds that allows an infinite/open upper bound. @@ -189,8 +200,8 @@ def from_db_value( if value is None: return None return ranges.HalfFiniteDatetimeRange( - start=localtime.as_localtime(value.lower), - end=localtime.as_localtime(value.upper) if value.upper else None, + start=self.localise(value.lower), + end=self.localise(value.upper) if value.upper else None, ) def to_python( @@ -201,8 +212,8 @@ def to_python( obj = json.loads(value) end = self.base_field.to_python(obj["end"]) return ranges.HalfFiniteDatetimeRange( - start=localtime.as_localtime(self.base_field.to_python(obj["start"])), - end=localtime.as_localtime(end) if end else None, + start=self.localise(self.base_field.to_python(obj["start"])), + end=self.localise(end) if end else None, ) def value_to_string(self, obj: models.Model) -> Optional[str]: From b0bcd4746f2ebc70f80b5da43d00eb7af6e88268 Mon Sep 17 00:00:00 2001 From: Thomas Da Costa Date: Tue, 29 Oct 2024 09:45:44 +0100 Subject: [PATCH 2/2] Add test demonstrating issue with local time --- tests/fields/postgres/test_ranges.py | 44 ++++++++++++++++++++++++++++ tests/models/models.py | 12 ++++++++ 2 files changed, 56 insertions(+) diff --git a/tests/fields/postgres/test_ranges.py b/tests/fields/postgres/test_ranges.py index 5babf8d..447138a 100644 --- a/tests/fields/postgres/test_ranges.py +++ b/tests/fields/postgres/test_ranges.py @@ -279,6 +279,28 @@ def test_timezone_conversions(self): assert obj.finite_datetime_range_utc.start.tzinfo == TZ_UTC assert obj.finite_datetime_range.start.tzinfo != TZ_MELB + def test_timezone_conversions_and_dst_issue(self): + TZ_UTC = zoneinfo.ZoneInfo("UTC") + + dst_missing_hour = ranges.FiniteDatetimeRange( + start=datetime.datetime(2021, 10, 31, 0, tzinfo=TZ_UTC), + end=datetime.datetime(2021, 10, 31, 1, tzinfo=TZ_UTC), + ) + utc_obj = models.FiniteDateTimeRangeUTCModel.objects.create( + finite_datetime_range=dst_missing_hour, + ) + local_obj = models.FiniteDateTimeRangeModel.objects.create( + finite_datetime_range=dst_missing_hour, + ) + + # No issue getting this object as the range is configurated as UTC + utc_obj.refresh_from_db() + + # Unable to get this object because the datetime (stored as UTC) is converted to a DST timezone + # and then both start and end == datetime.datetime(2021, 10, 31, 1,) raising a ValueError + with pytest.raises(ValueError): + local_obj.refresh_from_db() + class TestHalfFiniteDateTimeRangeField: def test_roundtrip(self): @@ -469,3 +491,25 @@ def test_timezone_conversions(self): assert obj.half_finite_datetime_range.start.tzinfo != TZ_MELB assert obj.half_finite_datetime_range_utc.start.tzinfo == TZ_UTC assert obj.half_finite_datetime_range_utc.start.tzinfo != TZ_MELB + + def test_timezone_conversions_and_dst_issue(self): + TZ_UTC = zoneinfo.ZoneInfo("UTC") + + dst_missing_hour = ranges.HalfFiniteDatetimeRange( + start=datetime.datetime(2021, 10, 31, 0, tzinfo=TZ_UTC), + end=datetime.datetime(2021, 10, 31, 1, tzinfo=TZ_UTC), + ) + utc_obj = models.HalfFiniteDateTimeRangeUTCModel.objects.create( + half_finite_datetime_range=dst_missing_hour, + ) + local_obj = models.HalfFiniteDateTimeRangeModel.objects.create( + half_finite_datetime_range=dst_missing_hour, + ) + + # No issue getting this object as the range is configurated as UTC + utc_obj.refresh_from_db() + + # Unable to get this object because the datetime (stored as UTC) is converted to a DST timezone + # and then both start and end == datetime.datetime(2021, 10, 31, 1,) raising a ValueError + with pytest.raises(ValueError): + local_obj.refresh_from_db() diff --git a/tests/models/models.py b/tests/models/models.py index 1ba35ec..e2162da 100644 --- a/tests/models/models.py +++ b/tests/models/models.py @@ -18,6 +18,12 @@ class FiniteDateTimeRangeModel(models.Model): ) +class FiniteDateTimeRangeUTCModel(models.Model): + finite_datetime_range = range_fields.FiniteDateTimeRangeField( + timezone=zoneinfo.ZoneInfo("UTC"), null=True + ) + + class HalfFiniteDateTimeRangeModel(models.Model): half_finite_datetime_range = range_fields.HalfFiniteDateTimeRangeField() half_finite_datetime_range_nullable = range_fields.HalfFiniteDateTimeRangeField( @@ -26,3 +32,9 @@ class HalfFiniteDateTimeRangeModel(models.Model): half_finite_datetime_range_utc = range_fields.HalfFiniteDateTimeRangeField( timezone=zoneinfo.ZoneInfo("UTC"), null=True ) + + +class HalfFiniteDateTimeRangeUTCModel(models.Model): + half_finite_datetime_range = range_fields.HalfFiniteDateTimeRangeField( + timezone=zoneinfo.ZoneInfo("UTC"), null=True + )