From dc1e5b65292ea2a6c726ac99db2eb9cf0cefd83e Mon Sep 17 00:00:00 2001 From: Squeaky Date: Wed, 29 May 2024 20:36:24 +0200 Subject: [PATCH 1/7] Vendor json_util from pymongo --- inbox/sqlalchemy_ext/json_util.py | 249 ++++++++++++++++++++++++++++++ inbox/sqlalchemy_ext/util.py | 12 +- 2 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 inbox/sqlalchemy_ext/json_util.py diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py new file mode 100644 index 000000000..e941ac565 --- /dev/null +++ b/inbox/sqlalchemy_ext/json_util.py @@ -0,0 +1,249 @@ +# Copyright 2009-2015 MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for using Python's :mod:`json` module with BSON documents. + +This module provides two helper methods `dumps` and `loads` that wrap the +native :mod:`json` methods and provide explicit BSON conversion to and from +json. This allows for specialized encoding and decoding of BSON documents +into `Mongo Extended JSON +`_'s *Strict* +mode. This lets you encode / decode BSON documents to JSON even when +they use special BSON types. + +Example usage (serialization): + +.. doctest:: + + >>> from bson import Binary, Code + >>> from bson.json_util import dumps + >>> dumps([{'foo': [1, 2]}, + ... {'bar': {'hello': 'world'}}, + ... {'code': Code("function x() { return 1; }")}, + ... {'bin': Binary("\x01\x02\x03\x04")}]) + '[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }", "$scope": {}}}, {"bin": {"$binary": "AQIDBA==", "$type": "00"}}]' + +Example usage (deserialization): + +.. doctest:: + + >>> from bson.json_util import loads + >>> loads('[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$scope": {}, "$code": "function x() { return 1; }"}}, {"bin": {"$type": "00", "$binary": "AQIDBA=="}}]') + [{u'foo': [1, 2]}, {u'bar': {u'hello': u'world'}}, {u'code': Code('function x() { return 1; }', {})}, {u'bin': Binary('...', 0)}] + +Alternatively, you can manually pass the `default` to :func:`json.dumps`. +It won't handle :class:`~bson.binary.Binary` and :class:`~bson.code.Code` +instances (as they are extended strings you can't provide custom defaults), +but it will be faster as there is less recursion. + +.. versionchanged:: 2.8 + The output format for :class:`~bson.timestamp.Timestamp` has changed from + '{"t": , "i": }' to '{"$timestamp": {"t": , "i": }}'. + This new format will be decoded to an instance of + :class:`~bson.timestamp.Timestamp`. The old format will continue to be + decoded to a python dict as before. Encoding to the old format is no longer + supported as it was never correct and loses type information. + Added support for $numberLong and $undefined - new in MongoDB 2.6 - and + parsing $date in ISO-8601 format. + +.. versionchanged:: 2.7 + Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef + instances. + +.. versionchanged:: 2.3 + Added dumps and loads helpers to automatically handle conversion to and + from json and supports :class:`~bson.binary.Binary` and + :class:`~bson.code.Code` +""" + +import base64 +import calendar +import collections +import datetime +import json +import re +import uuid + +from bson import EPOCH_AWARE, RE_TYPE, SON +from bson.binary import Binary +from bson.code import Code +from bson.dbref import DBRef +from bson.int64 import Int64 +from bson.max_key import MaxKey +from bson.min_key import MinKey +from bson.objectid import ObjectId +from bson.py3compat import PY3, iteritems, string_type, text_type +from bson.regex import Regex +from bson.timestamp import Timestamp +from bson.tz_util import utc + +_RE_OPT_TABLE = {"i": re.I, "l": re.L, "m": re.M, "s": re.S, "u": re.U, "x": re.X} + + +def dumps(obj, *args, **kwargs): + """Helper function that wraps :class:`json.dumps`. + + Recursive function that handles all BSON types including + :class:`~bson.binary.Binary` and :class:`~bson.code.Code`. + + .. versionchanged:: 2.7 + Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef + instances. + """ + return json.dumps(_json_convert(obj), *args, **kwargs) + + +def loads(s, *args, **kwargs): + """Helper function that wraps :class:`json.loads`. + + Automatically passes the object_hook for BSON type conversion. + """ + kwargs["object_hook"] = lambda dct: object_hook(dct) + return json.loads(s, *args, **kwargs) + + +def _json_convert(obj): + """Recursive helper method that converts BSON types so they can be + converted into json. + """ + if hasattr(obj, "iteritems") or hasattr(obj, "items"): # PY3 support + return SON(((k, _json_convert(v)) for k, v in iteritems(obj))) + elif hasattr(obj, "__iter__") and not isinstance(obj, (text_type, bytes)): + return list((_json_convert(v) for v in obj)) + try: + return default(obj) + except TypeError: + return obj + + +def object_hook(dct): + if "$oid" in dct: + return ObjectId(str(dct["$oid"])) + if "$ref" in dct: + return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None)) + if "$date" in dct: + dtm = dct["$date"] + # mongoexport 2.6 and newer + if isinstance(dtm, string_type): + aware = datetime.datetime.strptime( + dtm[:23], "%Y-%m-%dT%H:%M:%S.%f" + ).replace(tzinfo=utc) + offset = dtm[23:] + if not offset or offset == "Z": + # UTC + return aware + else: + if len(offset) == 5: + # Offset from mongoexport is in format (+|-)HHMM + secs = int(offset[1:3]) * 3600 + int(offset[3:]) * 60 + elif ":" in offset and len(offset) == 6: + # RFC-3339 format (+|-)HH:MM + hours, minutes = offset[1:].split(":") + secs = int(hours) * 3600 + int(minutes) * 60 + else: + # Not RFC-3339 compliant or mongoexport output. + raise ValueError("invalid format for offset") + if offset[0] == "-": + secs *= -1 + return aware - datetime.timedelta(seconds=secs) + # mongoexport 2.6 and newer, time before the epoch (SERVER-15275) + elif isinstance(dtm, collections.Mapping): + secs = float(dtm["$numberLong"]) / 1000.0 + # mongoexport before 2.6 + else: + secs = float(dtm) / 1000.0 + return EPOCH_AWARE + datetime.timedelta(seconds=secs) + if "$regex" in dct: + flags = 0 + # PyMongo always adds $options but some other tools may not. + for opt in dct.get("$options", ""): + flags |= _RE_OPT_TABLE.get(opt, 0) + return Regex(dct["$regex"], flags) + if "$minKey" in dct: + return MinKey() + if "$maxKey" in dct: + return MaxKey() + if "$binary" in dct: + if isinstance(dct["$type"], int): + dct["$type"] = "%02x" % dct["$type"] + subtype = int(dct["$type"], 16) + if subtype >= 0xFFFFFF80: # Handle mongoexport values + subtype = int(dct["$type"][6:], 16) + return Binary(base64.b64decode(dct["$binary"].encode()), subtype) + if "$code" in dct: + return Code(dct["$code"], dct.get("$scope")) + if "$uuid" in dct: + return uuid.UUID(dct["$uuid"]) + if "$undefined" in dct: + return None + if "$numberLong" in dct: + return Int64(dct["$numberLong"]) + if "$timestamp" in dct: + tsp = dct["$timestamp"] + return Timestamp(tsp["t"], tsp["i"]) + return dct + + +def default(obj): + # We preserve key order when rendering SON, DBRef, etc. as JSON by + # returning a SON for those types instead of a dict. + if isinstance(obj, ObjectId): + return {"$oid": str(obj)} + if isinstance(obj, DBRef): + return _json_convert(obj.as_doc()) + if isinstance(obj, datetime.datetime): + # TODO share this code w/ bson.py? + if obj.utcoffset() is not None: + obj = obj - obj.utcoffset() + millis = int(calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) + return {"$date": millis} + if isinstance(obj, (RE_TYPE, Regex)): + flags = "" + if obj.flags & re.IGNORECASE: + flags += "i" + if obj.flags & re.LOCALE: + flags += "l" + if obj.flags & re.MULTILINE: + flags += "m" + if obj.flags & re.DOTALL: + flags += "s" + if obj.flags & re.UNICODE: + flags += "u" + if obj.flags & re.VERBOSE: + flags += "x" + if isinstance(obj.pattern, text_type): + pattern = obj.pattern + else: + pattern = obj.pattern.decode("utf-8") + return SON([("$regex", pattern), ("$options", flags)]) + if isinstance(obj, MinKey): + return {"$minKey": 1} + if isinstance(obj, MaxKey): + return {"$maxKey": 1} + if isinstance(obj, Timestamp): + return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])} + if isinstance(obj, Code): + return SON([("$code", str(obj)), ("$scope", obj.scope)]) + if isinstance(obj, Binary): + return SON( + [ + ("$binary", base64.b64encode(obj).decode()), + ("$type", "%02x" % obj.subtype), + ] + ) + if PY3 and isinstance(obj, bytes): + return SON([("$binary", base64.b64encode(obj).decode()), ("$type", "00")]) + if isinstance(obj, uuid.UUID): + return {"$uuid": obj.hex} + raise TypeError("%r is not JSON serializable" % obj) diff --git a/inbox/sqlalchemy_ext/util.py b/inbox/sqlalchemy_ext/util.py index 60a0a5f73..79a40afc6 100644 --- a/inbox/sqlalchemy_ext/util.py +++ b/inbox/sqlalchemy_ext/util.py @@ -6,12 +6,7 @@ import weakref from typing import Any, MutableMapping, Optional, Tuple -from bson import EPOCH_NAIVE, json_util - -# Monkeypatch to not include tz_info in decoded JSON. -# Kind of a ridiculous solution, but works. -json_util.EPOCH_AWARE = EPOCH_NAIVE - +from bson import EPOCH_NAIVE from sqlalchemy import String, Text, event from sqlalchemy.engine import Engine from sqlalchemy.ext.mutable import Mutable @@ -20,8 +15,13 @@ from sqlalchemy.types import BINARY, TypeDecorator from inbox.logging import get_logger +from inbox.sqlalchemy_ext import json_util from inbox.util.encoding import base36decode, base36encode +# Monkeypatch to not include tz_info in decoded JSON. +# Kind of a ridiculous solution, but works. +json_util.EPOCH_AWARE = EPOCH_NAIVE + log = get_logger() From 05bc8c9414319f28248ffe7fd3e2c4e40f5f6725 Mon Sep 17 00:00:00 2001 From: Squeaky Date: Wed, 29 May 2024 20:55:07 +0200 Subject: [PATCH 2/7] Remove % formatting --- inbox/sqlalchemy_ext/json_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py index e941ac565..136405dfa 100644 --- a/inbox/sqlalchemy_ext/json_util.py +++ b/inbox/sqlalchemy_ext/json_util.py @@ -176,7 +176,7 @@ def object_hook(dct): return MaxKey() if "$binary" in dct: if isinstance(dct["$type"], int): - dct["$type"] = "%02x" % dct["$type"] + dct["$type"] = format(dct["$type"], "02x") subtype = int(dct["$type"], 16) if subtype >= 0xFFFFFF80: # Handle mongoexport values subtype = int(dct["$type"][6:], 16) @@ -239,11 +239,11 @@ def default(obj): return SON( [ ("$binary", base64.b64encode(obj).decode()), - ("$type", "%02x" % obj.subtype), + ("$type", format(obj.subtype, "02x")), ] ) if PY3 and isinstance(obj, bytes): return SON([("$binary", base64.b64encode(obj).decode()), ("$type", "00")]) if isinstance(obj, uuid.UUID): return {"$uuid": obj.hex} - raise TypeError("%r is not JSON serializable" % obj) + raise TypeError(f"{obj!r} is not JSON serializable") From 1e60b53500ba7f0709260e265a224204c15762ec Mon Sep 17 00:00:00 2001 From: Squeaky Date: Wed, 29 May 2024 22:08:11 +0200 Subject: [PATCH 3/7] Remove monkeypatch --- inbox/sqlalchemy_ext/json_util.py | 6 ++++-- inbox/sqlalchemy_ext/util.py | 5 ----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py index 136405dfa..1f0b95296 100644 --- a/inbox/sqlalchemy_ext/json_util.py +++ b/inbox/sqlalchemy_ext/json_util.py @@ -75,7 +75,7 @@ import re import uuid -from bson import EPOCH_AWARE, RE_TYPE, SON +from bson import RE_TYPE, SON from bson.binary import Binary from bson.code import Code from bson.dbref import DBRef @@ -88,6 +88,8 @@ from bson.timestamp import Timestamp from bson.tz_util import utc +EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0) + _RE_OPT_TABLE = {"i": re.I, "l": re.L, "m": re.M, "s": re.S, "u": re.U, "x": re.X} @@ -163,7 +165,7 @@ def object_hook(dct): # mongoexport before 2.6 else: secs = float(dtm) / 1000.0 - return EPOCH_AWARE + datetime.timedelta(seconds=secs) + return EPOCH_NAIVE + datetime.timedelta(seconds=secs) if "$regex" in dct: flags = 0 # PyMongo always adds $options but some other tools may not. diff --git a/inbox/sqlalchemy_ext/util.py b/inbox/sqlalchemy_ext/util.py index 79a40afc6..7cc85abd2 100644 --- a/inbox/sqlalchemy_ext/util.py +++ b/inbox/sqlalchemy_ext/util.py @@ -6,7 +6,6 @@ import weakref from typing import Any, MutableMapping, Optional, Tuple -from bson import EPOCH_NAIVE from sqlalchemy import String, Text, event from sqlalchemy.engine import Engine from sqlalchemy.ext.mutable import Mutable @@ -18,10 +17,6 @@ from inbox.sqlalchemy_ext import json_util from inbox.util.encoding import base36decode, base36encode -# Monkeypatch to not include tz_info in decoded JSON. -# Kind of a ridiculous solution, but works. -json_util.EPOCH_AWARE = EPOCH_NAIVE - log = get_logger() From 9f0ddee37f394ea4894f050a2b11c79c28903a71 Mon Sep 17 00:00:00 2001 From: Squeaky Date: Wed, 29 May 2024 22:18:02 +0200 Subject: [PATCH 4/7] Remove types we don't need --- inbox/sqlalchemy_ext/json_util.py | 94 +------------------------------ 1 file changed, 3 insertions(+), 91 deletions(-) diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py index 1f0b95296..484e45050 100644 --- a/inbox/sqlalchemy_ext/json_util.py +++ b/inbox/sqlalchemy_ext/json_util.py @@ -67,31 +67,17 @@ :class:`~bson.code.Code` """ -import base64 import calendar import collections import datetime import json -import re -import uuid - -from bson import RE_TYPE, SON -from bson.binary import Binary -from bson.code import Code -from bson.dbref import DBRef -from bson.int64 import Int64 -from bson.max_key import MaxKey -from bson.min_key import MinKey -from bson.objectid import ObjectId -from bson.py3compat import PY3, iteritems, string_type, text_type -from bson.regex import Regex -from bson.timestamp import Timestamp + +from bson import SON +from bson.py3compat import iteritems, string_type, text_type from bson.tz_util import utc EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0) -_RE_OPT_TABLE = {"i": re.I, "l": re.L, "m": re.M, "s": re.S, "u": re.U, "x": re.X} - def dumps(obj, *args, **kwargs): """Helper function that wraps :class:`json.dumps`. @@ -130,10 +116,6 @@ def _json_convert(obj): def object_hook(dct): - if "$oid" in dct: - return ObjectId(str(dct["$oid"])) - if "$ref" in dct: - return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None)) if "$date" in dct: dtm = dct["$date"] # mongoexport 2.6 and newer @@ -166,86 +148,16 @@ def object_hook(dct): else: secs = float(dtm) / 1000.0 return EPOCH_NAIVE + datetime.timedelta(seconds=secs) - if "$regex" in dct: - flags = 0 - # PyMongo always adds $options but some other tools may not. - for opt in dct.get("$options", ""): - flags |= _RE_OPT_TABLE.get(opt, 0) - return Regex(dct["$regex"], flags) - if "$minKey" in dct: - return MinKey() - if "$maxKey" in dct: - return MaxKey() - if "$binary" in dct: - if isinstance(dct["$type"], int): - dct["$type"] = format(dct["$type"], "02x") - subtype = int(dct["$type"], 16) - if subtype >= 0xFFFFFF80: # Handle mongoexport values - subtype = int(dct["$type"][6:], 16) - return Binary(base64.b64decode(dct["$binary"].encode()), subtype) - if "$code" in dct: - return Code(dct["$code"], dct.get("$scope")) - if "$uuid" in dct: - return uuid.UUID(dct["$uuid"]) - if "$undefined" in dct: - return None - if "$numberLong" in dct: - return Int64(dct["$numberLong"]) - if "$timestamp" in dct: - tsp = dct["$timestamp"] - return Timestamp(tsp["t"], tsp["i"]) return dct def default(obj): # We preserve key order when rendering SON, DBRef, etc. as JSON by # returning a SON for those types instead of a dict. - if isinstance(obj, ObjectId): - return {"$oid": str(obj)} - if isinstance(obj, DBRef): - return _json_convert(obj.as_doc()) if isinstance(obj, datetime.datetime): # TODO share this code w/ bson.py? if obj.utcoffset() is not None: obj = obj - obj.utcoffset() millis = int(calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) return {"$date": millis} - if isinstance(obj, (RE_TYPE, Regex)): - flags = "" - if obj.flags & re.IGNORECASE: - flags += "i" - if obj.flags & re.LOCALE: - flags += "l" - if obj.flags & re.MULTILINE: - flags += "m" - if obj.flags & re.DOTALL: - flags += "s" - if obj.flags & re.UNICODE: - flags += "u" - if obj.flags & re.VERBOSE: - flags += "x" - if isinstance(obj.pattern, text_type): - pattern = obj.pattern - else: - pattern = obj.pattern.decode("utf-8") - return SON([("$regex", pattern), ("$options", flags)]) - if isinstance(obj, MinKey): - return {"$minKey": 1} - if isinstance(obj, MaxKey): - return {"$maxKey": 1} - if isinstance(obj, Timestamp): - return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])} - if isinstance(obj, Code): - return SON([("$code", str(obj)), ("$scope", obj.scope)]) - if isinstance(obj, Binary): - return SON( - [ - ("$binary", base64.b64encode(obj).decode()), - ("$type", format(obj.subtype, "02x")), - ] - ) - if PY3 and isinstance(obj, bytes): - return SON([("$binary", base64.b64encode(obj).decode()), ("$type", "00")]) - if isinstance(obj, uuid.UUID): - return {"$uuid": obj.hex} raise TypeError(f"{obj!r} is not JSON serializable") From 6f000f4864f9b7b9a85ce3555d9692b338d8fa8f Mon Sep 17 00:00:00 2001 From: Squeaky Date: Wed, 29 May 2024 22:28:34 +0200 Subject: [PATCH 5/7] Drop bson dependency in json_util --- inbox/sqlalchemy_ext/json_util.py | 41 +++---------------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py index 484e45050..87daf72d6 100644 --- a/inbox/sqlalchemy_ext/json_util.py +++ b/inbox/sqlalchemy_ext/json_util.py @@ -68,14 +68,9 @@ """ import calendar -import collections import datetime import json -from bson import SON -from bson.py3compat import iteritems, string_type, text_type -from bson.tz_util import utc - EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0) @@ -105,9 +100,9 @@ def _json_convert(obj): """Recursive helper method that converts BSON types so they can be converted into json. """ - if hasattr(obj, "iteritems") or hasattr(obj, "items"): # PY3 support - return SON(((k, _json_convert(v)) for k, v in iteritems(obj))) - elif hasattr(obj, "__iter__") and not isinstance(obj, (text_type, bytes)): + if hasattr(obj, "items"): + return dict(((k, _json_convert(v)) for k, v in obj.items())) + elif hasattr(obj, "__iter__") and not isinstance(obj, (str, bytes)): return list((_json_convert(v) for v in obj)) try: return default(obj) @@ -118,35 +113,7 @@ def _json_convert(obj): def object_hook(dct): if "$date" in dct: dtm = dct["$date"] - # mongoexport 2.6 and newer - if isinstance(dtm, string_type): - aware = datetime.datetime.strptime( - dtm[:23], "%Y-%m-%dT%H:%M:%S.%f" - ).replace(tzinfo=utc) - offset = dtm[23:] - if not offset or offset == "Z": - # UTC - return aware - else: - if len(offset) == 5: - # Offset from mongoexport is in format (+|-)HHMM - secs = int(offset[1:3]) * 3600 + int(offset[3:]) * 60 - elif ":" in offset and len(offset) == 6: - # RFC-3339 format (+|-)HH:MM - hours, minutes = offset[1:].split(":") - secs = int(hours) * 3600 + int(minutes) * 60 - else: - # Not RFC-3339 compliant or mongoexport output. - raise ValueError("invalid format for offset") - if offset[0] == "-": - secs *= -1 - return aware - datetime.timedelta(seconds=secs) - # mongoexport 2.6 and newer, time before the epoch (SERVER-15275) - elif isinstance(dtm, collections.Mapping): - secs = float(dtm["$numberLong"]) / 1000.0 - # mongoexport before 2.6 - else: - secs = float(dtm) / 1000.0 + secs = float(dtm) / 1000.0 return EPOCH_NAIVE + datetime.timedelta(seconds=secs) return dct From 5e8a711963eec20be84d787803b846c3fe4de471 Mon Sep 17 00:00:00 2001 From: Squeaky Date: Thu, 30 May 2024 10:58:54 +0200 Subject: [PATCH 6/7] Cleanup docstrings --- inbox/sqlalchemy_ext/json_util.py | 73 +++---------------- inbox/sqlalchemy_ext/util.py | 4 +- ...consolidate_account_sync_status_columns.py | 3 +- 3 files changed, 14 insertions(+), 66 deletions(-) diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py index 87daf72d6..c3afc660b 100644 --- a/inbox/sqlalchemy_ext/json_util.py +++ b/inbox/sqlalchemy_ext/json_util.py @@ -15,56 +15,14 @@ """Tools for using Python's :mod:`json` module with BSON documents. This module provides two helper methods `dumps` and `loads` that wrap the -native :mod:`json` methods and provide explicit BSON conversion to and from -json. This allows for specialized encoding and decoding of BSON documents -into `Mongo Extended JSON +native :mod:`json` methods and provide explicit datetime.datetime conversion to and from +json. This is a very stripped version of `Mongo Extended JSON `_'s *Strict* -mode. This lets you encode / decode BSON documents to JSON even when -they use special BSON types. - -Example usage (serialization): - -.. doctest:: - - >>> from bson import Binary, Code - >>> from bson.json_util import dumps - >>> dumps([{'foo': [1, 2]}, - ... {'bar': {'hello': 'world'}}, - ... {'code': Code("function x() { return 1; }")}, - ... {'bin': Binary("\x01\x02\x03\x04")}]) - '[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }", "$scope": {}}}, {"bin": {"$binary": "AQIDBA==", "$type": "00"}}]' - -Example usage (deserialization): - -.. doctest:: - - >>> from bson.json_util import loads - >>> loads('[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$scope": {}, "$code": "function x() { return 1; }"}}, {"bin": {"$type": "00", "$binary": "AQIDBA=="}}]') - [{u'foo': [1, 2]}, {u'bar': {u'hello': u'world'}}, {u'code': Code('function x() { return 1; }', {})}, {u'bin': Binary('...', 0)}] - -Alternatively, you can manually pass the `default` to :func:`json.dumps`. -It won't handle :class:`~bson.binary.Binary` and :class:`~bson.code.Code` -instances (as they are extended strings you can't provide custom defaults), -but it will be faster as there is less recursion. - -.. versionchanged:: 2.8 - The output format for :class:`~bson.timestamp.Timestamp` has changed from - '{"t": , "i": }' to '{"$timestamp": {"t": , "i": }}'. - This new format will be decoded to an instance of - :class:`~bson.timestamp.Timestamp`. The old format will continue to be - decoded to a python dict as before. Encoding to the old format is no longer - supported as it was never correct and loses type information. - Added support for $numberLong and $undefined - new in MongoDB 2.6 - and - parsing $date in ISO-8601 format. - -.. versionchanged:: 2.7 - Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef - instances. - -.. versionchanged:: 2.3 - Added dumps and loads helpers to automatically handle conversion to and - from json and supports :class:`~bson.binary.Binary` and - :class:`~bson.code.Code` +mode handling just datetime.dateime fields because we don't serialize any +other types that are not handled by JSON. + +The original unstripped version of this module can be found at +https://github.com/mongodb/mongo-python-driver/blob/18328a909545ece6e1cd7e172e28271a59e367d5/bson/json_util.py. """ import calendar @@ -77,27 +35,19 @@ def dumps(obj, *args, **kwargs): """Helper function that wraps :class:`json.dumps`. - Recursive function that handles all BSON types including - :class:`~bson.binary.Binary` and :class:`~bson.code.Code`. - - .. versionchanged:: 2.7 - Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef - instances. + Recursive function that handles all datetime.datetime type. """ return json.dumps(_json_convert(obj), *args, **kwargs) def loads(s, *args, **kwargs): - """Helper function that wraps :class:`json.loads`. - - Automatically passes the object_hook for BSON type conversion. - """ + """Helper function that wraps :class:`json.loads`.""" kwargs["object_hook"] = lambda dct: object_hook(dct) return json.loads(s, *args, **kwargs) def _json_convert(obj): - """Recursive helper method that converts BSON types so they can be + """Recursive helper method that converts datetime.datetime type so it can be converted into json. """ if hasattr(obj, "items"): @@ -119,10 +69,7 @@ def object_hook(dct): def default(obj): - # We preserve key order when rendering SON, DBRef, etc. as JSON by - # returning a SON for those types instead of a dict. if isinstance(obj, datetime.datetime): - # TODO share this code w/ bson.py? if obj.utcoffset() is not None: obj = obj - obj.utcoffset() millis = int(calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) diff --git a/inbox/sqlalchemy_ext/util.py b/inbox/sqlalchemy_ext/util.py index 7cc85abd2..6307bf5db 100644 --- a/inbox/sqlalchemy_ext/util.py +++ b/inbox/sqlalchemy_ext/util.py @@ -164,8 +164,8 @@ def process_result_value( # http://bit.ly/1LbMnqu -# Can simply use this as is because though we use bson.json_util, loads() -# dumps() return standard Python dicts like the json.* equivalents +# Can simply use this as is because though we use inbox.sqlalchemy_ext.json_util, +# loads() dumps() return standard Python dicts like the json.* equivalents # (because these are simply called under the hood) class MutableDict(Mutable, dict): @classmethod diff --git a/migrations/versions/057_consolidate_account_sync_status_columns.py b/migrations/versions/057_consolidate_account_sync_status_columns.py index cbfead593..7311e2acf 100644 --- a/migrations/versions/057_consolidate_account_sync_status_columns.py +++ b/migrations/versions/057_consolidate_account_sync_status_columns.py @@ -12,7 +12,8 @@ import sqlalchemy as sa from alembic import op -from bson import json_util + +from inbox.sqlalchemy_ext import json_util def upgrade(): From e6f4eff9f81181e7d6895147cba87a2bc92e1f72 Mon Sep 17 00:00:00 2001 From: Squeaky Date: Thu, 30 May 2024 11:21:32 +0200 Subject: [PATCH 7/7] Remove pymongo requirement --- requirements/prod.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/prod.txt b/requirements/prod.txt index 15b58c3a2..43beb45f2 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -62,7 +62,6 @@ pycparser==2.21 pygments==2.17.2 pyinstrument==3.2.0 pyinstrument-cext==0.2.2 -pymongo==3.0 # For json_util in bson Pympler==0.9 PyNaCl==1.5.0 python-dateutil==2.8.2