Merge original repository changes

.bumpversion.cfg

@@ -1,7 +1,7 @@
 [bumpversion]
 commit = True
 tag = True
-current_version = 0.4.4
+current_version = 0.4.12
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
 serialize = 
 	{major}.{minor}.{patch}-{release}

README.md

@@ -67,9 +67,6 @@ The values returned by the search query will be automatically decoded into Pytho
 ```python
 >>> listing = search_result.data[0]
 
->>> listing.resource_key
-'20170104191513476022000000'
-
 >>> listing.data
 {
     'internal_listing_id': '20170104191513476022000000',
@@ -79,17 +76,16 @@ The values returned by the search query will be automatically decoded into Pytho
     'list_price': 250000,
     ...
 }
+
+>>> listing.data[listing.resource_class.resource.key_field]
+'20170104191513476022000000'
 ```
 
-Photos and other object types for a record can be retrieved directly from the record object. They
-can also be retrieved in bulk from the ObjectType object using the resource keys of the records.
+Photos can also be retrieved in bulk from the ObjectType object using the resource keys of the records.
 
 ```python
->>> listing.get_objects('HiRes', location=True)
-(Object(mime_type='image/jpeg', content_id='20170104191513476022000000', description='Front', object_id='1', url='...', preferred=True, data=None), ...)
-
 >>> all_photos = photo_object_type.get(
-    resource_keys=[listing.resource_key for listing in listings],
+    resource_keys=[listing.data[listing.resource_class.resource.key_field] for listing in listings],
     location=True,
 )
 
@@ -147,3 +143,23 @@ objects = client.get_object(
     location=True,
 )
 ```
+# Developing/Releasing
+To release a new version, use `bin/release <major|minor|patch>`
+
+This package is deployed to: https://pypi.org/manage/project/rets-python/releases/
+
+To deploy, you can try `bin/deploy`, but it may give you a SSL error. Alternatively, to deploy, see: https://packaging.python.org/tutorials/packaging-projects/ or in summary:
+
+
+Update version number in the following files (0.4.10 -> 0.4.11)
+* setup.py
+* build/lib/rets/__init__.py
+* rets/__init__.py
+
+
+```
+python3 -m pip install --user --upgrade setuptools wheel
+python3 setup.py sdist bdist_wheel
+python3 -m pip install --user --upgrade twine
+python3 -m twine upload dist/*
+```

rets/__init__.py

@@ -3,7 +3,7 @@
 from rets.http.data import Metadata, Object, SearchResult, SystemMetadata
 
 __title__ = 'rets'
-__version__ = '0.4.4'
+__version__ = '0.4.12'
 __author__ = 'Martin Liu <[email protected]>'
 __license__ = 'MIT License'
 

rets/client/decoder.py

@@ -29,7 +29,10 @@ def decode(self, rows: Sequence[dict]) -> Sequence[dict]:
         def decode_field(field: str, value: str) -> Any:
             if value == '':
                 return None
-            return decoders[field](value)
+            try:
+                return decoders[field](value)
+            except Exception as e:
+                raise ValueError(f"Error decoding field {field} with value {value}. Error: {e}") from e
 
         return tuple(OrderedDict((field, decode_field(field, value)) for field, value in row.items())
                      for row in rows)
@@ -68,12 +71,12 @@ def _get_decoder(data_type: str, interpretation: str, include_tz: bool = False):
 
 
 def _decode_datetime(value: str, include_tz: bool) -> datetime:
-    # Correct `0000-00-00 00:00:00` to `0000-00-00T00:00:00`
-    if re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$', value):
-        value = '%sT%s' % (value[0:10], value[11:])
     # Correct `0000-00-00` to `0000-00-00T00:00:00`
-    elif re.match(r'^\d{4}-\d{2}-\d{2}$', value):
+    if len(value) == 10:
         value = '%sT00:00:00' % value[0:10]
+    # Correct `0000-00-00 00:00:00` to `0000-00-00T00:00:00`
+    elif value[10] == ' ':
+        value = '%sT%s' % (value[0:10], value[11:])
 
     decoded = udatetime.from_string(value)
     if not include_tz:
@@ -113,4 +116,6 @@ def _decode_date(value: str, include_tz: bool) -> datetime:
     'Long': int,
     'Decimal': Decimal,
     'Number': int,
+    # Point is new "Edm.GeographyPoint" from RESO, look online for spec. Can store as Postgres Point, see https://bit.ly/2BDPgUS
+    'Point': str,
 }

rets/client/record.py

@@ -6,18 +6,11 @@
 class Record:
 
     def __init__(self, resource_class, data: dict):
-        self.resource = resource_class.resource
         self.resource_class = resource_class
-        self.resource_key = str(data[resource_class.resource.key_field])
         self.data = data
 
-    def get_objects(self, name: str, **kwargs) -> Sequence[Object]:
-        resource_object = self.resource.get_object_type(name)
-        return resource_object.get(self.resource_key, **kwargs)
-
     def __repr__(self) -> str:
-        return '<Record: %s:%s:%s>' % (
+        return '<Record: %s:%s>' % (
             self.resource_class.resource.name,
             self.resource_class.name,
-            self.resource_key,
         )

rets/client/resource_class.py

@@ -70,7 +70,7 @@ def search(self,
         return SearchResult(
             count=result.count,
             max_rows=result.max_rows,
-            data=tuple(Record(self, row) for row in rows),
+            data=tuple(Record(self, row) for row in rows) if rows else tuple(),
         )
 
     def _validate_query(self, query: Union[str, Mapping[str, str]]) -> str:

rets/http/client.py

@@ -30,11 +30,13 @@ def __init__(self,
                  capability_urls: str = None,
                  cookie_dict: dict = None,
                  use_get_method: bool = False,
+                 send_rets_ua_authorization: bool = True,
                  ):
         self._user_agent = user_agent
         self._user_agent_password = user_agent_password
         self._rets_version = rets_version
         self._use_get_method = use_get_method
+        self._send_rets_ua_authorization = send_rets_ua_authorization
 
         splits = urlsplit(login_url)
         self._base_url = urlunsplit((splits.scheme, splits.netloc, '', '', ''))
@@ -297,8 +299,9 @@ def _http_request(self, url: str, headers: dict = None, payload: dict = None) ->
             **(headers or {}),
             'User-Agent': self.user_agent,
             'RETS-Version': self.rets_version,
-            'RETS-UA-Authorization': self._rets_ua_authorization()
         }
+        if self._send_rets_ua_authorization:
+            request_headers['RETS-UA-Authorization'] = self._rets_ua_authorization()
 
         if self._use_get_method:
             if payload:

rets/http/parsers/parse.py

@@ -16,7 +16,14 @@
 
 def parse_xml(response: ResponseLike) -> etree.Element:
     encoding = response.encoding or DEFAULT_ENCODING
-    root = etree.fromstring(response.content.decode(encoding), parser=etree.XMLParser(recover=True))
+    try:
+        root = etree.fromstring(response.content.decode(encoding), parser=etree.XMLParser(recover=True))
+    except ValueError as e:
+        if str(e) == "Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.":
+            # parse bytes directly, rather than from string
+            root = etree.XML(response.content)
+        else:
+            raise e
 
     if root is None:
         raise RetsResponseError(response.content, response.headers)

rets/http/parsers/parse_object.py

@@ -89,6 +89,8 @@ def _parse_body_part(part: ResponseLike) -> Optional[Object]:
         except RetsApiError as e:
             if e.reply_code == 20403:  # No object found
                 return None
+            elif e.reply_code == 20407:  # Access to object not allowed
+                return None
             raise
 
     # All RETS responses _must_ have `Content-ID` and `Object-ID` headers.

setup.py

@@ -12,8 +12,7 @@
 install_requires = [
     'requests>=2.12.3',
     'requests-toolbelt>=0.7.0,!=0.9.0',
-    'udatetime==0.0.16',
-    'docopts',
+    'udatetime==0.0.17',
     'lxml>=4.3.0',
 ]
 
@@ -35,7 +34,7 @@
 
 setup(
     name='rets-python',
-    version='0.4.4',
+    version='0.4.12',
     description='rets-python',
     long_description=long_desc,
     author='Martin Liu',

tests/client/decoder_test.py

@@ -106,6 +106,8 @@ def test_decode_datetime():
     # digit, however udatetime only permits 3 or 6 digits.
     assert _decode_datetime('2017-01-02T03:04:05.600', True) == \
            datetime(2017, 1, 2, 3, 4, 5, 600000, tzinfo=timezone(timedelta(0)))
+    assert _decode_datetime('2020-10-12 10:46:54.146488', True) == \
+           datetime(2020, 10, 12, 10, 46, 54, 146488, tzinfo=timezone(timedelta(0)))
     assert _decode_datetime('2017-01-02T03:04:05Z', True) == \
            datetime(2017, 1, 2, 3, 4, 5, tzinfo=timezone(timedelta(0)))
     assert _decode_datetime('2017-01-02T03:04:05+00:00', True) == \
@@ -122,6 +124,8 @@ def test_decode_datetime():
            datetime(2017, 1, 2, 3, 4, 5)
     assert _decode_datetime('2017-01-02T03:04:05.600', False) == \
            datetime(2017, 1, 2, 3, 4, 5, 600000)
+    assert _decode_datetime('2017-01-02 03:04:05.600', False) == \
+           datetime(2017, 1, 2, 3, 4, 5, 600000)
     assert _decode_datetime('2017-01-02T03:04:05Z', False) == datetime(2017, 1, 2, 3, 4, 5)
     assert _decode_datetime('2017-01-02T03:04:05+00:00', False) == datetime(2017, 1, 2, 3, 4, 5)
     assert _decode_datetime('2017-01-02T03:04:05-00:00', False) == datetime(2017, 1, 2, 3, 4, 5)

tests/client/request_test.py

@@ -0,0 +1,35 @@
+from unittest.mock import MagicMock, call
+
+from rets.http.client import (
+    RetsHttpClient
+)
+
+
+def test_rets_ua_authorization_false():
+    send_auth = False
+
+    client = RetsHttpClient(login_url='test.url',
+                            username='user',
+                            password='pass',
+                            send_rets_ua_authorization=send_auth,
+                            )
+    client._session = MagicMock()
+    client._http_request(url='test.url')
+
+    assert client._send_rets_ua_authorization == send_auth
+
+    assert client._session.post.called
+    assert 'RETS-UA-Authorization' not in client._session.post.call_args_list[0][1]['headers']
+
+
+def test_rets_ua_authorization_default():
+    # by default, sends 'RETS-UA-Authorization'
+    client = RetsHttpClient(login_url='test.url',
+                            username='user',
+                            password='pass',
+                            )
+    client._session = MagicMock()
+    client._http_request(url='test.url')
+
+    assert client._session.post.called
+    assert 'RETS-UA-Authorization' in client._session.post.call_args_list[0][1]['headers']