From 5be5c523c94c16057b220d1feeb79fdb30b2b6eb Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Thu, 14 Jul 2022 21:47:03 +0700 Subject: [PATCH 01/12] ignore pycharm env --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7da223a7..4e4e89d2 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,9 @@ ENV/ env.bak/ venv.bak/ +# Jetbrains IDE +.idea/ + # Spyder project settings .spyderproject .spyproject From 17c71fa7144945c983da5b3da1a8ca72e6a8a2f6 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Mon, 18 Jul 2022 14:17:46 +0700 Subject: [PATCH 02/12] fix imports on test hash model --- tests/test_hash_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_hash_model.py b/tests/test_hash_model.py index 0a79aa6b..c0069dae 100644 --- a/tests/test_hash_model.py +++ b/tests/test_hash_model.py @@ -22,8 +22,8 @@ # We need to run this check as sync code (during tests) even in async mode # because we call it in the top-level module scope. -from redis_om import has_redisearch -from tests.conftest import py_test_mark_asyncio +from aredis_om import has_redisearch +from conftest import py_test_mark_asyncio if not has_redisearch(): pytestmark = pytest.mark.skip From ee3fd302bfc9f3a9ea661fdbde04332cd8cd6ad4 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Mon, 18 Jul 2022 16:38:46 +0700 Subject: [PATCH 03/12] delete when exists keys to delete --- aredis_om/model/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aredis_om/model/model.py b/aredis_om/model/model.py index 92bb6f9a..f59964c9 100644 --- a/aredis_om/model/model.py +++ b/aredis_om/model/model.py @@ -792,7 +792,10 @@ async def update(self, use_transaction=True, **field_values): async def delete(self): """Delete all matching records in this query.""" # TODO: Better response type, error detection - return await self.model.db().delete(*[m.key() for m in await self.all()]) + keys_to_delete = [m.key() for m in await self.all()] + if not keys_to_delete: + return 0 + return await self.model.db().delete(*keys_to_delete) async def __aiter__(self): if self._model_cache: From ea171a87beef2dbd847fb6841fba6ec91efddbb7 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Mon, 18 Jul 2022 20:49:25 +0700 Subject: [PATCH 04/12] count method --- aredis_om/model/model.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/aredis_om/model/model.py b/aredis_om/model/model.py index f59964c9..c6ef23d4 100644 --- a/aredis_om/model/model.py +++ b/aredis_om/model/model.py @@ -760,6 +760,25 @@ async def all(self, batch_size=10): return await query.execute() return await self.execute() + async def count(self): + args = ["FT.AGGREGATE", self.model.Meta.index_name, "*"] + if self.expression: + args += ["FILTER", *self.query] + args += [ + "APPLY", + "matched_terms()", + "AS", + "countable", + "GROUPBY", + "1", + "@countable", + "REDUCE", + "COUNT", + "0", + ] + raw_result = await self.model.db().execute_command(*args) + return sum([count for [count, _] in raw_result]) + def sort_by(self, *fields: str): if not fields: return self From a0dec58b11dfb6842e37076ab7fe8e0a24949d68 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Mon, 18 Jul 2022 20:49:37 +0700 Subject: [PATCH 05/12] test count method --- tests/test_find_count.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/test_find_count.py diff --git a/tests/test_find_count.py b/tests/test_find_count.py new file mode 100644 index 00000000..4ef93502 --- /dev/null +++ b/tests/test_find_count.py @@ -0,0 +1,41 @@ +import datetime +from typing import Optional + +from pydantic import EmailStr + +from aredis_om import Field, HashModel, Migrator + + +class Customer(HashModel): + first_name: str + last_name: str = Field(index=True) + email: EmailStr + join_date: datetime.date + age: int = Field(index=True) + bio: Optional[str] + + +andrew = Customer( + first_name="Andrew", + last_name="Brookins", + email="andrew.brookins@example.com", + join_date=datetime.date.today(), + age=38, +) + +await Migrator().run() + +await Customer.find().delete() + +andy = await andrew.save() + +customers = await Customer.find(Customer.last_name == "Brookins").all() +assert len(customers) == 1 + +await Migrator().run() + +customers_count = await Customer.find().count() +assert customers_count == 1 + +brookins_count = await Customer.find(Customer.last_name == "Brookins").count() +assert brookins_count == 1 From 730d3390eda3042023553f287faf3bccf2e3a3ac Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 10:45:15 +0700 Subject: [PATCH 06/12] add basic all test and count assertions --- tests/test_hash_model.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/test_hash_model.py b/tests/test_hash_model.py index c0069dae..f8886d4b 100644 --- a/tests/test_hash_model.py +++ b/tests/test_hash_model.py @@ -96,6 +96,18 @@ async def members(m): yield member1, member2, member3 +@py_test_mark_asyncio +async def test_all_query(members, m): + + actual = await m.Member.find().all() + assert all([member in actual for member in members]) + + actual_count = await m.Member.find().count() + assert actual_count == len(members) + + assert False # pls just fail + + @py_test_mark_asyncio async def test_exact_match_queries(members, m): member1, member2, member3 = members @@ -129,6 +141,11 @@ async def test_exact_match_queries(members, m): ).all() assert actual == [member2] + actual_count = await m.Member.find( + m.Member.first_name == "Kim", m.Member.last_name == "Brookins" + ).count() + assert actual_count == 1 + @py_test_mark_asyncio async def test_full_text_search_queries(members, m): @@ -162,16 +179,17 @@ async def test_recursive_query_resolution(members, m): async def test_tag_queries_boolean_logic(members, m): member1, member2, member3 = members - actual = await ( - m.Member.find( - (m.Member.first_name == "Andrew") & (m.Member.last_name == "Brookins") - | (m.Member.last_name == "Smith") - ) - .sort_by("age") - .all() - ) + find_query = m.Member.find( + (m.Member.first_name == "Andrew") & (m.Member.last_name == "Brookins") + | (m.Member.last_name == "Smith") + ).sort_by("age") + + actual = await find_query.all() assert actual == [member1, member3] + actual_count = await find_query.count() + assert actual_count == 2 + @py_test_mark_asyncio async def test_tag_queries_punctuation(m): From 33af9085223d3256cde813a5e95e1a557b68e3dc Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 10:45:54 +0700 Subject: [PATCH 07/12] fix import refs --- tests/conftest.py | 2 +- tests/test_hash_model.py | 2 +- tests/test_json_model.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9f067a38..77685361 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def key_prefix(request, redis): def cleanup_keys(request): # Always use the sync Redis connection with finalizer. Setting up an # async finalizer should work, but I'm not suer how yet! - from redis_om.connections import get_redis_connection as get_sync_redis + from aredis_om.connections import get_redis_connection as get_sync_redis # Increment for every pytest-xdist worker conn = get_sync_redis() diff --git a/tests/test_hash_model.py b/tests/test_hash_model.py index f8886d4b..dbae3bb7 100644 --- a/tests/test_hash_model.py +++ b/tests/test_hash_model.py @@ -23,7 +23,7 @@ # We need to run this check as sync code (during tests) even in async mode # because we call it in the top-level module scope. from aredis_om import has_redisearch -from conftest import py_test_mark_asyncio +from tests.conftest import py_test_mark_asyncio if not has_redisearch(): pytestmark = pytest.mark.skip diff --git a/tests/test_json_model.py b/tests/test_json_model.py index 8a114f9a..9fe8efc0 100644 --- a/tests/test_json_model.py +++ b/tests/test_json_model.py @@ -24,7 +24,7 @@ # We need to run this check as sync code (during tests) even in async mode # because we call it in the top-level module scope. -from redis_om import has_redis_json +from aredis_om import has_redis_json from tests.conftest import py_test_mark_asyncio if not has_redis_json(): From 15dd2ee12242584bf3a5ca705705bd40d26f68d1 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 10:46:54 +0700 Subject: [PATCH 08/12] count method returns int --- aredis_om/model/model.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/aredis_om/model/model.py b/aredis_om/model/model.py index c6ef23d4..29e9cbee 100644 --- a/aredis_om/model/model.py +++ b/aredis_om/model/model.py @@ -760,11 +760,11 @@ async def all(self, batch_size=10): return await query.execute() return await self.execute() - async def count(self): - args = ["FT.AGGREGATE", self.model.Meta.index_name, "*"] - if self.expression: - args += ["FILTER", *self.query] - args += [ + async def count(self) -> int: + args = [ + "FT.AGGREGATE", + self.model.Meta.index_name, + self.query, "APPLY", "matched_terms()", "AS", @@ -777,7 +777,11 @@ async def count(self): "0", ] raw_result = await self.model.db().execute_command(*args) - return sum([count for [count, _] in raw_result]) + print(raw_result, args) + try: + return int(raw_result[1][3].decode("utf-8", "ignore")) + except IndexError: + return 0 def sort_by(self, *fields: str): if not fields: From e9909a3f7d288a659e62a6391049880de5c28c53 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 10:48:33 +0700 Subject: [PATCH 09/12] expect bytestring --- tests/test_json_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_json_model.py b/tests/test_json_model.py index 9fe8efc0..b3a3ba3d 100644 --- a/tests/test_json_model.py +++ b/tests/test_json_model.py @@ -291,7 +291,7 @@ async def test_saves_many_explicit_transaction(address, m): async with m.Member.db().pipeline(transaction=True) as pipeline: await m.Member.add(members, pipeline=pipeline) assert result == [member1, member2] - assert await pipeline.execute() == ["OK", "OK"] + assert await pipeline.execute() == [b"OK", b"OK"] assert await m.Member.get(pk=member1.pk) == member1 assert await m.Member.get(pk=member2.pk) == member2 From 4d63c402571838d6e76e4abda6aea97cf7844157 Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 11:02:11 +0700 Subject: [PATCH 10/12] remove temp test file --- tests/test_find_count.py | 41 ---------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 tests/test_find_count.py diff --git a/tests/test_find_count.py b/tests/test_find_count.py deleted file mode 100644 index 4ef93502..00000000 --- a/tests/test_find_count.py +++ /dev/null @@ -1,41 +0,0 @@ -import datetime -from typing import Optional - -from pydantic import EmailStr - -from aredis_om import Field, HashModel, Migrator - - -class Customer(HashModel): - first_name: str - last_name: str = Field(index=True) - email: EmailStr - join_date: datetime.date - age: int = Field(index=True) - bio: Optional[str] - - -andrew = Customer( - first_name="Andrew", - last_name="Brookins", - email="andrew.brookins@example.com", - join_date=datetime.date.today(), - age=38, -) - -await Migrator().run() - -await Customer.find().delete() - -andy = await andrew.save() - -customers = await Customer.find(Customer.last_name == "Brookins").all() -assert len(customers) == 1 - -await Migrator().run() - -customers_count = await Customer.find().count() -assert customers_count == 1 - -brookins_count = await Customer.find(Customer.last_name == "Brookins").count() -assert brookins_count == 1 From 381ab0ebd4f26e022344cd9b7a012bbd43f3590e Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 11:02:23 +0700 Subject: [PATCH 11/12] sum the counts --- aredis_om/model/model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aredis_om/model/model.py b/aredis_om/model/model.py index 29e9cbee..4eb34f84 100644 --- a/aredis_om/model/model.py +++ b/aredis_om/model/model.py @@ -779,7 +779,9 @@ async def count(self) -> int: raw_result = await self.model.db().execute_command(*args) print(raw_result, args) try: - return int(raw_result[1][3].decode("utf-8", "ignore")) + return sum( + [int(result[3].decode("utf-8", "ignore")) for result in raw_result[1:]] + ) except IndexError: return 0 From efdca3cd18447b42b17c57955cd6ae4dfb0cca4d Mon Sep 17 00:00:00 2001 From: Tom Counsell Date: Tue, 19 Jul 2022 11:02:55 +0700 Subject: [PATCH 12/12] don't test sort_by with count --- tests/test_hash_model.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_hash_model.py b/tests/test_hash_model.py index dbae3bb7..baeb5ed2 100644 --- a/tests/test_hash_model.py +++ b/tests/test_hash_model.py @@ -105,8 +105,6 @@ async def test_all_query(members, m): actual_count = await m.Member.find().count() assert actual_count == len(members) - assert False # pls just fail - @py_test_mark_asyncio async def test_exact_match_queries(members, m): @@ -182,9 +180,9 @@ async def test_tag_queries_boolean_logic(members, m): find_query = m.Member.find( (m.Member.first_name == "Andrew") & (m.Member.last_name == "Brookins") | (m.Member.last_name == "Smith") - ).sort_by("age") + ) - actual = await find_query.all() + actual = await find_query.sort_by("age").all() assert actual == [member1, member3] actual_count = await find_query.count()