From 02ce8456683d998519f1bd33528240c733533c78 Mon Sep 17 00:00:00 2001 From: Carsten Rietz Date: Tue, 10 Dec 2024 09:13:42 +0100 Subject: [PATCH 1/2] docstore/memdocstore: #3508 allow nested slices query --- docstore/memdocstore/codec.go | 2 +- docstore/memdocstore/mem.go | 48 +++++++++++++++++++++----------- docstore/memdocstore/mem_test.go | 44 +++++++++++++++++++++++++++++ docstore/memdocstore/query.go | 23 +++++++++++---- 4 files changed, 95 insertions(+), 22 deletions(-) diff --git a/docstore/memdocstore/codec.go b/docstore/memdocstore/codec.go index 68c4d1db85..c2c5baca59 100644 --- a/docstore/memdocstore/codec.go +++ b/docstore/memdocstore/codec.go @@ -104,7 +104,7 @@ func decodeDoc(m storedDoc, ddoc driver.Document, fps [][]string) error { // (We don't need the key field because ddoc must already have it.) m2 = map[string]interface{}{} for _, fp := range fps { - val, err := getAtFieldPath(m, fp) + val, err := getAtFieldPath(m, fp, false) if err != nil { if gcerrors.Code(err) == gcerrors.NotFound { continue diff --git a/docstore/memdocstore/mem.go b/docstore/memdocstore/mem.go index a5470261b6..3e4ebeb01e 100644 --- a/docstore/memdocstore/mem.go +++ b/docstore/memdocstore/mem.go @@ -68,6 +68,12 @@ type Options struct { // When the collection is closed, its contents are saved to the file. Filename string + // AllowNestedSlicesQuery allows querying with nested slices. + // This makes the memdocstore more compatible with MongoDB, + // but other providers may not support this feature. + // See https://github.com/google/go-cloud/pull/3511 for more details. + AllowNestedSlicesQuery bool + // Call this function when the collection is closed. // For internal use only. onClose func() @@ -399,16 +405,34 @@ func (c *collection) checkRevision(arg driver.Document, current storedDoc) error // getAtFieldPath gets the value of m at fp. It returns an error if fp is invalid // (see getParentMap). -func getAtFieldPath(m map[string]interface{}, fp []string) (interface{}, error) { - m2, err := getParentMap(m, fp, false) - if err != nil { - return nil, err +func getAtFieldPath(m map[string]interface{}, fp []string, nested bool) (result interface{}, err error) { + + var get func(m interface{}, name string) interface{} + get = func(m interface{}, name string) interface{} { + switch concrete := m.(type) { + case map[string]interface{}: + return concrete[name] + case []interface{}: + if !nested { + return nil + } + result := []interface{}{} + for _, e := range concrete { + result = append(result, get(e, name)) + } + return result + } + return nil } - v, ok := m2[fp[len(fp)-1]] - if ok { - return v, nil + result = m + for _, k := range fp { + next := get(result, k) + if next == nil { + return nil, gcerr.Newf(gcerr.NotFound, nil, "field %s not found", strings.Join(fp, ".")) + } + result = next } - return nil, gcerr.Newf(gcerr.NotFound, nil, "field %s not found", fp) + return result, nil } // setAtFieldPath sets m's value at fp to val. It creates intermediate maps as @@ -422,14 +446,6 @@ func setAtFieldPath(m map[string]interface{}, fp []string, val interface{}) erro return nil } -// Delete the value from m at the given field path, if it exists. -func deleteAtFieldPath(m map[string]interface{}, fp []string) { - m2, _ := getParentMap(m, fp, false) // ignore error - if m2 != nil { - delete(m2, fp[len(fp)-1]) - } -} - // getParentMap returns the map that directly contains the given field path; // that is, the value of m at the field path that excludes the last component // of fp. If a non-map is encountered along the way, an InvalidArgument error is diff --git a/docstore/memdocstore/mem_test.go b/docstore/memdocstore/mem_test.go index 2129f62ef7..f2b3051346 100644 --- a/docstore/memdocstore/mem_test.go +++ b/docstore/memdocstore/mem_test.go @@ -16,6 +16,7 @@ package memdocstore import ( "context" + "io" "os" "path/filepath" "testing" @@ -129,6 +130,49 @@ func TestUpdateAtomic(t *testing.T) { } } +func TestQueryNested(t *testing.T) { + ctx := context.Background() + + count := func(iter *docstore.DocumentIterator) (c int) { + doc := docmap{} + for { + if err := iter.Next(ctx, doc); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + c++ + } + return c + } + + dc, err := newCollection(drivertest.KeyField, nil, &Options{AllowNestedSlicesQuery: true}) + if err != nil { + t.Fatal(err) + } + coll := docstore.NewCollection(dc) + defer coll.Close() + + doc := docmap{drivertest.KeyField: "TestQueryNested", + "list": []any{docmap{"a": "A"}}, + "map": docmap{"b": "B"}, + dc.RevisionField(): nil, + } + if err := coll.Put(ctx, doc); err != nil { + t.Fatal(err) + } + + got := count(coll.Query().Where("list.a", "=", "A").Get(ctx)) + if got != 1 { + t.Errorf("got %v docs when filtering by list.a, want 1", got) + } + got = count(coll.Query().Where("map.b", "=", "B").Get(ctx)) + if got != 1 { + t.Errorf("got %v docs when filtering by map.b, want 1", got) + } +} + func TestSortDocs(t *testing.T) { newDocs := func() []storedDoc { return []storedDoc{ diff --git a/docstore/memdocstore/query.go b/docstore/memdocstore/query.go index 419017b993..9a660ffe50 100644 --- a/docstore/memdocstore/query.go +++ b/docstore/memdocstore/query.go @@ -37,7 +37,7 @@ func (c *collection) RunGetQuery(_ context.Context, q *driver.Query) (driver.Doc var resultDocs []storedDoc for _, doc := range c.docs { - if filtersMatch(q.Filters, doc) { + if filtersMatch(q.Filters, doc, c.opts.AllowNestedSlicesQuery) { resultDocs = append(resultDocs, doc) } } @@ -74,17 +74,17 @@ func (c *collection) RunGetQuery(_ context.Context, q *driver.Query) (driver.Doc }, nil } -func filtersMatch(fs []driver.Filter, doc storedDoc) bool { +func filtersMatch(fs []driver.Filter, doc storedDoc, nested bool) bool { for _, f := range fs { - if !filterMatches(f, doc) { + if !filterMatches(f, doc, nested) { return false } } return true } -func filterMatches(f driver.Filter, doc storedDoc) bool { - docval, err := getAtFieldPath(doc, f.FieldPath) +func filterMatches(f driver.Filter, doc storedDoc, nested bool) bool { + docval, err := getAtFieldPath(doc, f.FieldPath, nested) // missing or bad field path => no match if err != nil { return false @@ -138,6 +138,19 @@ func compare(x1, x2 interface{}) (int, bool) { } return -1, true } + if v1.Kind() == reflect.Slice { + for i := 0; i < v1.Len(); i++ { + if c, ok := compare(x2, v1.Index(i).Interface()); ok { + if !ok { + return 0, false + } + if c == 0 { + return 0, true + } + } + } + return -1, true + } if v1.Kind() == reflect.String && v2.Kind() == reflect.String { return strings.Compare(v1.String(), v2.String()), true } From e16b6c9c34eb778c6779abb87f435216b394a024 Mon Sep 17 00:00:00 2001 From: Carsten Rietz Date: Thu, 12 Dec 2024 15:23:21 +0100 Subject: [PATCH 2/2] docstore/memdocstore: #3508 first review renamed option to AllowNestedSliceQueries --- docstore/memdocstore/mem.go | 17 ++++++++--------- docstore/memdocstore/mem_test.go | 25 ++++++++++++++++++++++++- docstore/memdocstore/query.go | 5 +++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/docstore/memdocstore/mem.go b/docstore/memdocstore/mem.go index 3e4ebeb01e..9b68d0da32 100644 --- a/docstore/memdocstore/mem.go +++ b/docstore/memdocstore/mem.go @@ -68,11 +68,11 @@ type Options struct { // When the collection is closed, its contents are saved to the file. Filename string - // AllowNestedSlicesQuery allows querying with nested slices. + // AllowNestedSliceQueries allows querying with nested slices. // This makes the memdocstore more compatible with MongoDB, // but other providers may not support this feature. // See https://github.com/google/go-cloud/pull/3511 for more details. - AllowNestedSlicesQuery bool + AllowNestedSliceQueries bool // Call this function when the collection is closed. // For internal use only. @@ -405,18 +405,17 @@ func (c *collection) checkRevision(arg driver.Document, current storedDoc) error // getAtFieldPath gets the value of m at fp. It returns an error if fp is invalid // (see getParentMap). -func getAtFieldPath(m map[string]interface{}, fp []string, nested bool) (result interface{}, err error) { - - var get func(m interface{}, name string) interface{} - get = func(m interface{}, name string) interface{} { +func getAtFieldPath(m map[string]any, fp []string, nested bool) (result any, err error) { + var get func(m any, name string) any + get = func(m any, name string) any { switch concrete := m.(type) { - case map[string]interface{}: + case map[string]any: return concrete[name] - case []interface{}: + case []any: if !nested { return nil } - result := []interface{}{} + var result []any for _, e := range concrete { result = append(result, get(e, name)) } diff --git a/docstore/memdocstore/mem_test.go b/docstore/memdocstore/mem_test.go index f2b3051346..d393e44c8e 100644 --- a/docstore/memdocstore/mem_test.go +++ b/docstore/memdocstore/mem_test.go @@ -147,7 +147,7 @@ func TestQueryNested(t *testing.T) { return c } - dc, err := newCollection(drivertest.KeyField, nil, &Options{AllowNestedSlicesQuery: true}) + dc, err := newCollection(drivertest.KeyField, nil, &Options{AllowNestedSliceQueries: true}) if err != nil { t.Fatal(err) } @@ -157,6 +157,9 @@ func TestQueryNested(t *testing.T) { doc := docmap{drivertest.KeyField: "TestQueryNested", "list": []any{docmap{"a": "A"}}, "map": docmap{"b": "B"}, + "listOfMaps": []any{docmap{"id": "1"}, docmap{"id": "2"}, docmap{"id": "3"}}, + "mapOfLists": docmap{"ids": []any{"1", "2", "3"}}, + "deep": []any{docmap{"nesting": []any{docmap{"of": docmap{"elements": "yes"}}}}}, dc.RevisionField(): nil, } if err := coll.Put(ctx, doc); err != nil { @@ -167,10 +170,30 @@ func TestQueryNested(t *testing.T) { if got != 1 { t.Errorf("got %v docs when filtering by list.a, want 1", got) } + got = count(coll.Query().Where("list.a", "=", "missing").Get(ctx)) + if got != 0 { + t.Errorf("got %v docs when filtering by list.a, want 0", got) + } got = count(coll.Query().Where("map.b", "=", "B").Get(ctx)) if got != 1 { t.Errorf("got %v docs when filtering by map.b, want 1", got) } + got = count(coll.Query().Where("listOfMaps.id", "=", "1").Get(ctx)) + if got != 1 { + t.Errorf("got %v docs when filtering by listOfMaps.id, want 1", got) + } + got = count(coll.Query().Where("listOfMaps.id", "=", "123").Get(ctx)) + if got != 0 { + t.Errorf("got %v docs when filtering by listOfMaps.id, want 0", got) + } + got = count(coll.Query().Where("mapOfLists.ids", "=", "1").Get(ctx)) + if got != 1 { + t.Errorf("got %v docs when filtering by listOfMaps.1, want 1", got) + } + got = count(coll.Query().Where("deep.nesting.of.elements", "=", "yes").Get(ctx)) + if got != 1 { + t.Errorf("got %v docs when filtering by deep.nesting.of.elements, want 1", got) + } } func TestSortDocs(t *testing.T) { diff --git a/docstore/memdocstore/query.go b/docstore/memdocstore/query.go index 9a660ffe50..ee0a08f8a4 100644 --- a/docstore/memdocstore/query.go +++ b/docstore/memdocstore/query.go @@ -37,7 +37,7 @@ func (c *collection) RunGetQuery(_ context.Context, q *driver.Query) (driver.Doc var resultDocs []storedDoc for _, doc := range c.docs { - if filtersMatch(q.Filters, doc, c.opts.AllowNestedSlicesQuery) { + if filtersMatch(q.Filters, doc, c.opts.AllowNestedSliceQueries) { resultDocs = append(resultDocs, doc) } } @@ -138,6 +138,8 @@ func compare(x1, x2 interface{}) (int, bool) { } return -1, true } + // for AllowNestedSliceQueries + // when querying for x2 in the document and x1 is a list of values we only need one value to match if v1.Kind() == reflect.Slice { for i := 0; i < v1.Len(); i++ { if c, ok := compare(x2, v1.Index(i).Interface()); ok { @@ -149,7 +151,6 @@ func compare(x1, x2 interface{}) (int, bool) { } } } - return -1, true } if v1.Kind() == reflect.String && v2.Kind() == reflect.String { return strings.Compare(v1.String(), v2.String()), true