JSON field support

Author: bastianjoel

pkg/meta/collections.go

@@ -16,17 +16,24 @@ type Collection struct {
 
 // CollectionRelation describes a related collection
 type CollectionRelation struct {
-	Type       string                         `json:"type"`
-	Collection *string                        `json:"collection,omitempty"`
-	Fields     map[string]*CollectionRelation `json:"fields"`
+	Type       string                         `yaml:"type"`
+	Collection *string                        `yaml:"collection,omitempty"`
+	Fields     map[string]*CollectionRelation `yaml:"fields"`
+}
+
+// CollectionSearchableConfig contains per field config of a collection
+type CollectionSearchableConfig struct {
+	Type     *string `yaml:"type,omitempty"`
+	Analyzer *string `yaml:"analyzer,omitempty"`
 }
 
 // CollectionDescription is the collection format for search filters
 type CollectionDescription struct {
-	Searchable []string                       `yaml:"searchable"`
-	Additional []string                       `yaml:"additional"`
-	Contains   []string                       `yaml:"contains,omitempty"`
-	Relations  map[string]*CollectionRelation `yaml:"relations,omitempty"`
+	Searchable       []string                               `yaml:"searchable"`
+	SearchableConfig map[string]*CollectionSearchableConfig `yaml:"searchable_config,omitempty"`
+	Additional       []string                               `yaml:"additional"`
+	Contains         []string                               `yaml:"contains,omitempty"`
+	Relations        map[string]*CollectionRelation         `yaml:"relations,omitempty"`
 }
 
 // Collections is part of the meta model.

pkg/meta/filters.go

@@ -9,11 +9,12 @@ import (
 
 // Filter is part of the meta model.
 type Filter struct {
-	Name       string
-	Items      []string
-	Additional []string
-	Contains   map[string]struct{}
-	Relations  map[string]*CollectionRelation
+	Name        string
+	Items       []string
+	ItemsConfig map[string]*CollectionSearchableConfig
+	Additional  []string
+	Contains    map[string]struct{}
+	Relations   map[string]*CollectionRelation
 }
 
 // FilterKey is part of the meta model.
@@ -65,11 +66,12 @@ func (fs *Filters) UnmarshalYAML(value *yaml.Node) error {
 		}
 
 		*fs = append(*fs, Filter{
-			Name:       s.Name,
-			Items:      fsm[s].Searchable,
-			Additional: fsm[s].Additional,
-			Relations:  relations,
-			Contains:   contains,
+			Name:        s.Name,
+			Items:       fsm[s].Searchable,
+			ItemsConfig: fsm[s].SearchableConfig,
+			Additional:  fsm[s].Additional,
+			Relations:   relations,
+			Contains:    contains,
 		})
 	}
 	return nil
@@ -101,11 +103,16 @@ func (fs Filters) Retain(verbose bool) func(string, string, *Member) bool {
 	keep := map[key]struct{}{}
 	additional := map[key]struct{}{}
 	relations := map[key]*CollectionRelation{}
+	config := map[key]*CollectionSearchableConfig{}
 	for _, m := range fs {
 		for _, f := range m.Items {
 			keep[key{rel: m.Name, field: f}] = struct{}{}
 		}
 
+		for f, data := range m.ItemsConfig {
+			config[key{rel: m.Name, field: f}] = data
+		}
+
 		for _, f := range m.Additional {
 			additional[key{rel: m.Name, field: f}] = struct{}{}
 		}
@@ -119,6 +126,14 @@ func (fs Filters) Retain(verbose bool) func(string, string, *Member) bool {
 			m.Relation = relations[key{rel: rk, field: fk}]
 		}
 
+		if c, ok := config[key{rel: rk, field: fk}]; ok {
+			if c.Type != nil {
+				m.Type = *c.Type
+			}
+
+			m.Analyzer = c.Analyzer
+		}
+
 		if _, ok := additional[key{rel: rk, field: fk}]; ok {
 			m.Searchable = false
 			return true

pkg/meta/member.go

@@ -24,6 +24,7 @@ type Member struct {
 	RestrictionMode       string              `yaml:"restriction_mode"`
 	Required              bool                `yaml:"required"`
 	Searchable            bool                `yaml:"-"`
+	Analyzer              *string             `yaml:"-"`
 	Relation              *CollectionRelation `yaml:"-"`
 	Order                 int32               `yaml:"-"`
 }

pkg/search/textindex.go

@@ -168,20 +168,31 @@ func buildIndexMapping(collections meta.Collections) mapping.IndexMapping {
 		docMapping.AddFieldMappingsAt("_bleve_type", keywordFieldMapping)
 		for fname, cf := range col.Fields {
 			if cf.Searchable {
-				switch cf.Type {
-				case "HTMLStrict", "HTMLPermissive":
-					docMapping.AddFieldMappingsAt(fname, htmlFieldMapping)
-				case "string", "text":
-					docMapping.AddFieldMappingsAt(fname, textFieldMapping)
-					docMapping.AddFieldMappingsAt("_"+fname+"_original", simpleFieldMapping)
-				case "generic-relation":
-					docMapping.AddFieldMappingsAt(fname, keywordFieldMapping)
-				case "relation", "number":
-					docMapping.AddFieldMappingsAt(fname, numberFieldMapping)
-				case "number[]":
-					docMapping.AddFieldMappingsAt(fname, numberFieldMapping)
-				default:
-					log.Errorf("unsupport type %q on field %s\n", cf.Type, fname)
+				if cf.Analyzer == nil {
+					switch cf.Type {
+					case "HTMLStrict", "HTMLPermissive":
+						docMapping.AddFieldMappingsAt(fname, htmlFieldMapping)
+					case "string", "text":
+						docMapping.AddFieldMappingsAt(fname, textFieldMapping)
+						docMapping.AddFieldMappingsAt("_"+fname+"_original", simpleFieldMapping)
+					case "generic-relation":
+						docMapping.AddFieldMappingsAt(fname, keywordFieldMapping)
+					case "relation", "number":
+						docMapping.AddFieldMappingsAt(fname, numberFieldMapping)
+					case "number[]":
+						docMapping.AddFieldMappingsAt(fname, numberFieldMapping)
+					default:
+						log.Errorf("unsupport type %q on field %s\n", cf.Type, fname)
+					}
+				} else {
+					switch *cf.Analyzer {
+					case "html":
+						docMapping.AddFieldMappingsAt(fname, htmlFieldMapping)
+					case "simple":
+						docMapping.AddFieldMappingsAt(fname, simpleFieldMapping)
+					default:
+						log.Errorf("unsupported Analyzer %q on field %s\n", *cf.Analyzer, fname)
+					}
 				}
 			}
 		}
@@ -220,6 +231,12 @@ func (bt bleveType) fill(fields map[string]*meta.Member, data []byte) {
 				}
 			}, fname)
 			continue
+		case "json-int-string-map":
+			bt[fname] = []string{}
+			jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
+				bt[fname] = append(bt[fname].([]string), string(value))
+				return nil
+			}, fname)
 		default:
 			if v, _, _, err := jsonparser.Get(data, fname); err == nil {
 				bt[fname] = v

search.yml

@@ -84,6 +84,11 @@ motion:
     - text
     - reason
     - meeting_id
+    - amendment_paragraphs
+  searchable_config:
+    amendment_paragraphs:
+      type: json-int-string-map
+      analyzer: html
   additional:
     - id
     - sequential_number
@@ -219,6 +224,13 @@ user:
     - organization_management_level
     - meeting_ids
     - owner_id
+  searchable_config:
+    first_name:
+      analyzer: simple
+    last_name:
+      analyzer: simple
+    email:
+      analyzer: simple
   additional:
     - id
   relations: