Merge pull request guardian#4401 from bbc/t2118-fuzzy-search

T2118 fuzzy search

Author: andrew-nowak

dev/script/generate-config/service-config.js

@@ -142,6 +142,9 @@ function getMediaApiConfig(config) {
         |es6.replicas=${config.es6.replicas}
         |quota.store.key="rcs-quota.json"
         |security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}"
+        |search.fuzziness={
+        | enabled=true
+        |}
         |metrics.request.enabled=false
         |syndication.review.useRuntimeFieldsFix=true
         |`;

media-api/app/lib/MediaApiConfig.scala

@@ -31,6 +31,15 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla
   val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key")
   val cloudFrontKeyPairId: Option[String]           = stringOpt("cloudfront.keypair.id")
 
+  val fuzzySearchEnabled: Boolean = boolean("search.fuzziness.enabled")
+  val fuzzySearchPrefixLength: Int = intOpt("search.fuzziness.prefixLength").getOrElse(1)
+  val fuzzySearchEditDistance: String = stringOpt("search.fuzziness.editDistance") match {
+    case Some(editDistance) if editDistance.toIntOption.isDefined => editDistance
+    case Some(editDistance) if editDistance.contains("AUTO:") => editDistance  //<- for non-default AUTO word boundaries
+    case _ => "AUTO"
+  }
+  val fuzzyMaxExpansions: Int = intOpt("search.fuzziness.maxExpansions").getOrElse(50)
+
   val rootUri: String = services.apiBaseUri
   val kahunaUri: String = services.kahunaBaseUri
   val cropperUri: String = services.cropperBaseUri

media-api/app/lib/elasticsearch/QueryBuilder.scala

@@ -26,10 +26,21 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc
   private def multiMatchPhraseQuery(value: String, fields: Seq[String]): MultiMatchQuery =
     ElasticDsl.multiMatchQuery(value).fields(fields).matchType(MultiMatchQueryBuilderType.PHRASE)
 
+  private def multiMatchWordQuery(value: String, fields: Seq[String]): MultiMatchQuery = {
+    val multiMatchQuery = ElasticDsl.multiMatchQuery(value).fields(fields).operator(Operator.AND)
+
+    if (config.fuzzySearchEnabled) {
+      multiMatchQuery.matchType(MultiMatchQueryBuilderType.BEST_FIELDS)
+        .fuzziness(config.fuzzySearchEditDistance)
+        .maxExpansions(config.fuzzyMaxExpansions)
+        .prefixLength(config.fuzzySearchPrefixLength)
+    } else {
+      multiMatchQuery.matchType(MultiMatchQueryBuilderType.CROSS_FIELDS)
+    }
+  }
+
   private def makeMultiQuery(value: Value, fields: Seq[String]): MultiMatchQuery = value match {
-    case Words(value) => ElasticDsl.multiMatchQuery(value).fields(fields).
-      operator(Operator.AND).
-      matchType(MultiMatchQueryBuilderType.CROSS_FIELDS)
+    case Words(value) => multiMatchWordQuery(value, fields)
     case Phrase(string) => multiMatchPhraseQuery(string, fields)
     // That's OK, we only do date queries on a single field at a time
     case e => throw InvalidQuery(s"Cannot do multiQuery on $e")

media-api/test/lib/elasticsearch/QueryBuilderTest.scala

@@ -23,8 +23,10 @@ class QueryBuilderTest extends AnyFunSpec with Matchers with ConditionFixtures w
 
   val matchFields: Seq[String] = Seq("afield", "anothermatchfield")
 
+  private val commonConfigurations = USED_CONFIGS_IN_TEST ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap
+
   private val mediaApiConfig = new MediaApiConfig(GridConfigResources(
-    Configuration.from(USED_CONFIGS_IN_TEST ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap),
+    Configuration.from(commonConfigurations),
     null,
     new ApplicationLifecycle {
       override def addStopHook(hook: () => Future[_]): Unit = {}
@@ -132,6 +134,28 @@ class QueryBuilderTest extends AnyFunSpec with Matchers with ConditionFixtures w
       multiMatchClause.`type` shouldBe Some(MultiMatchQueryBuilderType.CROSS_FIELDS)
     }
 
+    it("any field words queries should be applied to all of the match fields with best fields type and fuzziness, operator and analyzers set") {
+      val mediaApiConfigWithFuzzySearch = new MediaApiConfig(GridConfigResources(
+        Configuration.from(commonConfigurations ++ Map("search.fuzziness.enabled" -> true)),
+        null,
+        new ApplicationLifecycle {
+          override def addStopHook(hook: () => Future[_]): Unit = {}
+          override def stop(): Future[_] = Future.successful(())
+        }
+      ))
+      val queryBuilder = new QueryBuilder(matchFields, () => Nil, mediaApiConfigWithFuzzySearch)
+      val query = queryBuilder.makeQuery(List(anyFieldWordsCondition)).asInstanceOf[BoolQuery]
+
+      query.must.size shouldBe 1
+      val multiMatchClause = query.must.head.asInstanceOf[MultiMatchQuery]
+      multiMatchClause.text shouldBe "cats dogs"
+      multiMatchClause.fields.map(_.field) shouldBe matchFields
+      multiMatchClause.operator shouldBe Some(Operator.AND)
+      multiMatchClause.`type` shouldBe Some(MultiMatchQueryBuilderType.BEST_FIELDS)
+      multiMatchClause.fuzziness shouldBe defined
+      multiMatchClause.fuzziness shouldBe Some("AUTO")
+    }
+
     it("multiple field queries should query against the requested fields only") {
       val query = queryBuilder.makeQuery(List(multipleFieldWordsCondition)).asInstanceOf[BoolQuery]