diff --git a/README.md b/README.md index 5e1a436c..7528d578 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,36 @@ when starting the application. For example: Remember this command needs to be run from the website module, that is `{parent_directory}/website` directory. +### Search functionality + +The search API is defined in the `searchapi` module. We currently have two implementations for the search api: +1. ElasticSearch - Implemented in the `elasticsearch-module` module +2. JPA based search - Implemented in the `jpa-search-module` module + +The `jpa-search-module` module is used in the `website` module which represents the website running at www.yorubaname.com + +If you want to use `elasticsearch` module then remove the following section in the pom.xml for `website` module: + +``` + + ${project.groupId} + jpa-search-module + ${project.version} + +``` + +with + +``` + + ${project.groupId} + elasticsearch-module + ${project.version} + +``` + +The `elasticsearch-module` needs to be configured. This is explained in the next session. + ### Configuring ElasticSearch Properties The ElasticSearch module does not require the installation of ElasticSearch as it will run with an embedded ElasticSearch instance: diff --git a/bootstrap/pom.xml b/bootstrap/pom.xml index a6ae241a..d62a583a 100644 --- a/bootstrap/pom.xml +++ b/bootstrap/pom.xml @@ -21,7 +21,7 @@ ${project.groupId} - elasticsearch-module + search-api 0.0.1-SNAPSHOT diff --git a/bootstrap/src/main/java/org/oruko/dictionary/util/DataImporter.java b/bootstrap/src/main/java/org/oruko/dictionary/util/DataImporter.java index 7abbe12d..8e5a985a 100644 --- a/bootstrap/src/main/java/org/oruko/dictionary/util/DataImporter.java +++ b/bootstrap/src/main/java/org/oruko/dictionary/util/DataImporter.java @@ -1,6 +1,5 @@ package org.oruko.dictionary.util; -import org.oruko.dictionary.elasticsearch.ElasticSearchService; import org.oruko.dictionary.model.GeoLocation; import org.oruko.dictionary.model.NameEntry; import org.oruko.dictionary.model.State; @@ -30,9 +29,6 @@ public class DataImporter { @Autowired private GeoLocationRepository geoLocationRepository; - @Autowired - private ElasticSearchService elasticSearchService; - @Autowired private NameEntryRepository nameEntryRepository; @@ -49,7 +45,6 @@ public void initializeData() { */ if (host.equalsIgnoreCase("localhost")) { List nameEntries = initializeDb(); - initializeElastic(nameEntries); } } @@ -111,7 +106,15 @@ private List initializeDb() { bolanle.setExtendedMeaning("This is extended dummy meaning for Bọ́lánlé"); bolanle.setGeoLocation(Arrays.asList(new GeoLocation("IBADAN", "NWY"))); bolanle.setEtymology(etymology); - bolanle.setState(State.NEW); + bolanle.setState(State.PUBLISHED); + + + NameEntry bimpe = new NameEntry("Bimpe"); + bimpe.setMeaning("This is dummy meaning for Bimpe"); + bimpe.setExtendedMeaning("This is extended dummy meaning for Bimpe"); + bimpe.setGeoLocation(Arrays.asList(new GeoLocation("IBADAN", "NWY"))); + bimpe.setEtymology(etymology); + bimpe.setState(State.PUBLISHED); NameEntry ade0 = new NameEntry("Ade"); ade0.setMeaning("This is dummy meaning for ade"); @@ -174,6 +177,7 @@ private List initializeDb() { nameEntryRepository.save(tola); nameEntryRepository.save(dadepo); nameEntryRepository.save(bolanle); + nameEntryRepository.save(bimpe); nameEntryRepository.save(ade0); nameEntryRepository.save(ade1); nameEntryRepository.save(ade2); @@ -186,10 +190,6 @@ private List initializeDb() { ade0, ade1, ade2, ade3, ade4, omowumi, omolabi); } - private void initializeElastic(List nameEntries) { - elasticSearchService.bulkIndexName(nameEntries); - } - private void initGeoLocation() { // North-West Yoruba (NWY): Abẹokuta, Ibadan, Ọyọ, Ogun and Lagos (Eko) areas // Central Yoruba (CY): Igbomina, Yagba, Ilésà, Ifẹ, Ekiti, Akurẹ, Ẹfọn, and Ijẹbu areas. diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index d5391ee3..9216d452 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -12,6 +12,11 @@ elasticsearch-module + + ${project.groupId} + search-api + ${project.version} + org.apache.lucene diff --git a/elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/ElasticSearchService.java b/elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/ElasticSearchService.java index 07438fde..53da937a 100644 --- a/elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/ElasticSearchService.java +++ b/elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/ElasticSearchService.java @@ -17,6 +17,8 @@ import org.elasticsearch.node.Node; import org.elasticsearch.search.SearchHit; import org.oruko.dictionary.model.NameEntry; +import org.oruko.dictionary.search.api.IndexOperationStatus; +import org.oruko.dictionary.search.api.SearchService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,9 +26,10 @@ import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Service; import org.springframework.util.FileCopyUtils; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; +import javax.annotation.PostConstruct; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -35,7 +38,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.PostConstruct; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -45,7 +47,7 @@ * @author Dadepo Aderemi. */ @Service -public class ElasticSearchService { +public class ElasticSearchService implements SearchService { private Logger logger = LoggerFactory.getLogger(ElasticSearchService.class); @@ -86,24 +88,37 @@ public boolean isElasticSearchNodeAvailable() { * @param nameQuery the name * @return the nameEntry as a Map or null if name not found */ - public Map getByName(String nameQuery) { + @Override + public NameEntry getByName(String nameQuery) { SearchResponse searchResponse = exactSearchByName(nameQuery); SearchHit[] hits = searchResponse.getHits().getHits(); if (hits.length == 1) { - return hits[0].getSource(); + return sourceToNameEntry(hits[0].getSource()); } else { return null; } } + private NameEntry sourceToNameEntry(Map source) { + String valueAsString = null; + try { + valueAsString = mapper.writeValueAsString(source); + return mapper.readValue(valueAsString, NameEntry.class); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + /** * For searching the index for a name * * @param searchTerm the search term * @return the list of entries found */ - public Set> search(String searchTerm) { + @Override + public Set search(String searchTerm) { /** * 1. First do a exact search. If found return result. If not go to 2. * 2. Do a search with ascii-folding. If found return result, if not go to 3 @@ -116,13 +131,13 @@ public Set> search(String searchTerm) { * 7. Do a full text search against extendedMeaning */ - final Set> result = new LinkedHashSet<>(); + final Set result = new LinkedHashSet<>(); // 1. exact search SearchResponse searchResponse = exactSearchByName(searchTerm); if (searchResponse.getHits().getHits().length >= 1) { Stream.of(searchResponse.getHits().getHits()).forEach(hit -> { - result.add(hit.getSource()); + result.add(sourceToNameEntry(hit.getSource())); }); if (result.size() == 1) { @@ -134,7 +149,7 @@ public Set> search(String searchTerm) { searchResponse = exactSearchByNameAsciiFolded(searchTerm); if (searchResponse.getHits().getHits().length >= 1) { Stream.of(searchResponse.getHits().getHits()).forEach(hit -> { - result.add(hit.getSource()); + result.add(sourceToNameEntry(hit.getSource())); }); if (result.size() == 1) { @@ -147,7 +162,7 @@ public Set> search(String searchTerm) { searchResponse = prefixFilterSearch(searchTerm, false); if (searchResponse.getHits().getHits().length >= 1) { Stream.of(searchResponse.getHits().getHits()).forEach(hit -> { - result.add(hit.getSource()); + result.add(sourceToNameEntry(hit.getSource())); }); return result; @@ -162,27 +177,28 @@ public Set> search(String searchTerm) { * TODO Should revisit */ MultiMatchQueryBuilder searchSpec = QueryBuilders.multiMatchQuery(searchTerm, - "name.autocomplete", - "meaning", - "extendedMeaning", - "variants"); + "name.autocomplete", + "meaning", + "extendedMeaning", + "variants"); SearchResponse tempSearchAll = client.prepareSearch(esConfig.getIndexName()) - .setQuery(searchSpec) - .setSize(20) - .execute() - .actionGet(); + .setQuery(searchSpec) + .setSize(20) + .execute() + .actionGet(); Stream.of(tempSearchAll.getHits().getHits()).forEach(hit -> { - result.add(hit.getSource()); + result.add(sourceToNameEntry(hit.getSource())); }); return result; } - public Set> listByAlphabet(String alphabetQuery) { - final Set> result = new LinkedHashSet<>(); + @Override + public Set listByAlphabet(String alphabetQuery) { + final Set result = new LinkedHashSet<>(); final SearchResponse searchResponse = prefixFilterSearch(alphabetQuery, true); final SearchHit[] hits = searchResponse.getHits().getHits(); @@ -190,7 +206,7 @@ public Set> listByAlphabet(String alphabetQuery) { Collections.reverse(searchHits); searchHits.forEach(hit -> { - result.add(hit.getSource()); + result.add(sourceToNameEntry(hit.getSource())); }); return result; @@ -202,8 +218,9 @@ public Set> listByAlphabet(String alphabetQuery) { * @param query the query * @return the list of partial matches */ - public List autocomplete(String query) { - final List result = new ArrayList(); + @Override + public Set autocomplete(String query) { + final Set result = new LinkedHashSet<>(); SearchResponse tempSearchAll = partialSearchByName(query); @@ -214,6 +231,18 @@ public List autocomplete(String query) { return result; } + @Override + public Integer getSearchableNames() { + try { + CountResponse response = client.prepareCount(esConfig.getIndexName()) + .setQuery(matchAllQuery()) + .execute() + .actionGet(); + return Math.toIntExact(response.getCount()); + } catch (Exception e) { + return 0; + } + } /** * Add a {@link org.oruko.dictionary.model.NameEntry} into ElasticSearch index @@ -225,16 +254,16 @@ public IndexOperationStatus indexName(NameEntry entry) { if (!isElasticSearchNodeAvailable()) { return new IndexOperationStatus(false, - "Index attempt unsuccessful. You do not have an elasticsearch node running"); + "Index attempt unsuccessful. You do not have an elasticsearch node running"); } try { String entryAsJson = mapper.writeValueAsString(entry); String name = entry.getName(); client.prepareIndex(esConfig.getIndexName(), esConfig.getDocumentType(), name.toLowerCase()) - .setSource(entryAsJson) - .execute() - .actionGet(); + .setSource(entryAsJson) + .execute() + .actionGet(); return new IndexOperationStatus(true, name + " indexed successfully"); } catch (JsonProcessingException e) { @@ -244,6 +273,7 @@ public IndexOperationStatus indexName(NameEntry entry) { } + @Override public IndexOperationStatus bulkIndexName(List entries) { if (entries.size() == 0) { @@ -252,7 +282,7 @@ public IndexOperationStatus bulkIndexName(List entries) { if (!isElasticSearchNodeAvailable()) { return new IndexOperationStatus(false, - "Index attempt unsuccessful. You do not have an elasticsearch node running"); + "Index attempt unsuccessful. You do not have an elasticsearch node running"); } BulkRequestBuilder bulkRequest = client.prepareBulk(); @@ -261,9 +291,9 @@ public IndexOperationStatus bulkIndexName(List entries) { String entryAsJson = mapper.writeValueAsString(entry); String name = entry.getName(); IndexRequestBuilder request = client.prepareIndex(esConfig.getIndexName(), - esConfig.getDocumentType(), - name.toLowerCase()) - .setSource(entryAsJson); + esConfig.getDocumentType(), + name.toLowerCase()) + .setSource(entryAsJson); bulkRequest.add(request); } catch (JsonProcessingException e) { logger.debug("Error while building bulk indexing operation", e); @@ -285,34 +315,34 @@ public IndexOperationStatus bulkIndexName(List entries) { * @param name the name to delete from the index * @return true | false depending on if deleting operation was successful */ - public IndexOperationStatus deleteFromIndex(String name) { + public IndexOperationStatus removeFromIndex(String name) { if (!isElasticSearchNodeAvailable()) { return new IndexOperationStatus(false, - "Delete unsuccessful. You do not have an elasticsearch node running"); + "Delete unsuccessful. You do not have an elasticsearch node running"); } DeleteResponse response = deleteName(name); return new IndexOperationStatus(response.isFound(), name + " deleted from index"); } - public IndexOperationStatus bulkDeleteFromIndex(List entries) { + public IndexOperationStatus bulkRemoveByNameFromIndex(List entries) { if (entries.size() == 0) { return new IndexOperationStatus(false, "Cannot index an empty list"); } if (!isElasticSearchNodeAvailable()) { return new IndexOperationStatus(false, - "Delete unsuccessful. You do not have an elasticsearch node running"); + "Delete unsuccessful. You do not have an elasticsearch node running"); } BulkRequestBuilder bulkRequest = client.prepareBulk(); entries.forEach(entry -> { DeleteRequestBuilder request = client.prepareDelete(esConfig.getIndexName(), - esConfig.getDocumentType(), - entry.toLowerCase()); + esConfig.getDocumentType(), + entry.toLowerCase()); bulkRequest.add(request); }); @@ -339,9 +369,9 @@ private DeleteResponse deleteName(String name) { //TODO revisit. Omo returns Omowunmi and Owolabi. Ideal this should return just one result private SearchResponse exactSearchByName(String nameQuery) { return client.prepareSearch(esConfig.getIndexName()) - .setPostFilter(FilterBuilders.termFilter("name", nameQuery.toLowerCase())) - .execute() - .actionGet(); + .setPostFilter(FilterBuilders.termFilter("name", nameQuery.toLowerCase())) + .execute() + .actionGet(); } private SearchResponse prefixFilterSearch(String nameQuery, boolean getAll) { @@ -353,26 +383,26 @@ private SearchResponse prefixFilterSearch(String nameQuery, boolean getAll) { } return client.prepareSearch(esConfig.getIndexName()) - .setPostFilter(FilterBuilders.prefixFilter("name", nameQuery.toLowerCase())) - .setSize(resultSet) - .execute() - .actionGet(); + .setPostFilter(FilterBuilders.prefixFilter("name", nameQuery.toLowerCase())) + .setSize(resultSet) + .execute() + .actionGet(); } private SearchResponse exactSearchByNameAsciiFolded(String nameQuery) { return client.prepareSearch(esConfig.getIndexName()) - .setQuery(QueryBuilders.matchQuery("name.asciifolded", nameQuery.toLowerCase())) - .setSize(20) - .execute() - .actionGet(); + .setQuery(QueryBuilders.matchQuery("name.asciifolded", nameQuery.toLowerCase())) + .setSize(20) + .execute() + .actionGet(); } private SearchResponse partialSearchByName(String nameQuery) { return client.prepareSearch(esConfig.getIndexName()) - .setQuery(QueryBuilders.matchQuery("name.autocomplete", nameQuery.toLowerCase())) - .setSize(20) - .execute() - .actionGet(); + .setQuery(QueryBuilders.matchQuery("name.autocomplete", nameQuery.toLowerCase())) + .setSize(20) + .execute() + .actionGet(); } // On start up, creates an index for the application if one does not @@ -394,38 +424,32 @@ private void buildElasticSearchClient() { try { boolean exists = this.client.admin().indices() - .prepareExists(esConfig.getIndexName()) - .execute() - .actionGet() - .isExists(); + .prepareExists(esConfig.getIndexName()) + .execute() + .actionGet() + .isExists(); if (!exists) { CreateIndexResponse createIndexResponse = this.client.admin().indices() - .prepareCreate(esConfig.getIndexName()) - .setSettings(indexSettings) - .addMapping(esConfig.getDocumentType(), - mapping).execute() - .actionGet(); + .prepareCreate(esConfig.getIndexName()) + .setSettings(indexSettings) + .addMapping(esConfig.getDocumentType(), + mapping).execute() + .actionGet(); logger.info("Created {} and added {} to type {} status was {} at startup", - esConfig.getIndexName(), mapping, esConfig.getDocumentType(), - createIndexResponse.isAcknowledged()); + esConfig.getIndexName(), mapping, esConfig.getDocumentType(), + createIndexResponse.isAcknowledged()); } } catch (Exception e) { logger.error("ElasticSearch not running", e); } } - public long getCount() { - try { - CountResponse response = client.prepareCount(esConfig.getIndexName()) - .setQuery(matchAllQuery()) - .execute() - .actionGet(); - return response.getCount(); - } catch (Exception e) { - return 0; - } + + @Override + public IndexOperationStatus bulkRemoveFromIndex(List nameEntries) { + throw new NotImplementedException(); } } diff --git a/elasticsearch/src/test/java/org/oruko/dictionary/elasticsearch/ElasticSearchServiceTest.java b/elasticsearch/src/test/java/org/oruko/dictionary/elasticsearch/ElasticSearchServiceTest.java index 1ed6dd02..800bc834 100644 --- a/elasticsearch/src/test/java/org/oruko/dictionary/elasticsearch/ElasticSearchServiceTest.java +++ b/elasticsearch/src/test/java/org/oruko/dictionary/elasticsearch/ElasticSearchServiceTest.java @@ -4,16 +4,18 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.junit.*; +import org.junit.Before; +import org.junit.Test; +import org.oruko.dictionary.model.NameEntry; +import org.oruko.dictionary.search.api.SearchService; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; /** - * Integration test for {@link ElasticSearchService} + * * Created by Dadepo Aderemi. */ @@ -21,7 +23,7 @@ public class ElasticSearchServiceTest extends ElasticsearchIntegrationTest { private String dictionary = "dictionary"; private ESConfig esConfig; - ElasticSearchService elasticSearchService; + SearchService searchService; private class TestNode implements org.elasticsearch.node.Node { @@ -70,7 +72,7 @@ public void setup() throws IOException { flushAndRefresh(); - elasticSearchService = new ElasticSearchService(new TestNode(), esConfig); + searchService = new ElasticSearchService(new TestNode(), esConfig); } @Test @@ -82,8 +84,8 @@ public void testGetByName() throws IOException { flushAndRefresh(); - Map result = elasticSearchService.getByName("jamo"); - assertEquals("jamo", result.get("name")); + NameEntry jamo = searchService.getByName("jamo"); + assertEquals("jamo", jamo.getName()); } } diff --git a/jpasearch/pom.xml b/jpasearch/pom.xml new file mode 100644 index 00000000..9835fb6f --- /dev/null +++ b/jpasearch/pom.xml @@ -0,0 +1,24 @@ + + + + dictionary + org.yorubaname + 0.0.1-SNAPSHOT + + 4.0.0 + + jpa-search-module + + + + + + ${project.groupId} + search-api + ${project.version} + + + + \ No newline at end of file diff --git a/jpasearch/src/main/java/org/oruko/dictionary/search/jpa/JpaSearchService.java b/jpasearch/src/main/java/org/oruko/dictionary/search/jpa/JpaSearchService.java new file mode 100644 index 00000000..3e202cdd --- /dev/null +++ b/jpasearch/src/main/java/org/oruko/dictionary/search/jpa/JpaSearchService.java @@ -0,0 +1,142 @@ +package org.oruko.dictionary.search.jpa; + +import org.oruko.dictionary.model.NameEntry; +import org.oruko.dictionary.model.State; +import org.oruko.dictionary.model.repository.NameEntryRepository; +import org.oruko.dictionary.search.api.IndexOperationStatus; +import org.oruko.dictionary.search.api.SearchService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +public class JpaSearchService implements SearchService { + private NameEntryRepository nameEntryRepository; + + @Autowired + public JpaSearchService(NameEntryRepository nameEntryRepository) { + this.nameEntryRepository = nameEntryRepository; + } + + @Override + public NameEntry getByName(String nameQuery) { + return nameEntryRepository.findByNameAndState(nameQuery, State.PUBLISHED); + } + + @Override + public Set search(String searchTerm) { + Set possibleFound = new LinkedHashSet<>(); + /** + * The following approach should be taken: + * + * 1. First do a exact search. If found return result. If not go to 2. + * 2. Do a search with ascii-folding. If found return result, if not go to 3 + * 3. Do a prefix search. If found, return result, if not go to 4 + * 4. Do a search based on partial match. Irrespective of outcome, proceed to 4 + * 4a - Using nGram? + * 4b - ? + * 5. Do a full text search against other variants. Irrespective of outcome, proceed to 6 + * 6. Do a full text search against meaning. Irrespective of outcome, proceed to 7 + * 7. Do a full text search against extendedMeaning + */ + NameEntry exactFound = nameEntryRepository.findByNameAndState(searchTerm, State.PUBLISHED); + if (exactFound != null) { + return Collections.singleton(exactFound); + } + Set startingWithSearchTerm = nameEntryRepository.findByNameStartingWithAndState(searchTerm, State.PUBLISHED); + if (startingWithSearchTerm != null && startingWithSearchTerm.size() > 0) { + return startingWithSearchTerm; + } + + possibleFound.addAll(nameEntryRepository.findNameEntryByNameContainingAndState(searchTerm, State.PUBLISHED)); + possibleFound.addAll(nameEntryRepository.findNameEntryByVariantsContainingAndState(searchTerm, State.PUBLISHED)); + possibleFound.addAll(nameEntryRepository.findNameEntryByMeaningContainingAndState(searchTerm, State.PUBLISHED)); + possibleFound.addAll(nameEntryRepository.findNameEntryByExtendedMeaningContainingAndState(searchTerm, State.PUBLISHED)); + + return possibleFound; + } + + @Override + public Set listByAlphabet(String alphabetQuery) { + return nameEntryRepository.findByNameStartingWithAndState(alphabetQuery, State.PUBLISHED); + } + + @Override + public Set autocomplete(String query) { + Set names = new LinkedHashSet<>(); + Set nameToReturn = new LinkedHashSet<>(); + // TODO calling the db in a for loop might not be a terribly good idea. Revist + for (int i=2; i otherParts = nameEntryRepository.findNameEntryByNameContainingAndState(query, State.PUBLISHED); + names.addAll(otherParts); + names.forEach(name -> { + nameToReturn.add(name.getName()); + }); + return nameToReturn; + } + + @Override + public Integer getSearchableNames() { + return nameEntryRepository.countByState(State.PUBLISHED); + } + + @Override + public IndexOperationStatus bulkIndexName(List entries) { + if (entries.size() == 0) { + return new IndexOperationStatus(false, "Cannot index an empty list"); + } + nameEntryRepository.save(entries); + return new IndexOperationStatus(true, "Bulk indexing successfully. Indexed the following names " + + String.join(",", entries.stream().map(NameEntry::getName).collect(Collectors.toList()))); + } + + public IndexOperationStatus removeFromIndex(String name) { + NameEntry foundName = nameEntryRepository.findByNameAndState(name, State.PUBLISHED); + if (foundName == null) { + return new IndexOperationStatus(false, "Published Name not found"); + } + foundName.setState(State.UNPUBLISHED); + nameEntryRepository.save(foundName); + return new IndexOperationStatus(true, name + " removed from index"); + } + + @Override + public IndexOperationStatus bulkRemoveByNameFromIndex(List names) { + if (names.size() == 0) { + return new IndexOperationStatus(false, "Cannot index an empty list"); + } + List nameEntries = names.stream().map(name -> nameEntryRepository.findByNameAndState(name, State.PUBLISHED)) + .collect(Collectors.toList()); + + + List namesUnpublished = nameEntries.stream().map(name -> { + name.setState(State.UNPUBLISHED); + return name; + }).collect(Collectors.toList()); + + nameEntryRepository.save(namesUnpublished); + return new IndexOperationStatus(true, "Successfully. " + + "Removed the following names from search index " + + String.join(",", names)); + } + + @Override + public IndexOperationStatus bulkRemoveFromIndex(List nameEntries) { + List namesUnpublished = nameEntries.stream() + .peek(name -> name.setState(State.UNPUBLISHED)) + .collect(Collectors.toList()); + + nameEntryRepository.save(namesUnpublished); + return new IndexOperationStatus(true, "Successfully. " + + "Removed the following names from search index " + + String.join(",", nameEntries.stream().map(NameEntry::getName).collect(Collectors.toList()))); + } +} diff --git a/model/src/main/java/org/oruko/dictionary/model/AbstractNameEntry.java b/model/src/main/java/org/oruko/dictionary/model/AbstractNameEntry.java index 005c5324..b23b2050 100644 --- a/model/src/main/java/org/oruko/dictionary/model/AbstractNameEntry.java +++ b/model/src/main/java/org/oruko/dictionary/model/AbstractNameEntry.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; import org.oruko.dictionary.model.repository.Etymology; -import java.time.LocalDateTime; -import java.util.List; import javax.persistence.Column; import javax.persistence.ElementCollection; import javax.persistence.EnumType; @@ -18,6 +16,8 @@ import javax.persistence.JoinColumn; import javax.persistence.ManyToMany; import javax.persistence.MappedSuperclass; +import java.time.LocalDateTime; +import java.util.List; /** * 1. Name @@ -92,9 +92,6 @@ public abstract class AbstractNameEntry { @ElementCollection protected List etymology; - @Column - protected Boolean isIndexed = false; - @Column @Enumerated(EnumType.STRING) protected State state; @@ -253,14 +250,6 @@ public void setTags(String tags) { this.tags = tags; } - public Boolean getIndexed() { - return isIndexed; - } - - public void setIndexed(Boolean published) { - this.isIndexed = published; - } - public LocalDateTime getUpdatedAt() { return updatedAt; } diff --git a/model/src/main/java/org/oruko/dictionary/model/State.java b/model/src/main/java/org/oruko/dictionary/model/State.java index e5d51064..1af65205 100644 --- a/model/src/main/java/org/oruko/dictionary/model/State.java +++ b/model/src/main/java/org/oruko/dictionary/model/State.java @@ -15,6 +15,10 @@ public enum State { * Name has been indexed in the search engine, thus publish and users can find it when they search for it */ PUBLISHED, + /** + * A name that was initially published but has been removed from the index + */ + UNPUBLISHED, /** * A name that has been published (indexed into the search engine) was modified, thus needs to be re-indexed */ diff --git a/model/src/main/java/org/oruko/dictionary/model/repository/NameEntryRepository.java b/model/src/main/java/org/oruko/dictionary/model/repository/NameEntryRepository.java index a1aa056f..b92c9842 100644 --- a/model/src/main/java/org/oruko/dictionary/model/repository/NameEntryRepository.java +++ b/model/src/main/java/org/oruko/dictionary/model/repository/NameEntryRepository.java @@ -5,8 +5,9 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; -import java.util.List; import javax.transaction.Transactional; +import java.util.List; +import java.util.Set; /** * Created by dadepo on 2/4/15. @@ -36,4 +37,14 @@ public interface NameEntryRepository extends JpaRepository { * @return list of {@link NameEntry} */ List findByState(State state); + Set findByNameStartingWithAndState(String alphabet, State state); + Set findNameEntryByNameContainingAndState(String name, State state); + Set findNameEntryByVariantsContainingAndState(String name, State state); + Set findNameEntryByMeaningContainingAndState(String name, State state); + Set findNameEntryByExtendedMeaningContainingAndState(String name, State state); + NameEntry findByNameAndState(String name, State state); + + + Integer countByState(State state); + Boolean deleteByNameAndState(String name, State state); } diff --git a/pom.xml b/pom.xml index b2ffca69..4302f001 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,8 @@ elasticsearch website event + searchapi + jpasearch pom diff --git a/searchapi/pom.xml b/searchapi/pom.xml new file mode 100644 index 00000000..a3216dc8 --- /dev/null +++ b/searchapi/pom.xml @@ -0,0 +1,26 @@ + + + + dictionary + org.yorubaname + 0.0.1-SNAPSHOT + + 4.0.0 + + search-api + + + + + + + ${project.groupId} + model-module + ${project.version} + + + + + \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/IndexOperationStatus.java b/searchapi/src/main/java/org/oruko/dictionary/search/api/IndexOperationStatus.java similarity index 91% rename from elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/IndexOperationStatus.java rename to searchapi/src/main/java/org/oruko/dictionary/search/api/IndexOperationStatus.java index b340c724..747f5a34 100644 --- a/elasticsearch/src/main/java/org/oruko/dictionary/elasticsearch/IndexOperationStatus.java +++ b/searchapi/src/main/java/org/oruko/dictionary/search/api/IndexOperationStatus.java @@ -1,4 +1,4 @@ -package org.oruko.dictionary.elasticsearch; +package org.oruko.dictionary.search.api; /** * For communicating the status of operation on the index diff --git a/searchapi/src/main/java/org/oruko/dictionary/search/api/SearchService.java b/searchapi/src/main/java/org/oruko/dictionary/search/api/SearchService.java new file mode 100644 index 00000000..d98e564a --- /dev/null +++ b/searchapi/src/main/java/org/oruko/dictionary/search/api/SearchService.java @@ -0,0 +1,50 @@ +package org.oruko.dictionary.search.api; + +import org.oruko.dictionary.model.NameEntry; + +import java.util.List; +import java.util.Set; + +public interface SearchService { + /** + * For getting an entry from the search index by name + * + * @param nameQuery the name + * @return the nameEntry or null if name not found + */ + NameEntry getByName(String nameQuery); + + /** + * For searching the name entries for a name + * + * + * + * @param searchTerm the search term + * @return the list of entries found + */ + Set search(String searchTerm); + + /** + * Return all the names which starts with the given alphabet + * + * @param alphabetQuery the given alphabet + * + * @return the list of names that starts with the given alphabet + */ + Set listByAlphabet(String alphabetQuery); + + /** + * For getting the list of partial matches for autocomplete + * + * @param query the query + * @return the list of partial matches + */ + Set autocomplete(String query); + + Integer getSearchableNames(); + + IndexOperationStatus bulkIndexName(List entries); + IndexOperationStatus removeFromIndex(String name); + IndexOperationStatus bulkRemoveByNameFromIndex(List name); + IndexOperationStatus bulkRemoveFromIndex(List nameEntries); +} diff --git a/webapi/pom.xml b/webapi/pom.xml index 98d44bbe..2726dfb1 100644 --- a/webapi/pom.xml +++ b/webapi/pom.xml @@ -22,7 +22,7 @@ ${project.groupId} - elasticsearch-module + search-api ${project.version} diff --git a/webapi/src/main/java/org/oruko/dictionary/web/NameEntryService.java b/webapi/src/main/java/org/oruko/dictionary/web/NameEntryService.java index 58abd01f..56859695 100644 --- a/webapi/src/main/java/org/oruko/dictionary/web/NameEntryService.java +++ b/webapi/src/main/java/org/oruko/dictionary/web/NameEntryService.java @@ -339,6 +339,4 @@ private boolean namePresentAsVariant(String name) { return false; }); } - - } diff --git a/webapi/src/main/java/org/oruko/dictionary/web/event/NameDeletedEventHandler.java b/webapi/src/main/java/org/oruko/dictionary/web/event/NameDeletedEventHandler.java index c079df12..585ef5fc 100644 --- a/webapi/src/main/java/org/oruko/dictionary/web/event/NameDeletedEventHandler.java +++ b/webapi/src/main/java/org/oruko/dictionary/web/event/NameDeletedEventHandler.java @@ -2,20 +2,23 @@ import com.google.common.eventbus.AllowConcurrentEvents; import com.google.common.eventbus.Subscribe; -import org.oruko.dictionary.elasticsearch.ElasticSearchService; import org.oruko.dictionary.events.NameDeletedEvent; +import org.oruko.dictionary.search.api.SearchService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; /** - * Handler for {@link org.oruko.dictionary.events.NameDeletedEvent} + * Handler for {@link NameDeletedEvent} * @author Dadepo Aderemi. */ @Component public class NameDeletedEventHandler { + // TODO should not be hardwiring a bean here + @Qualifier("jpaSearchService") @Autowired - ElasticSearchService elasticSearchService; + SearchService nameSearchService; @Autowired RecentIndexes recentIndexes; @Autowired @@ -26,7 +29,7 @@ public class NameDeletedEventHandler { public void listen(NameDeletedEvent event) { // Handle when a name is deleted try { - elasticSearchService.deleteFromIndex(event.getName()); + nameSearchService.removeFromIndex(event.getName()); recentIndexes.remove(event.getName()); recentSearches.remove(event.getName()); } catch (Exception e) { diff --git a/webapi/src/main/java/org/oruko/dictionary/web/rest/NameApi.java b/webapi/src/main/java/org/oruko/dictionary/web/rest/NameApi.java index 980770e3..130aef71 100644 --- a/webapi/src/main/java/org/oruko/dictionary/web/rest/NameApi.java +++ b/webapi/src/main/java/org/oruko/dictionary/web/rest/NameApi.java @@ -32,22 +32,22 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import javax.validation.Valid; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import java.util.Collections; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.validation.Valid; /** * End point for inserting and retrieving NameDto Entries @@ -151,21 +151,17 @@ public ResponseEntity> getMetaData() { * result set from. 0 if none is given * @param countParam a {@link Integer} the number of names to return. 50 is none is given * @return the list of {@link org.oruko.dictionary.model.NameEntry} - * @throws JsonProcessingException JSON processing exception */ @RequestMapping(value = "/v1/names", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public List getAllNames(@RequestParam("page") final Optional pageParam, @RequestParam("count") final Optional countParam, @RequestParam("all") final Optional all, @RequestParam("submittedBy") final Optional submittedBy, - @RequestParam("state") final Optional state, - @RequestParam(value = "indexed", required = false) final Optional indexed) - throws JsonProcessingException { + @RequestParam("state") final Optional state) { - List names = new ArrayList<>(); List allNameEntries; - if (all.isPresent() && all.get() == true) { + if (all.isPresent() && all.get()) { if (state.isPresent()) { allNameEntries = entryService.loadAllByState(state); } else { @@ -175,16 +171,14 @@ public List getAllNames(@RequestParam("page") final Optional allNameEntries = entryService.loadByState(state, pageParam, countParam); } - names.addAll(allNameEntries); - - // for filtering based on whether entry has been indexed - Predicate filterBasedOnIndex = name -> indexed.map(aBoolean -> name.getIndexed().equals(aBoolean)).orElse(true); + List names = new ArrayList<>(allNameEntries); // for filtering based on value of submitBy - Predicate filterBasedOnSubmitBy = name -> submittedBy.map(s -> name.getSubmittedBy().trim().equalsIgnoreCase(s.trim())).orElse(true); + Predicate filterBasedOnSubmitBy = (name) -> submittedBy + .map(s -> name.getSubmittedBy().trim().equalsIgnoreCase(s.trim())) + .orElse(true); return names.stream() - .filter(filterBasedOnIndex) .filter(filterBasedOnSubmitBy) .collect(Collectors.toCollection(ArrayList::new)); diff --git a/webapi/src/main/java/org/oruko/dictionary/web/rest/SearchApi.java b/webapi/src/main/java/org/oruko/dictionary/web/rest/SearchApi.java index 6024bc47..f42e37e8 100644 --- a/webapi/src/main/java/org/oruko/dictionary/web/rest/SearchApi.java +++ b/webapi/src/main/java/org/oruko/dictionary/web/rest/SearchApi.java @@ -1,12 +1,12 @@ package org.oruko.dictionary.web.rest; -import org.oruko.dictionary.elasticsearch.ElasticSearchService; -import org.oruko.dictionary.elasticsearch.IndexOperationStatus; import org.oruko.dictionary.events.EventPubService; import org.oruko.dictionary.events.NameIndexedEvent; import org.oruko.dictionary.events.NameSearchedEvent; import org.oruko.dictionary.model.NameEntry; import org.oruko.dictionary.model.State; +import org.oruko.dictionary.search.api.IndexOperationStatus; +import org.oruko.dictionary.search.api.SearchService; import org.oruko.dictionary.web.NameEntryService; import org.oruko.dictionary.web.event.RecentIndexes; import org.oruko.dictionary.web.event.RecentSearches; @@ -36,7 +36,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * Handler for search functionality @@ -49,8 +48,8 @@ public class SearchApi { private Logger logger = LoggerFactory.getLogger(SearchApi.class); - private NameEntryService entryService; - private ElasticSearchService elasticSearchService; + private NameEntryService nameEntryService; + private SearchService searchService; private RecentSearches recentSearches; private RecentIndexes recentIndexes; private EventPubService eventPubService; @@ -58,18 +57,19 @@ public class SearchApi { /** * Public constructor for {@link SearchApi} * - * @param entryService service layer for interacting with name entries - * @param elasticSearchService service layer for elastic search functions + * @param nameEntryService service layer for interacting with name entries * @param recentSearches object holding the recent searches in memory * @param recentIndexes object holding the recent index names in memory */ @Autowired - public SearchApi(EventPubService eventPubService, NameEntryService entryService, - ElasticSearchService elasticSearchService, RecentSearches recentSearches, + public SearchApi(EventPubService eventPubService, + NameEntryService nameEntryService, + SearchService searchService, + RecentSearches recentSearches, RecentIndexes recentIndexes) { this.eventPubService = eventPubService; - this.entryService = entryService; - this.elasticSearchService = elasticSearchService; + this.nameEntryService = nameEntryService; + this.searchService = searchService; this.recentSearches = recentSearches; this.recentIndexes = recentIndexes; } @@ -83,7 +83,7 @@ public SearchApi(EventPubService eventPubService, NameEntryService entryService, @RequestMapping(value = "/meta", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> getMetaData() { Map metaData = new HashMap<>(); - metaData.put("totalPublishedNames", elasticSearchService.getCount()); + metaData.put("totalPublishedNames", searchService.getSearchableNames()); return new ResponseEntity<>(metaData, HttpStatus.OK); } @@ -94,42 +94,44 @@ public ResponseEntity> getMetaData() { */ @RequestMapping(value = {"/", ""}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public Set> search(@RequestParam(value = "q", required = true) String searchTerm, - HttpServletRequest request) { + public Set search(@RequestParam(value = "q", required = true) String searchTerm, + HttpServletRequest request) { - Set> name = elasticSearchService.search(searchTerm); - if (name != null - && name.size() == 1 - && name.stream().allMatch(result -> result.get("name").equals(searchTerm))) { + Set foundNames = searchService.search(searchTerm); + if (foundNames != null + && foundNames.size() == 1 + && foundNames.stream().allMatch(result -> result.getName().equals(searchTerm))) { eventPubService.publish(new NameSearchedEvent(searchTerm, request.getRemoteAddr())); } - return name; + return foundNames; } @RequestMapping(value = "/autocomplete", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public List getAutocomplete(@RequestParam(value = "q") Optional searchQuery) { - if (!searchQuery.isPresent()) { - return Collections.emptyList(); + public Set getAutocomplete(@RequestParam(value = "q") Optional searchQuery) { + if (!searchQuery.isPresent() || searchQuery.get().length() < 2) { + return Collections.emptySet(); } String query = searchQuery.get(); - return elasticSearchService.autocomplete(query); + return searchService.autocomplete(query); } @RequestMapping(value = "/alphabet/{alphabet}", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public Set> getByAlphabet(@PathVariable Optional alphabet) { - return alphabet.map(s -> elasticSearchService.listByAlphabet(s)).orElseGet(Collections::emptySet); - + public Set getByAlphabet(@PathVariable Optional alphabet) { + if (!alphabet.isPresent()) { + return Collections.emptySet(); + } + return searchService.listByAlphabet(alphabet.get()); } @RequestMapping(value = "/{searchTerm}", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public Map findByName(@PathVariable String searchTerm, HttpServletRequest request) { + public NameEntry findByName(@PathVariable String searchTerm, HttpServletRequest request) { - Map name = elasticSearchService.getByName(searchTerm); + NameEntry name = searchService.getByName(searchTerm); if (name != null) { eventPubService.publish(new NameSearchedEvent(searchTerm, request.getRemoteAddr())); @@ -175,31 +177,24 @@ public Map allActivity() { * Endpoint to index a NameEntry sent in as JSON string. * * @param entry the {@link NameEntry} representation of the JSON String. - * @return a {@link org.springframework.http.ResponseEntity} representing the status of the operation. + * @return a {@link ResponseEntity} representing the status of the operation. */ @RequestMapping(value = "/indexes", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> indexEntry(@Valid NameEntry entry) { Map response = new HashMap<>(); - NameEntry nameEntry = entryService.loadName(entry.getName()); + NameEntry nameEntry = nameEntryService.loadName(entry.getName()); if (nameEntry == null) { response.put("message", "Cannot index entry. Name " + entry.getName() + " not in the database"); return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST); } - IndexOperationStatus indexOperationStatus = elasticSearchService.indexName(entry); - boolean isIndexed = indexOperationStatus.getStatus(); - String message = indexOperationStatus.getMessage(); - if (isIndexed) { - publishNameIsIndexed(nameEntry); - nameEntry.setIndexed(true); - nameEntry.setState(State.PUBLISHED); - entryService.saveName(nameEntry); - response.put("message", message); - return new ResponseEntity<>(response, HttpStatus.CREATED); - } - return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST); + nameEntry.setState(State.PUBLISHED); + nameEntryService.saveName(nameEntry); + publishNameIsIndexed(nameEntry); + response.put("message", "Name is now searchable"); + return new ResponseEntity<>(response, HttpStatus.CREATED); } private void publishNameIsIndexed(NameEntry nameEntry) { @@ -210,34 +205,27 @@ private void publishNameIsIndexed(NameEntry nameEntry) { * Endpoint that takes a name, looks it up in the repository and index the entry found * * @param name the name - * @return a {@link org.springframework.http.ResponseEntity} representing the status of the operation + * @return a {@link ResponseEntity} representing the status of the operation */ @RequestMapping(value = "/indexes/{name}", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> indexEntryByName(@PathVariable String name) { Map response = new HashMap<>(); - NameEntry nameEntry = entryService.loadName(name); + NameEntry nameEntry = nameEntryService.loadName(name); if (nameEntry == null) { // name requested to be indexed not in the database - response.put("message", name + " not found in the repository so not indexed"); + response.put("message", + name+" not found in the repository so not indexed"); return new ResponseEntity<>(response, HttpStatus.INTERNAL_SERVER_ERROR); } - IndexOperationStatus indexOperationStatus = elasticSearchService.indexName(nameEntry); - boolean isIndexed = indexOperationStatus.getStatus(); - - response.put("message", indexOperationStatus.getMessage()); + publishNameIsIndexed(nameEntry); + nameEntry.setState(State.PUBLISHED); + nameEntryService.saveName(nameEntry); + response.put("message", name + " has been published"); - if (isIndexed) { - publishNameIsIndexed(nameEntry); - nameEntry.setIndexed(true); - nameEntry.setState(State.PUBLISHED); - entryService.saveName(nameEntry); - return new ResponseEntity<>(response, HttpStatus.CREATED); - } - - return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST); + return new ResponseEntity<>(response, HttpStatus.CREATED); } @@ -247,7 +235,7 @@ public ResponseEntity> indexEntryByName(@PathVariable String * It allows for batch indexing of names * * @param names the array of names - * @return a {@link org.springframework.http.ResponseEntity} representing the status of the operation + * @return a {@link ResponseEntity} representing the status of the operation */ @RequestMapping(value = "/indexes/batch", method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE, @@ -258,7 +246,7 @@ public ResponseEntity> batchIndexEntriesByName(@RequestBody List notFound = new ArrayList<>(); Arrays.stream(names).forEach(name -> { - NameEntry entry = entryService.loadName(name); + NameEntry entry = nameEntryService.loadName(name); if (entry == null) { notFound.add(name); } else { @@ -272,14 +260,20 @@ public ResponseEntity> batchIndexEntriesByName(@RequestBody return new ResponseEntity<>(response, HttpStatus.INTERNAL_SERVER_ERROR); } - IndexOperationStatus indexOperationStatus = elasticSearchService.bulkIndexName(nameEntries); - updateIsIndexFlag(nameEntries, true, indexOperationStatus); + IndexOperationStatus indexOperationStatus = searchService.bulkIndexName(nameEntries); + response.put("message", indexOperationStatus.getMessage()); + + if (!indexOperationStatus.getStatus()) { + return new ResponseEntity<>(response, HttpStatus.INTERNAL_SERVER_ERROR); + } + for (NameEntry nameEntry : nameEntries) { publishNameIsIndexed(nameEntry); nameEntry.setState(State.PUBLISHED); - entryService.saveName(nameEntry); + nameEntryService.saveName(nameEntry); } - return returnStatusMessage(notFound, indexOperationStatus); + + return new ResponseEntity<>(response, HttpStatus.CREATED); } @@ -287,23 +281,22 @@ public ResponseEntity> batchIndexEntriesByName(@RequestBody * Endpoint used to remove a name from the index. * * @param name the name to remove from the index. - * @return a {@link org.springframework.http.ResponseEntity} representing the status of the operation. + * @return a {@link ResponseEntity} representing the status of the operation. */ @RequestMapping(value = "/indexes/{name}", method = RequestMethod.DELETE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> deleteFromIndex(@PathVariable String name) { - IndexOperationStatus indexOperationStatus = elasticSearchService.deleteFromIndex(name); + IndexOperationStatus indexOperationStatus = searchService.removeFromIndex(name); boolean deleted = indexOperationStatus.getStatus(); String message = indexOperationStatus.getMessage(); Map response = new HashMap<>(); response.put("message", message); if (deleted) { - NameEntry nameEntry = entryService.loadName(name); + NameEntry nameEntry = nameEntryService.loadName(name); if (nameEntry != null) { - nameEntry.setIndexed(false); nameEntry.setState(State.NEW); - entryService.saveName(nameEntry); + nameEntryService.saveName(nameEntry); } return new ResponseEntity<>(response, HttpStatus.OK); } @@ -315,7 +308,7 @@ public ResponseEntity> deleteFromIndex(@PathVariable String * Endpoint used to remove a list of names from the index. * * @param names the names to remove from the index. - * @return a {@link org.springframework.http.ResponseEntity} representing the status of the operation. + * @return a {@link ResponseEntity} representing the status of the operation. */ @RequestMapping(value = "/indexes/batch", method = RequestMethod.DELETE, consumes = MediaType.APPLICATION_JSON_VALUE, @@ -327,7 +320,7 @@ public ResponseEntity> batchDeleteFromIndex(@RequestBody Str List notFound = new ArrayList<>(); Arrays.stream(names).forEach(name -> { - NameEntry entry = entryService.loadName(name); + NameEntry entry = nameEntryService.loadName(name); if (entry == null) { notFound.add(name); } else { @@ -341,22 +334,10 @@ public ResponseEntity> batchDeleteFromIndex(@RequestBody Str return new ResponseEntity<>(response, HttpStatus.INTERNAL_SERVER_ERROR); } - IndexOperationStatus indexOperationStatus = elasticSearchService.bulkDeleteFromIndex(found); - updateIsIndexFlag(nameEntries, false, indexOperationStatus); + IndexOperationStatus indexOperationStatus = searchService.bulkRemoveFromIndex(nameEntries); return returnStatusMessage(notFound, indexOperationStatus); } - private void updateIsIndexFlag(List nameEntries, boolean flag, - IndexOperationStatus indexOperationStatus) { - if (indexOperationStatus.getStatus()) { - List updatedNames = nameEntries.stream().map(entry -> { - entry.setIndexed(flag); - return entry; - }).collect(Collectors.toList()); - - entryService.saveNames(updatedNames); - } - } private ResponseEntity> returnStatusMessage(List notFound, IndexOperationStatus indexOperationStatus) { diff --git a/webapi/src/test/java/org/oruko/dictionary/web/rest/NameApiTest.java b/webapi/src/test/java/org/oruko/dictionary/web/rest/NameApiTest.java index 4a60a963..5014bafe 100644 --- a/webapi/src/test/java/org/oruko/dictionary/web/rest/NameApiTest.java +++ b/webapi/src/test/java/org/oruko/dictionary/web/rest/NameApiTest.java @@ -93,18 +93,6 @@ public void test_get_all_names_via_get() throws Exception { .andExpect(status().isOk()); } - @Test - public void test_get_all_names_filtered_by_is_indexed() throws Exception { - testNameEntry.setIndexed(true); - anotherTestNameEntry.setIndexed(false); - when(entryService.loadAllNames()).thenReturn(Arrays.asList(testNameEntry, anotherTestNameEntry)); - - mockMvc.perform(get("/v1/names?all=true&indexed=true")) - .andExpect(jsonPath("$", hasSize(1))) - .andExpect(jsonPath("$[0].name", is("test-entry"))) - .andExpect(status().isOk()); - } - @Test public void test_get_all_names_filtered_by_is_submitted_by() throws Exception { testNameEntry.setSubmittedBy("test"); diff --git a/webapi/src/test/java/org/oruko/dictionary/web/rest/SearchApiTest.java b/webapi/src/test/java/org/oruko/dictionary/web/rest/SearchApiTest.java index 58cfb540..d85754b5 100644 --- a/webapi/src/test/java/org/oruko/dictionary/web/rest/SearchApiTest.java +++ b/webapi/src/test/java/org/oruko/dictionary/web/rest/SearchApiTest.java @@ -1,20 +1,24 @@ package org.oruko.dictionary.web.rest; -import org.junit.*; +import org.junit.Before; +import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.*; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import org.oruko.dictionary.elasticsearch.ElasticSearchService; import org.oruko.dictionary.events.EventPubService; +import org.oruko.dictionary.search.api.SearchService; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import java.util.Collections; - import static org.hamcrest.CoreMatchers.is; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; /** * Created by Dadepo Aderemi. @@ -27,7 +31,7 @@ public class SearchApiTest extends AbstractApiTest { MockMvc mockMvc; @Mock - ElasticSearchService searchService; + SearchService searchService; @Mock EventPubService eventPubService; @@ -40,12 +44,12 @@ public void setUp() { @Test public void testMetadata() throws Exception { - when(searchService.getCount()).thenReturn(3L); + when(searchService.getSearchableNames()).thenReturn(3); mockMvc.perform(get("/v1/search/meta")) .andExpect(jsonPath("$.totalPublishedNames", is(3))) .andExpect(status().isOk()); - verify(searchService).getCount(); + verify(searchService).getSearchableNames(); } @Test @@ -72,11 +76,10 @@ public void test_auto_complete_with_empty_query() throws Exception { @Test public void testFindByName_NameNotFound() throws Exception { - when(searchService.getByName("query")).thenReturn(Collections.emptyMap()); - mockMvc.perform(get("/v1/search/query")) - .andExpect(jsonPath("$").isEmpty()) - .andExpect(status().isOk()); + when(searchService.getByName("searchTerm")).thenReturn(null); + mockMvc.perform(get("/v1/search/searchTerm")) + .andExpect(content().string("")) + .andExpect(status().isOk()); } - } diff --git a/website/pom.xml b/website/pom.xml index 806bd570..ca492441 100644 --- a/website/pom.xml +++ b/website/pom.xml @@ -24,6 +24,12 @@ + + ${project.groupId} + jpa-search-module + ${project.version} + + ${project.groupId} bootstrap-module diff --git a/website/src/main/java/org/oruko/dictionary/DictionaryApplication.java b/website/src/main/java/org/oruko/dictionary/DictionaryApplication.java index 6b5c8989..eb29df95 100644 --- a/website/src/main/java/org/oruko/dictionary/DictionaryApplication.java +++ b/website/src/main/java/org/oruko/dictionary/DictionaryApplication.java @@ -98,7 +98,6 @@ public ReloadableResourceBundleMessageSource messageSource() { return source; } - @Bean public net.sf.ehcache.CacheManager ecacheManager() { CacheConfiguration allNames = new CacheConfiguration(); @@ -127,7 +126,6 @@ public net.sf.ehcache.CacheManager ecacheManager() { return net.sf.ehcache.CacheManager.newInstance(config); } - @Bean public CacheManager cacheManager() { return new EhCacheCacheManager(ecacheManager()); } diff --git a/website/src/main/java/org/oruko/dictionary/website/ApiService.java b/website/src/main/java/org/oruko/dictionary/website/ApiService.java index 8cce30aa..985fc2ea 100644 --- a/website/src/main/java/org/oruko/dictionary/website/ApiService.java +++ b/website/src/main/java/org/oruko/dictionary/website/ApiService.java @@ -1,5 +1,6 @@ package org.oruko.dictionary.website; +import org.oruko.dictionary.model.NameEntry; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; @@ -32,8 +33,8 @@ public List> getAllNames() { } @Cacheable("querySearchResult") - public Map getName(String nameQuery) { - return restTemplate.getForObject(APIPATH + "/search/" + nameQuery, Map.class); + public NameEntry getName(String nameQuery) { + return restTemplate.getForObject(APIPATH + "/search/" + nameQuery, NameEntry.class); } @Cacheable("names") diff --git a/website/src/main/java/org/oruko/dictionary/website/SearchResultController.java b/website/src/main/java/org/oruko/dictionary/website/SearchResultController.java index 17f62fc7..9467c2b2 100644 --- a/website/src/main/java/org/oruko/dictionary/website/SearchResultController.java +++ b/website/src/main/java/org/oruko/dictionary/website/SearchResultController.java @@ -1,6 +1,7 @@ package org.oruko.dictionary.website; import org.apache.commons.lang3.StringUtils; +import org.oruko.dictionary.model.NameEntry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Controller; @@ -52,7 +53,7 @@ public String showEntry(@PathVariable String nameEntry, Model map) { map.addAttribute("title", "Name Entry"); map.addAttribute("host", host); if (!map.containsAttribute("name")) { - final Map name = apiService.getName(nameEntry); + final NameEntry name = apiService.getName(nameEntry); if (name == null) { // no single entry found for query, return to view where search result can be displayed return "redirect:/entries?q=" + nameEntry; @@ -85,7 +86,6 @@ public String searchNameQuery(@RequestParam(value = "q",required = false) String return "redirect:/entries/"+nameQuery; } - Collections.reverse(names); map.addAttribute("query", nameQuery); map.addAttribute("names", names); diff --git a/website/src/main/resources/website/home.hbs b/website/src/main/resources/website/home.hbs index b5fde7c5..b049755c 100644 --- a/website/src/main/resources/website/home.hbs +++ b/website/src/main/resources/website/home.hbs @@ -69,8 +69,8 @@ - - + diff --git a/website/src/main/resources/website/singleresult.hbs b/website/src/main/resources/website/singleresult.hbs index 7fb89d62..f95fad85 100644 --- a/website/src/main/resources/website/singleresult.hbs +++ b/website/src/main/resources/website/singleresult.hbs @@ -9,7 +9,7 @@ - {{normalize name.name }} + {{name.name }} {{message "lang.pronunciation"}} @@ -32,7 +32,7 @@ - {{message "lang.meaningof"}} {{normalize name.name }} + {{message "lang.meaningof"}} {{name.name }} {{ name.meaning }}
{{message "lang.pronunciation"}} @@ -32,7 +32,7 @@
{{ name.meaning }}