Merge pull request #1000 from AtlasOfLivingAustralia/feature/biocolle…

…ct-1627

issue with stepped shape file

grails-app/controllers/au/org/ala/ecodata/AdminController.groovy

@@ -528,6 +528,17 @@ class AdminController {
         render reports as JSON
     }
 
+    @AlaSecured("ROLE_ADMIN")
+    def indexProjectDependencies() {
+        if(params.projectId){
+            List projects = params.projectId.split(',')?.toList()
+            elasticSearchService.indexDependenciesOfProjects( projects )
+            render text: [message: 'indexing completed'] as JSON, contentType: 'application/json'
+        } else {
+            render(status: HttpStatus.SC_BAD_REQUEST, text: 'projectId must be provided')
+        }
+    }
+
     /**
      * a test function to index a project.
      * @return
@@ -543,7 +554,7 @@ class AdminController {
                         Map projectMap = elasticSearchService.prepareProjectForHomePageIndex(project)
                         elasticSearchService.indexDoc(projectMap, HOMEPAGE_INDEX)
                     } catch (Exception e) {
-                        log.error("Unable to index projewt: " + project?.projectId, e)
+                        log.error("Unable to index project: " + project?.projectId, e)
                         e.printStackTrace();
                     }
                 }

grails-app/services/au/org/ala/ecodata/ActivityService.groovy

@@ -2,7 +2,9 @@ package au.org.ala.ecodata
 
 import au.org.ala.ecodata.metadata.OutputMetadata
 import au.org.ala.ecodata.metadata.OutputModelProcessor
-import com.mongodb.BasicDBObject
+import com.mongodb.client.MongoCollection
+import com.mongodb.client.model.Filters
+import org.bson.conversions.Bson
 import org.grails.datastore.mapping.query.api.BuildableCriteria
 
 import javax.persistence.PessimisticLockException
@@ -83,11 +85,13 @@ class ActivityService {
      * @param action the action to be performed on each Activity.
      * @return
      */
-    def doWithAllActivities(Closure action) {
+    def doWithAllActivities(Closure action, List<Bson> filters = [], int batchSize = 100) {
         // Due to various memory & performance issues with GORM mongo plugin 1.3, this method uses the native API.
-        def collection = Activity.getCollection()
-        BasicDBObject query = new BasicDBObject('status', ACTIVE)
-        def results = collection.find(query).batchSize(100)
+        MongoCollection collection = Activity.getCollection()
+        Bson query = Filters.eq("status", ACTIVE)
+        filters.add(query)
+        query = Filters.and(filters)
+        def results = collection.find(query).batchSize(batchSize)
 
         results.each { dbObject ->
             action.call(dbObject)

grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy

@@ -15,6 +15,8 @@
 
 package au.org.ala.ecodata
 
+
+import com.mongodb.client.model.Filters
 import grails.converters.JSON
 import grails.core.GrailsApplication
 import grails.util.Environment
@@ -68,7 +70,6 @@ import static au.org.ala.ecodata.ElasticIndex.*
 import static au.org.ala.ecodata.Status.DELETED
 import static grails.async.Promises.task
 import static org.elasticsearch.index.query.QueryBuilders.*
-
 /**
  * ElasticSearch service. This service is responsible for indexing documents as well as handling searches (queries).
  *
@@ -781,6 +782,104 @@ class ElasticSearchService {
 
     }
 
+    def indexDependenciesOfProjects(List projectIds) {
+        log.info("Started indexing of projects: ${projectIds}")
+        projectIds?.each { projectId ->
+            log.debug("Started indexing assets of project: ${projectId}")
+            indexDependenciesOfProject(projectId)
+            log.debug("Indexed assets of project: ${projectId}")
+        }
+
+        log.info("Completed indexing of projects: ${projectIds}")
+    }
+
+    def indexDependenciesOfProject (String projectId) {
+        int batchSize = 50
+        log.debug "Indexing project"
+        Project.withNewSession { session ->
+            Project project = Project.findByProjectIdAndStatusNotEqual(projectId, DELETED)
+            try {
+                Map projectMap = prepareProjectForHomePageIndex(project)
+                indexDoc(projectMap, HOMEPAGE_INDEX)
+            }
+            catch (Exception e) {
+                log.error("Unable to index project:  " + project?.projectId, e)
+            }
+
+            log.debug "Indexing sites"
+            int count = 0
+            siteService.doWithAllSites({ siteMap ->
+                siteMap["className"] = Site.class.name
+                try {
+                    siteMap = prepareSiteForIndexing(siteMap, false)
+                    if (siteMap) {
+                        indexDoc(siteMap, DEFAULT_INDEX)
+                    }
+                }
+                catch (Exception e) {
+                    log.error("Unable index site: " + siteMap?.siteId, e)
+                }
+                count++
+                if (count % 1000 == 0) {
+                    session.clear()
+                    log.info("Processed " + count + " sites")
+                }
+            }, [Filters.eq('projectId', projectId)], batchSize)
+
+            if (project.organisationId) {
+                log.debug "Indexing organisations of project"
+                organisationService.doWithAllOrganisations ({ Map org ->
+                    try {
+                        prepareOrganisationForIndexing(org)
+                        indexDoc(org, DEFAULT_INDEX)
+                    }
+                    catch (Exception e) {
+                        log.error("Unable to index organisation: " + org?.organisationId, e)
+                    }
+                }, [Filters.eq('organisationId', project.organisationId)], batchSize)
+            }
+
+            log.debug "Indexing activities"
+            count = 0
+            activityService.doWithAllActivities({ Map activity ->
+                try {
+                    activity = prepareActivityForIndexing(activity)
+                    indexDoc(activity, activity?.projectActivityId || activity?.isWorks ? PROJECT_ACTIVITY_INDEX : DEFAULT_INDEX)
+                }
+                catch (Exception e) {
+                    log.error("Unable to index activity: " + activity?.activityId, e)
+                }
+
+                count++
+                if (count % 1000 == 0) {
+                    session.clear()
+                    log.info("Processed " + count + " activities")
+                }
+            }, [Filters.eq('projectId', projectId)], batchSize)
+
+            log.debug "Indexing documents"
+            count = 0
+            Document.findAllByProjectIdAndStatusNotEqual(projectId, DELETED, [batchSize: batchSize]).each { Document document ->
+                try {
+                    Map doc = documentService.toMap(document)
+                    doc = prepareDocumentForIndexing(doc)
+                    if (doc) {
+                        indexDoc(doc, DEFAULT_INDEX)
+                    }
+                }
+                catch (Exception e) {
+                    log.error("Unable to index document: " + doc?.documentId, e)
+                }
+
+                count++
+                if (count % 100 == 0) {
+                    session.clear()
+                    log.info("Processed " + count + " documents")
+                }
+            }
+        }
+    }
+
     /**
      * Index all documents. Index is cleared first.
      */

grails-app/services/au/org/ala/ecodata/OrganisationService.groovy

@@ -1,5 +1,6 @@
 package au.org.ala.ecodata
 
+import com.mongodb.client.MongoCollection
 import com.mongodb.client.model.Filters
 import grails.validation.ValidationException
 import org.bson.conversions.Bson
@@ -179,14 +180,14 @@ class OrganisationService {
      * @param action the action to be performed on each Organisation.
      * @param filters list of filters
      */
-    void doWithAllOrganisations(Closure action, List<Filters> filters = []) {
+    void doWithAllOrganisations(Closure action, List<Bson> filters = [], int batchSize = 100) {
         // Due to various memory & performance issues with GORM mongo plugin 1.3, this method uses the native API.
-        def collection = Organisation.getCollection()
+        MongoCollection collection = Organisation.getCollection()
         //DBObject siteQuery = new QueryBuilder().start('status').notEquals(DELETED).get()
         Bson query = Filters.ne("status", DELETED)
         filters.add(query)
         query = Filters.and(filters)
-        def results = collection.find(query).batchSize(100)
+        def results = collection.find(query).batchSize(batchSize)
 
         results.each { dbObject ->
             action.call(dbObject)

grails-app/services/au/org/ala/ecodata/SiteService.groovy

@@ -6,6 +6,7 @@ import com.mongodb.client.MongoCollection
 import com.mongodb.client.MongoCursor
 import com.mongodb.client.model.Filters
 import grails.converters.JSON
+import org.bson.conversions.Bson
 import org.geotools.geojson.geom.GeometryJSON
 import org.grails.datastore.mapping.core.Session
 import org.grails.datastore.mapping.engine.event.EventType
@@ -645,8 +646,16 @@ class SiteService {
     }
 
     def geometryForPid(pid) {
-        def url = "${grailsApplication.config.getProperty('spatial.baseUrl')}/ws/shape/geojson/${pid}"
-        webService.getJson(url)
+        // getting wkt since spatial portal geojson api is not returning all precision points of lat and lng.
+        def url = "${grailsApplication.config.getProperty('spatial.baseUrl')}${spatialService.WKT_SHAPE_URL_PREFIX}${pid}"
+        def wkt = webService.get(url, false)
+        if (wkt instanceof String) {
+            return GeometryUtils.wktToGeoJson(wkt)
+        }
+        else {
+            log.error("No geometry for pid: ${pid}")
+            return null
+        }
     }
 
     def populateLocationMetadataForSite(Map site, List<String> fids = null) {
@@ -818,10 +827,13 @@ class SiteService {
      * at once.
      * @param action the action to be performed on each Activity.
      */
-    void doWithAllSites(Closure action, Integer max = null) {
+    void doWithAllSites(Closure action, List<Bson> filters = [], int batchSize = 100) {
 
         MongoCollection collection = Site.getCollection()
-        def results = collection.find(Filters.ne('status', DELETED)).batchSize(100)
+        Bson query = Filters.ne("status", DELETED)
+        filters.add(query)
+        query = Filters.and(filters)
+        def results = collection.find(query).batchSize(batchSize)
 
         results.each { dbObject ->
             action.call(dbObject)

grails-app/services/au/org/ala/ecodata/SpatialService.groovy

@@ -22,6 +22,7 @@ class SpatialService {
     static final String INTERSECTION_AREA = "intersectionAreaByFacets"
     final String GEOJSON_INTERSECT_URL_PREFIX = "/ws/intersect/geojson/"
     final String WKT_INTERSECT_URL_PREFIX = "/ws/intersect/wkt/"
+    final String WKT_SHAPE_URL_PREFIX = "/ws/shapes/wkt/"
 
     final String PID_INTERSECT_URL_PREFIX = "/ws/intersect/object/"
     final String LOOKUP_TABLE_PATH = "/data/pidIntersectCache.json"

src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy

@@ -1,5 +1,6 @@
 package au.org.ala.ecodata
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import grails.converters.JSON
 import org.apache.commons.logging.Log
 import org.apache.commons.logging.LogFactory
@@ -26,6 +27,14 @@ class GeometryUtils {
     static CoordinateReferenceSystem sourceCRS = CRS.decode("EPSG:4326", true)
     static GeometryFactory geometryFactory = new GeometryFactory()
 
+    static Map wktToGeoJson(String wkt, int decimals = 20) {
+        WKTReader wktReader = new WKTReader()
+        Geometry geom = wktReader.read(wkt)
+        String geoJSON = new GeometryJSON(decimals).toString(geom)
+        ObjectMapper mapper = new ObjectMapper()
+        return mapper.readValue(geoJSON, Map)
+    }
+
     static String wktToMultiPolygonWkt(String wkt) {
         Geometry geom = new WKTReader().read(wkt)
 

src/test/groovy/au/org/ala/ecodata/GeometryUtilsSpec.groovy

@@ -80,4 +80,30 @@ class GeometryUtilsSpec extends Specification {
         features[3].properties.type == 1
 
     }
+
+    def "wktToGeoJson should correctly convert WKT to GeoJSON"() {
+        given: "A WKT string representing a point"
+        String wkt = "POINT (30 10)"
+
+        when:
+        Map result = GeometryUtils.wktToGeoJson(wkt)
+
+        then:
+        result != null
+        result.type == "Point"
+        result.coordinates == [30.0, 10.0]
+    }
+
+    def "wktToGeoJson should handle precision with the specified number of decimals"() {
+        given: "A WKT string representing a geometry with decimals"
+        String wkt = "POINT (30.1234567890123456789 10.1234567890123456789)"
+
+        when:
+        Map result = GeometryUtils.wktToGeoJson(wkt, 5)
+
+        then: "The result has the coordinates rounded to the specified number of decimals"
+        result != null
+        result.type == "Point"
+        result.coordinates == [30.12346, 10.12346]
+    }
 }

src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy

@@ -202,7 +202,7 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
 
 
         then:
-        1 * webServiceMock.getJson(_) >>  [type:'Polygon', coordinates: [[137, -34], [137,-35], [136, -35], [136, -34], [137, -34]]]
+        1 * webServiceMock.get(_, _) >>  "POLYGON ((137 -34, 137 -35, 136 -35, 136 -34, 137 -34))"
         1 * spatialServiceMock.intersectPid('cl123', null, null) >> [state:'state1', test:'test']
 
         def site = Site.findBySiteId (result.siteId)