From e66fef5e8f43a9896f24eaf91abeb8235e0ddfd8 Mon Sep 17 00:00:00 2001 From: Andrew Piechota <46939468+andrewcp54@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:19:37 -0600 Subject: [PATCH] Mirror mishap, update mapillary w/ v4 functionality (#68) --- backend/__init__.py | 7 +- backend/services/mapillary_service.py | 151 +++++++------------------- 2 files changed, 39 insertions(+), 119 deletions(-) diff --git a/backend/__init__.py b/backend/__init__.py index 3689c33c2b..b220962559 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -287,7 +287,7 @@ def add_api_endpoints(app): from backend.api.system.image_upload import SystemImageUploadRestAPI # Mapillary API endpoint - from backend.api.mapillary.mapillary import MapillaryTasksAPI, SequencesAsGPX + from backend.api.mapillary.mapillary import MapillaryTasksAPI # Projects REST endpoint api.add_resource(ProjectsAllAPI, format_url("projects/"), methods=["GET"]) @@ -559,11 +559,6 @@ def add_api_endpoints(app): methods=["GET", "POST"], ) - # Mapillary Projects as sequences - api.add_resource( - SequencesAsGPX, format_url("projects//sequences-as-gpx") - ) - # Annotations REST endoints api.add_resource( AnnotationsRestAPI, diff --git a/backend/services/mapillary_service.py b/backend/services/mapillary_service.py index 902676f015..b7a131b0d6 100644 --- a/backend/services/mapillary_service.py +++ b/backend/services/mapillary_service.py @@ -1,10 +1,7 @@ -import json import datetime import requests from shapely.geometry import shape, mapping, LineString from shapely.ops import linemerge -import xml.etree.ElementTree as ET -from backend.models.postgis.task import Task import os import math import mapbox_vector_tile @@ -64,7 +61,7 @@ def __init__( geom: Optional[List[dict[str, Any]]] = None, download_additional: IntFlag = Bounds.NONE, cutoff_keys: Optional[Set[Tuple[str, Bounds]]] = None, - client_id: Optional[str] = "", + api: Optional[dict[str, str]] = {}, ): self.x_coordinate = x_coordinate self.y_coordinate = y_coordinate @@ -76,13 +73,16 @@ def __init__( self.tile_bounds = BBox.tile_latlon_bounds( self.x_coordinate, self.y_coordinate, self.zoom ) + self.api = api # build tile's mapillary url from properties - # TODO update for mapillary v4 - self.url = ( - "https://tiles3.mapillary.com/v0.1/" - + f"{self.zoom}/{self.x_coordinate}/{self.y_coordinate}.mvt" - + (f"?client_id={client_id}" if client_id else "") - ) + if self.api: + self.url = ( + api["tile_base"] + + api["coverage_tiles"] + + f"{self.zoom}/{self.x_coordinate}/{self.y_coordinate}" + + "?access_token=" + + api["access_token"] + ) def __repr__(self) -> str: return f"{self.zoom}{os.path.sep}{self.x_coordinate}{os.path.sep}{self.y_coordinate}" @@ -130,7 +130,7 @@ def compute_cutoff_keys(self, tile_list): and (self.zoom, self.x_coordinate - 1, self.y_coordinate) not in tile_list ): - keys.add((f["properties"]["key"], Bounds.WEST)) + keys.add((f["properties"]["id"], Bounds.WEST)) self.download_additional = self.download_additional | Bounds.WEST if ( any( @@ -140,7 +140,7 @@ def compute_cutoff_keys(self, tile_list): and (self.zoom, self.x_coordinate + 1, self.y_coordinate) not in tile_list ): - keys.add((f["properties"]["key"], Bounds.EAST)) + keys.add((f["properties"]["id"], Bounds.EAST)) self.download_additional = self.download_additional | Bounds.EAST if ( any( @@ -150,7 +150,7 @@ def compute_cutoff_keys(self, tile_list): and (self.zoom, self.x_coordinate, self.y_coordinate + 1) not in tile_list ): - keys.add((f["properties"]["key"], Bounds.SOUTH)) + keys.add((f["properties"]["id"], Bounds.SOUTH)) self.download_additional = self.download_additional | Bounds.SOUTH if ( any( @@ -160,7 +160,7 @@ def compute_cutoff_keys(self, tile_list): and (self.zoom, self.x_coordinate, self.y_coordinate - 1) not in tile_list ): - keys.add((f["properties"]["key"], Bounds.NORTH)) + keys.add((f["properties"]["id"], Bounds.NORTH)) self.download_additional = self.download_additional | Bounds.NORTH self.cutoff_keys = keys @@ -250,7 +250,7 @@ def geom_in_bounds(feature, tile, bounding_box): def bbox_to_tiles( - bbox: BBox, zoom: int, client_id: Optional[str] = "" + bbox: BBox, zoom: int, mapillary_api: dict[str, str] ) -> Generator[Tile, None, None]: """Convert a bbox to a series of tiles""" tile_lower_left: Tile = Tile.lat_lon_to_tile(bbox.minLat, bbox.minLon, zoom) @@ -264,7 +264,7 @@ def bbox_to_tiles( x_coordinate: int = tile_lower_left.x_coordinate while y_coordinate >= tile_upper_right.y_coordinate: while x_coordinate <= tile_upper_right.x_coordinate: - yield Tile(x_coordinate, y_coordinate, zoom, client_id=client_id) + yield Tile(x_coordinate, y_coordinate, zoom, api=mapillary_api) x_coordinate += 1 y_coordinate -= 1 x_coordinate = tile_lower_left.x_coordinate @@ -273,11 +273,9 @@ def bbox_to_tiles( def deduplicate_data(geometry): """ Merge all valid LineString data with same key into one contiguous MultiLineString/LineString """ for feature in geometry: - current_key = feature["properties"]["key"] + current_key = feature["properties"]["id"] dup_features = [ - f - for f in geometry - if f["properties"]["key"] == current_key and f != feature + f for f in geometry if f["properties"]["id"] == current_key and f != feature ] if len(dup_features) > 0: dup_shapes = [shape(f["geometry"]) for f in dup_features] @@ -319,7 +317,7 @@ def compute_tasks(geom): ).total_seconds() ) <= MAPILLARY_TIME_CORRELATION - and f["properties"]["key"] != feature["properties"]["key"] + and f["properties"]["id"] != feature["properties"]["id"] ) ]: # TODO possibly find any additional geometry that is within the @@ -341,7 +339,7 @@ def compute_tasks(geom): geojson.Feature( geometry=mapping(shapely_geom), properties={ - "mapillary": [t["properties"]["key"] for t in task], + "mapillary": [t["properties"]["id"] for t in task], "x": task[0]["properties"]["x"], "y": task[0]["properties"]["y"], "zoom": task[0]["properties"]["zoom"], @@ -352,7 +350,7 @@ def compute_tasks(geom): return project_tasks -def fetch_additional_tiles(project_tiles, client_id): +def fetch_additional_tiles(project_tiles, mapillary_api): project_tile_coordinates = [p_t.get_tile_coordinates() for p_t in project_tiles] additional_tiles = set() for t in project_tiles: @@ -364,7 +362,7 @@ def fetch_additional_tiles(project_tiles, client_id): t.x_coordinate, t.y_coordinate + 1, t.zoom, - client_id=client_id, + api=mapillary_api, ), tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.SOUTH]), ) @@ -376,7 +374,7 @@ def fetch_additional_tiles(project_tiles, client_id): t.x_coordinate, t.y_coordinate - 1, t.zoom, - client_id=client_id, + api=mapillary_api, ), tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.NORTH]), ) @@ -388,7 +386,7 @@ def fetch_additional_tiles(project_tiles, client_id): t.x_coordinate - 1, t.y_coordinate, t.zoom, - client_id=client_id, + api=mapillary_api, ), tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.WEST]), ) @@ -400,7 +398,7 @@ def fetch_additional_tiles(project_tiles, client_id): t.x_coordinate + 1, t.y_coordinate, t.zoom, - client_id=client_id, + api=mapillary_api, ), tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.EAST]), ) @@ -410,12 +408,12 @@ def fetch_additional_tiles(project_tiles, client_id): r = requests.get(tile.url) if r and r.status_code == 200: decoded_data = mapbox_vector_tile.decode(r.content) - if "mapillary-sequences" in decoded_data: - decoded_seq = decoded_data["mapillary-sequences"] + if "sequence" in decoded_data: + decoded_seq = decoded_data["sequence"] tile.set_extent(int(decoded_seq["extent"])) tile_geom = [] for f in decoded_seq["features"]: - if f["properties"]["key"] in keys: + if f["properties"]["id"] in keys: tile_geom.append(f) if ( len(tile_geom) > 0 @@ -429,7 +427,7 @@ def fetch_additional_tiles(project_tiles, client_id): tile.compute_cutoff_keys(project_tile_coordinates) if any(tile.download_additional for tile, keys in additional_tiles): - fetch_additional_tiles(project_tiles, client_id) + fetch_additional_tiles(project_tiles, mapillary_api) class MapillaryService: @@ -442,11 +440,11 @@ def getMapillarySequences(parameters: dict): raise ValueError("parameters must have at least a start or an end time") start_epoch, end_epoch = None, None - # Kaart org key - org_keys = ["O0K377md1CVrVkzGu4PV5z"] + # Kaart org id + org_ids = [1805883732926354] if "organization_key" in parameters: - org_keys.extend(parameters["organization_key"].split(",")) + org_ids.extend(parameters["organization_key"].split(",")) if "start_time" in parameters: start_epoch = datetime.datetime.strptime( @@ -462,7 +460,7 @@ def getMapillarySequences(parameters: dict): # BBox expects minLat, minLon, maxLat, maxLon bbox = BBox(bbox_list[1], bbox_list[0], bbox_list[3], bbox_list[2]) z = 14 - initial_tiles = bbox_to_tiles(bbox, z, MAPILLARY_API["clientId"]) + initial_tiles = bbox_to_tiles(bbox, z, MAPILLARY_API) # TODO async this? project_tiles = [] @@ -470,8 +468,8 @@ def getMapillarySequences(parameters: dict): r = requests.get(tile.url) if r and r.status_code == 200: decoded_data = mapbox_vector_tile.decode(r.content) - if "mapillary-sequences" in decoded_data: - decoded_seq = decoded_data["mapillary-sequences"] + if "sequence" in decoded_data: + decoded_seq = decoded_data["sequence"] tile.set_extent(int(decoded_seq["extent"])) tile_geom = [] for f in decoded_seq["features"]: @@ -497,8 +495,8 @@ def getMapillarySequences(parameters: dict): ) ): if ( - "organization_key" in f["properties"] - and f["properties"]["organization_key"] in org_keys + "organization_id" in f["properties"] + and f["properties"]["organization_id"] in org_ids and geom_in_bounds(f, tile, bbox) ): tile_geom.append(f) @@ -515,7 +513,7 @@ def getMapillarySequences(parameters: dict): ) if any(tile.download_additional for tile in project_tiles): - fetch_additional_tiles(project_tiles, MAPILLARY_API["clientId"]) + fetch_additional_tiles(project_tiles, MAPILLARY_API) project_geom = [] for t in project_tiles: @@ -525,76 +523,3 @@ def getMapillarySequences(parameters: dict): deduplicated_geom = deduplicate_data(project_geom) project_tasks = compute_tasks(deduplicated_geom) return geojson.FeatureCollection(project_tasks) - - @staticmethod - def getSequencesAsGPX(project_id: int, task_ids_str: str): - MAPILLARY_API = current_app.config["MAPILLARY_API"] - url = ( - MAPILLARY_API["base"] - + "sequences/{}?client_id=" - + MAPILLARY_API["clientId"] - ) - headers = {"Accept": "application/gpx+xml"} - - timestamp = datetime.datetime.utcnow() - - XMLNS_NAMESPACE = "http://www.topografix.com/GPX/1/1" - XMLNS = "{%s}" % XMLNS_NAMESPACE - XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" - XSI = "{%s}" % XSI_NAMESPACE - - root = ET.Element( - "gpx", - attrib=dict( - xmlns=XMLNS_NAMESPACE, - version="1.1", - creator="Kaart Tasking Manager", - ), - ) - root.set(XMLNS + "xsi", XSI_NAMESPACE) - root.set( - XSI + "schemaLocation", - "http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx .xsd", - ) - - # Create GPX Metadata element - metadata = ET.Element("metadata") - link = ET.SubElement( - metadata, - "link", - attrib=dict(href="https://github.com/kaartgroup/tasking-manager"), - ) - ET.SubElement(link, "text").text = "Kaart Tasking Manager" - ET.SubElement(metadata, "time").text = timestamp.isoformat() - root.append(metadata) - - # Create trk element - trk = ET.Element("trk") - root.append(trk) - ET.SubElement( - trk, "name" - ).text = f"Task for project {project_id}. Do not edit outside of this area!" - - # Create trkseg element - trkseg = ET.Element("trkseg") - trk.append(trkseg) - - if task_ids_str is not None: - task_ids = map(int, task_ids_str.split(",")) - tasks = Task.get_tasks(project_id, task_ids) - if not tasks or tasks.count() == 0: - raise NotFound() - else: - tasks = Task.get_all_tasks(project_id) - if not tasks or len(tasks) == 0: - raise NotFound() - - for task in tasks: - key = json.loads(task.extra_properties)["mapillary"]["sequence_key"] - gpx = requests.get(url.format(key), headers=headers).content - root2 = ET.fromstring(gpx) - for trkpt in root2.iter("{http://www.topografix.com/GPX/1/1}trkpt"): - ET.SubElement(trkseg, "trkpt", attrib=trkpt.attrib) - - sequences_gpx = ET.tostring(root, encoding="utf8") - return sequences_gpx