Skip to content

Commit

Permalink
Mirror mishap, update mapillary w/ v4 functionality (#68)
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrewcp54
andrewcp54 committed Jun 17, 2021
1 parent d1389be commit a8295e9
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 119 deletions.
7 changes: 1 addition & 6 deletions backend/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ def add_api_endpoints(app):
from backend.api.system.image_upload import SystemImageUploadRestAPI

# Mapillary API endpoint
from backend.api.mapillary.mapillary import MapillaryTasksAPI, SequencesAsGPX
from backend.api.mapillary.mapillary import MapillaryTasksAPI

# Projects REST endpoint
api.add_resource(ProjectsAllAPI, format_url("projects/"), methods=["GET"])
Expand Down Expand Up @@ -559,11 +559,6 @@ def add_api_endpoints(app):
methods=["GET", "POST"],
)

# Mapillary Projects as sequences
api.add_resource(
SequencesAsGPX, format_url("projects/<int:project_id>/sequences-as-gpx")
)

# Annotations REST endoints
api.add_resource(
AnnotationsRestAPI,
Expand Down
151 changes: 38 additions & 113 deletions backend/services/mapillary_service.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import json
import datetime
import requests
from shapely.geometry import shape, mapping, LineString
from shapely.ops import linemerge
import xml.etree.ElementTree as ET
from backend.models.postgis.task import Task
import os
import math
import mapbox_vector_tile
Expand Down Expand Up @@ -64,7 +61,7 @@ def __init__(
geom: Optional[List[dict[str, Any]]] = None,
download_additional: IntFlag = Bounds.NONE,
cutoff_keys: Optional[Set[Tuple[str, Bounds]]] = None,
client_id: Optional[str] = "",
api: Optional[dict[str, str]] = {},
):
self.x_coordinate = x_coordinate
self.y_coordinate = y_coordinate
Expand All @@ -76,13 +73,16 @@ def __init__(
self.tile_bounds = BBox.tile_latlon_bounds(
self.x_coordinate, self.y_coordinate, self.zoom
)
self.api = api
# build tile's mapillary url from properties
# TODO update for mapillary v4
self.url = (
"https://tiles3.mapillary.com/v0.1/"
+ f"{self.zoom}/{self.x_coordinate}/{self.y_coordinate}.mvt"
+ (f"?client_id={client_id}" if client_id else "")
)
if self.api:
self.url = (
api["tile_base"]
+ api["coverage_tiles"]
+ f"{self.zoom}/{self.x_coordinate}/{self.y_coordinate}"
+ "?access_token="
+ api["access_token"]
)

def __repr__(self) -> str:
return f"{self.zoom}{os.path.sep}{self.x_coordinate}{os.path.sep}{self.y_coordinate}"
Expand Down Expand Up @@ -130,7 +130,7 @@ def compute_cutoff_keys(self, tile_list):
and (self.zoom, self.x_coordinate - 1, self.y_coordinate)
not in tile_list
):
keys.add((f["properties"]["key"], Bounds.WEST))
keys.add((f["properties"]["id"], Bounds.WEST))
self.download_additional = self.download_additional | Bounds.WEST
if (
any(
Expand All @@ -140,7 +140,7 @@ def compute_cutoff_keys(self, tile_list):
and (self.zoom, self.x_coordinate + 1, self.y_coordinate)
not in tile_list
):
keys.add((f["properties"]["key"], Bounds.EAST))
keys.add((f["properties"]["id"], Bounds.EAST))
self.download_additional = self.download_additional | Bounds.EAST
if (
any(
Expand All @@ -150,7 +150,7 @@ def compute_cutoff_keys(self, tile_list):
and (self.zoom, self.x_coordinate, self.y_coordinate + 1)
not in tile_list
):
keys.add((f["properties"]["key"], Bounds.SOUTH))
keys.add((f["properties"]["id"], Bounds.SOUTH))
self.download_additional = self.download_additional | Bounds.SOUTH
if (
any(
Expand All @@ -160,7 +160,7 @@ def compute_cutoff_keys(self, tile_list):
and (self.zoom, self.x_coordinate, self.y_coordinate - 1)
not in tile_list
):
keys.add((f["properties"]["key"], Bounds.NORTH))
keys.add((f["properties"]["id"], Bounds.NORTH))
self.download_additional = self.download_additional | Bounds.NORTH
self.cutoff_keys = keys

Expand Down Expand Up @@ -250,7 +250,7 @@ def geom_in_bounds(feature, tile, bounding_box):


def bbox_to_tiles(
bbox: BBox, zoom: int, client_id: Optional[str] = ""
bbox: BBox, zoom: int, mapillary_api: dict[str, str]
) -> Generator[Tile, None, None]:
"""Convert a bbox to a series of tiles"""
tile_lower_left: Tile = Tile.lat_lon_to_tile(bbox.minLat, bbox.minLon, zoom)
Expand All @@ -264,7 +264,7 @@ def bbox_to_tiles(
x_coordinate: int = tile_lower_left.x_coordinate
while y_coordinate >= tile_upper_right.y_coordinate:
while x_coordinate <= tile_upper_right.x_coordinate:
yield Tile(x_coordinate, y_coordinate, zoom, client_id=client_id)
yield Tile(x_coordinate, y_coordinate, zoom, api=mapillary_api)
x_coordinate += 1
y_coordinate -= 1
x_coordinate = tile_lower_left.x_coordinate
Expand All @@ -273,11 +273,9 @@ def bbox_to_tiles(
def deduplicate_data(geometry):
""" Merge all valid LineString data with same key into one contiguous MultiLineString/LineString """
for feature in geometry:
current_key = feature["properties"]["key"]
current_key = feature["properties"]["id"]
dup_features = [
f
for f in geometry
if f["properties"]["key"] == current_key and f != feature
f for f in geometry if f["properties"]["id"] == current_key and f != feature
]
if len(dup_features) > 0:
dup_shapes = [shape(f["geometry"]) for f in dup_features]
Expand Down Expand Up @@ -319,7 +317,7 @@ def compute_tasks(geom):
).total_seconds()
)
<= MAPILLARY_TIME_CORRELATION
and f["properties"]["key"] != feature["properties"]["key"]
and f["properties"]["id"] != feature["properties"]["id"]
)
]:
# TODO possibly find any additional geometry that is within the
Expand All @@ -341,7 +339,7 @@ def compute_tasks(geom):
geojson.Feature(
geometry=mapping(shapely_geom),
properties={
"mapillary": [t["properties"]["key"] for t in task],
"mapillary": [t["properties"]["id"] for t in task],
"x": task[0]["properties"]["x"],
"y": task[0]["properties"]["y"],
"zoom": task[0]["properties"]["zoom"],
Expand All @@ -352,7 +350,7 @@ def compute_tasks(geom):
return project_tasks


def fetch_additional_tiles(project_tiles, client_id):
def fetch_additional_tiles(project_tiles, mapillary_api):
project_tile_coordinates = [p_t.get_tile_coordinates() for p_t in project_tiles]
additional_tiles = set()
for t in project_tiles:
Expand All @@ -364,7 +362,7 @@ def fetch_additional_tiles(project_tiles, client_id):
t.x_coordinate,
t.y_coordinate + 1,
t.zoom,
client_id=client_id,
api=mapillary_api,
),
tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.SOUTH]),
)
Expand All @@ -376,7 +374,7 @@ def fetch_additional_tiles(project_tiles, client_id):
t.x_coordinate,
t.y_coordinate - 1,
t.zoom,
client_id=client_id,
api=mapillary_api,
),
tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.NORTH]),
)
Expand All @@ -388,7 +386,7 @@ def fetch_additional_tiles(project_tiles, client_id):
t.x_coordinate - 1,
t.y_coordinate,
t.zoom,
client_id=client_id,
api=mapillary_api,
),
tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.WEST]),
)
Expand All @@ -400,7 +398,7 @@ def fetch_additional_tiles(project_tiles, client_id):
t.x_coordinate + 1,
t.y_coordinate,
t.zoom,
client_id=client_id,
api=mapillary_api,
),
tuple([k[0] for k in t.cutoff_keys if k[1] == Bounds.EAST]),
)
Expand All @@ -410,12 +408,12 @@ def fetch_additional_tiles(project_tiles, client_id):
r = requests.get(tile.url)
if r and r.status_code == 200:
decoded_data = mapbox_vector_tile.decode(r.content)
if "mapillary-sequences" in decoded_data:
decoded_seq = decoded_data["mapillary-sequences"]
if "sequence" in decoded_data:
decoded_seq = decoded_data["sequence"]
tile.set_extent(int(decoded_seq["extent"]))
tile_geom = []
for f in decoded_seq["features"]:
if f["properties"]["key"] in keys:
if f["properties"]["id"] in keys:
tile_geom.append(f)
if (
len(tile_geom) > 0
Expand All @@ -429,7 +427,7 @@ def fetch_additional_tiles(project_tiles, client_id):
tile.compute_cutoff_keys(project_tile_coordinates)

if any(tile.download_additional for tile, keys in additional_tiles):
fetch_additional_tiles(project_tiles, client_id)
fetch_additional_tiles(project_tiles, mapillary_api)


class MapillaryService:
Expand All @@ -442,11 +440,11 @@ def getMapillarySequences(parameters: dict):
raise ValueError("parameters must have at least a start or an end time")

start_epoch, end_epoch = None, None
# Kaart org key
org_keys = ["O0K377md1CVrVkzGu4PV5z"]
# Kaart org id
org_ids = [1805883732926354]

if "organization_key" in parameters:
org_keys.extend(parameters["organization_key"].split(","))
org_ids.extend(parameters["organization_key"].split(","))

if "start_time" in parameters:
start_epoch = datetime.datetime.strptime(
Expand All @@ -462,16 +460,16 @@ def getMapillarySequences(parameters: dict):
# BBox expects minLat, minLon, maxLat, maxLon
bbox = BBox(bbox_list[1], bbox_list[0], bbox_list[3], bbox_list[2])
z = 14
initial_tiles = bbox_to_tiles(bbox, z, MAPILLARY_API["clientId"])
initial_tiles = bbox_to_tiles(bbox, z, MAPILLARY_API)

# TODO async this?
project_tiles = []
for tile in initial_tiles:
r = requests.get(tile.url)
if r and r.status_code == 200:
decoded_data = mapbox_vector_tile.decode(r.content)
if "mapillary-sequences" in decoded_data:
decoded_seq = decoded_data["mapillary-sequences"]
if "sequence" in decoded_data:
decoded_seq = decoded_data["sequence"]
tile.set_extent(int(decoded_seq["extent"]))
tile_geom = []
for f in decoded_seq["features"]:
Expand All @@ -497,8 +495,8 @@ def getMapillarySequences(parameters: dict):
)
):
if (
"organization_key" in f["properties"]
and f["properties"]["organization_key"] in org_keys
"organization_id" in f["properties"]
and f["properties"]["organization_id"] in org_ids
and geom_in_bounds(f, tile, bbox)
):
tile_geom.append(f)
Expand All @@ -515,7 +513,7 @@ def getMapillarySequences(parameters: dict):
)

if any(tile.download_additional for tile in project_tiles):
fetch_additional_tiles(project_tiles, MAPILLARY_API["clientId"])
fetch_additional_tiles(project_tiles, MAPILLARY_API)

project_geom = []
for t in project_tiles:
Expand All @@ -525,76 +523,3 @@ def getMapillarySequences(parameters: dict):
deduplicated_geom = deduplicate_data(project_geom)
project_tasks = compute_tasks(deduplicated_geom)
return geojson.FeatureCollection(project_tasks)

@staticmethod
def getSequencesAsGPX(project_id: int, task_ids_str: str):
MAPILLARY_API = current_app.config["MAPILLARY_API"]
url = (
MAPILLARY_API["base"]
+ "sequences/{}?client_id="
+ MAPILLARY_API["clientId"]
)
headers = {"Accept": "application/gpx+xml"}

timestamp = datetime.datetime.utcnow()

XMLNS_NAMESPACE = "http://www.topografix.com/GPX/1/1"
XMLNS = "{%s}" % XMLNS_NAMESPACE
XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance"
XSI = "{%s}" % XSI_NAMESPACE

root = ET.Element(
"gpx",
attrib=dict(
xmlns=XMLNS_NAMESPACE,
version="1.1",
creator="Kaart Tasking Manager",
),
)
root.set(XMLNS + "xsi", XSI_NAMESPACE)
root.set(
XSI + "schemaLocation",
"http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx .xsd",
)

# Create GPX Metadata element
metadata = ET.Element("metadata")
link = ET.SubElement(
metadata,
"link",
attrib=dict(href="https://github.com/kaartgroup/tasking-manager"),
)
ET.SubElement(link, "text").text = "Kaart Tasking Manager"
ET.SubElement(metadata, "time").text = timestamp.isoformat()
root.append(metadata)

# Create trk element
trk = ET.Element("trk")
root.append(trk)
ET.SubElement(
trk, "name"
).text = f"Task for project {project_id}. Do not edit outside of this area!"

# Create trkseg element
trkseg = ET.Element("trkseg")
trk.append(trkseg)

if task_ids_str is not None:
task_ids = map(int, task_ids_str.split(","))
tasks = Task.get_tasks(project_id, task_ids)
if not tasks or tasks.count() == 0:
raise NotFound()
else:
tasks = Task.get_all_tasks(project_id)
if not tasks or len(tasks) == 0:
raise NotFound()

for task in tasks:
key = json.loads(task.extra_properties)["mapillary"]["sequence_key"]
gpx = requests.get(url.format(key), headers=headers).content
root2 = ET.fromstring(gpx)
for trkpt in root2.iter("{http://www.topografix.com/GPX/1/1}trkpt"):
ET.SubElement(trkseg, "trkpt", attrib=trkpt.attrib)

sequences_gpx = ET.tostring(root, encoding="utf8")
return sequences_gpx

0 comments on commit a8295e9

Please sign in to comment.