Job output transform

Makefile

@@ -17,6 +17,7 @@ DOCKER_REPO ?= pavics/weaver
 # guess OS (Linux, Darwin,...)
 OS_NAME := $(shell uname -s 2>/dev/null || echo "unknown")
 CPU_ARCH := $(shell uname -m 2>/dev/null || uname -p 2>/dev/null || echo "unknown")
+SUDO ?=
 
 # conda
 CONDA_CMD      ?= __EMPTY__
@@ -228,10 +229,10 @@ conda-env-export:		## export the conda environment
 install: install-all    ## alias for 'install-all' target
 
 .PHONY: install-run
-install-run: conda-install install-sys install-pkg install-raw 	## install requirements and application to run locally
+install-run: conda-install install-sys install-pkg install-raw install-dev install-transform ## install requirements and application to run locally
 
 .PHONY: install-all
-install-all: conda-install install-sys install-pkg install-pip install-dev  ## install application with all dependencies
+install-all: conda-install install-sys install-pkg install-pip install-dev install-transform ## install application with all dependencies
 
 .PHONY: install-doc
 install-doc: install-pip	## install documentation dependencies
@@ -274,7 +275,7 @@ install-raw:	## install without any requirements or dependencies (suppose everyt
 install-npm:	## install npm package manager and dependencies if they cannot be found
 	@[ -f "$(shell which npm)" ] || ( \
 		echo "Binary package manager npm not found. Attempting to install it."; \
-		apt-get install npm \
+		$(SUDO) apt-get install npm \
 	)
 
 .PHONY: install-npm-stylelint
@@ -291,6 +292,16 @@ install-npm-remarklint: install-npm		## install remark-lint dependency for 'chec
 		npm install --save-dev \
 	)
 
+.PHONY: install-transform
+install-transform: install-cairo-dependencies       # install-transform dependencies
+
+.PHONY: install-cairo-dependencies
+install-cairo-dependencies:   ## install required dependencies for Transformer
+	@[ -f "$(shell which cairo)" ] || ( \
+		echo "Binary package manager cairo not found. Attempting to install it."; \
+		$(SUDO) apt-get install libpangocairo-1.0-0 \
+	)
+
 .PHONY: install-dev-npm
 install-dev-npm: install-npm install-npm-remarklint install-npm-remarklint  ## install all npm development dependencies
 
@@ -833,6 +844,7 @@ docker-clean:  ## remove all built docker images (only matching current/latest v
 	docker rmi -f "$(APP_NAME):latest" || true
 	docker rmi -f "$(APP_NAME):base" || true
 
+
 ## -- Launchers targets --------------------------------------------------------------------------------------------- ##
 
 .PHONY: start

config/weaver.ini.example

@@ -257,4 +257,4 @@ level = NOTSET
 formatter = generic
 
 [formatter_generic]
-format = [%(asctime)s] %(levelname)-8.8s [%(threadName)s][%(name)s] %(message)s
+format = [%(asctime)s] %(levelname)-8.8s [%(threadName)s][%(name)s] %(message)s

docker/Dockerfile-base

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         g++ \
         git \
         nodejs \
+        libpangocairo-1.0-0 \
     && pip install --no-cache-dir --upgrade -r requirements-sys.txt \
     && pip install --no-cache-dir -r requirements.txt \
     && pip install --no-cache-dir -e ${APP_DIR} \

requirements-dev.txt

@@ -41,3 +41,13 @@ typing_extensions
 WebTest
 wsgiproxy
 WSGIProxy2
+
+# transformer
+fpdf
+Pillow
+pandas
+cairosvg
+multipagetiff
+# gdal
+rasterio
+python-magic

requirements.txt

@@ -58,6 +58,7 @@ gunicorn>=22
 # even more reduced dependency constraints (https://github.com/vinitkumar/json2xml/pull/195)
 json2xml==4.1.0
 jsonschema>=3.0.1
+
 # FIXME: kombu for pymongo>=4 not yet released as 5.3.0 (only pre-releases available)
 # - https://github.com/crim-ca/weaver/issues/386
 # - https://github.com/celery/kombu/pull/1536
@@ -124,5 +125,16 @@ urllib3==1.26.19 ; python_version < "3.10"  # pyup: ignore
 urlmatch
 xmltodict
 webob
+werkzeug>2
+
+# transformer
+fpdf
+Pillow
+pandas
+cairosvg
+multipagetiff
+# gdal
+rasterio
+python-magic
 werkzeug>=3.0.3,<3.1
 zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability

tests/functional/test_wps_package.py

@@ -4325,8 +4325,6 @@ def test_execute_single_output_multipart_accept_link(self):
             },
         }
 
-    # FIXME: implement (https://github.com/crim-ca/weaver/pull/548)
-    @pytest.mark.xfail(reason="not implemented")
     def test_execute_single_output_multipart_accept_alt_format(self):
         """
         Validate the returned contents combining an ``Accept`` header as ``multipart`` and a ``format`` in ``outputs``.
@@ -4381,23 +4379,25 @@ def test_execute_single_output_multipart_accept_alt_format(self):
         output_json_as_yaml = yaml.safe_dump({"data": "test"})
         results_body = self.fix_result_multipart_indent(f"""
             --{boundary}
+            Content-Disposition: attachment; name="output_json"; filename="result.yml"
             Content-Type: {ContentType.APP_YAML}
+            Content-Location: {out_url}/{job_id}/output_json/result.yml
             Content-ID: <output_json@{job_id}>
-            Content-Length: 12
+            Content-Length: 11
 
             {output_json_as_yaml}
             --{boundary}--
         """)
         results_text = self.remove_result_multipart_variable(results.text)
         assert results.content_type.startswith(ContentType.MULTIPART_MIXED)
-        assert results_text == results_body
+        for line1, line2 in zip(results_text.splitlines(), results_body.splitlines()):
+            assert line1 == line2
         outputs = self.app.get(f"/jobs/{job_id}/outputs", params={"schema": JobInputsOutputsSchema.OGC_STRICT})
         assert outputs.content_type.startswith(ContentType.APP_JSON)
         assert outputs.json["outputs"] == {
-            "output_data": "test",
             "output_json": {
-                "href": f"{out_url}/{job_id}/output_json/output.yml",
-                "type": ContentType.APP_YAML,
+                "href": f"{out_url}/{job_id}/output_json/result.json",
+                "type": ContentType.APP_JSON,
             },
         }
 
@@ -4407,8 +4407,6 @@ def test_execute_single_output_multipart_accept_alt_format(self):
         assert result_json.content_type == ContentType.APP_JSON
         assert result_json.text == "{\"data\":\"test\"}"
 
-    # FIXME: implement (https://github.com/crim-ca/weaver/pull/548)
-    @pytest.mark.xfail(reason="not implemented")
     def test_execute_single_output_response_document_alt_format_yaml(self):
         proc = "EchoResultsTester"
         p_id = self.fully_qualified_test_process_name(proc)
@@ -4457,27 +4455,29 @@ def test_execute_single_output_response_document_alt_format_yaml(self):
         output_json_as_yaml = yaml.safe_dump({"data": "test"})
         results_body = self.fix_result_multipart_indent(f"""
             --{boundary}
+            Content-Disposition: attachment; name="output_json"; filename="result.yml"
             Content-Type: {ContentType.APP_YAML}
+            Content-Location: {out_url}/{job_id}/output_json/result.yml
             Content-ID: <output_json@{job_id}>
-            Content-Length: 12
+            Content-Length: 11
 
             {output_json_as_yaml}
             --{boundary}--
         """)
         results_text = self.remove_result_multipart_variable(results.text)
         assert results.content_type.startswith(ContentType.MULTIPART_MIXED)
-        assert results_text == results_body
+        for line1, line2 in zip(results_text.splitlines(), results_body.splitlines()):
+            assert line1 == line2
+
         outputs = self.app.get(f"/jobs/{job_id}/outputs", params={"schema": JobInputsOutputsSchema.OGC_STRICT})
         assert outputs.content_type.startswith(ContentType.APP_JSON)
         assert outputs.json["outputs"] == {
-            "output_data": "test",
             "output_json": {
-                "href": f"{out_url}/{job_id}/output_json/output.yml",
-                "type": ContentType.APP_YAML,
+                "href": f"{out_url}/{job_id}/output_json/result.json",
+                "type": ContentType.APP_JSON,
             },
         }
 
-        # FIXME: implement (https://github.com/crim-ca/weaver/pull/548)
         # validate the results can be obtained with the "real" representation
         result_json = self.app.get(f"/jobs/{job_id}/results/output_json", headers=self.json_headers)
         assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}"
@@ -4548,12 +4548,11 @@ def test_execute_single_output_response_document_alt_format_json_raw_literal(sel
             },
         }
 
-        # FIXME: add check of direct request of output (https://github.com/crim-ca/weaver/pull/548)
         # validate the results can be obtained with the "real" representation
-        # result_json = self.app.get(f"/jobs/{job_id}/results/output_json", headers=self.json_headers)
-        # assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}"
-        # assert result_json.content_type == ContentType.APP_JSON
-        # assert result_json.json == {"data": "test"}
+        result_json = self.app.get(f"/jobs/{job_id}/results/output_json", headers=self.json_headers)
+        assert result_json.status_code == 200, f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}"
+        assert result_json.content_type == ContentType.APP_JSON
+        assert result_json.json == {"data": "test"}
 
     def test_execute_single_output_response_document_default_format_json_special(self):
         """

tests/resources/__init__.py

@@ -10,6 +10,7 @@
     from weaver.typedefs import JSON
 
 RESOURCES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ""))
+TRANSFORM_PATH = os.path.join(RESOURCES_PATH, "transform")
 EXAMPLES_PATH = os.path.join(WEAVER_MODULE_DIR, "wps_restapi/examples")
 FUNCTIONAL_APP_PKG = os.path.abspath(os.path.join(RESOURCES_PATH, "../functional/application-packages"))
 
@@ -62,6 +63,8 @@
     TEST_REMOTE_SERVER_URL, WPS_NO_INPUTS_ID
 )
 
+WPS_transform_RESSOURCES = os.path.join(RESOURCES_PATH, "transform")
+
 
 def _load_path(file_path, text=False, xml=False):
     # type: (str, bool, bool) -> Union[JSON, xml_util.XML, str]