feat: go-yq

test_wrappers.py

@@ -7158,3 +7158,20 @@ def test_orthanq(run):
             "out/calls_virus",
         ],
     )
+
+def test_go_yq(run):
+    run(
+        "utils/go-yq",
+        [
+            "snakemake",
+            "--cores",
+            "1",
+            "--use-conda",
+            "-F",
+            "concat.yaml",
+            "updated.yaml",
+            "evaluated.yaml",
+            "table.json",
+        ],
+    )
+

utils/go-yq/environment.linux-64.pin.txt

@@ -0,0 +1,10 @@
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: linux-64
+# created-by: conda 25.9.1
+@EXPLICIT
+https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
+https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_15.conda#a90d6983da0757f4c09bb8fcfaf34e71
+https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_15.conda#a5d86b0496174a412d531eac03af9174
+https://conda.anaconda.org/conda-forge/linux-64/go-yq-4.49.2-h280c20c_0.conda#8ce54444fc979c3b3b8c940bab203a4c

utils/go-yq/environment.yaml

@@ -0,0 +1,5 @@
+channels:
+  - conda-forge 
+dependencies:
+  - go-yq =4.49.2
+

utils/go-yq/meta.yaml

@@ -0,0 +1,28 @@
+name: go-yq
+url: https://github.com/mikefarah/yq
+description: >
+  A lightweight and portable command-line YAML, JSON and XML processor.
+  yq uses jq like syntax but works with yaml files as well as json, xml, 
+  properties, csv and tsv.
+authors:
+  - Thibault Dayris
+input:
+  - Optional path to one or multiple file(s) to process
+output:
+  - Path to one or multiple output file(s)
+params:
+  - extra: Optional arguments for `yq` command, besides `--null-input`.
+  - expression: Optional expression to evaluate.
+  - subcommand: Optional subcommand for `yq`
+notes: |
+  This tool does not support multithreading.
+
+  Format are mostly auto-detected using `--output-format` and `--input-format`.
+  However, you may want to provide formats through `params.extra` in case of
+  ambiguity (e.g. `uri`, `base64`).
+
+  Through file splitting, files name expression are not automatically inferred.
+  User should provide the proper `-s/--split-exp` parameters in `params.extra`.
+
+
+

utils/go-yq/test/Snakefile

@@ -0,0 +1,79 @@
+rule test_yq_concat:
+    input:
+        ["bazquux.yaml", "foobar.yaml"],
+    output:
+        "concat.yaml",
+    threads: 1
+    log:
+        "test_yq_concat.log",
+    params:
+        extra="",
+        subcommand="",
+        expression="",
+    wrapper:
+        "master/utils/go-yq"
+
+
+rule test_yq_simple_yaml_update:
+    input:
+        "foobar.yaml",
+    output:
+        "updated.yaml",
+    threads: 1
+    log:
+        "test_yq_simple_yaml_update.log",
+    params:
+        extra="",
+        subcommand="",
+        expression='.[].foo = "foobar"',
+    wrapper:
+        "master/utils/go-yq"
+
+
+rule test_yq_evaluate_yaml_content:
+    input:
+        "eval.yaml",
+    output:
+        "evaluated.yaml",
+    threads: 1
+    log:
+        "test_yq_evaluate_yaml_content.log",
+    params:
+        extra="",
+        subcommand="",
+        expression="eval(.pathExp)",
+    wrapper:
+        "master/utils/go-yq"
+
+
+rule test_yq_split_yaml:
+    input:
+        "concat.yaml",
+    output:
+        "foo_bar.yaml",
+        "foo_bar2.yaml",
+    threads: 1
+    log:
+        "test_yq_split_yaml.log",
+    params:
+        extra="-s '\"user_\" + .name'",
+        subcommand="",
+        expression=".[]",
+    wrapper:
+        "master/utils/go-yq"
+
+
+rule test_yq_format_conversion:
+    input:
+        "table.csv",
+    output:
+        "table.json",
+    threads: 1
+    log:
+        "test_yq_format_conversion.log",
+    params:
+        extra="",
+        subcommand="",
+        expression="",
+    wrapper:
+        "master/utils/go-yq"

utils/go-yq/test/bazquux.yaml

@@ -0,0 +1 @@
+baz: quux

utils/go-yq/test/eval.yaml

@@ -0,0 +1,5 @@
+pathExp: .a.b[] | select(.name == "cat")
+a:
+  b:
+    - name: dog
+    - name: cat

utils/go-yq/test/foobar.yaml

@@ -0,0 +1,4 @@
+- foo: bar
+  boo: far
+- foo: bar2
+  boo: far2

utils/go-yq/test/table.csv

@@ -0,0 +1,3 @@
+name,type
+Bobo,dog
+Fifi,cat

utils/go-yq/wrapper.py

@@ -0,0 +1,79 @@
+# coding: utf-8
+
+"""Snakemake wrapper for go-yq"""
+
+import re
+import shlex
+import warnings
+
+from snakemake_wrapper_utils.snakemake import is_arg, get_format
+from snakemake.shell import shell
+
+
+def detect_fmt(files, long_option, short_option, extra) -> str:
+    """Try to detect input/output formats"""
+    formats = [get_format(f) for f in files]
+    accepted_formats = {
+        "yml": "yaml",
+        "yaml": "yaml",
+        "json": "json",
+        "properties": "props",
+        "csv": "csv",
+        "tsv": "tsv",
+        "toml": "toml",
+        "sh": "shell",
+        "lua": "lua",
+        "ini": "ini",
+        "xml": "xml",
+    }
+
+    # Ensure user did not provide any format information
+    if not (is_arg(long_option, extra) or is_arg(short_option, extra)):
+        # Check unique format in input and output separately
+        if len(set(formats)) > 1:
+            raise ValueError(
+                f"The following files: {files}, should have the same extension. "
+                f"Got {set(formats)} instead."
+            )
+
+        # Trying to detect output format from file extension
+        # since yq defaults to yaml output and writes information
+        # in standard output.
+        fmt = accepted_formats.get(formats[0])
+        if fmt:
+            return f" {long_option} {fmt}"
+
+    return ""
+
+
+# Enhance extra parameters if needed
+extra = snakemake.params.get("extra", "")
+extra += detect_fmt(snakemake.input, "--input-format", "-p", extra)
+extra += detect_fmt(snakemake.output, "--output-format", "-o", extra)
+
+subcommand = snakemake.params.get("subcommand", "")
+# yq/jq expression should be quoted
+expression = shlex.quote(snakemake.params.get("expression", ""))
+
+# Handle the case user creates a file from command line
+infile = snakemake.input
+if len(infile) == 0:
+    infile = " --null-input "
+
+# Handling file splitting
+outfile = snakemake.output
+if len(snakemake.output) > 1:
+    if not (is_arg("--split-exp", extra) or is_arg("-s", extra)):
+        raise ValueError(
+            "File splitting is not automatically handled. Please provide "
+            "adequate '-s/--split-exp' in 'params.extra'."
+        )
+    log = snakemake.log_fmt_shell(stdout=True, stderr=True)
+    outfile = ""
+elif len(snakemake.outfile) == 1:
+    log = snakemake.log_fmt_shell(stdout=False, stderr=True)
+    outfile = f"> {snakemake.output[0]}"
+else:
+    raise ValueError("Inplace modifications are not handled by this wrapper.")
+
+shell("yq {subcommand} {extra} {expression} {infile} {outfile} {log}")