From c2b08d7e182a1fe96e4439eb6e9645ae20e12e6d Mon Sep 17 00:00:00 2001 From: Benoit Date: Thu, 23 Jan 2025 14:03:32 +0100 Subject: [PATCH 1/4] [feature] add bash sanity checks --- .../test/resources/sanity-checks/bash.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 plugin-script-shell/src/test/resources/sanity-checks/bash.yaml diff --git a/plugin-script-shell/src/test/resources/sanity-checks/bash.yaml b/plugin-script-shell/src/test/resources/sanity-checks/bash.yaml new file mode 100644 index 00000000..825ac8db --- /dev/null +++ b/plugin-script-shell/src/test/resources/sanity-checks/bash.yaml @@ -0,0 +1,23 @@ +id: bash +namespace: qa + +tasks: + - id: bash_simple_command + type: io.kestra.core.tasks.scripts.Bash + commands: + - 'echo "Hello Kestra - execution id: {{ execution.id }}"' + + - id: bash_with_outputs + type: io.kestra.core.tasks.scripts.Bash + outputFiles: + - hello + commands: + - echo "Hello Kestra" >> {{ outputFiles.hello }} + + - id: bash_with_input + type: io.kestra.core.tasks.scripts.Bash + inputFiles: + script.sh: | + echo 'Hello Kestra! from script.sh' + commands: + - /bin/bash script.sh \ No newline at end of file From 361e017e8c314a80be4cdb8b6fc906e6eeaaff5c Mon Sep 17 00:00:00 2001 From: Benoit Date: Thu, 23 Jan 2025 14:14:45 +0100 Subject: [PATCH 2/4] [feature] add shell sanity checks --- .../test/resources/sanity-checks/shell.yaml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 plugin-script-shell/src/test/resources/sanity-checks/shell.yaml diff --git a/plugin-script-shell/src/test/resources/sanity-checks/shell.yaml b/plugin-script-shell/src/test/resources/sanity-checks/shell.yaml new file mode 100644 index 00000000..1a41ffc0 --- /dev/null +++ b/plugin-script-shell/src/test/resources/sanity-checks/shell.yaml @@ -0,0 +1,40 @@ +id: shell +namespace: qa + +tasks: + - id: shell_script + type: io.kestra.plugin.scripts.shell.Script + outputFiles: + - hello.txt + script: | + echo "{{ execution.id }} - Test with Docker runner" + echo "Hello Kestra!" >> hello.txt + + - id: shell_script_process_runner + type: io.kestra.plugin.scripts.shell.Script + taskRunner: + type: io.kestra.plugin.core.runner.Process + targetOS: AUTO + outputFiles: + - hello.txt + script: | + echo "{{ execution.id }} - Test with Process runner" + echo "Hello Kestra!" >> hello.txt + + - id: shell_command + type: io.kestra.plugin.scripts.shell.Commands + outputFiles: + - hello.txt + commands: + - echo "{{ execution.id }} - Test with Docker runner" + - echo "Hello Kestra!" >> hello.txt + + - id: shell_command_process_runner + type: io.kestra.plugin.scripts.shell.Commands + taskRunner: + type: io.kestra.plugin.core.runner.Process + outputFiles: + - hello.txt + commands: + - echo "{{ execution.id }} - Test with Docker runner" + - echo "Hello Kestra!" >> hello.txt \ No newline at end of file From c14e3b3a9f744bda817b78de140306883380dac2 Mon Sep 17 00:00:00 2001 From: Benoit Date: Thu, 23 Jan 2025 14:16:16 +0100 Subject: [PATCH 3/4] [feature] add python sanity cehcks --- .../test/resources/sanity-checks/python.yaml | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 plugin-script-python/src/test/resources/sanity-checks/python.yaml diff --git a/plugin-script-python/src/test/resources/sanity-checks/python.yaml b/plugin-script-python/src/test/resources/sanity-checks/python.yaml new file mode 100644 index 00000000..995eb759 --- /dev/null +++ b/plugin-script-python/src/test/resources/sanity-checks/python.yaml @@ -0,0 +1,131 @@ +id: python +namespace: qa + +inputs: + - id: dataset_uri + type: STRING + displayName: Dataset URI + defaults: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv + + - id: discount_amount + type: FLOAT + displayName: Discount Amount + description: By default, it's set to 0 (no discount). + min: 0 + max: 1 + defaults: 0.2 + +tasks: + - id: python_script + type: io.kestra.plugin.scripts.python.Script + script: | + print("Hello, World from a Docker Task Runner") + + - id: python_script_process + type: io.kestra.plugin.scripts.python.Script + taskRunner: + type: io.kestra.plugin.core.runner.Process + script: | + print("Hello, World from a Process Task Runner") + + - id: python_commands + type: io.kestra.plugin.scripts.python.Commands + inputFiles: + main.py: | + print("Hello, World from the Commands Task") + commands: + - python main.py + + - id: python_commands_process + type: io.kestra.plugin.scripts.python.Commands + taskRunner: + type: io.kestra.plugin.core.runner.Process + inputFiles: + main.py: | + print("Hello, World from the Commands Task and Process Task Runner") + commands: + - python main.py + + - id: python_script_logs + type: io.kestra.plugin.scripts.python.Script + allowFailure: true + allowWarning: true + beforeCommands: + - pip install kestra + script: | + import time + from kestra import Kestra + + logger = Kestra.logger() + + logger.debug("DEBUG is used for diagnostic info.") + time.sleep(0.5) + + logger.info("INFO confirms normal operation.") + time.sleep(0.5) + + logger.warning("WARNING signals something unexpected.") + time.sleep(0.5) + + logger.error("ERROR indicates a serious issue.") + time.sleep(0.5) + + logger.critical("CRITICAL means a severe failure.") + + - id: python_script_metrics + type: io.kestra.plugin.scripts.python.Script + beforeCommands: + - pip install kestra + script: | + from kestra import Kestra + import requests + import time + + start = time.perf_counter() + requests.get("{{ inputs.dataset_uri }}") + end = time.perf_counter() + + Kestra.timer('duration', end - start) + + - id: python_script_outputs + type: io.kestra.plugin.scripts.python.Script + containerImage: ghcr.io/kestra-io/pydata:latest + beforeCommands: + - pip install kestra + outputFiles: + - processed_orders.csv + script: | + import pandas as pd + from kestra import Kestra + + df = pd.read_csv('{{ inputs.dataset_uri }}') + total_revenue = df['total'].sum() + Kestra.outputs({"total": total_revenue}) + if {{ inputs.discount_amount }} > 0: + df['discounted_total'] = df['total'] * (1 - {{ inputs.discount_amount }}) + df.to_csv('processed_orders.csv') + + - id: python_commands_etl_outputs + type: io.kestra.plugin.scripts.python.Commands + containerImage: ghcr.io/kestra-io/pydata:latest + beforeCommands: + - pip install kestra + inputFiles: + main.py: | + import pandas as pd + from kestra import Kestra + import os + + df = pd.read_csv(os.environ['DATASET_URI']) + total_revenue = df['total'].sum() + Kestra.outputs({"total": total_revenue}) + if float(os.environ['DISCOUNT_AMOUNT']) > 0: + df['discounted_total'] = df['total'] * (1 - float(os.environ['DISCOUNT_AMOUNT'])) + df.to_csv('processed_orders.csv') + outputFiles: + - processed_orders.csv + env: + DATASET_URI: "{{ inputs.dataset_uri }}" + DISCOUNT_AMOUNT: "{{ inputs.discount_amount }}" + commands: + - python main.py From ae486d00fc0eb76810a67cb00eac01a2385157b1 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Tue, 28 Jan 2025 11:51:41 +0100 Subject: [PATCH 4/4] feat: add runner to run sanity checks flows --- .../plugin/scripts/python/RunnerTest.java | 21 ++++++++++++++ .../plugin/scripts/shell/RunnerTest.java | 28 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 plugin-script-python/src/test/java/io/kestra/plugin/scripts/python/RunnerTest.java create mode 100644 plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/RunnerTest.java diff --git a/plugin-script-python/src/test/java/io/kestra/plugin/scripts/python/RunnerTest.java b/plugin-script-python/src/test/java/io/kestra/plugin/scripts/python/RunnerTest.java new file mode 100644 index 00000000..efa64d35 --- /dev/null +++ b/plugin-script-python/src/test/java/io/kestra/plugin/scripts/python/RunnerTest.java @@ -0,0 +1,21 @@ +package io.kestra.plugin.scripts.python; + +import io.kestra.core.junit.annotations.ExecuteFlow; +import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.executions.Execution; +import io.kestra.core.models.flows.State; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +@KestraTest(startRunner = true) +class RunnerTest { + @Test + @ExecuteFlow("sanity-checks/python.yaml") + void python(Execution execution) { + assertThat(execution.getTaskRunList(), hasSize(8)); + assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS)); + } +} diff --git a/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/RunnerTest.java b/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/RunnerTest.java new file mode 100644 index 00000000..6ee66a4a --- /dev/null +++ b/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/RunnerTest.java @@ -0,0 +1,28 @@ +package io.kestra.plugin.scripts.shell; + +import io.kestra.core.junit.annotations.ExecuteFlow; +import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.executions.Execution; +import io.kestra.core.models.flows.State; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +@KestraTest(startRunner = true) +class RunnerTest { + @Test + @ExecuteFlow("sanity-checks/bash.yaml") + void bash(Execution execution) { + assertThat(execution.getTaskRunList(), hasSize(3)); + assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS)); + } + + @Test + @ExecuteFlow("sanity-checks/shell.yaml") + void shell(Execution execution) { + assertThat(execution.getTaskRunList(), hasSize(4)); + assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS)); + } +}