Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add bash sanity checks #207

Merged
merged 4 commits into from
Jan 30, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package io.kestra.plugin.scripts.python;

import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.State;
import org.junit.jupiter.api.Test;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;

@KestraTest(startRunner = true)
class RunnerTest {
@Test
@ExecuteFlow("sanity-checks/python.yaml")
void python(Execution execution) {
assertThat(execution.getTaskRunList(), hasSize(8));
assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS));
}
}
131 changes: 131 additions & 0 deletions plugin-script-python/src/test/resources/sanity-checks/python.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
id: python
namespace: qa

inputs:
- id: dataset_uri
type: STRING
displayName: Dataset URI
defaults: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv

- id: discount_amount
type: FLOAT
displayName: Discount Amount
description: By default, it's set to 0 (no discount).
min: 0
max: 1
defaults: 0.2

tasks:
- id: python_script
type: io.kestra.plugin.scripts.python.Script
script: |
print("Hello, World from a Docker Task Runner")

- id: python_script_process
type: io.kestra.plugin.scripts.python.Script
taskRunner:
type: io.kestra.plugin.core.runner.Process
script: |
print("Hello, World from a Process Task Runner")

- id: python_commands
type: io.kestra.plugin.scripts.python.Commands
inputFiles:
main.py: |
print("Hello, World from the Commands Task")
commands:
- python main.py

- id: python_commands_process
type: io.kestra.plugin.scripts.python.Commands
taskRunner:
type: io.kestra.plugin.core.runner.Process
inputFiles:
main.py: |
print("Hello, World from the Commands Task and Process Task Runner")
commands:
- python main.py

- id: python_script_logs
type: io.kestra.plugin.scripts.python.Script
allowFailure: true
allowWarning: true
beforeCommands:
- pip install kestra
script: |
import time
from kestra import Kestra

logger = Kestra.logger()

logger.debug("DEBUG is used for diagnostic info.")
time.sleep(0.5)

logger.info("INFO confirms normal operation.")
time.sleep(0.5)

logger.warning("WARNING signals something unexpected.")
time.sleep(0.5)

logger.error("ERROR indicates a serious issue.")
time.sleep(0.5)

logger.critical("CRITICAL means a severe failure.")

- id: python_script_metrics
type: io.kestra.plugin.scripts.python.Script
beforeCommands:
- pip install kestra
script: |
from kestra import Kestra
import requests
import time

start = time.perf_counter()
requests.get("{{ inputs.dataset_uri }}")
end = time.perf_counter()

Kestra.timer('duration', end - start)

- id: python_script_outputs
type: io.kestra.plugin.scripts.python.Script
containerImage: ghcr.io/kestra-io/pydata:latest
beforeCommands:
- pip install kestra
outputFiles:
- processed_orders.csv
script: |
import pandas as pd
from kestra import Kestra

df = pd.read_csv('{{ inputs.dataset_uri }}')
total_revenue = df['total'].sum()
Kestra.outputs({"total": total_revenue})
if {{ inputs.discount_amount }} > 0:
df['discounted_total'] = df['total'] * (1 - {{ inputs.discount_amount }})
df.to_csv('processed_orders.csv')

- id: python_commands_etl_outputs
type: io.kestra.plugin.scripts.python.Commands
containerImage: ghcr.io/kestra-io/pydata:latest
beforeCommands:
- pip install kestra
inputFiles:
main.py: |
import pandas as pd
from kestra import Kestra
import os

df = pd.read_csv(os.environ['DATASET_URI'])
total_revenue = df['total'].sum()
Kestra.outputs({"total": total_revenue})
if float(os.environ['DISCOUNT_AMOUNT']) > 0:
df['discounted_total'] = df['total'] * (1 - float(os.environ['DISCOUNT_AMOUNT']))
df.to_csv('processed_orders.csv')
outputFiles:
- processed_orders.csv
env:
DATASET_URI: "{{ inputs.dataset_uri }}"
DISCOUNT_AMOUNT: "{{ inputs.discount_amount }}"
commands:
- python main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package io.kestra.plugin.scripts.shell;

import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.State;
import org.junit.jupiter.api.Test;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;

@KestraTest(startRunner = true)
class RunnerTest {
@Test
@ExecuteFlow("sanity-checks/bash.yaml")
void bash(Execution execution) {
assertThat(execution.getTaskRunList(), hasSize(3));
assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS));
}

@Test
@ExecuteFlow("sanity-checks/shell.yaml")
void shell(Execution execution) {
assertThat(execution.getTaskRunList(), hasSize(4));
assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS));
}
}
23 changes: 23 additions & 0 deletions plugin-script-shell/src/test/resources/sanity-checks/bash.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
id: bash
namespace: qa

tasks:
- id: bash_simple_command
type: io.kestra.core.tasks.scripts.Bash
commands:
- 'echo "Hello Kestra - execution id: {{ execution.id }}"'

- id: bash_with_outputs
type: io.kestra.core.tasks.scripts.Bash
outputFiles:
- hello
commands:
- echo "Hello Kestra" >> {{ outputFiles.hello }}

- id: bash_with_input
type: io.kestra.core.tasks.scripts.Bash
inputFiles:
script.sh: |
echo 'Hello Kestra! from script.sh'
commands:
- /bin/bash script.sh
40 changes: 40 additions & 0 deletions plugin-script-shell/src/test/resources/sanity-checks/shell.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
id: shell
namespace: qa

tasks:
- id: shell_script
type: io.kestra.plugin.scripts.shell.Script
outputFiles:
- hello.txt
script: |
echo "{{ execution.id }} - Test with Docker runner"
echo "Hello Kestra!" >> hello.txt

- id: shell_script_process_runner
type: io.kestra.plugin.scripts.shell.Script
taskRunner:
type: io.kestra.plugin.core.runner.Process
targetOS: AUTO
outputFiles:
- hello.txt
script: |
echo "{{ execution.id }} - Test with Process runner"
echo "Hello Kestra!" >> hello.txt

- id: shell_command
type: io.kestra.plugin.scripts.shell.Commands
outputFiles:
- hello.txt
commands:
- echo "{{ execution.id }} - Test with Docker runner"
- echo "Hello Kestra!" >> hello.txt

- id: shell_command_process_runner
type: io.kestra.plugin.scripts.shell.Commands
taskRunner:
type: io.kestra.plugin.core.runner.Process
outputFiles:
- hello.txt
commands:
- echo "{{ execution.id }} - Test with Docker runner"
- echo "Hello Kestra!" >> hello.txt
Loading