Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New search endpoint #6910

Merged
merged 1 commit into from
Feb 6, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 250 additions & 0 deletions core/src/main/java/io/kestra/core/models/QueryFilter.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,250 @@
package io.kestra.core.models;

import io.kestra.core.utils.Enums;
import lombok.Builder;

import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;

@Builder
public record QueryFilter(
Field field,
Op operation,
Object value
) {
public enum Op {
EQUALS("$eq"),
NOT_EQUALS("$ne"),
GREATER_THAN("$gte"),
LESS_THAN("$lte"),
IN("$in"),
NOT_IN("$notIn"),
STARTS_WITH("$startsWith"),
ENDS_WITH("$endsWith"),
CONTAINS("$contains"),
REGEX("$regex");

private static final Map<String, Op> BY_VALUE = Arrays.stream(values())
.collect(Collectors.toMap(Op::value, Function.identity()));

private final String value;

Op(String value) {
this.value = value;
}

public static Op fromString(String value) {
return Enums.fromString(value, BY_VALUE, "operation");
}

public String value() {
return value;
}
}

public enum Field {
QUERY("q") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.REGEX);
}
},
SCOPE("scope") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS);
}
},
NAMESPACE("namespace") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.REGEX);
}
},
LABELS("labels") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS);
}
},
FLOW_ID("flowId") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.IN, Op.NOT_IN);
}
},
START_DATE("startDate") {
@Override
public List<Op> supportedOp() {
return List.of(Op.GREATER_THAN, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS);
}
},
END_DATE("endDate") {
@Override
public List<Op> supportedOp() {
return List.of(Op.GREATER_THAN, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS);
}
},
STATE("state") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.IN, Op.NOT_IN);
}
},
TIME_RANGE("timeRange") {
aeSouid marked this conversation as resolved.
Show resolved Hide resolved
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH,
Op.ENDS_WITH, Op.IN, Op.NOT_IN, Op.REGEX);
}
},
TRIGGER_EXECUTION_ID("triggerExecutionId") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.IN, Op.NOT_IN);
}
},
TRIGGER_ID("triggerId") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.IN, Op.NOT_IN);
}
},
CHILD_FILTER("childFilter") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS);
}
},
WORKER_ID("workerId") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.IN, Op.NOT_IN);
}
},
EXISTING_ONLY("existingOnly") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS);
}
},
MIN_LEVEL("level") {
@Override
public List<Op> supportedOp() {
return List.of(Op.EQUALS, Op.NOT_EQUALS);
}
};

private static final Map<String, Field> BY_VALUE = Arrays.stream(values())
.collect(Collectors.toMap(Field::value, Function.identity()));

public abstract List<Op> supportedOp();

private final String value;

Field(String value) {
this.value = value;
}

public static Field fromString(String value) {
return Enums.fromString(value, BY_VALUE, "field");
}

public String value() {
return value;
}
}


public enum Resource {
FLOW {
@Override
public List<Field> supportedField() {
return List.of(Field.LABELS, Field.NAMESPACE, Field.QUERY, Field.SCOPE);
}
},
NAMESPACE {
@Override
public List<Field> supportedField() {
return List.of(Field.EXISTING_ONLY);
}
},
EXECUTION {
@Override
public List<Field> supportedField() {
return List.of(
Field.QUERY, Field.SCOPE, Field.FLOW_ID, Field.START_DATE, Field.END_DATE, Field.TIME_RANGE,
Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER,
Field.NAMESPACE
);
}
},
LOG {
@Override
public List<Field> supportedField() {
return List.of(Field.NAMESPACE, Field.START_DATE, Field.END_DATE,
Field.FLOW_ID, Field.TRIGGER_ID, Field.MIN_LEVEL
);
}
},
TASK {
@Override
public List<Field> supportedField() {
return List.of(Field.NAMESPACE, Field.QUERY, Field.END_DATE, Field.FLOW_ID, Field.START_DATE,
Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER
);
}
},
TEMPLATE {
@Override
public List<Field> supportedField() {
return List.of(Field.NAMESPACE, Field.QUERY);
}
},
TRIGGER {
@Override
public List<Field> supportedField() {
return List.of(Field.QUERY, Field.NAMESPACE, Field.WORKER_ID, Field.FLOW_ID
);
}
};

public abstract List<Field> supportedField();

/**
* Converts {@code Resource} enums to a list of {@code ResourceField},
* including fields and their supported operations.
*
* @return List of {@code ResourceField} with resource names, fields, and operations.
*/
public static List<ResourceField> asResourceList() {
return Arrays.stream(values())
.map(Resource::toResourceField)
.toList();
}

private static ResourceField toResourceField(Resource resource) {
List<FieldOp> fieldOps = resource.supportedField().stream()
.map(Resource::toFieldInfo)
.toList();
return new ResourceField(resource.name().toLowerCase(), fieldOps);
}

private static FieldOp toFieldInfo(Field field) {
List<Operation> operations = field.supportedOp().stream()
.map(Resource::toOperation)
.toList();
return new FieldOp(field.name().toLowerCase(), field.value(), operations);
}

private static Operation toOperation(Op op) {
return new Operation(op.name(), op.value());
}
}

public record ResourceField(String name, List<FieldOp> fields) {}
public record FieldOp(String name, String value, List<Operation> operations) {}
public record Operation(String name, String value) {}

}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.kestra.core.repositories;

import io.kestra.core.models.QueryFilter;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.executions.statistics.DailyExecutionStatistics;
Expand Down Expand Up @@ -59,19 +60,9 @@ default Optional<Execution> findById(String tenantId, String id) {

ArrayListTotal<Execution> find(
Pageable pageable,
@Nullable String query,
@Nullable String tenantId,
@Nullable List<FlowScope> scope,
@Nullable String namespace,
@Nullable String flowId,
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
@Nullable List<State.Type> state,
@Nullable Map<String, String> labels,
@Nullable String triggerExecutionId,
@Nullable ChildFilter childFilter
@Nullable List<QueryFilter> filters
);

default Flux<Execution> find(
@Nullable String query,
@Nullable String tenantId,
Expand Down Expand Up @@ -103,18 +94,11 @@ Flux<Execution> find(
boolean allowDeleted
);


ArrayListTotal<TaskRun> findTaskRun(
Pageable pageable,
@Nullable String query,
@Nullable String tenantId,
@Nullable String namespace,
@Nullable String flowId,
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
@Nullable List<State.Type> states,
@Nullable Map<String, String> labels,
@Nullable String triggerExecutionId,
@Nullable ChildFilter childFilter
List<QueryFilter> filters
);

Execution delete(Execution execution);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.kestra.core.repositories;

import io.kestra.core.models.QueryFilter;
import io.kestra.core.models.SearchResult;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
Expand Down Expand Up @@ -143,6 +144,12 @@ ArrayListTotal<Flow> find(
@Nullable Map<String, String> labels
);

ArrayListTotal<Flow> find(
Pageable pageable,
@Nullable String tenantId,
@Nullable List<QueryFilter> filters
);

List<FlowWithSource> findWithSource(
@Nullable String query,
@Nullable String tenantId,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.kestra.core.repositories;

import io.kestra.core.models.QueryFilter;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.LogEntry;
import io.kestra.core.models.executions.statistics.LogStatistics;
Expand Down Expand Up @@ -76,15 +77,9 @@ public interface LogRepositoryInterface extends SaveRepositoryInterface<LogEntry

ArrayListTotal<LogEntry> find(
Pageable pageable,
@Nullable String query,
@Nullable String tenantId,
@Nullable String namespace,
@Nullable String flowId,
@Nullable String triggerId,
@Nullable Level minLevel,
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate
);
List<QueryFilter> filters
);

Flux<LogEntry> findAsync(
@Nullable String tenantId,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.kestra.core.repositories;

import io.kestra.core.models.QueryFilter;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.triggers.Trigger;
import io.kestra.core.models.triggers.TriggerContext;
Expand Down Expand Up @@ -29,6 +30,7 @@ public interface TriggerRepositoryInterface {
Trigger lock(String triggerUid, Function<Trigger, Trigger> function);

ArrayListTotal<Trigger> find(Pageable from, String query, String tenantId, String namespace, String flowId, String workerId);
ArrayListTotal<Trigger> find(Pageable from, String tenantId, List<QueryFilter> filters);

/**
* Counts the total number of triggers.
Expand Down
16 changes: 16 additions & 0 deletions core/src/main/java/io/kestra/core/utils/Enums.java
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,22 @@ public static <T extends Enum<T>> Set<T> allExcept(final @NotNull Class<T> enumT
.collect(Collectors.toSet());
}

/**
* Converts a string to its corresponding enum value based on a provided mapping.
*
* @param value The string representation of the enum value.
* @param mapping A map of string values to enum constants.
* @param typeName A descriptive name of the enum type (used in error messages).
* @param <T> The type of the enum.
* @return The corresponding enum constant.
* @throws IllegalArgumentException If the string does not match any enum value.
*/
public static <T extends Enum<T>> T fromString(String value, Map<String, T> mapping, String typeName) {
return Optional.ofNullable(mapping.get(value))
.orElseThrow(() -> new IllegalArgumentException(
"Unsupported %s '%s'. Expected one of: %s".formatted(typeName, value, mapping.keySet())
));
}

private Enums() {
}
Expand Down
Loading
Loading