Skip to content

Commit

Permalink
refactor: migrate to dynamic properties
Browse files Browse the repository at this point in the history 
migrate ExcelToIon
migrate IonToExcel
  • Loading branch information
@mgabelle
mgabelle committed Dec 6, 2024
1 parent 75ab12b commit 4c014a6
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 56 deletions.
52 changes: 26 additions & 26 deletions src/main/java/io/kestra/plugin/serdes/excel/ExcelToIon.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

import com.github.pjfanning.xlsx.StreamingReader;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.runners.RunContext;
Expand Down Expand Up @@ -40,7 +42,7 @@
@Example(
full = true,
title = "Convert an Excel file to the Ion format.",
code = """
code = """
id: excel_to_ion
namespace: company.team
Expand All @@ -67,45 +69,40 @@ public class ExcelToIon extends Task implements RunnableTask<ExcelToIon.Output>
@Schema(
title = "The sheets title to be included"
)
@PluginProperty
private List<String> sheetsTitle;
private Property<List<String>> sheetsTitle;

@Schema(
title = "The name of a supported character set"
)
@PluginProperty
@Builder.Default
private String charset = "UTF-8";
private Property<String> charset = Property.of("UTF-8");

@Schema(
title = "Determines how values should be rendered in the output",
description = "Possible values: FORMATTED_VALUE, UNFORMATTED_VALUE, FORMULA"
)
@PluginProperty
@Builder.Default
private ValueRender valueRender = ValueRender.UNFORMATTED_VALUE;
private Property<ValueRender> valueRender = Property.of(ValueRender.UNFORMATTED_VALUE);

@Schema(
title = "How dates, times, and durations should be represented in the output",
description = "Possible values: SERIAL_NUMBER, FORMATTED_STRING"
)
@Builder.Default
@PluginProperty
private DateTimeRender dateTimeRender = DateTimeRender.UNFORMATTED_VALUE;
private Property<DateTimeRender> dateTimeRender = Property.of(DateTimeRender.UNFORMATTED_VALUE);

@Schema(
title = "Whether the first row should be treated as the header"
)
@PluginProperty
@Builder.Default
private boolean header = true;
private Property<Boolean> header = Property.of(true);

@Schema(
title = "Specifies if empty rows should be skipped"
)
@PluginProperty
@Builder.Default
private boolean skipEmptyRows = false;
private Property<Boolean> skipEmptyRows = Property.of(false);

@Schema(
title = "Number of lines to skip at the start of the file. Useful if a table has a title and explanation in the first few rows"
Expand All @@ -124,7 +121,7 @@ public Output run(RunContext runContext) throws Exception {
List<Sheet> sheets = new ArrayList<>();
workbook.sheetIterator().forEachRemaining(sheets::add);

List<String> includedSheetsTitle = ListUtils.emptyOnNull(this.sheetsTitle)
List<String> includedSheetsTitle = ListUtils.emptyOnNull(runContext.render(this.sheetsTitle).asList(String.class))
.stream()
.map(throwFunction(runContext::render))
.toList();
Expand All @@ -140,6 +137,9 @@ public Output run(RunContext runContext) throws Exception {
AtomicInteger rowsCount = new AtomicInteger();

AtomicInteger skipped = new AtomicInteger();
var renderedValueRender = runContext.render(this.valueRender).as(ValueRender.class).orElseThrow();
var renderedSkipEmptyRowsValue = runContext.render(this.skipEmptyRows).as(Boolean.class).orElseThrow();
var renderedDateTimeRender = runContext.render(this.dateTimeRender).as(DateTimeRender.class).orElseThrow();
for (Sheet sheet : selectedSheets) {
List<Object> values = new ArrayList<>();

Expand All @@ -153,15 +153,15 @@ public Output run(RunContext runContext) throws Exception {
for (int i = row.getFirstCellNum(); i < row.getLastCellNum(); i++) {
Cell cell = row.getCell(i);
if (cell != null) {
if (this.valueRender.equals(ValueRender.FORMATTED_VALUE)) {
extractValue(rowValues, cell);
} else if (this.valueRender.equals(ValueRender.FORMULA)) {
if (renderedValueRender.equals(ValueRender.FORMATTED_VALUE)) {
extractValue(rowValues, cell, renderedSkipEmptyRowsValue, renderedDateTimeRender);
} else if (renderedValueRender.equals(ValueRender.FORMULA)) {
switch (cell.getCachedFormulaResultType()) {
case NUMERIC -> rowValues.add(convertNumeric(cell));
case NUMERIC -> rowValues.add(convertNumeric(cell, renderedDateTimeRender));
case STRING -> rowValues.add(cell.getRichStringCellValue().getString());
}
} else {
extractValue(rowValues, cell);
extractValue(rowValues, cell, renderedSkipEmptyRowsValue, renderedDateTimeRender);
}
}
}
Expand All @@ -180,19 +180,19 @@ public Output run(RunContext runContext) throws Exception {
}
}

public void extractValue(List<Object> rowValues, Cell cell) {
public void extractValue(List<Object> rowValues, Cell cell, boolean skipEmptyRowsValue, DateTimeRender dateTimeRender) {
switch (cell.getCellType()) {
case STRING -> rowValues.add(cell.getStringCellValue());
case BOOLEAN -> rowValues.add(cell.getBooleanCellValue());
case NUMERIC -> rowValues.add(convertNumeric(cell));
case NUMERIC -> rowValues.add(convertNumeric(cell, dateTimeRender));
case FORMULA -> {
switch (cell.getCachedFormulaResultType()){
case NUMERIC -> rowValues.add(convertNumeric(cell));
case NUMERIC -> rowValues.add(convertNumeric(cell, dateTimeRender));
case STRING -> rowValues.add(cell.getRichStringCellValue().getString());
}
}
case BLANK -> {
if (!this.skipEmptyRows) {
if (!skipEmptyRowsValue) {
rowValues.add(cell.getStringCellValue());
}
}
Expand All @@ -201,8 +201,8 @@ public void extractValue(List<Object> rowValues, Cell cell) {
}
}

private URI convertToIon(RunContext runContext, List<Object> values) throws IOException {
if (header) {
private URI convertToIon(RunContext runContext, List<Object> values) throws IOException, IllegalVariableEvaluationException {
if (runContext.render(header).as(Boolean.class).orElseThrow()) {
List<Object> headers = (List<Object>) values.remove(0);
List<Object> convertedSheet = new LinkedList<>();

Expand All @@ -224,9 +224,9 @@ private URI convertToIon(RunContext runContext, List<Object> values) throws IOEx
return runContext.storage().putFile(this.store(runContext, values));
}

private Object convertNumeric(Cell cell) {
private Object convertNumeric(Cell cell, DateTimeRender dateTimeRender) {
if(DateUtil.isCellDateFormatted(cell)) {
return switch (this.dateTimeRender) {
return switch (dateTimeRender) {
case SERIAL_NUMBER -> cell.getNumericCellValue();
case FORMATTED_STRING -> {
DataFormatter dataFormatter = new DataFormatter();
Expand Down
42 changes: 21 additions & 21 deletions src/main/java/io/kestra/plugin/serdes/excel/IonToExcel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.FileSerde;
Expand Down Expand Up @@ -49,7 +50,7 @@
@Example(
full = true,
title = "Download a CSV file and convert it to the Excel file format.",
code = """
code = """
id: ion_to_excel
namespace: company.team
Expand Down Expand Up @@ -93,7 +94,7 @@
- id: write
type: io.kestra.plugin.serdes.excel.IonToExcel
from:
from:
Sheet_1: "{{ outputs.convert1.uri }}"
Sheet_2: "{{ outputs.convert2.uri }}"
"""
Expand All @@ -115,32 +116,28 @@ public class IonToExcel extends AbstractTextWriter implements RunnableTask<IonTo
defaultValue = "UTF-8"
)
@Builder.Default
@PluginProperty
private String charset = "UTF-8";
private Property<String> charset = Property.of("UTF-8");

@Schema(
title = "The sheet title to be used when writing data to an Excel spreadsheet",
defaultValue = "Sheet"
)
@Builder.Default
@PluginProperty
private String sheetsTitle = "Sheet";
private Property<String> sheetsTitle = Property.of("Sheet");

@Schema(
title = "Whether header should be written as the first line"
)
@Builder.Default
@PluginProperty
private boolean header = true;
private Property<Boolean> header = Property.of(true);

@Schema(
title = "Whether styles should be applied to format values",
description = "Excel is limited to 64000 styles per document, and styles are applied on every date, " +
"removed this options when you have a lots of values."
)
@Builder.Default
@PluginProperty
private boolean styles = true;
private Property<Boolean> styles = Property.of(true);

@Override
public Output run(RunContext runContext) throws Exception {
Expand All @@ -151,7 +148,7 @@ public Output run(RunContext runContext) throws Exception {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(1)) {
File tempFile = runContext.workingDir().createTempFile(".xlsx").toFile();

lineCount = writeQuery(runContext, sheetsTitle, runContext.render(fromStr), tempFile, workbook);
lineCount = writeQuery(runContext, runContext.render(sheetsTitle).as(String.class).orElseThrow(), runContext.render(fromStr), tempFile, workbook);

return Output.builder()
.uri(runContext.storage().putFile(tempFile))
Expand Down Expand Up @@ -184,9 +181,12 @@ private Long writeQuery(RunContext runContext, String title, String from, File t
Long lineCount;

try (
Reader reader = new BufferedReader(new InputStreamReader(runContext.storage().getFile(fromUri), Charset.forName(this.charset)), FileSerde.BUFFER_SIZE);
Reader reader = new BufferedReader(new InputStreamReader(runContext.storage().getFile(fromUri), Charset.forName(runContext.render(this.charset).as(String.class).orElseThrow())), FileSerde.BUFFER_SIZE);
OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile), FileSerde.BUFFER_SIZE)
) {
var renderedHeaderValue = runContext.render(this.header).as(Boolean.class).orElseThrow();
var renderedStyles =runContext.render(this.styles).as(Boolean.class).orElseThrow();

SXSSFSheet sheet = workbook.createSheet(title);
Flux<Object> flowable = FileSerde.readAll(reader)
.doOnNext(new Consumer<>() {
Expand All @@ -199,13 +199,13 @@ public void accept(Object row) {
if (row instanceof List casted) {
SXSSFRow xssfRow = sheet.createRow(rowAt++);

if (header) {
if (renderedHeaderValue) {
throw new IllegalArgumentException("Invalid data of type List with header");
}

int cellAt = 0;
for (final Object value : casted) {
createCell(workbook, sheet, value, xssfRow, cellAt);
createCell(workbook, sheet, value, xssfRow, cellAt, renderedStyles);
++cellAt;
}

Expand All @@ -214,18 +214,18 @@ public void accept(Object row) {

if (!first) {
this.first = true;
if (header) {
if (renderedHeaderValue) {
int cellAt = 0;
for (final Object value : casted.keySet()) {
createCell(workbook, sheet, value, xssfRow, cellAt++);
createCell(workbook, sheet, value, xssfRow, cellAt++, renderedStyles);
}
xssfRow = sheet.createRow(rowAt++);
}
}

int cellAt = 0;
for (final Object value : casted.values()) {
createCell(workbook, sheet, value, xssfRow, cellAt++);
createCell(workbook, sheet, value, xssfRow, cellAt++, renderedStyles);
}
}
}
Expand All @@ -242,7 +242,7 @@ public void accept(Object row) {
return lineCount;
}

private void createCell(Workbook workbook, Sheet sheet, Object value, SXSSFRow xssfRow, int rowNumber) {
private void createCell(Workbook workbook, Sheet sheet, Object value, SXSSFRow xssfRow, int rowNumber, boolean styles) {
SXSSFCell cell = xssfRow.createCell(rowNumber);

switch (value) {
Expand Down Expand Up @@ -271,23 +271,23 @@ private void createCell(Workbook workbook, Sheet sheet, Object value, SXSSFRow x
cell.setCellType(CellType.NUMERIC);
}
case LocalDate date -> {
if (this.styles) {
if (styles) {
sheet.setDefaultColumnStyle(cell.getColumnIndex(), getCellStyle(workbook));
}

cell.setCellValue(DateUtil.getExcelDate(date));
cell.setCellType(CellType.NUMERIC);
}
case LocalDateTime date -> {
if (this.styles) {
if (styles) {
sheet.setDefaultColumnStyle(cell.getColumnIndex(), getCellStyle(workbook));
}

cell.setCellValue(DateUtil.getExcelDate(date));
cell.setCellType(CellType.NUMERIC);
}
case Instant instant -> {
if (this.styles) {
if (styles) {
sheet.setDefaultColumnStyle(cell.getColumnIndex(), getCellStyle(workbook));
}

Expand Down
9 changes: 5 additions & 4 deletions src/test/java/io/kestra/plugin/serdes/excel/ExcelToIonTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.google.common.collect.ImmutableMap;
import com.google.common.io.CharStreams;
import io.kestra.core.models.property.Property;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.storages.StorageInterface;
import io.kestra.core.utils.TestsUtils;
Expand Down Expand Up @@ -43,7 +44,7 @@ void ion() throws Exception {
.id(ExcelToIonTest.class.getSimpleName())
.type(ExcelToIon.class.getName())
.from(source.toString())
.header(true)
.header(Property.of(true))
.build();
ExcelToIon.Output ionOutput = reader.run(TestsUtils.mockRunContext(runContextFactory, reader, ImmutableMap.of()));

Expand All @@ -66,14 +67,14 @@ void multiSheets() throws Exception {
.id(ExcelToIonTest.class.getSimpleName())
.type(ExcelToIon.class.getName())
.from(source.toString())
.sheetsTitle(
.sheetsTitle(Property.of(
List.of(
"Worksheet_1",
"Worksheet_2",
"Worksheet_3"
)
)
.header(true)
))
.header(Property.of(true))
.build();

ExcelToIon.Output ionOutput = reader.run(
Expand Down
Loading

0 comments on commit 4c014a6

Please sign in to comment.