Skip to content

Commit

Permalink
Introduce custom compiler option
Browse files Browse the repository at this point in the history
  • Loading branch information
RustedBones committed Oct 15, 2024
1 parent 3dfa5e0 commit ba3dc45
Show file tree
Hide file tree
Showing 16 changed files with 171 additions and 117 deletions.
23 changes: 12 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,35 @@ For instance, add the following lines to `project/plugins.sbt`:

```
addSbtPlugin("com.github.sbt" % "sbt-avro" % "3.5.0")
// Java sources compiled with one version of Avro might be incompatible with a
// different version of the Avro library. Therefore we specify the compiler
// version here explicitly.
libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.12.0"
```

Add the library dependency to `build.sbt`:
Enable the plugin in your `build.sbt` and select the avro version to use:

```
libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion
enablePlugins(SbtAvro)
avroVersion := "1.12.0"
```

## Config

An `avro` configuration will be added to the project. Libraries defined with this scope will be loaded by the sbt plugin
to generate the avro classes.

## Settings

| Name | Default | Description |
|:-------------------------------------------|:----------------------------------------------|:----------------------------------------------------------------------------------------|
| `avroSource` | `sourceDirectory` / `avro` | Source directory with `*.avsc`, `*.avdl` and `*.avpr` files. |
| `avroSources` | `sourceDirectory` / `avro` | Source directories with `*.avsc`, `*.avdl` and `*.avpr` files. |
| `avroAdditionalDependencies` | `avro-compiler` % `avro`, `avro` % `compile` | Additional dependencies to be added to library dependencies. |
| `avroSpecificRecords` | `Seq.empty` | List of avro generated classes to recompile with current avro version and settings. |
| `avroSchemaParserBuilder` | `DefaultSchemaParserBuilder.default()` | `.avsc` schema parser builder |
| `avroUnpackDependencies` / `includeFilter` | All avro specifications | Avro specification files from dependencies to unpack |
| `avroUnpackDependencies` / `excludeFilter` | Hidden files | Avro specification files from dependencies to exclude from unpacking |
| `avroUnpackDependencies` / `target` | `sourceManaged` / `avro` / `$config` | Target directory for schemas packaged in the dependencies |
| `avroGenerate` / `target` | `sourceManaged` / `compiled_avro` / `$config` | Source directory for generated `.java` files. |
| `avroDependencyIncludeFilter` | `source` typed `avro` classifier artifacts | Dependencies containing avro schema to be unpacked for generation |
| `avroIncludes` | `Seq()` | Paths with extra `*.avsc` files to be included in compilation. |
| `packageAvro` / `artifactClassifier` | `Some("avro")` | Classifier for avro artifact |
| `packageAvro` / `publishArtifact` | `false` | Enable / Disable avro artifact publishing |
| `avroCompiler` | `com.github.sbt.avro.AvroCompilerBridge` | Sbt avro compiler class. |
| `avroStringType` | `CharSequence` | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. |
| `avroUseNamespace` | `false` | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`. |
| `avroFieldVisibility` | `public` | Field Visibility for the properties. Possible values: `private`, `public`. |
Expand All @@ -72,7 +73,7 @@ If you depend on an artifact with previously generated avro java classes with st
you can recompile them with `String` by also adding the following

```sbt
Compile / avroSpecificRecords += classOf[com.example.MyAvroRecord] // lib must be declared in project/plugins.sbt
Compile / avroSpecificRecords += "com.example.MyAvroRecord" // lib must be added in the avro scope
```

## Packaging Avro files
Expand Down
8 changes: 8 additions & 0 deletions api/src/main/java/com/github/sbt/avro/AvroCompiler.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@
import java.io.File;

public interface AvroCompiler {

void setStringType(String stringType);
void setFieldVisibility(String fieldVisibility);
void setUseNamespace(boolean useNamespace);
void setEnableDecimalLogicalType(boolean enableDecimalLogicalType);
void setCreateSetters(boolean createSetters);
void setOptionalGetters(boolean optionalGetters);

void recompile(Class<?>[] records, File target) throws Exception;
void compileIdls(File[] idls, File target) throws Exception;
void compileAvscs(AvroFileRef[] avscs, File target) throws Exception;
Expand Down
69 changes: 38 additions & 31 deletions bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import org.apache.avro.Schema;
import org.apache.avro.specific.SpecificRecord;
import xsbti.Logger;

import org.apache.avro.Protocol;
import org.apache.avro.compiler.idl.Idl;
Expand All @@ -16,55 +15,63 @@

public class AvroCompilerBridge implements AvroCompiler {

private final Logger logger;
private final StringType stringType;
private final FieldVisibility fieldVisibility;
private final boolean useNamespace;
private final boolean enableDecimalLogicalType;
private final boolean createSetters;
private final boolean optionalGetters;

public AvroCompilerBridge(
Logger logger,
String stringType,
String fieldVisibility,
boolean useNamespace,
boolean enableDecimalLogicalType,
boolean createSetters,
boolean optionalGetters
) {
this.logger = logger;
private StringType stringType;
private FieldVisibility fieldVisibility;
private boolean useNamespace;
private boolean enableDecimalLogicalType;
private boolean createSetters;
private boolean optionalGetters;

protected Schema.Parser createParser() {
return new Schema.Parser();
}

@Override
public void setStringType(String stringType) {
this.stringType = StringType.valueOf(stringType);
}

@Override
public void setFieldVisibility(String fieldVisibility) {
this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility);
}

@Override
public void setUseNamespace(boolean useNamespace) {
this.useNamespace = useNamespace;
}

@Override
public void setEnableDecimalLogicalType(boolean enableDecimalLogicalType) {
this.enableDecimalLogicalType = enableDecimalLogicalType;
}

@Override
public void setCreateSetters(boolean createSetters) {
this.createSetters = createSetters;
this.optionalGetters = optionalGetters;
}

protected Schema.Parser createParser() {
return new Schema.Parser();
@Override
public void setOptionalGetters(boolean optionalGetters) {
this.optionalGetters = optionalGetters;
}

@Override
public void recompile(Class<?>[] records, File target) throws Exception {
AvscFilesCompiler compiler = new AvscFilesCompiler();
AvscFilesCompiler compiler = new AvscFilesCompiler(this::createParser);
compiler.setStringType(stringType);
compiler.setFieldVisibility(fieldVisibility);
compiler.setUseNamespace(useNamespace);
compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
compiler.setCreateSetters(createSetters);
if (AvroVersion.getMinor() > 8) {
compiler.setGettersReturnOptional(optionalGetters);
}
if (AvroVersion.getMinor() > 9) {
compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
}
compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");

Set<Class<? extends SpecificRecord>> classes = new HashSet<>();
for (Class<?> record : records) {
logger.info(() -> "Recompiling Avro record: " + record.getName());
System.out.println("Recompiling Avro record: " + record.getName());
classes.add((Class<? extends SpecificRecord>) record);
}
compiler.compileClasses(classes, target);
Expand All @@ -73,7 +80,7 @@ public void recompile(Class<?>[] records, File target) throws Exception {
@Override
public void compileIdls(File[] idls, File target) throws Exception {
for (File idl : idls) {
logger.info(() -> "Compiling Avro IDL " + idl);
System.out.println("Compiling Avro IDL " + idl);
Idl parser = new Idl(idl);
Protocol protocol = parser.CompilationUnit();
SpecificCompiler compiler = new SpecificCompiler(protocol);
Expand All @@ -93,7 +100,7 @@ public void compileIdls(File[] idls, File target) throws Exception {

@Override
public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {
AvscFilesCompiler compiler = new AvscFilesCompiler();
AvscFilesCompiler compiler = new AvscFilesCompiler(this::createParser);
compiler.setStringType(stringType);
compiler.setFieldVisibility(fieldVisibility);
compiler.setUseNamespace(useNamespace);
Expand All @@ -109,7 +116,7 @@ public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {

Set<AvroFileRef> files = new HashSet<>();
for (AvroFileRef ref: avscs) {
logger.info(() -> "Compiling Avro schema: " + ref.getFile());
System.out.println("Compiling Avro schema: " + ref.getFile());
files.add(ref);
}
compiler.compileFiles(Set.of(avscs), target);
Expand All @@ -118,7 +125,7 @@ public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {
@Override
public void compileAvprs(File[] avprs, File target) throws Exception {
for (File avpr : avprs) {
logger.info(() -> "Compiling Avro protocol " + avpr);
System.out.println("Compiling Avro protocol " + avpr);
Protocol protocol = Protocol.parse(avpr);
SpecificCompiler compiler = new SpecificCompiler(protocol);
compiler.setStringType(stringType);
Expand Down
10 changes: 6 additions & 4 deletions bridge/src/main/java/com/github/sbt/avro/AvscFilesCompiler.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@

public class AvscFilesCompiler {

private final Supplier<Schema.Parser> parserSupplier;
private Schema.Parser schemaParser;

private String templateDirectory;
private GenericData.StringType stringType;
private SpecificCompiler.FieldVisibility fieldVisibility;
Expand All @@ -26,9 +28,9 @@ public class AvscFilesCompiler {
private Boolean optionalGettersForNullableFieldsOnly;
private Map<AvroFileRef, Exception> compileExceptions;

public AvscFilesCompiler() {
// this.builder = builder;
this.schemaParser = new Schema.Parser(); //builder.build();
public AvscFilesCompiler(Supplier<Schema.Parser> parserSupplier) {
this.parserSupplier = parserSupplier;
this.schemaParser = parserSupplier.get();
}

public void compileFiles(Set<AvroFileRef> files, File outputDirectory) {
Expand Down Expand Up @@ -151,7 +153,7 @@ private boolean tryCompile(File src, Schema schema, File outputDirectory) {
private Schema.Parser stashParser() {
// on failure Schema.Parser changes cache state.
// We want last successful state.
Schema.Parser parser = new Schema.Parser(); // builder.build();
Schema.Parser parser = parserSupplier.get();
Set<String> predefinedTypes = parser.getTypes().keySet();
Map<String, Schema> compiledTypes = schemaParser.getTypes();
compiledTypes.keySet().removeAll(predefinedTypes);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.github.sbt.avro

import com.github.sbt.avro.test.{TestSpecificRecord, TestSpecificRecordParent}
import org.apache.avro.Schema
import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility
import org.apache.avro.generic.GenericData.StringType
import org.apache.avro.specific.SpecificRecord
Expand All @@ -16,7 +17,7 @@ class AvscFilesCompilerSpec extends Specification {
val targetDir = Files.createTempDirectory("sbt-avro-compiler-bridge").toFile
val packageDir = new File(targetDir, "com/github/sbt/avro/test")

val compiler = new AvscFilesCompiler()
val compiler = new AvscFilesCompiler(() => new Schema.Parser())
compiler.setUseNamespace(false)
compiler.setStringType(StringType.CharSequence)
compiler.setFieldVisibility(FieldVisibility.PRIVATE)
Expand Down
1 change: 0 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ lazy val `sbt-avro-compiler-bridge`: Project = project
autoScalaLibrary := false,
libraryDependencies ++= Seq(
Dependencies.Provided.AvroCompiler,
Dependencies.Provided.SbtUtilInterface,
Dependencies.Test.Specs2Core,
)
)
Expand Down
Loading

0 comments on commit ba3dc45

Please sign in to comment.