Make parquet module compatible with Avro 1.8 and 1.11 (#633)

* Downgrade Apache Avro and fix code * Fix PR * added compatibility checks into build * Found API that works for all avro versions * clean plugins file
spotify · Oct 28, 2022 · b540882 · b540882
1 parent 6b536a9
commit b540882
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 10 deletions.
diff --git a/build.sbt b/build.sbt
@@ -406,6 +406,10 @@ lazy val parquet: Project = project
       "org.apache.hadoop" % "hadoop-client" % hadoopVersion % Provided,
       "org.apache.parquet" % "parquet-avro" % parquetVersion % Provided,
       "org.apache.parquet" % "parquet-hadoop" % parquetVersion % Provided
+    ),
+    dependencyOverrides ++= Seq(
+      "org.apache.avro" % "avro" % avroVersion % Provided,
+      "org.apache.avro" % "avro" % avroVersion % Test
     )
   )
   .dependsOn(

diff --git a/parquet/src/main/scala/magnolify/parquet/SchemaUtil.scala b/parquet/src/main/scala/magnolify/parquet/SchemaUtil.scala
@@ -26,7 +26,7 @@ object SchemaUtil {
 
   object Union {
     def unapply(schema: AvroSchema): Option[Seq[AvroSchema]] =
-      if (schema.isUnion) Some(schema.getTypes.asScala.toSeq) else None
+      if (schema.getType == AvroSchema.Type.UNION) Some(schema.getTypes.asScala.toSeq) else None
   }
 
   object Array {
@@ -69,13 +69,14 @@ object SchemaUtil {
           f.defaultVal()
         )
       }
-      AvroSchema.createRecord(
+      val record = AvroSchema.createRecord(
         schema.getName,
         rootDoc.orNull,
         schema.getNamespace,
-        schema.isError,
-        updatedFields.result().asJava
+        schema.isError
       )
+      record.setFields(updatedFields.result().asJava)
+      record
     case _ =>
       schema
   }

diff --git a/parquet/src/test/scala/magnolify/parquet/AvroParquetSuite.scala b/parquet/src/test/scala/magnolify/parquet/AvroParquetSuite.scala
@@ -20,9 +20,9 @@ import java.time._
 import cats._
 import magnolify.cats.auto._
 import magnolify.cats.TestEq._
-import magnolify.avro._
+import magnolify.avro.AvroType
+import magnolify.shared.doc
 import magnolify.avro.unsafe._
-import magnolify.parquet._
 import magnolify.parquet.unsafe._
 import magnolify.parquet.ParquetArray.AvroCompat._
 import magnolify.parquet.util.AvroSchemaComparer

diff --git a/parquet/src/test/scala/magnolify/parquet/SchemaEvolutionSuite.scala b/parquet/src/test/scala/magnolify/parquet/SchemaEvolutionSuite.scala
@@ -88,7 +88,7 @@ object SchemaEvolutionSuite {
 
   val (location1ProjectionSchema, user1ProjectionSchema): (AvroSchema, AvroSchema) = {
     def id = new AvroSchema.Field("id", AvroSchema.create(AvroSchema.Type.LONG), "", null)
-    def name = new AvroSchema.Field("name", nullableString, "")
+    def name = new AvroSchema.Field("name", nullableString, "", null)
 
     def country =
       new AvroSchema.Field("country", AvroSchema.create(AvroSchema.Type.STRING), "", null)

diff --git a/parquet/src/test/scala/magnolify/parquet/util/AvroSchemaComparer.scala b/parquet/src/test/scala/magnolify/parquet/util/AvroSchemaComparer.scala
@@ -86,9 +86,6 @@ object AvroSchemaComparer {
       ) ++ require(
         s1.getType == s2.getType,
         s"$path 'type' are different '${s1.getType}' != '${s2.getType}'"
-      ) ++ require(
-        s1.isNullable == s2.isNullable,
-        s"$path 'isNullable' are different '${s1.isNullable}' != '${s2.isNullable}'"
       ) ++ require(
         s1.getDoc == s2.getDoc,
         s"$path 'doc' are different '${s1.getDoc}' != '${s2.getDoc}'"