From f4295062d7380139961bef9c5d9b347ef29f1fac Mon Sep 17 00:00:00 2001 From: Daniel Mantovani Date: Thu, 10 Mar 2022 14:32:23 -0300 Subject: [PATCH 1/6] upgrade default scala version to 2.12 --- README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index 65d6e94..18a8ee1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ libraryDependencies += "com.github.music-of-the-ainur" %% "quenya-dsl" % "1.2.0- To run in spark-shell: ``` -spark-shell --packages "com.github.music-of-the-ainur:quenya-dsl_2.11:1.2.0-$SPARK_VERSION" +spark-shell --packages "com.github.music-of-the-ainur:quenya-dsl_2.12:1.2.0-$SPARK_VERSION" ``` ## Introduction @@ -199,14 +199,6 @@ You can _alias_ using the fully qualified name using ```printDsl(df,true)```, yo | DoubleType | FloatType | ByteType | IntegerType | LongType | ShortType ``` -## Requirements - -| Software | Version | -|--------------|-----------| -| Java | 8 | -| Scala | 2.11/2.12 | -| Apache Spark | 2.4 | - ## Author Daniel Mantovani [daniel.mantovani@modak.com](mailto:daniel.mantovani@modak.com) From 8475f26191f53f2eeada674785f68e9ad477c1ac Mon Sep 17 00:00:00 2001 From: chntanpallik1999 Date: Mon, 21 Mar 2022 12:53:34 +0530 Subject: [PATCH 2/6] Updated README.md --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 18a8ee1..65f44fe 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,8 @@ Output: ## DSL Generator -You can generate a DSL based on a DataFrame: +### printDsl +You can generate and print a DSL based on a DataFrame: ```scala import com.github.music.of.the.ainur.quenya.QuenyaDSL @@ -148,6 +149,18 @@ val quenyaDsl = QuenyaDSL quenyaDsl.printDsl(df) ``` +### getDsl +You can generate and asssign a DSL to variable based on a DataFrame: + +```scala +import com.github.music.of.the.ainur.quenya.QuenyaDSL + +val df:DataFrame = ... +val quenyaDsl = QuenyaDSL +val dsl = quenyaDsl.getDsl(df) +``` + + json: ``` { From 426a4242612cc4a82f15fd26b31dc7900f0c5f47 Mon Sep 17 00:00:00 2001 From: Daniel de Oliveira Mantovani Date: Thu, 13 Oct 2022 11:33:41 -0300 Subject: [PATCH 3/6] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49f1f60..dc31bd5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Adding Quenya DSL dependency to your sbt build: ``` -libraryDependencies += "com.github.music-of-the-ainur" %% "quenya-dsl" % "1.2.2-3.3" +libraryDependencies += "com.github.music-of-the-ainur" %% "quenya-dsl" % "1.2.2-$SPARK_VERSION" ``` To run in spark-shell: From 819273a5f330242c92be63bc8c557ea4205b043c Mon Sep 17 00:00:00 2001 From: ChinthapallyAkanksha Date: Tue, 25 Oct 2022 16:59:25 +0530 Subject: [PATCH 4/6] Added data.json file and modified the CombinatorParser.scala and Test.scala --- .../workflows/quneys-dsl-githubactions.yml | 4 +- .../quenya/compiler/CombinatorParser.scala | 32 ++-- src/test/resources/data.json | 50 +++++++ .../data/complexData.parquet/._SUCCESS.crc | Bin 0 -> 8 bytes ...-ae41-9a6734d3ab36-c000.snappy.parquet.crc | Bin 0 -> 40 bytes .../data/complexData.parquet/_SUCCESS | 0 ...4a07-ae41-9a6734d3ab36-c000.snappy.parquet | Bin 0 -> 3736 bytes .../music/of/the/ainur/quenya/Test.scala | 137 +++++++++++++----- 8 files changed, 170 insertions(+), 53 deletions(-) create mode 100644 src/test/resources/data.json create mode 100644 src/test/resources/data/complexData.parquet/._SUCCESS.crc create mode 100644 src/test/resources/data/complexData.parquet/.part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet.crc create mode 100644 src/test/resources/data/complexData.parquet/_SUCCESS create mode 100644 src/test/resources/data/complexData.parquet/part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet diff --git a/.github/workflows/quneys-dsl-githubactions.yml b/.github/workflows/quneys-dsl-githubactions.yml index 822860b..7e7447e 100644 --- a/.github/workflows/quneys-dsl-githubactions.yml +++ b/.github/workflows/quneys-dsl-githubactions.yml @@ -28,8 +28,8 @@ jobs: cache: sbt - name: Build and test scala version run: | - PGPASSWORD="postgres" psql -c 'create database almaren;' -U postgres -h localhost - PGPASSWORD="postgres" psql -c "ALTER USER postgres PASSWORD 'foo' ;" -U postgres -h localhost + PGPASSWORD="postgres" psql -c 'create database almaren;' -U postgres -h localhost + PGPASSWORD="postgres" psql -c "ALTER USER postgres PASSWORD 'postgres' ;" -U postgres -h localhost PGPASSWORD="postgres" psql -c 'create role runner;' -U postgres -h localhost PGPASSWORD="postgres" psql -c 'ALTER ROLE "runner" WITH LOGIN SUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION;' -U postgres -h localhost sbt ++2.12.10 test diff --git a/src/main/scala/com/github/music/of/the/ainur/quenya/compiler/CombinatorParser.scala b/src/main/scala/com/github/music/of/the/ainur/quenya/compiler/CombinatorParser.scala index b89a2ee..efe7581 100644 --- a/src/main/scala/com/github/music/of/the/ainur/quenya/compiler/CombinatorParser.scala +++ b/src/main/scala/com/github/music/of/the/ainur/quenya/compiler/CombinatorParser.scala @@ -6,7 +6,6 @@ import scala.util.parsing.combinator._ /* Spark DSL Backus-Naur form. - ::= \{"[\r\n]*".r \} ::= "[\s\t]*".r ::= "a-zA-Z0-9_.".r [ element ] @@ -15,9 +14,8 @@ import scala.util.parsing.combinator._ <@> ::= @ <$> ::= $ : ::= "0-9a-zA-Z_".r - ::= BinaryType | BooleanType | StringType | TimestampType | DecimalType + ::= BinaryType | BooleanType | StringType | TimestampType | DecimalType | DoubleType | FloatType | ByteType | IntegerType | LongType | ShortType - */ private[quenya] trait CombinatorParser { val parser = ParserQuenyaDsl @@ -30,9 +28,16 @@ private[quenya] trait CombinatorParser { } } -object ParserQuenyaDsl extends JavaTokenParsers { +trait ParserUtil { + def removeLiteral(content: String, literal: String): String = { + if (content.head.toString == literal && content.last.toString == literal) + content.substring(1, content.length - 1) + else content + } +} +object ParserQuenyaDsl extends JavaTokenParsers with ParserUtil { override val skipWhitespace = false - + def dsl: Parser[List[Statement]] = repsep(expression,"""[\n\r]*""".r) ^^ (List() ++ _ ) def expression: Parser[Statement] = precedence ~ col ~ operator ^^ { case prec ~ cl ~ op => @@ -45,16 +50,21 @@ object ParserQuenyaDsl extends JavaTokenParsers { case al:String => Statement(prec,cl,AT,al) } } - def precedence: Parser[Int] = """[\t\s]*""".r ^^ (prec => prec.replaceAll(" ","\t").count(_ == '\t')) - def col: Parser[StateSelect] = """[0-9A-Za-z._]+""".r ~ opt(element) ^^ { - case a ~ Some(b) => StateSelect(a,b) - case a ~ None => StateSelect(a,None) + def precedence: Parser[Int] = """^[\t\s]*""".r ^^ (prec => prec.replaceAll(" ","\t").count(_ == '\t')) + def col: Parser[StateSelect] = """[\w.]+|`[\w. \-:;$]+`""".r ~ opt(element) ^^ { + case a ~ Some(b) => createStateSelect(a,b) + case a ~ None => createStateSelect(a,None) } + private def createStateSelect(name: String, element: Option[String]): StateSelect = + StateSelect(removeLiteral(name,"`"),element) + def element: Parser[Option[String]] = "[" ~> opt("""\d+""".r) <~ "]" def operator: Parser[Any] = at | dollar def at: Parser[String] = "@" ~> alias def dollar : Parser[Any] = "$" ~> alias ~ opt(":") ~ datatype - def alias : Parser[String] = "[0-9a-zA-Z_]+".r + def alias : Parser[String] = """\w+|`[\w \-:;$]+`""".r ^^ { + alias => removeLiteral(alias,"`") + } def datatype : Parser[Option[DataType]] = ("BinaryType" ^^ (dt => Some(BinaryType)) | "FloatType" ^^ (dt => Some(FloatType)) | "ByteType" ^^ (dt => Some(ByteType)) @@ -67,4 +77,4 @@ object ParserQuenyaDsl extends JavaTokenParsers { | "DoubleType" ^^ (dt => Some(DoubleType)) | "ShortType" ^^ (dt => Some(ShortType)) | "" ^^ (dt => None)) -} +} \ No newline at end of file diff --git a/src/test/resources/data.json b/src/test/resources/data.json new file mode 100644 index 0000000..63f1191 --- /dev/null +++ b/src/test/resources/data.json @@ -0,0 +1,50 @@ +{ + "Coffee": { + "sub region": [ + { + "id": 1, + "full name": "John Doe" + }, + { + "id": 2, + "name": "Don Joeh" + } + ], + "country": { + "id": 2, + "company": "ACME" + } + }, + "brewing": { + "sub-region": [ + { + "id": 1, + "name": "John Doe" + }, + { + "id": 2, + "name": "Don Joeh" + } + ], + "world:country": { + "id": 2, + "company": "ACME" + } + }, + "brewing2": { + "sub;region": [ + { + "id": 1, + "name": "John Doe" + }, + { + "id": 2, + "name": "Don Joeh" + } + ], + "world;country": { + "id": 2, + "company": "ACME" + } + } +} \ No newline at end of file diff --git a/src/test/resources/data/complexData.parquet/._SUCCESS.crc b/src/test/resources/data/complexData.parquet/._SUCCESS.crc new file mode 100644 index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c GIT binary patch literal 8 PcmYc;N@ieSU}69O2$TUk literal 0 HcmV?d00001 diff --git a/src/test/resources/data/complexData.parquet/.part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet.crc b/src/test/resources/data/complexData.parquet/.part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..826d9313e66c28b74684f74e13fbbb82f54ced09 GIT binary patch literal 40 ycmV+@0N4Lxa$^7h00IEgO`FSg6u3u(p!~G-PX4A--wvp8>~oa;*3LBx*f|PitP@HA literal 0 HcmV?d00001 diff --git a/src/test/resources/data/complexData.parquet/_SUCCESS b/src/test/resources/data/complexData.parquet/_SUCCESS new file mode 100644 index 0000000..e69de29 diff --git a/src/test/resources/data/complexData.parquet/part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet b/src/test/resources/data/complexData.parquet/part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6c29ec5b8e8ed97a21ccdf26a5371a0aaf2aa84a GIT binary patch literal 3736 zcmd5{1qwOVL%Ni1gq=yzGLY;vp={9z;AWdnmdr1%v-LlQ%C(Q`@8-3?X^* z-v9Uef4u)ka^>>tam?c&4&kll-5=dE*oAq71O-0O<(FQ286)fP=dTXF^eM39u?Gu8 zzy$(94v`&#zW2e+RR>JY$m@78!U2H)^w;j+9>R`nc2CG+C-~DG^Eiyh@xvRe7$zu?t|NRkE2sB-KaFJtr?XaZTKvn%G1|FlM%YGt#wDXEMhvy(jNLIu> z&RI4&JWF?7&hmgQH+ekY61eSlmfIeOq3(5-4X?wnFF3NhK4*8wZ_7fI9={T(tmx&p z7b}@%qp0H-pxDp4q>+r7%Na&2V-{ER(%M<{ssc7+9*)MGqgtt2UW?_kHY8N3r5DS_ zQr;{r=4ypPtfUu>cHl4*a6qH5H*SLt$>Vg{Sk0G~78x!@l|A1&OTk&p#;4l#5g&w> z@$q&ehVt3C?FXz9w&iYuOi(ET_4e?a&u7PTNDRt~{6L*ka6rBwczC%xRM;g3#j5z8 zI;RSXT^;tPh|XEHVm~1{N}3^dk@iZykoe0e4vA0vSVQU_r(nLte>mQjin91Fh^g;$ z3QIS(0%cATvfUm^{}{!hcZ%Uj6$(bprTm?*rDhAV=gr#u? zzX+y71Gh{Emd+b@;Crqv`1K&bpIPt&fKwy7lfxl6=q9*B2(|)5|0d5u*E@qOnh!bg z8VADaH$xo(q^+hifONM<0KX%uzYdT9-gdLBI4VIc-`xGB-ebajcbKGJ-OJ)fip8QO$yWEOk31c*qFt^t zR&=+A3!7u0=HW1jRfDR5G$EB}sUf-5@OC9foq%qQkh|R%nBR8MwuN(FQ0v0U!}&R( z{dR=dvJ115<{Yhj$DUnZmp$JT;4{mf9Y2ee)gGK+?fBI{eWwdngXq~B1p~!T{bEoR z#NYiPm>W?^GRsSvzM@0N)~x2OyD#3CQO0QcwWjCX zV|4vlb;O`#bp2XKr|RGg&{Y_?KX0H@3<|Y|JOdBt_TU8M=L18Z#O=2>khIV$+E}q1 zZdG|WuEi5tB781g%NMfY>1iW5k-0c=DVe-DWn?ZSGRaI%pUNaB)6+&SnM}l!iA3ho V $"csv.age" && - $"dsl.LastName" <=> $"csv.LastName" && - $"dsl.nameOne" <=> $"csv.nameOne" && - $"dsl.nickNames" <=> $"csv.nickNames" && - $"dsl.race" <=> $"csv.race" && - $"dsl.weapon" <=> $"csv.weapon","leftanti").count() + test(dslDf, csvDf, "Test for Dsl") - test("data should be exactly the same") { - assert(diff == 0) - } - - - val dslMatch = """age$age:LongType + // Test case for getDsl method + val dslMatch = + """age$age:LongType name.LastName$LastName:StringType name.nameOne$nameOne:StringType name.nickNames@nickNames @@ -66,13 +53,83 @@ race$race:StringType weapon@weapon weapon$weapon:StringType""" - test("DSL generate") { + test("Test for DSL Generate") { assert(dslMatch == quenyaDsl.getDsl(df)) } + //Test case for Dsl containing special characters + val complexJsonDf = spark.read.option("multiLine", "true").json("src/test/resources/data.json") + + // For columns containing special characters like ;,:,- and spaces etc , need to include them in `` + + val complexDsl = quenyaDsl.compile( + """Coffee.country.company$`coffee-company`:StringType + |Coffee.country.id$`coffee-country-id`:LongType + |`Coffee.sub region`@`sub:region` + | `sub region.full name`$`sub_region_full-name`:StringType + | `sub region.id`$`sub_region-id`:LongType + | `sub region.name`$`sub$region-name`:StringType + |`brewing.sub-region`@`sub-region` + | `sub-region.id`$`brewing_sub_region:id`:LongType + | `sub-region.name`$`brewing:sub-region:name`:StringType + |`brewing.world:country.company`$company1:StringType + |`brewing.world:country.id`$id1:LongType + |`brewing2.sub;region`@`sub;region` + | `sub;region.id2`$id2:LongType + | `sub;region.name`$name2:StringType + |`brewing2.world;country.company2`$company:StringType + |`brewing2.world;country.id`$`id$f`:LongType""".stripMargin) + + val complexJsonData = quenyaDsl.execute(complexDsl, complexJsonDf) + complexJsonData.coalesce(1).write.mode("overwrite").parquet("src/test/resources/data/complexData.parquet") + + val complexDataParquet = spark.read.parquet("src/test/resources/data/complexData.parquet") + test(complexJsonData, complexDataParquet, "Test for Complex Dsl with Special Characters") + + def test(df1: DataFrame, df2: DataFrame, name: String): Unit = { + testCount(df1, df2, name) + testCompare(df1, df2, name) + } + + def testCount(df1: DataFrame, df2: DataFrame, name: String): Unit = { + val count1 = df1.count() + val count2 = df2.count() + val count3 = spark.emptyDataFrame.count() + test(s"Count Test:$name should match of count: ${df1.count()}") { + assert(count1 == count2) + } + test(s"Count Test:$name should not match") { + assert(count1 != count3) + } + } + + // Doesn't support nested type and we don't need it :) + def testCompare(df1: DataFrame, df2: DataFrame, name: String): Unit = { + val diff = compare(df1, df2) + test(s"Compare Test:$name should be zero") { + assert(diff == 0) + } + test(s"Compare Test:$name, should not be able to join") { + assertThrows[AnalysisException] { + compare(df2, spark.emptyDataFrame) + } + } + } + + private def compare(df1: DataFrame, df2: DataFrame): Long = { + val diff = df1.as("df1").join(df2.as("df2"), joinExpression(df1), "leftanti") + diff.show(false) + diff.count + } + + private def joinExpression(df1: DataFrame): Column = + df1.schema.fields + .map(field => col(s"df1.${field.name}") <=> col(s"df2.${field.name}")) + .reduce((col1, col2) => col1.and(col2)) + after { spark.stop() } -} +} \ No newline at end of file From 803adfb4aeb5b7c4cd4c83053b976a1109ed64ef Mon Sep 17 00:00:00 2001 From: ChinthapallyAkanksha Date: Tue, 25 Oct 2022 18:04:24 +0530 Subject: [PATCH 5/6] Added data.json file and modified ComparatorParser.scala and Test.scala --- README.md | 69 +++++++++++++----- build.sbt | 6 ++ src/test/resources/data.csv | 8 +- ...-ae41-9a6734d3ab36-c000.snappy.parquet.crc | Bin 40 -> 0 bytes ...-9514-c1c7010257da-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...295-9514-c1c7010257da-c000.snappy.parquet} | Bin 3736 -> 3736 bytes 6 files changed, 61 insertions(+), 22 deletions(-) delete mode 100644 src/test/resources/data/complexData.parquet/.part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet.crc create mode 100644 src/test/resources/data/complexData.parquet/.part-00000-f0e17416-ae5a-4295-9514-c1c7010257da-c000.snappy.parquet.crc rename src/test/resources/data/complexData.parquet/{part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet => part-00000-f0e17416-ae5a-4295-9514-c1c7010257da-c000.snappy.parquet} (78%) diff --git a/README.md b/README.md index 8515aab..8da7cbe 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Quenya DSL +# Quenya-DSL [![Build Status](https://github.com/music-of-the-ainur/quenya-dsl/actions/workflows/quneys-dsl-githubactions.yml/badge.svg)](https://github.com/music-of-the-ainur/quenya-dsl/actions/workflows/quneys-dsl-githubactions.yml) -Adding Quenya DSL dependency to your sbt build: +Adding Quenya-DSL dependency to your sbt build: ``` libraryDependencies += "com.github.music-of-the-ainur" %% "quenya-dsl" % "1.2.2-3.3" @@ -14,15 +14,11 @@ To run in spark-shell: spark-shell --packages "com.github.music-of-the-ainur:quenya-dsl_2.13:1.2.2-3.3" ``` -### Connector Usage -#### Maven / Ivy Package Usage -The connector is also available from the -[Maven Central](https://mvnrepository.com/artifact/com.github.music-of-the-ainur) -repository. It can be used using the `--packages` option or the -`spark.jars.packages` configuration property. Use the following value +Quenya-Dsl is available in [Maven Central](https://mvnrepository.com/artifact/com.github.music-of-the-ainur) +repository. -| version | Connector Artifact | +| versions | Connector Artifact | |----------------------------|-----------------------------------------------------------| | Spark 3.3.x and scala 2.13 | `com.github.music-of-the-ainur:quenya-dsl_2.13:1.2.2-3.3` | | Spark 3.3.x and scala 2.12 | `com.github.music-of-the-ainur:quenya-dsl_2.12:1.2.2-3.3` | @@ -32,7 +28,7 @@ repository. It can be used using the `--packages` option or the | Spark 2.4.x and scala 2.11 | `com.github.music-of-the-ainur:quenya-dsl_2.11:1.2.2-2.4` | ## Introduction -Quenya DSL(Domain Specific Language) is a language that simplifies the task to parser complex semi-structured data. +Quenya-DSL(Domain Specific Language) is a language that simplifies the task to parser complex semi-structured data. ```scala @@ -155,7 +151,7 @@ Output: ## DSL Generator -You can generate a DSL based on a DataFrame: +You can generate the DSL from an existing DataFrame: ```scala import com.github.music.of.the.ainur.quenya.QuenyaDSL @@ -201,6 +197,50 @@ weapon@weapon You can _alias_ using the fully qualified name using ```printDsl(df,true)```, you should turn on in case of name conflict. +## How to Handle Special Characters + + + +Use the literal backtick **``** to handle special characters like space,semicolon,hyphen and colon. +Example: + + + +json: +``` +{ + "name":{ + "name One":"Mithrandir", + "Last-Name":"Olórin", + "nick:Names":[ + "Gandalf the Grey", + "Gandalf the White" + ] + }, + "race":"Maiar", + "age":"immortal", + "weapon;name":[ + "Glamdring", + "Narya", + "Wizard Staff" + ] +} +``` + + + +DSL: +``` +age$age:StringType +`name.Last-Name`$`Last-Name`:StringType +`name.name One`$`name-One`:StringType +`name.nick:Names`@`nick:Names` + `nick:Names`$`nick:Names`:StringType +race$race:StringType +`weapon;name`@`weapon;name` + `weapon;name`$`weapon_name`:StringType +``` + ## Backus–Naur form ``` @@ -216,13 +256,6 @@ You can _alias_ using the fully qualified name using ```printDsl(df,true)```, yo | DoubleType | FloatType | ByteType | IntegerType | LongType | ShortType ``` -## Requirements - -| Software | Version | -|--------------|-----------| -| Java | 8 | -| Scala | 2.11/2.12 | -| Apache Spark | 2.4 | ## Author Daniel Mantovani [daniel.mantovani@modak.com](mailto:daniel.mantovani@modak.com) diff --git a/build.sbt b/build.sbt index 5617c48..276290d 100644 --- a/build.sbt +++ b/build.sbt @@ -32,6 +32,12 @@ ThisBuild / developers := List( name = "Daniel Mantovani", email = "daniel.mantovani@modakanalytics.com", url = url("https://github.com/music-of-the-ainur") + ), + Developer( + id = "ChinthapallyAkanksha", + name = "Akanksha Chinthapally", + email = "akanksha.chinthapally@modak.com", + url = url("https://github.com/music-of-the-ainur") ) ) diff --git a/src/test/resources/data.csv b/src/test/resources/data.csv index 886052d..e39b9dd 100644 --- a/src/test/resources/data.csv +++ b/src/test/resources/data.csv @@ -1,10 +1,10 @@ age,LastName,nameOne,nickNames,race,weapon -3500,Olórin,Mithrandir,Gandalf the Grey,Maiar,Glamdring -3500,Olórin,Mithrandir,Gandalf the Grey,Maiar,Narya -3500,Olórin,Mithrandir,Gandalf the Grey,Maiar,Wizard Staff +3500,Olórin,Mithrandir,Gandalf the Grey,Maiar,Glamdring +3500,Olórin,Mithrandir,Gandalf the Grey,Maiar,Narya +3500,Olórin,Mithrandir,Gandalf the Grey,Maiar,Wizard Staff 4500,"",Ilmarë,"",Ainur,Powers of the Ainur 3500,"",Morgoth,Bauglir,Ainur,Powers of the Ainur 3500,"",Morgoth,Bauglir,Ainur,Grond 3500,"",Morgoth,Bauglir,Ainur,Mace 3500,"",Morgoth,Bauglir,Ainur,Sword -3500,"",Manwë,"King of Arda,",Ainur,Powers of the Ainur +3500,"",Manwë,"King of Arda,",Ainur,Powers of the Ainur \ No newline at end of file diff --git a/src/test/resources/data/complexData.parquet/.part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet.crc b/src/test/resources/data/complexData.parquet/.part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet.crc deleted file mode 100644 index 826d9313e66c28b74684f74e13fbbb82f54ced09..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40 ycmV+@0N4Lxa$^7h00IEgO`FSg6u3u(p!~G-PX4A--wvp8>~oa;*3LBx*f|PitP@HA diff --git a/src/test/resources/data/complexData.parquet/.part-00000-f0e17416-ae5a-4295-9514-c1c7010257da-c000.snappy.parquet.crc b/src/test/resources/data/complexData.parquet/.part-00000-f0e17416-ae5a-4295-9514-c1c7010257da-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..10efee331210626585c35ae3b774a11fb37e0ce3 GIT binary patch literal 40 ycmV+@0N4Lxa$^7h00IEgO`FSg6u3uOZa4Fo!Sk&K+<266>~oa;*3LBx*f|PPI1+#W literal 0 HcmV?d00001 diff --git a/src/test/resources/data/complexData.parquet/part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet b/src/test/resources/data/complexData.parquet/part-00000-f0e17416-ae5a-4295-9514-c1c7010257da-c000.snappy.parquet similarity index 78% rename from src/test/resources/data/complexData.parquet/part-00000-ba1f50c8-81ef-4a07-ae41-9a6734d3ab36-c000.snappy.parquet rename to src/test/resources/data/complexData.parquet/part-00000-f0e17416-ae5a-4295-9514-c1c7010257da-c000.snappy.parquet index 6c29ec5b8e8ed97a21ccdf26a5371a0aaf2aa84a..4a93a077f809896f7742dfe3b8430244fd34c1f5 100644 GIT binary patch delta 106 zcmbOsJ41HEPF5xsw#mC$<3N-&TQL&@+vEvsE+Fa}n=ObEWw&Q#VUv^)oy^Z8IysL$ p5v1}Wdpd}+;)nuK(>PK=D*tjAgLp=qp&(u-r#XPK=D*tjAgLp=qp&(u-r#X Date: Tue, 25 Oct 2022 18:14:43 +0530 Subject: [PATCH 6/6] Updated quenya dsl to support special characters for spark-3.3 branch --- ...{quneys-dsl-githubactions.yml => quenya-dsl-githubactions.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{quneys-dsl-githubactions.yml => quenya-dsl-githubactions.yml} (100%) diff --git a/.github/workflows/quneys-dsl-githubactions.yml b/.github/workflows/quenya-dsl-githubactions.yml similarity index 100% rename from .github/workflows/quneys-dsl-githubactions.yml rename to .github/workflows/quenya-dsl-githubactions.yml