diff --git a/aerojoin-example/build.sbt b/aerojoin-example/build.sbt new file mode 100644 index 0000000..2896545 --- /dev/null +++ b/aerojoin-example/build.sbt @@ -0,0 +1,28 @@ +import scala.util.Properties + +name := "aerojoin-example" + +version := "1.2.0" + +scalaVersion := "2.12.11" +def sysPropOrDefault(propName:String,default:String):String = Option(System.getProperty(propName)).getOrElse(default) + +val sparkVer = sysPropOrDefault("version","2.4.4") +val hadoopVer = "2.7.3" +val asClientVer = "4.4.13" +val asSparkConnectorVer = "2.3.0" +val sparkHome = Properties.envOrElse("SPARK_HOME", "/opt/spark") + + +libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % sparkVer % "provided", + "org.apache.spark" %% "spark-sql" % sparkVer , + "org.apache.hadoop" % "hadoop-common" % hadoopVer , + "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVer % Provided, + "com.aerospike" %% "aerospike-spark" % "provided" from s"file://${sparkHome}/jars/aerospike-spark-assembly-${asSparkConnectorVer}.jar", + "com.aerospike" % "aerospike-client" % asClientVer, + +) +resolvers ++= Seq("Local Maven" at Path.userHome.asFile.toURI.toURL + ".m2/repository") +resolvers += "Local Maven Repository" at "file://"+Path.userHome.absolutePath+"/.m2/repository" +publishTo := Some(Resolver.file("file", new File(Path.userHome.absolutePath+"/.m2/repository"))) diff --git a/src/main/scala/com/aerospike/spark/aeroJoinExample.scala b/aerojoin-example/src/main/scala/com/aerospike/spark/aeroJoinExample.scala similarity index 95% rename from src/main/scala/com/aerospike/spark/aeroJoinExample.scala rename to aerojoin-example/src/main/scala/com/aerospike/spark/aeroJoinExample.scala index 99d0490..2230a07 100644 --- a/src/main/scala/com/aerospike/spark/aeroJoinExample.scala +++ b/aerojoin-example/src/main/scala/com/aerospike/spark/aeroJoinExample.scala @@ -1,6 +1,5 @@ package com.aerospike.spark -import com.typesafe.scalalogging.slf4j.LazyLogging import org.apache.spark.SparkConf import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} import org.apache.spark.sql.{Row, SaveMode, SparkSession} @@ -11,12 +10,12 @@ import org.apache.spark.sql.{Row, SaveMode, SparkSession} * * Prereqs: A working Aerospike Connect for Spark and an Aerospike server running on default port on localhost with at least 1 namespace named "Business" */ -object aeroJoinExample extends LazyLogging with Serializable { +object aeroJoinExample extends Serializable { val conf: SparkConf = new SparkConf() .setAppName("AeroJoin") .set("aerospike.seedhost", "localhost") .set("aerospike.port", "3000") - .set("aerospike.namespace", "Business") + .set("aerospike.namespace", "test") val session: SparkSession = SparkSession.builder() .config(conf) diff --git a/build.sbt b/build.sbt deleted file mode 100755 index 260caec..0000000 --- a/build.sbt +++ /dev/null @@ -1,86 +0,0 @@ -import sbtassembly.MergeStrategy._ -import scala.util.Properties - -name := "spark-aerojoin-example" - -version := "1.0" - -organization := "com.aerospike" -mainClass in (Compile, run) := Some("com.aerospike.spark.aeroJoinExample") -mainClass in assembly := Some("com.aerospike.spark.aeroJoinExample") - -crossScalaVersions := Seq("2.10.6", "2.11.8", "2.12.0") - -scalaVersion := "2.11.8" - -val sparkHome = Properties.envOrElse("RAF_HOME", "/opt/spark") - -javacOptions ++= Seq("-source", "1.7", "-target", "1.7") - -parallelExecution in test := false - -fork in test := true - -libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-core" % "2.2.0", - "org.apache.spark" %% "spark-sql" % "2.2.0", - "org.apache.spark" %% "spark-mllib" % "2.2.0", - "org.apache.spark" %% "spark-streaming" % "2.2.0", - "com.ning" % "async-http-client" % "1.9.10", - - "com.databricks" %% "spark-csv" % "1.5.0", - "com.aerospike" %% "aerospike-spark" % "provided" from s"file://${sparkHome}/jars/aerospike-spark-assembly-1.0.0.jar", - "com.aerospike" % "aerospike-helper-java" % "1.2.2", - "com.aerospike" % "aerospike-client" % "4.0.8", - "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", - "org.scalatest" %% "scalatest" % "2.2.1" % Test, - "joda-time" % "joda-time" % "2.9.9" % Test -) - -resolvers ++= Seq("Local Maven" at Path.userHome.asFile.toURI.toURL + ".m2/repository") -resolvers += "Local Maven Repository" at "file://"+Path.userHome.absolutePath+"/.m2/repository" -publishTo := Some(Resolver.file("file", new File(Path.userHome.absolutePath+"/.m2/repository"))) - -cancelable in Global := true - -assemblyMergeStrategy in assembly := { - case x if Assembly.isConfigFile(x) => - MergeStrategy.concat - case PathList(ps @ _*) if Assembly.isReadme(ps.last) || Assembly.isLicenseFile(ps.last) => - MergeStrategy.rename - case PathList("META-INF", "maven","com.aerospike","aerospike-client", "pom.properties") => - MergeStrategy.discard - case PathList("META-INF", "maven","com.aerospike","aerospike-client", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","org.slf4j","slf4j-api", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","com.fasterxml.jackson.core","jackson-annotations", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","com.fasterxml.jackson.core","jackson-core", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","com.fasterxml.jackson.core","jackson-databind", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","commons-logging","commons-logging", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","com.google.guava","guava", "pom.xml") => - MergeStrategy.discard - case PathList("META-INF", "maven","jline","jline", "pom.xml") => - MergeStrategy.discard - case PathList(ps @ _*) if ps.last endsWith "pom.properties" => - MergeStrategy.discard - case PathList("META-INF", xs @ _*) => - xs.map(_.toLowerCase) match { - case ("manifest.mf" :: Nil) | ("index.list" :: Nil) | ("dependencies" :: Nil) => - MergeStrategy.discard - case ps @ (x :: _) if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") => - MergeStrategy.discard - case "plexus" :: _ => - MergeStrategy.discard - case "services" :: _ => - MergeStrategy.filterDistinctLines - case ("spring.schemas" :: Nil) | ("spring.handlers" :: Nil) => - MergeStrategy.filterDistinctLines - case _ => MergeStrategy.deduplicate - } - case _ => MergeStrategy.first -} diff --git a/project/assembly.sbt b/project/assembly.sbt deleted file mode 100644 index d6540b8..0000000 --- a/project/assembly.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.8")