diff --git a/.bsp/sbt.json b/.bsp/sbt.json deleted file mode 100644 index 35c64c6..0000000 --- a/.bsp/sbt.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"sbt","version":"1.6.2","bspVersion":"2.0.0-M5","languages":["scala"],"argv":["/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java","-Xms100m","-Xmx100m","-classpath","/home/badri/.local/share/JetBrains/IdeaIC2021.3/Scala/launcher/sbt-launch.jar","-Dsbt.script=/usr/bin/sbt","xsbt.boot.Boot","-bsp"]} \ No newline at end of file diff --git a/.github/workflows/http.almaren-githubactions.yml b/.github/workflows/http.almaren-githubactions.yml new file mode 100644 index 0000000..04c3161 --- /dev/null +++ b/.github/workflows/http.almaren-githubactions.yml @@ -0,0 +1,49 @@ +name: HTTP.Almaren +on: [push, pull_request] + +jobs: + Build: + runs-on: ubuntu-20.04 + services: + postgres: + image: postgres:13.4 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name : Check out repository code + uses: actions/checkout@v2 + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8 + cache: sbt + - name: Setup web environment + run: | + curl -L http://cpanmin.us | perl - --sudo App::cpanminus + cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib) + cpanm Mojolicious + cpanm JSON::Parse + perl src/test/resources/script/mock_api.pl daemon -m dev -l http://\*:3000 & + - name: Build and test scala version + run: | + PGPASSWORD="postgres" psql -c 'create database almaren;' -U postgres -h localhost + PGPASSWORD="postgres" psql -c "ALTER USER postgres PASSWORD 'foo' ;" -U postgres -h localhost + PGPASSWORD="postgres" psql -c 'create role runner;' -U postgres -h localhost + PGPASSWORD="postgres" psql -c 'ALTER ROLE "runner" WITH LOGIN SUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION;' -U postgres -h localhost + sbt ++2.12.15 test + sbt ++2.13.9 test + rm -rf "$HOME/.ivy2/local" || true + find $HOME/Library/Caches/Coursier/v1 -name "ivydata-*.properties" -delete || true + find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete || true + find $HOME/.cache/coursier/v1 -name "ivydata-*.properties" -delete || true + find $HOME/.sbt -name "*.lock" -delete || true + killall -9 perl \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 406127c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: scala - -jdk: openjdk8 - -scala: - - 2.12.15 - -cache: - directories: - - $HOME/.cache/coursier - - $HOME/.ivy2/cache - - $HOME/.sbt - -before_cache: - - sudo apt-get -y install perl - - rm -fv $HOME/.ivy2/.sbt.ivy.lock - - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete - - find $HOME/.sbt -name "*.lock" -print -delete - -services: - - postgresql - - -before_script: - - cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib) - - cpanm Mojolicious - - cpanm JSON::Parse - - perl src/test/resources/script/mock_api.pl daemon -m dev -l http://\*:3000 & - -after_script: - - killall -9 perl - -script: - - sbt +test - - - diff --git a/README.md b/README.md index c4d5177..2600f53 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,24 @@ # HTTP Connector -[![Build Status](https://travis-ci.com/modakanalytics/http.almaren.svg?token=TEB3zRDqVUuChez9334q&branch=master)](https://travis-ci.com/modakanalytics/http.almaren) +[![Build Status](https://github.com/music-of-the-ainur/http.almaren/actions/workflows/http.almaren-githubactions.yml/badge.svg)](https://github.com/music-of-the-ainur/http.almaren/actions/workflows/http.almaren-githubactions.yml) + +To add http.almaren dependency to your sbt build: ``` -libraryDependencies += "com.github.music-of-the-ainur" %% "http-almaren" % "1.2.4-$SPARK_VERSION" +libraryDependencies += "com.github.music-of-the-ainur" %% "http-almaren" % "1.2.5-3.3" ``` +To run in spark-shell: +For scala-version(2.12): ``` -spark-shell --master "local[*]" --packages "com.github.music-of-the-ainur:almaren-framework_2.12:0.9.4-$SPARK_VERSION,com.github.music-of-the-ainur:http-almaren_2.12:1.2.4-$SPARK_VERSION" +spark-shell --master "local[*]" --packages "com.github.music-of-the-ainur:almaren-framework_2.12:0.9.8-3.3,com.github.music-of-the-ainur:http-almaren_2.12:1.2.5-3.3" +``` +For scala-version(2.13): +``` +spark-shell --master "local[*]" --packages "com.github.music-of-the-ainur:almaren-framework_2.13:0.9.8-3.3,com.github.music-of-the-ainur:http-almaren_2.13:1.2.5-3.3" ``` -## Table of Contents +## Table of Contents +- [Maven / Ivy Package Usage](#maven--ivy-package-usage) - [Methods](#methods) * [HTTP](#http) + [Example](#example) @@ -31,6 +40,23 @@ spark-shell --master "local[*]" --packages "com.github.music-of-the-ainur:almare - [Batch Delimiter](#batch-delimiter) - [Examples](#examples) + +#### Maven / Ivy Package Usage +The connector is also available from the +[Maven Central](https://mvnrepository.com/artifact/com.github.music-of-the-ainur) +repository. It can be used using the `--packages` option or the +`spark.jars.packages` configuration property. Use the following value + +| version | Connector Artifact | +|----------------------------|-------------------------------------------------------------| +| Spark 3.3.x and scala 2.13 | `com.github.music-of-the-ainur:http-almaren_2.13:1.2.5-3.3` | +| Spark 3.3.x and scala 2.12 | `com.github.music-of-the-ainur:http-almaren_2.12:1.2.5-3.3` | +| Spark 3.2.x and scala 2.12 | `com.github.music-of-the-ainur:http-almaren_2.12:1.2.5-3.2` | +| Spark 3.1.x and scala 2.12 | `com.github.music-of-the-ainur:http-almaren_2.12:1.2.5-3.1` | +| Spark 2.4.x and scala 2.12 | `com.github.music-of-the-ainur:http-almaren_2.12:1.2.5-2.4` | +| Spark 2.4.x and scala 2.11 | `com.github.music-of-the-ainur:http-almaren_2.11:1.2.5-2.4` | + + ## Methods ### HTTP diff --git a/build.sbt b/build.sbt index c173d0a..ec2eb83 100644 --- a/build.sbt +++ b/build.sbt @@ -2,10 +2,12 @@ ThisBuild / name := "http.almaren" ThisBuild / organization := "com.github.music-of-the-ainur" lazy val scala212 = "2.12.15" +lazy val scala213 = "2.13.9" -ThisBuild / scalaVersion := scala212 +crossScalaVersions := Seq(scala212,scala213) +ThisBuild / scalaVersion := scala213 -val sparkVersion = "3.2.1" +val sparkVersion = "3.3.0" val majorVersionReg = "([0-9]+\\.[0-9]+).{0,}".r val majorVersionReg(majorVersion) = sparkVersion @@ -15,10 +17,10 @@ scalacOptions ++= Seq("-deprecation", "-feature") libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % sparkVersion % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - "com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.4-${majorVersion}" % "provided", - "com.lihaoyi" %% "requests" % "0.7.0", - "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0", - "org.scalatest" %% "scalatest" % "3.0.5" % "test" + "com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.8-${majorVersion}" % "provided", + "com.lihaoyi" %% "requests" % "0.7.1", + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", + "org.scalatest" %% "scalatest" % "3.2.14" % "test" ) enablePlugins(GitVersioning) diff --git a/src/test/scala/com/github/music/of/the/ainur/almaren/http/Test.scala b/src/test/scala/com/github/music/of/the/ainur/almaren/http/Test.scala index 59c1c39..dfc29f3 100644 --- a/src/test/scala/com/github/music/of/the/ainur/almaren/http/Test.scala +++ b/src/test/scala/com/github/music/of/the/ainur/almaren/http/Test.scala @@ -7,8 +7,9 @@ import com.github.music.of.the.ainur.almaren.Almaren import com.github.music.of.the.ainur.almaren.builder.Core.Implicit import com.github.music.of.the.ainur.almaren.http.HTTPConn.HTTPImplicit import org.apache.spark.sql.Row +import org.scalatest.funsuite.AnyFunSuite -class Test extends FunSuite with BeforeAndAfter { +class Test extends AnyFunSuite with BeforeAndAfter { val almaren = Almaren("http-almaren") val spark: SparkSession = almaren.spark