Merge pull request #117 from szeiger/sjsonnet-perf-improvements

Sjsonnet performance improvements

bench/src/main/scala/sjsonnet/MainBenchmark.scala

@@ -0,0 +1,44 @@
+package sjsonnet
+
+import java.io.{OutputStream, PrintStream}
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra._
+
+import scala.collection.mutable.ArrayBuffer
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@Fork(4)
+@Threads(1)
+@Warmup(iterations = 30)
+@Measurement(iterations = 40)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+class MainBenchmark {
+
+  val mainArgs = Array[String](
+    "../../universe2/rulemanager/deploy/rulemanager.jsonnet",
+    "-J", "../../universe2",
+    "-J", "../../universe2/mt-shards/dev/az-westus-c2",
+  )
+
+  val dummyOut = new PrintStream(new OutputStream {
+    def write(b: Int): Unit = ()
+    override def write(b: Array[Byte]): Unit = ()
+    override def write(b: Array[Byte], off: Int, len: Int): Unit = ()
+  })
+
+  @Benchmark
+  def main(bh: Blackhole): Unit = {
+    bh.consume(SjsonnetMain.main0(
+      mainArgs,
+      collection.mutable.HashMap.empty,
+      System.in,
+      dummyOut,
+      System.err,
+      os.pwd,
+      None
+    ))
+  }
+}

build.sbt

@@ -0,0 +1,55 @@
+val sjsonnetVersion = "0.3.3"
+
+scalaVersion in Global := "2.13.4"
+
+cancelable in Global := true
+
+lazy val main = (project in file("sjsonnet"))
+  .settings(
+    scalacOptions in Compile ++= Seq("-opt:l:inline", "-opt-inline-from:sjsonnet.**"),
+    fork in Test := true,
+    baseDirectory in Test := (baseDirectory in ThisBuild).value,
+    libraryDependencies ++= Seq(
+      "com.lihaoyi" %% "fastparse" % "2.3.1",
+      "com.lihaoyi" %% "pprint" % "0.6.1",
+      "com.lihaoyi" %% "ujson" % "1.3.7",
+      "com.lihaoyi" %% "scalatags" % "0.9.3",
+      "com.lihaoyi" %% "os-lib" % "0.7.2",
+      "com.lihaoyi" %% "mainargs" % "0.2.0",
+      "org.scala-lang.modules" %% "scala-collection-compat" % "2.4.0",
+      "org.tukaani" % "xz" % "1.8",
+    ),
+    libraryDependencies ++= Seq(
+      "com.lihaoyi" %% "utest" % "0.7.7",
+    ).map(_ % "test"),
+    testFrameworks += new TestFramework("utest.runner.Framework"),
+    (unmanagedSourceDirectories in Compile) := Seq(
+      baseDirectory.value / "src",
+      baseDirectory.value / "src-jvm",
+      baseDirectory.value / "src-jvm-native",
+    ),
+    (unmanagedSourceDirectories in Test) := Seq(
+      baseDirectory.value / "test/src",
+      baseDirectory.value / "test/src-jvm",
+      baseDirectory.value / "test/src-jvm-native",
+    ),
+    (unmanagedResourceDirectories in Test) := Seq(
+      baseDirectory.value / "test/resources",
+    ),
+    (sourceGenerators in Compile) += Def.task {
+      val file = (Compile / sourceManaged).value / "jsonnet" / "Version.scala"
+      IO.write(file,
+        s"""package sjsonnet
+           |object Version{
+           |  val version = "${sjsonnetVersion}"
+           |}
+           |""".stripMargin)
+      Seq(file)
+    }.taskValue
+  )
+
+lazy val bench = (project in file("bench"))
+  .dependsOn(main)
+  .enablePlugins(JmhPlugin)
+  .settings(
+  )

build.sc

@@ -107,13 +107,14 @@ class SjsonnetModule(val crossScalaVersion: String) extends Module {
     def ivyDeps = super.ivyDeps() ++ Agg(
       ivy"org.tukaani:xz::1.8"
     )
-    def compileIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
-    def scalacOptions = Seq("-P:acyclic:force")
-    def scalacPluginIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
+    def scalacOptions = Seq("-opt:l:inline", "-opt-inline-from:sjsonnet.**")
+    //def compileIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
+    //def scalacOptions = Seq("-P:acyclic:force")
+    //def scalacPluginIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
     object test extends Tests with CrossTests{
-      def compileIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
-      def scalacOptions = Seq("-P:acyclic:force")
-      def scalacPluginIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
+      //def compileIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
+      //def scalacOptions = Seq("-P:acyclic:force")
+      //def scalacPluginIvyDeps = Agg( ivy"com.lihaoyi::acyclic:0.2.0")
       def forkOptions = Seq("-Xss100m")
       def sources = T.sources(
         millSourcePath / "src",

project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.3.10

project/plugins.sbt

@@ -0,0 +1,2 @@
+//addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.7")
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.3")

sjsonnet/server/src/sjsonnet/SjsonnetServerMain.scala

@@ -24,7 +24,7 @@ trait SjsonnetServerMain[T]{
             wd: os.Path): (Boolean, Option[T])
 }
 
-object SjsonnetServerMain extends SjsonnetServerMain[collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]]]{
+object SjsonnetServerMain extends SjsonnetServerMain[collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]]]{
   def main(args0: Array[String]): Unit = {
     // Disable SIGINT interrupt signal in the Mill server.
     //
@@ -45,7 +45,7 @@ object SjsonnetServerMain extends SjsonnetServerMain[collection.mutable.Map[Stri
     ).run()
   }
   def main0(args: Array[String],
-            stateCache: Option[collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]]],
+            stateCache: Option[collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]]],
             mainInteractive: Boolean,
             stdin: InputStream,
             stdout: PrintStream,
@@ -55,7 +55,7 @@ object SjsonnetServerMain extends SjsonnetServerMain[collection.mutable.Map[Stri
             wd: os.Path) = {
 
     val stateCache2 = stateCache.getOrElse{
-      val p = collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]]()
+      val p = collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]]()
       this.stateCache = Some(p)
       p
     }

sjsonnet/src-js/java/util/BitSet.scala

@@ -0,0 +1,18 @@
+package java.util
+
+import java.util.stream.IntStream
+
+import scala.collection.mutable.{BitSet => SBitSet}
+import scala.collection.JavaConverters._
+
+class BitSet(_initialSizeDummy: Int) extends java.lang.Cloneable {
+  val bs: SBitSet = SBitSet.empty
+  def isEmpty: Boolean = bs.isEmpty
+  override def clone(): AnyRef = super.clone()
+  def cardinality(): Int = bs.size
+  def stream(): IntStream = IntStream.of(bs.toArray: _*)
+  def get(i: Int): Boolean = bs.apply(i)
+  def set(i: Int): Unit = bs += i
+  def clear(): Unit = bs.clear()
+  def andNot(other: BitSet): Unit = bs.&~=(other.bs)
+}

sjsonnet/src-js/sjsonnet/BitSetUtils.scala

@@ -0,0 +1,7 @@
+package sjsonnet
+
+import java.util.BitSet
+
+object BitSetUtils {
+  def iterator(bs: BitSet): Iterator[Int] = bs.bs.iterator
+}

sjsonnet/src-js/sjsonnet/SjsonnetMain.scala

@@ -6,7 +6,7 @@ import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel}
 
 @JSExportTopLevel("SjsonnetMain")
 object SjsonnetMain {
-  def createParseCache() = collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]]()
+  def createParseCache() = collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]]()
   @JSExport
   def interpret(text: String,
                 extVars: js.Any,
@@ -15,7 +15,7 @@ object SjsonnetMain {
                 importer: js.Function2[String, String, js.Array[String]],
                 preserveOrder: Boolean = false): js.Any = {
     val interp = new Interpreter(
-      mutable.Map.empty,
+      mutable.HashMap.empty,
       ujson.WebJson.transform(extVars, ujson.Value).obj.toMap,
       ujson.WebJson.transform(tlaVars, ujson.Value).obj.toMap,
       JsVirtualPath(wd0),
@@ -51,7 +51,7 @@ case class JsVirtualPath(path: String) extends Path{
 
   def /(s: String): Path = JsVirtualPath(path + "/" + s)
 
-  def renderOffsetStr(offset: Int, loadedFileContents: mutable.Map[Path, Array[Int]]): String = {
+  def renderOffsetStr(offset: Int, loadedFileContents: mutable.HashMap[Path, Array[Int]]): String = {
     path + ":" + offset
   }
 }

sjsonnet/src-jvm-native/sjsonnet/OsPath.scala

@@ -10,8 +10,14 @@ case class OsPath(p: os.Path) extends Path{
   def last: String = p.last
   def /(s: String): Path = OsPath(p / s)
 
+  override def equals(other: Any): Boolean = other match {
+    case OsPath(p2) => p == p2
+    case _ => false
+  }
+
+  override def hashCode: Int = p.hashCode()
 
-  def renderOffsetStr(offset: Int, loadedFileContents: mutable.Map[Path, Array[Int]]): String = {
+  def renderOffsetStr(offset: Int, loadedFileContents: mutable.HashMap[Path, Array[Int]]): String = {
     val offsetStr =
       if (p.toString.contains("(materialize)")) ""
       else {

sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala

@@ -10,7 +10,7 @@ import scala.util.Try
 import scala.util.control.NonFatal
 
 object SjsonnetMain {
-  def createParseCache() = collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]]()
+  def createParseCache() = collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]]()
   def resolveImport(searchRoots0: Seq[Path], allowedInputs: Option[Set[os.Path]] = None)(wd: Path, str: String) = {
     (wd +: searchRoots0)
       .flatMap(base => os.FilePath(str) match {
@@ -25,12 +25,12 @@ object SjsonnetMain {
       .flatMap(p => try Some((OsPath(p), os.read(p))) catch{case NonFatal(_) => None})
   }
   def main(args: Array[String]): Unit = {
-    val exitCode = main0(
+    var exitCode = main0(
       args match {
         case Array(s, _*) if s == "-i" || s == "--interactive" => args.tail
         case _ => args
       },
-      collection.mutable.Map.empty,
+      collection.mutable.HashMap.empty,
       System.in,
       System.out,
       System.err,
@@ -41,7 +41,7 @@ object SjsonnetMain {
   }
 
   def main0(args: Array[String],
-            parseCache: collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]],
+            parseCache: collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]],
             stdin: InputStream,
             stdout: PrintStream,
             stderr: PrintStream,
@@ -124,7 +124,7 @@ object SjsonnetMain {
 
   def mainConfigured(file: String,
                      config: Config,
-                     parseCache: collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]],
+                     parseCache: collection.mutable.HashMap[(Path, String), fastparse.Parsed[(Expr, FileScope)]],
                      wd: os.Path,
                      allowedInputs: Option[Set[os.Path]] = None,
                      importer: Option[(Path, String) => Option[os.Path]] = None): Either[String, String] = {

sjsonnet/src-jvm/sjsonnet/BitSetUtils.scala

@@ -0,0 +1,17 @@
+package sjsonnet
+
+import java.util.BitSet
+
+import scala.collection.mutable.ArrayBuffer
+
+object BitSetUtils {
+  def iterator(bs: BitSet): Iterator[Int] = {
+    val b = ArrayBuffer.empty[Int]
+    var i = bs.nextSetBit(0)
+    while(i  > 0) {
+      b += i
+      i = bs.nextSetBit(i+1)
+    }
+    b.iterator
+  }
+}

sjsonnet/src-native/java/util/BitSet.scala

@@ -0,0 +1,18 @@
+package java.util
+
+import java.util.stream.IntStream
+
+import scala.collection.mutable.{BitSet => SBitSet}
+import scala.collection.JavaConverters._
+
+class BitSet(_initialSizeDummy: Int) extends java.lang.Cloneable {
+  val bs: SBitSet = SBitSet.empty
+  def isEmpty: Boolean = bs.isEmpty
+  override def clone(): AnyRef = super.clone()
+  def cardinality(): Int = bs.size
+  def stream(): IntStream = IntStream.of(bs.toArray: _*)
+  def get(i: Int): Boolean = bs.apply(i)
+  def set(i: Int): Unit = bs += i
+  def clear(): Unit = bs.clear()
+  def andNot(other: BitSet): Unit = bs.&~=(other.bs)
+}

sjsonnet/src-native/sjsonnet/BitSetUtils.scala

@@ -0,0 +1,7 @@
+package sjsonnet
+
+import java.util.BitSet
+
+object BitSetUtils {
+  def iterator(bs: BitSet): Iterator[Int] = bs.bs.iterator
+}