Redesign Classpaths so that they could lazily read their files.

[sjrd]

The implementation in jdk.ClasspathLoaders and
nodejs.ClasspathLoaders still actually reads everything eagerly. However, a third-party implementation that wants to read things differently could now do so.

[bishabosha]

here is a test suite for wrapping a dotc classpath, works well if you copy it to the jvm test sources (and a library dependency on scala3-compiler):

package tastyquery.jdk.classpaths

import tastyquery.Classpaths.Classpath
import tastyquery.Classpaths.ClasspathEntry
import tastyquery.Classpaths.PackageData
import tastyquery.Classpaths.ClassData
import tastyquery.Contexts.Context
import tastyquery.Contexts.ctx

import dotty.tools.io.ClassPath as DottyClassPath
import dotty.tools.dotc.core.Contexts.ContextBase
import dotty.tools.dotc.core.Contexts.Context as DottyContext
import dotty.tools.dotc.core.Contexts.inContext
import dotty.tools.dotc.ScalacCommand
import dotty.tools.io.AbstractFile
import dotty.tools.dotc.classpath.PackageEntry
import dotty.tools.dotc.Compiler
import dotty.tools.dotc.classpath.AggregateClassPath
import dotty.tools.io.FileZipArchive

object LazyClasspathSuite:
  private val TestClassPathEnvVar = "TASTY_TEST_CLASSPATH"

  private lazy val classpathEntries: List[String] =
    System.getenv(TestClassPathEnvVar).nn.split(';').toList

  def nonJrtEntries: List[String] =
    classpathEntries.filterNot:
      case s"jrt:/modules/$module/" => true
      case _ => false

class LazyClasspathSuite extends munit.FunSuite:

  def runContext: DottyContext =
    val cp = LazyClasspathSuite.nonJrtEntries.mkString(java.io.File.pathSeparator.nn)
    val ictx = setup(Array("-classpath", cp), ContextBase().initialCtx)
    val _ = Compiler().newRun(using ictx) // side effect: initializes the classpath
    ictx

  def setup(args: Array[String], rootCtx: DottyContext): DottyContext = {
    val ictx = rootCtx.fresh
    val summary = ScalacCommand.distill(args, ictx.settings)(ictx.settingsState)(using ictx)
    ictx.setSettings(summary.sstate)
    ictx
  }

  enum ForceEvent:
    case Package(name: String)
    case Class(name: String)
    case Tasty(name: String)

  class DotcEntry(debugName: String, cp: DottyClassPath, callback: ForceEvent => Unit) extends ClasspathEntry:
    override def toString(): String = debugName

    def sibling(cls: AbstractFile): Option[AbstractFile] =
      val dir = cls match
        case cls: FileZipArchive#Entry => cls.parent
        case cls: AbstractFile => cls.container
      val name = cls.name.stripSuffix(".class") + ".tasty"
      Option(dir.lookupName(name, directory = false))

    lazy val _packages: List[DotcPackageData] =
      def loadClasses(pkg: PackageEntry): List[DotcClassData] =
        cp.classes(pkg.name).toList.map(cls =>
          val classRaw = cls.binary
          val tastyRaw = classRaw.flatMap(sibling)
          DotcClassData(s"$debugName:${pkg.name}.${cls.name}", cls.name, classRaw, tastyRaw, callback)
        )

      def loadPackage(pkg: PackageEntry): DotcPackageData =
        val name = pkg.name
        DotcPackageData(s"$debugName:$name", name, () => loadClasses(pkg), callback)
      def loadSubPackages(name: String): List[DotcPackageData] =
        cp.packages(name).toList.flatMap(pkg => loadPackage(pkg) :: loadSubPackages(pkg.name))
      loadSubPackages("")

    override def listAllPackages(): List[DotcPackageData] = _packages

  class DotcPackageData(val debugName: String, override val dotSeparatedName: String, fetchClasses: () => List[DotcClassData], callback: ForceEvent => Unit) extends PackageData:
    override def toString(): String = debugName

    lazy val _classes =
      try fetchClasses()
      finally callback(ForceEvent.Package(debugName))

    lazy val _byName = _classes.map(cls => cls.binaryName -> cls).toMap

    override def listAllClassDatas(): List[DotcClassData] = _classes

    override def getClassDataByBinaryName(binaryName: String): Option[DotcClassData] = _byName.get(binaryName)

  class DotcClassData(val debugName: String, override val binaryName: String, cls: Option[AbstractFile], tsty: Option[AbstractFile], callback: ForceEvent => Unit) extends ClassData {
    override def toString(): String = debugName

    override def readClassFileBytes(): IArray[Byte] =
      try IArray.unsafeFromArray(cls.get.toByteArray)
      finally callback(ForceEvent.Class(debugName))

    override def hasClassFile: Boolean = cls.exists(_.exists)

    override def readTastyFileBytes(): IArray[Byte] =
      try IArray.unsafeFromArray(tsty.get.toByteArray)
      finally callback(ForceEvent.Tasty(debugName))

    override def hasTastyFile: Boolean = tsty.exists(_.exists)

  }

  def makeClasspath(callback: ForceEvent => Unit): Classpath =
    inContext(runContext): ctx ?=>
      def flattenClasspath(cp: DottyClassPath): List[DottyClassPath] = cp match
        case ag: AggregateClassPath => ag.aggregates.flatMap(flattenClasspath).toList
        case _ => List(cp)
      flattenClasspath(ctx.base.platform.classPath).map(cp => DotcEntry(cp.asClassPathString, cp, callback))

  test("load lazy from dotty classpath") {
    val events = collection.mutable.ListBuffer.empty[ForceEvent]
    val classpath = makeClasspath: event =>
      println(s"received event: $event")
      events += event
    given Context = Context.initialize(classpath)

    // scala 2 classes
    val TryClass = ctx.findTopLevelClass("scala.util.Try")
    val SuccessClass = ctx.findTopLevelClass("scala.util.Success")
    assert(SuccessClass.isSubClass(TryClass), clue(events.toList))

    // scala 3 classes
    val TupleClass = ctx.findTopLevelClass("scala.Tuple")
    val ConsClass = ctx.findTopLevelClass("scala.*:")
    assert(ConsClass.isSubClass(TupleClass), clue(events.toList))

    // java classes
    val ListClass = ctx.findTopLevelClass("java.util.List")
    val ArrayListClass = ctx.findTopLevelClass("java.util.ArrayList")
    assert(ArrayListClass.isSubClass(ListClass), clue(events.toList))
  }