diff --git a/build.sbt b/build.sbt index 98aadb711..a777322da 100644 --- a/build.sbt +++ b/build.sbt @@ -83,7 +83,9 @@ lazy val infra = project "io.circe" %% "circe-core", "io.circe" %% "circe-generic", "io.circe" %% "circe-parser" - ).map(_ % V.circe), + ).map(_ % V.circe) ++ Seq( + "io.circe" %% "circe-generic-extras" + ).map(_ % V.circeGenericExtra), Elasticsearch.settings(defaultPort = 9200), Postgres.settings(Compile, defaultPort = 5432, database = "scaladex"), javaOptions ++= { @@ -243,6 +245,7 @@ lazy val V = new { val nscalaTime = "2.32.0" val scalatest = "3.2.19" val circe = "0.14.9" + val circeGenericExtra = "0.14.4" val json4s = "4.0.7" val coursier = "2.1.6" } diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala index c99b153c4..8ca9f21f3 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala @@ -32,7 +32,7 @@ case class Artifact( fullScalaVersion: Option[SemanticVersion], scaladocUrl: Option[Url], versionScheme: Option[String], - developers: Seq[Contributor] + developers: Seq[Contributor] = Seq.empty ) { val binaryVersion: BinaryVersion = BinaryVersion(platform, language) val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode) diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala index fa77e018c..598fdce44 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala @@ -44,10 +44,13 @@ case class Project( image = githubInfo.flatMap(_.logo).orElse(Some(Url("https://index.scala-lang.org/assets/img/scaladex-brand.svg"))) ) - def scaladoc(artifact: Artifact): Option[LabeledLink] = - settings.customScalaDoc - .map(DocumentationPattern("Scaladoc", _).eval(artifact)) - .orElse(artifact.defaultScaladoc.map(LabeledLink("Scaladoc", _))) + def scaladoc(artifact: Artifact): Option[LabeledLink] = artifact.scaladocUrl + .map(_.labeled("Scaladoc")) + .orElse( + settings.customScalaDoc + .map(DocumentationPattern("Scaladoc", _).eval(artifact)) + .orElse(artifact.defaultScaladoc.map(LabeledLink("Scaladoc", _))) + ) private def globalDocumentation: Seq[LabeledLink] = settings.customScalaDoc.flatMap(DocumentationPattern("Scaladoc", _).asGlobal).toSeq ++ diff --git a/modules/infra/src/main/scala/scaladex/infra/Codecs.scala b/modules/infra/src/main/scala/scaladex/infra/Codecs.scala index fbffab666..09886cc98 100644 --- a/modules/infra/src/main/scala/scaladex/infra/Codecs.scala +++ b/modules/infra/src/main/scala/scaladex/infra/Codecs.scala @@ -3,6 +3,8 @@ package scaladex.infra import java.time.Instant import io.circe._ +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.deriveConfiguredCodec import io.circe.generic.semiauto._ import scaladex.core.model._ import scaladex.core.model.search.GithubInfoDocument @@ -44,7 +46,8 @@ object Codecs { implicit val languageCodec: Codec[Language] = fromString(_.label, Language.fromLabel(_).get) implicit val resolverCodec: Codec[Resolver] = deriveCodec implicit val licenseCodec: Codec[License] = fromString(_.shortName, License.allByShortName.apply) - implicit val artifactCodec: Codec[Artifact] = deriveCodec + implicit val customConfig: Configuration = Configuration.default.withDefaults + implicit val artifactCodec: Codec[Artifact] = deriveConfiguredCodec[Artifact] implicit val scopeCodec: Codec[ArtifactDependency.Scope] = fromString(_.value, ArtifactDependency.Scope.apply) implicit val mavenRefCodec: Codec[Artifact.MavenReference] = deriveCodec @@ -62,4 +65,5 @@ object Codecs { private def fromString[A](encode: A => String, decode: String => A): Codec[A] = Codec.from(Decoder[String].map(decode), Encoder[String].contramap(encode)) + } diff --git a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala index 672b98b4f..8dd8d690d 100644 --- a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala +++ b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala @@ -41,13 +41,13 @@ object ArtifactTable { val isLatestVersion: String = "is_latest_version" def insertIfNotExist(artifact: Artifact): ConnectionIO[Int] = - insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease)) + insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease, artifact)) def insertIfNotExist(artifacts: Seq[Artifact]): ConnectionIO[Int] = - insertIfNotExist.updateMany(artifacts.map(a => (a, a.version.isSemantic, a.version.isPreRelease))) + insertIfNotExist.updateMany(artifacts.map(a => (a, a.version.isSemantic, a.version.isPreRelease, a))) - private[sql] val insertIfNotExist: Update[(Artifact, Boolean, Boolean)] = - insertOrUpdateRequest(table, fields ++ versionFields, mavenReferenceFields) + private[sql] val insertIfNotExist: Update[(Artifact, Boolean, Boolean, Artifact)] = + insertOrUpdateRequest(table, fields ++ versionFields, mavenReferenceFields, fields) val count: Query0[Long] = selectRequest(table, Seq("COUNT(*)")) diff --git a/modules/server/src/main/assets/css/partials/_project.scss b/modules/server/src/main/assets/css/partials/_project.scss index 533bf5428..0e66d2d5e 100644 --- a/modules/server/src/main/assets/css/partials/_project.scss +++ b/modules/server/src/main/assets/css/partials/_project.scss @@ -431,4 +431,15 @@ display: block; } } + + .developers { + a { + color: $gray; + &:hover, + &:focus { + color: $brand-primary; + text-decoration: none; + } + } + } } diff --git a/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala b/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala index eefbc786e..f2bf88041 100644 --- a/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala +++ b/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala @@ -65,7 +65,14 @@ class AdminPage(env: Env, adminService: AdminService) { redirect(Uri("/admin"), StatusCodes.SeeOther) } } + } ~ + post { + path("tasks" / Task.updateMavenArtifacts.name) { + adminService.updateMavenArtifacts(user) + redirect(Uri("/admin"), StatusCodes.SeeOther) + } } + case _ => complete(StatusCodes.Forbidden, view.html.forbidden(env, user)) } diff --git a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala index 21b30efce..f1a37cd75 100644 --- a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala @@ -137,6 +137,13 @@ class AdminService( tasks = tasks :+ task } + def updateMavenArtifacts(user: UserState): Unit = { + val task = TaskRunner.run(Task.updateMavenArtifacts, user.info.login, input = Seq.empty) { () => + sonatypeSynchronizer.updateAllArtifacts() + } + tasks = tasks :+ task + } + private def updateProjectCreationDate(): Future[String] = for { creationDates <- database.computeAllProjectsCreationDates() diff --git a/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala b/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala index 1173f7215..39b687d7d 100644 --- a/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala @@ -33,6 +33,29 @@ class SonatypeService( } yield s"Inserted ${result.sum} missing poms" } + private def findAndIndexMissingArtifacts( + groupId: GroupId, + artifactId: ArtifactId, + knownRefs: Set[MavenReference] + ): Future[Int] = + for { + versions <- sonatypeService.getAllVersions(groupId, artifactId) + mavenReferences = versions.map(v => + MavenReference(groupId = groupId.value, artifactId = artifactId.value, version = v.toString) + ) + missingVersions = mavenReferences.filterNot(knownRefs) + _ = if (missingVersions.nonEmpty) + logger.warn(s"${missingVersions.size} artifacts are missing for ${groupId.value}:${artifactId.value}") + missingPomFiles <- missingVersions.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence + publishResult <- missingPomFiles.flatten.mapSync { + case (mavenRef, (pomFile, creationDate)) => + publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None) + } + } yield publishResult.count { + case PublishResult.Success => true + case _ => false + } + def findMissing(): Future[String] = for { mavenReferenceFromDatabase <- database.getAllMavenReferences().map(_.toSet) @@ -41,12 +64,6 @@ class SonatypeService( result <- groupIds.mapSync(g => findAndIndexMissingArtifacts(g, None, mavenReferenceFromDatabase)) } yield s"Inserted ${result.sum} missing poms" - def syncOne(groupId: GroupId, artifactNameOpt: Option[Artifact.Name]): Future[String] = - for { - mavenReferenceFromDatabase <- database.getAllMavenReferences() - result <- findAndIndexMissingArtifacts(groupId, artifactNameOpt, mavenReferenceFromDatabase.toSet) - } yield s"Inserted ${result} poms" - private def findAndIndexMissingArtifacts( groupId: GroupId, artifactNameOpt: Option[Artifact.Name], @@ -61,26 +78,25 @@ class SonatypeService( .mapSync(id => findAndIndexMissingArtifacts(groupId, id, knownRefs)) } yield result.sum - private def findAndIndexMissingArtifacts( - groupId: GroupId, - artifactId: ArtifactId, - knownRefs: Set[MavenReference] - ): Future[Int] = + def syncOne(groupId: GroupId, artifactNameOpt: Option[Artifact.Name]): Future[String] = for { - versions <- sonatypeService.getAllVersions(groupId, artifactId) - mavenReferences = versions.map(v => - MavenReference(groupId = groupId.value, artifactId = artifactId.value, version = v.toString) - ) - missingVersions = mavenReferences.filterNot(knownRefs) - _ = if (missingVersions.nonEmpty) - logger.warn(s"${missingVersions.size} artifacts are missing for ${groupId.value}:${artifactId.value}") - missingPomFiles <- missingVersions.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence - publishResult <- missingPomFiles.flatten.mapSync { - case (mavenRef, (pomFile, creationDate)) => - publishProcess.publishPom(mavenRef.toString, pomFile, creationDate, None) - } - } yield publishResult.count { - case PublishResult.Success => true - case _ => false + mavenReferenceFromDatabase <- database.getAllMavenReferences() + result <- findAndIndexMissingArtifacts(groupId, artifactNameOpt, mavenReferenceFromDatabase.toSet) + } yield s"Inserted $result poms" + + def updateAllArtifacts(): Future[String] = + for { + mavenReferences <- database.getAllMavenReferences() + _ = logger.info(s"${mavenReferences.size} artifacts will be synced for new metadata.") + publishResult <- mavenReferences.mapSync(updateArtifact) + successCount = publishResult.count(_ == PublishResult.Success) + failedCount = publishResult.size - successCount + } yield s"Synced $successCount poms, while $failedCount poms failed to update." + + private def updateArtifact(ref: MavenReference): Future[PublishResult] = + sonatypeService.getPomFile(ref).map(ref -> _).flatMap { + case (mavenRef, Some((pomFile, creationDate))) => + publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None) + case _ => Future.successful(PublishResult.InvalidPom) } } diff --git a/modules/template/src/main/scala/scaladex/view/Task.scala b/modules/template/src/main/scala/scaladex/view/Task.scala index 78d7be6b4..f3ca5ebcc 100644 --- a/modules/template/src/main/scala/scaladex/view/Task.scala +++ b/modules/template/src/main/scala/scaladex/view/Task.scala @@ -24,6 +24,11 @@ object Task { "Update the Github info of an existing project" ) + val updateMavenArtifacts: Task = Task( + "update-maven-artifact", + "Download all pom files to update existing artifacts with new fields" + ) + case class Status(name: String, user: String, start: Instant, input: Seq[(String, String)], state: State) { def fromNow: FiniteDuration = TimeUtils.toFiniteDuration(start, Instant.now()) } diff --git a/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html b/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html index f790bc265..6b1a0db37 100644 --- a/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html +++ b/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html @@ -22,7 +22,7 @@

Background jobs

@jobs.map { case (job, status) => - +
@job.name every @job.frequency.shortPrint

@job.description

@@ -120,6 +120,16 @@
@Task.updateGithubInfo.name
+ + +
@Task.updateMavenArtifacts.name
+

@Task.updateMavenArtifacts.description

+ + +
+ +
+

Task History

diff --git a/modules/template/src/main/twirl/scaladex/view/project/artifact.scala.html b/modules/template/src/main/twirl/scaladex/view/project/artifact.scala.html index 57490cdb5..3ec0bc502 100644 --- a/modules/template/src/main/twirl/scaladex/view/project/artifact.scala.html +++ b/modules/template/src/main/twirl/scaladex/view/project/artifact.scala.html @@ -52,6 +52,13 @@

@info("Group ID") { @artifact.groupId } @info("Artifact ID") { @artifact.artifactId } @info("Version") { @artifact.version } + @artifact.versionScheme.map { scheme => + @info("Version Scheme"){ + + @scheme + + } + } @info("Release Date") { @artifact.releaseDateFormat } @info("Licenses") { @for(license <- artifact.licenses) { @@ -62,6 +69,7 @@

@info("Files") { View all } } @artifact.fullScalaVersion.map{ version => @info("Full Scala Version") { @version.toString }} + @developers @installBox(InstallTab.allOf(artifact, project.settings.cliArtifacts)) @@ -97,7 +105,21 @@

Documentation

} } - +@developers = { + @if(artifact.developers.nonEmpty){ + @info("Developers"){ +
+ @for(developer <- artifact.developers) { + + + @developer.name + | + + } +
+ } + } +} @scastieBox = { @if(artifact.scastieURL.nonEmpty) {