Skip to content

Commit 05f6a9c

Browse files
lihaoyilihaoyi
committed
.
1 parent 6c0e912 commit 05f6a9c

File tree

35 files changed

+372
-248
lines changed

35 files changed

+372
-248
lines changed

‎core/api/src/mill/api/Evaluator.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import mill.api.*
55
import mill.api.daemon.Watchable
66
import mill.api.BuildCtx
77
import mill.api.daemon.internal.{EvaluatorApi, TaskApi}
8-
import mill.api.internal.{Located, Resolved, RootModule0}
8+
import mill.api.internal.{AppendLocated, Located, Resolved, RootModule0}
99
import upickle.core.BufferedValue
1010
import scala.util.DynamicVariable
1111
import scala.collection.mutable
@@ -33,7 +33,7 @@ trait Evaluator extends AutoCloseable with EvaluatorApi {
3333
private[mill] def effectiveThreadCount: Int
3434
private[mill] def offline: Boolean
3535
private[mill] def useFileLocks: Boolean = false
36-
private[mill] def staticBuildOverrides: Map[String, Located[BufferedValue]] = Map()
36+
private[mill] def staticBuildOverrides: Map[String, AppendLocated[BufferedValue]] = Map()
3737
def withBaseLogger(newBaseLogger: Logger): Evaluator
3838

3939
def resolveSegments(

‎core/api/src/mill/api/internal/Located.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@ package mill.api.internal
22

33
case class Located[T](path: os.Path, index: Int, value: T)
44

5+
/** Located with an additional flag indicating whether to append to super */
6+
case class AppendLocated[T](located: Located[T], append: Boolean) {
7+
def path: os.Path = located.path
8+
def index: Int = located.index
9+
def value: T = located.value
10+
}
11+
512
object Located {
613
class UpickleReader[T](path: os.Path)(implicit r: upickle.Reader[T])
714
extends upickle.Reader[Located[T]] {

‎core/eval/src/mill/eval/EvaluatorImpl.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ final class EvaluatorImpl(
125125

126126
def validateModuleOverrides(allModules: Seq[ModuleCtx.Wrapper]): Seq[Result.Failure] = {
127127
val scriptBuildOverrides = allModules.flatMap(_.moduleDynamicBuildOverrides)
128+
.map { case (k, v) => k -> mill.api.internal.AppendLocated(v, append = false) }
128129
val allBuildOverrides = staticBuildOverrides ++ scriptBuildOverrides
129130

130131
allModules.flatMap { module =>

‎core/eval/src/mill/eval/SelectiveExecutionImpl.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ object SelectiveExecutionImpl {
213213
val allBuildOverrides =
214214
evaluator.staticBuildOverrides ++
215215
transitiveNamed.flatMap(_.ctx.enclosingModule.moduleDynamicBuildOverrides)
216+
.map { case (k, v) => k -> mill.api.internal.AppendLocated(v, append = false) }
216217

217218
val results: Map[Task.Named[?], mill.api.Result[Val]] = transitiveNamed
218219
.collect { case task: Task.Input[_] =>

‎core/exec/src/mill/exec/GroupExecution.scala‎

Lines changed: 82 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,32 @@ trait GroupExecution {
4242
def getEvaluator: () => EvaluatorApi
4343
def staticBuildOverrideFiles: Map[java.nio.file.Path, String]
4444

45-
import mill.api.internal.Located
46-
val staticBuildOverrides: Map[String, Located[BufferedValue]] = staticBuildOverrideFiles
45+
import mill.api.internal.{Located, AppendLocated}
46+
47+
/** Extract append flag and actual value from wrapper object if present */
48+
private def unwrapAppendMarker(v: BufferedValue): (BufferedValue, Boolean) = {
49+
v match {
50+
case obj: BufferedValue.Obj =>
51+
val kvMap = obj.value0.collect { case (BufferedValue.Str(k, _), v) =>
52+
k.toString -> v
53+
}.toMap
54+
(kvMap.get("__mill_append__"), kvMap.get("__mill_values__")) match {
55+
case (Some(BufferedValue.True(_)), Some(values)) => (values, true)
56+
case _ => (v, false)
57+
}
58+
case _ => (v, false)
59+
}
60+
}
61+
62+
val staticBuildOverrides: Map[String, AppendLocated[BufferedValue]] = staticBuildOverrideFiles
4763
.flatMap { case (path0, rawText) =>
4864
val path = os.Path(path0)
4965
val headerDataReader = mill.api.internal.HeaderData.headerDataReader(path)
5066

5167
def rec(
5268
segments: Seq[String],
5369
bufValue: upickle.core.BufferedValue
54-
): Seq[(String, Located[BufferedValue])] = {
70+
): Seq[(String, AppendLocated[BufferedValue])] = {
5571
val upickle.core.BufferedValue.Obj(kvs, _, _) = bufValue
5672
val (rawKvs, nested) = kvs.partitionMap { case (upickle.core.BufferedValue.Str(k, i), v) =>
5773
k.toString.split(" +") match {
@@ -60,7 +76,7 @@ trait GroupExecution {
6076
}
6177
}
6278

63-
val currentResults: Seq[(String, Located[BufferedValue])] =
79+
val currentResults: Seq[(String, AppendLocated[BufferedValue])] =
6480
BufferedValue.transform(
6581
BufferedValue.Obj(
6682
rawKvs.map { case (k, i, v) => (BufferedValue.Str(k, i), v) }.to(mutable.ArrayBuffer),
@@ -71,11 +87,15 @@ trait GroupExecution {
7187
)
7288
.rest
7389
.map { case (k, v) =>
74-
(segments ++ Seq(k.value)).mkString(".") -> Located(path, k.index, v)
90+
val (actualValue, append) = unwrapAppendMarker(v)
91+
(segments ++ Seq(k.value)).mkString(".") -> AppendLocated(
92+
Located(path, k.index, actualValue),
93+
append
94+
)
7595
}
7696
.toSeq
7797

78-
val nestedResults: Seq[(String, Located[BufferedValue])] = nested.flatten.toSeq
98+
val nestedResults: Seq[(String, AppendLocated[BufferedValue])] = nested.flatten.toSeq
7999

80100
currentResults ++ nestedResults
81101
}
@@ -190,14 +210,16 @@ trait GroupExecution {
190210
val paths = ExecutionPaths.resolve(out, labelled.ctx.segments)
191211
val dynamicBuildOverride = labelled.ctx.enclosingModule.moduleDynamicBuildOverrides
192212
staticBuildOverrides.get(labelled.ctx.segments.render)
193-
.orElse(dynamicBuildOverride.get(labelled.ctx.segments.render)) match {
213+
.orElse(dynamicBuildOverride.get(labelled.ctx.segments.render).map(loc =>
214+
AppendLocated(loc, append = false)
215+
)) match {
194216

195-
case Some(jsonData) =>
196-
lazy val originalText = os.read(jsonData.path)
217+
case Some(appendLocated) =>
218+
lazy val originalText = os.read(appendLocated.path)
197219
lazy val strippedText = originalText.replace("\n//|", "\n")
198220
lazy val lookupLineSuffix = fastparse
199221
.IndexedParserInput(strippedText)
200-
.prettyIndex(jsonData.index)
222+
.prettyIndex(appendLocated.index)
201223
.takeWhile(_ != ':') // split off column since it's not that useful
202224

203225
val (execRes, serializedPaths) =
@@ -208,30 +230,68 @@ trait GroupExecution {
208230

209231
(
210232
ExecResult.Failure(
211-
s"Build header config in ${jsonData.path.relativeTo(workspace)}:$lookupLineSuffix conflicts with task defined " +
233+
s"Build header config in ${appendLocated.path.relativeTo(workspace)}:$lookupLineSuffix conflicts with task defined " +
212234
s"in ${os.Path(labelled.ctx.fileName).relativeTo(workspace)}:${labelled.ctx.lineNum}"
213235
),
214236
Nil
215237
)
216238
} else {
217239
// apply build override
218240
try {
241+
// If !append tag is present, evaluate the task to get parent value
242+
val parentValue: Option[Seq[Any]] = if (appendLocated.append) {
243+
val taskInputValues = labelled.inputs
244+
.map(results(_))
245+
.collect { case ExecResult.Success((v, _)) => v }
246+
247+
if (taskInputValues.length != labelled.inputs.length) None
248+
else {
249+
val (multiLogger, _) = resolveLogger(Some(paths).map(_.log), logger)
250+
val destCreator = new GroupExecution.DestCreator(Some(paths))
251+
val args = new mill.api.TaskCtx.Impl(
252+
args = taskInputValues.map(_.value).toIndexedSeq,
253+
dest0 = () => destCreator.makeDest(),
254+
log = multiLogger,
255+
env = env,
256+
reporter = zincProblemReporter,
257+
testReporter = testReporter,
258+
workspace = workspace,
259+
_systemExitWithReason = systemExit,
260+
fork = executionContext,
261+
jobs = effectiveThreadCount,
262+
offline = offline,
263+
useFileLocks = useFileLocks
264+
)
265+
labelled.evaluate(args) match {
266+
case Result.Success(v) => Some(v.asInstanceOf[Seq[Any]])
267+
case _ => None
268+
}
269+
}
270+
} else None
271+
219272
val (resultData, serializedPaths) = PathRef.withSerializedPaths {
220273
PathRef.currentOverrideModulePath.withValue(
221274
labelled.ctx.enclosingModule.moduleCtx.millSourcePath
222275
) {
223-
upickle.read[Any](interpolateEnvVarsInJson(jsonData.value))(
224-
using labelled.readWriterOpt.get.asInstanceOf[upickle.Reader[Any]]
225-
)
276+
val yamlValue =
277+
upickle.read[Any](interpolateEnvVarsInJson(appendLocated.value))(
278+
using labelled.readWriterOpt.get.asInstanceOf[upickle.Reader[Any]]
279+
)
280+
// Merge parent value with YAML value if !append was present
281+
parentValue match {
282+
case Some(parent) =>
283+
(parent ++ yamlValue.asInstanceOf[Seq[Any]]).asInstanceOf[Any]
284+
case None => yamlValue
285+
}
226286
}
227287
}
228288

229289
// Write build header override JSON to meta `.json` file to support `show`
230290
writeCacheJson(
231291
paths.meta,
232-
upickle.core.BufferedValue.transform(jsonData.value, ujson.Value),
292+
upickle.core.BufferedValue.transform(appendLocated.value, ujson.Value),
233293
resultData.##,
234-
inputsHash + jsonData.value.##
294+
inputsHash + appendLocated.value.##
235295
)
236296

237297
(ExecResult.Success(Val(resultData), resultData.##), serializedPaths)
@@ -240,14 +300,18 @@ trait GroupExecution {
240300
// Try to get more specific index from AbortException if available
241301
val errorIndex = e.getCause match {
242302
case abort: upickle.core.AbortException => abort.index
243-
case _ => jsonData.value.index
303+
case _ => appendLocated.value.index
244304
}
245305

246306
val msg = s"Failed de-serializing config override: ${e.getCause.getMessage}"
247307
(
248308
ExecResult.Failure(
249309
s"Failed de-serializing config override: ${e.getCause.getMessage}",
250-
Some(Result.Failure(msg, path = jsonData.path.toNIO, index = errorIndex))
310+
Some(Result.Failure(
311+
msg,
312+
path = appendLocated.path.toNIO,
313+
index = errorIndex
314+
))
251315
),
252316
Nil
253317
)

‎core/internal/src/mill/internal/Util.scala‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -242,16 +242,45 @@ object Util {
242242
objVisitor.visitEnd(index)
243243

244244
case sequence: SequenceNode =>
245-
val arrVisitor = v.visitArray(sequence.getValue.size(), index)
246-
.asInstanceOf[upickle.core.ArrVisitor[Any, J]]
247-
for (item <- sequence.getValue.asScala) {
248-
val itemResult = rec(item, arrVisitor.subVisitor)
249-
arrVisitor.visitValue(
250-
itemResult,
251-
item.getStartMark.map(_.getIndex.intValue()).orElse(0)
252-
)
245+
// Check for !append tag - if present, wrap in marker object
246+
val hasAppendTag = sequence.getTag.getValue == "!append"
247+
if (hasAppendTag) {
248+
// Wrap array in object: {"__mill_append__": true, "__mill_values__": [...]}
249+
val objVisitor = v.visitObject(2, jsonableKeys = true, index)
250+
.asInstanceOf[upickle.core.ObjVisitor[Any, J]]
251+
252+
// Add __mill_append__ key
253+
val appendKeyVisitor = objVisitor.visitKey(index)
254+
objVisitor.visitKeyValue(appendKeyVisitor.visitString("__mill_append__", index))
255+
objVisitor.visitValue(objVisitor.subVisitor.visitTrue(index), index)
256+
257+
// Add __mill_values__ key with the actual array
258+
val valuesKeyVisitor = objVisitor.visitKey(index)
259+
objVisitor.visitKeyValue(valuesKeyVisitor.visitString("__mill_values__", index))
260+
val arrVisitor =
261+
objVisitor.subVisitor.visitArray(sequence.getValue.size(), index)
262+
.asInstanceOf[upickle.core.ArrVisitor[Any, Any]]
263+
for (item <- sequence.getValue.asScala) {
264+
val itemResult = rec(item, arrVisitor.subVisitor)
265+
arrVisitor.visitValue(
266+
itemResult,
267+
item.getStartMark.map(_.getIndex.intValue()).orElse(0)
268+
)
269+
}
270+
objVisitor.visitValue(arrVisitor.visitEnd(index), index)
271+
objVisitor.visitEnd(index)
272+
} else {
273+
val arrVisitor = v.visitArray(sequence.getValue.size(), index)
274+
.asInstanceOf[upickle.core.ArrVisitor[Any, J]]
275+
for (item <- sequence.getValue.asScala) {
276+
val itemResult = rec(item, arrVisitor.subVisitor)
277+
arrVisitor.visitValue(
278+
itemResult,
279+
item.getStartMark.map(_.getIndex.intValue()).orElse(0)
280+
)
281+
}
282+
arrVisitor.visitEnd(index)
253283
}
254-
arrVisitor.visitEnd(index)
255284
}
256285
} catch {
257286
case e: upickle.core.Abort =>

‎libs/init/buildgen/src/mill/main/buildgen/BuildGen.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ object BuildGen {
193193
.reduce(parentModule(_, _, "ProjectBaseModule", defaultSupertypes))
194194
.copy(children =
195195
extendingModules.flatMap(_.children.filter(isTestModule))
196-
.reduceOption(parentModule(_, _, "Tests", defaultTestSupertypes)).toSeq
196+
.reduceOption(parentModule(_, _, "ProjectBaseTests", defaultTestSupertypes)).toSeq
197197
)
198198
val packages0 =
199199
packages.map(pkg => pkg.copy(module = recExtendModule(pkg.module, baseModule)))

0 commit comments

Comments
 (0)