Skip to content

Commit 73f8c7a

Browse files
lihaoyilihaoyi
committed
.
1 parent b4d5569 commit 73f8c7a

File tree

9 files changed

+138
-33
lines changed

9 files changed

+138
-33
lines changed

‎core/api/src/mill/api/Evaluator.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import mill.api.*
55
import mill.api.daemon.Watchable
66
import mill.api.BuildCtx
77
import mill.api.daemon.internal.{EvaluatorApi, TaskApi}
8-
import mill.api.internal.{AppendLocated, Located, Resolved, RootModule0}
8+
import mill.api.internal.{Located, Resolved, RootModule0}
99
import upickle.core.BufferedValue
1010
import scala.util.DynamicVariable
1111
import scala.collection.mutable
@@ -33,7 +33,7 @@ trait Evaluator extends AutoCloseable with EvaluatorApi {
3333
private[mill] def effectiveThreadCount: Int
3434
private[mill] def offline: Boolean
3535
private[mill] def useFileLocks: Boolean = false
36-
private[mill] def staticBuildOverrides: Map[String, AppendLocated[BufferedValue]] = Map()
36+
private[mill] def staticBuildOverrides: Map[String, Located[BufferedValue]] = Map()
3737
def withBaseLogger(newBaseLogger: Logger): Evaluator
3838

3939
def resolveSegments(

‎core/api/src/mill/api/Module.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ trait Module extends Module.BaseClass with ModuleCtx.Wrapper with ModuleApi {
4545
private[mill] val moduleLinearized: Seq[Class[?]] =
4646
OverrideMapping.computeLinearization(this.getClass)
4747

48-
private[mill] def moduleDynamicBuildOverrides: Map[String, internal.AppendLocated[BufferedValue]] =
48+
private[mill] def moduleDynamicBuildOverrides: Map[String, internal.Located[BufferedValue]] =
4949
Map()
5050
}
5151

‎core/api/src/mill/api/ModuleCtx.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ object ModuleCtx extends LowPriCtx {
4646
def moduleSegments: Segments = moduleCtx.segments
4747
def moduleCtx: ModuleCtx
4848
private[mill] def moduleLinearized: Seq[Class[?]]
49-
private[mill] def moduleDynamicBuildOverrides: Map[String, internal.AppendLocated[BufferedValue]] =
49+
private[mill] def moduleDynamicBuildOverrides: Map[String, internal.Located[BufferedValue]] =
5050
Map()
5151
}
5252

‎core/api/src/mill/api/ScriptModule.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ trait ScriptModule extends ExternalModule {
2525
.rest
2626
.map { case (k, v) =>
2727
val newKey = (moduleSegments ++ mill.api.Segment.Label(k.value)).render
28-
val (actualValue, append) = internal.AppendLocated.unwrapAppendMarker(v)
29-
(newKey, internal.AppendLocated(internal.Located(scriptConfig.scriptFile, k.index, actualValue), append))
28+
val (actualValue, append) = internal.Located.unwrapAppendMarker(v)
29+
(newKey, internal.Located(scriptConfig.scriptFile, k.index, actualValue, append))
3030
}
3131
}
3232
@experimental

‎core/api/src/mill/api/internal/HeaderData.scala‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,9 @@ package mill.api.internal
22

33
private[mill] case class HeaderData(
44
`extends`: Located[OneOrMore[Located[String]]] = Located(null, -1, OneOrMore(Nil)),
5-
moduleDeps: AppendLocated[Seq[Located[String]]] =
6-
AppendLocated(Located(null, -1, Nil), append = false),
7-
compileModuleDeps: AppendLocated[Seq[Located[String]]] =
8-
AppendLocated(Located(null, -1, Nil), append = false),
9-
runModuleDeps: AppendLocated[Seq[Located[String]]] =
10-
AppendLocated(Located(null, -1, Nil), append = false),
5+
moduleDeps: Located[Seq[Located[String]]] = Located(null, -1, Nil),
6+
compileModuleDeps: Located[Seq[Located[String]]] = Located(null, -1, Nil),
7+
runModuleDeps: Located[Seq[Located[String]]] = Located(null, -1, Nil),
118
@upickle.implicits.flatten rest: Map[Located[String], upickle.core.BufferedValue]
129
)
1310
private[mill] object HeaderData {
@@ -20,8 +17,8 @@ private[mill] object HeaderData {
2017
def headerDataReader(path: os.Path) = {
2118
implicit def locatedReader[T: upickle.Reader]: Located.UpickleReader[T] =
2219
new Located.UpickleReader[T](path)
23-
implicit def appendLocatedReader[T: upickle.Reader]: AppendLocated.UpickleReader[T] =
24-
new AppendLocated.UpickleReader[T](path)
20+
implicit def appendLocatedReader[T: upickle.Reader]: Located.AppendUpickleReader[T] =
21+
new Located.AppendUpickleReader[T](path)
2522
upickle.macroR[HeaderData]
2623
}
2724
}

‎core/api/src/mill/api/internal/Located.scala‎

Lines changed: 113 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,38 @@
11
package mill.api.internal
22

3-
case class Located[T](path: os.Path, index: Int, value: T)
3+
case class Located[T](path: os.Path, index: Int, value: T, append: Boolean = false)
44

55
object Located {
6+
import upickle.core.BufferedValue
7+
8+
/** Marker key indicating append mode in YAML `!append` tag wrapper objects */
9+
val AppendMarkerKey = "__mill_append__"
10+
/** Marker key containing the actual values in YAML `!append` tag wrapper objects */
11+
val ValuesMarkerKey = "__mill_values__"
12+
13+
/**
14+
* Extract append flag and actual value from a BufferedValue that may contain
15+
* the marker object (produced by YAML `!append` tag parsing).
16+
* Returns (actualValue, appendFlag).
17+
*/
18+
def unwrapAppendMarker(v: BufferedValue): (BufferedValue, Boolean) = {
19+
v match {
20+
case obj: BufferedValue.Obj =>
21+
val kvMap = obj.value0.collect { case (BufferedValue.Str(k, _), v) =>
22+
k.toString -> v
23+
}.toMap
24+
(kvMap.get(AppendMarkerKey), kvMap.get(ValuesMarkerKey)) match {
25+
case (Some(BufferedValue.True(_)), Some(values)) => (values, true)
26+
case _ => (v, false)
27+
}
28+
case _ => (v, false)
29+
}
30+
}
31+
632
class UpickleReader[T](path: os.Path)(implicit r: upickle.Reader[T])
733
extends upickle.Reader[Located[T]] {
8-
private def wrap(index: Int, v: T): Located[T] = Located(path, index, v)
34+
private def wrap(index: Int, v: T, append: Boolean = false): Located[T] =
35+
Located(path, index, v, append)
936

1037
def visitArray(length: Int, index: Int): upickle.core.ArrVisitor[Any, Located[T]] = {
1138
val delegate = r.visitArray(length, index)
@@ -46,4 +73,88 @@ object Located {
4673
def visitExt(tag: Byte, bytes: Array[Byte], offset: Int, len: Int, index: Int) =
4774
wrap(index, r.visitExt(tag, bytes, offset, len, index))
4875
}
76+
77+
/**
78+
* Upickle reader that detects the `!append` marker object
79+
* and extracts the append flag along with the actual value.
80+
*/
81+
class AppendUpickleReader[T](path: os.Path)(implicit r: upickle.Reader[T])
82+
extends upickle.Reader[Located[T]] {
83+
private def wrap(index: Int, v: T, append: Boolean): Located[T] =
84+
Located(path, index, v, append)
85+
86+
// For arrays without !append marker - just wrap with append=false
87+
def visitArray(length: Int, index: Int): upickle.core.ArrVisitor[Any, Located[T]] = {
88+
val delegate = r.visitArray(length, index)
89+
new upickle.core.ArrVisitor[Any, Located[T]] {
90+
def subVisitor = delegate.subVisitor
91+
def visitValue(v: Any, index: Int): Unit = delegate.visitValue(v, index)
92+
def visitEnd(idx: Int) = wrap(index, delegate.visitEnd(idx), append = false)
93+
}
94+
}
95+
96+
// For objects - check if it's an !append marker object
97+
def visitObject(length: Int, jsonableKeys: Boolean, index: Int)
98+
: upickle.core.ObjVisitor[Any, Located[T]] = {
99+
new upickle.core.ObjVisitor[Any, Located[T]] {
100+
private var currentKey: String = ""
101+
private var hasAppendMarker = false
102+
private var valuesResult: Option[T] = None
103+
private val startIndex = index
104+
105+
// Delegate for parsing the inner values array
106+
private val valuesReader = r
107+
108+
def subVisitor: upickle.core.Visitor[?, ?] = currentKey match {
109+
case AppendMarkerKey => upickle.core.NoOpVisitor
110+
case ValuesMarkerKey => valuesReader
111+
case _ => upickle.core.NoOpVisitor
112+
}
113+
114+
def visitKey(index: Int): upickle.core.Visitor[?, ?] = upickle.core.StringVisitor
115+
116+
def visitKeyValue(s: Any): Unit = {
117+
currentKey = s.toString
118+
}
119+
120+
def visitValue(v: Any, index: Int): Unit = {
121+
currentKey match {
122+
case AppendMarkerKey => hasAppendMarker = true
123+
case ValuesMarkerKey => valuesResult = Some(v.asInstanceOf[T])
124+
case _ => ()
125+
}
126+
}
127+
128+
def visitEnd(idx: Int): Located[T] = {
129+
if (hasAppendMarker && valuesResult.isDefined) {
130+
wrap(startIndex, valuesResult.get, append = true)
131+
} else {
132+
// Not an append marker object - this shouldn't happen for moduleDeps
133+
// but fall back to empty value
134+
wrap(startIndex, r.visitArray(0, startIndex).visitEnd(startIndex), append = false)
135+
}
136+
}
137+
}
138+
}
139+
140+
def visitNull(index: Int) = wrap(index, r.visitNull(index), append = false)
141+
def visitFalse(index: Int) = wrap(index, r.visitFalse(index), append = false)
142+
def visitTrue(index: Int) = wrap(index, r.visitTrue(index), append = false)
143+
def visitFloat64StringParts(s: CharSequence, decIndex: Int, expIndex: Int, index: Int) =
144+
wrap(index, r.visitFloat64StringParts(s, decIndex, expIndex, index), append = false)
145+
def visitFloat64(d: Double, index: Int) = wrap(index, r.visitFloat64(d, index), append = false)
146+
def visitFloat32(d: Float, index: Int) = wrap(index, r.visitFloat32(d, index), append = false)
147+
def visitInt32(i: Int, index: Int) = wrap(index, r.visitInt32(i, index), append = false)
148+
def visitInt64(i: Long, index: Int) = wrap(index, r.visitInt64(i, index), append = false)
149+
def visitUInt64(i: Long, index: Int) = wrap(index, r.visitUInt64(i, index), append = false)
150+
def visitFloat64String(s: String, index: Int) =
151+
wrap(index, r.visitFloat64String(s, index), append = false)
152+
def visitString(s: CharSequence, index: Int) =
153+
wrap(index, r.visitString(s, index), append = false)
154+
def visitChar(s: Char, index: Int) = wrap(index, r.visitChar(s, index), append = false)
155+
def visitBinary(bytes: Array[Byte], offset: Int, len: Int, index: Int) =
156+
wrap(index, r.visitBinary(bytes, offset, len, index), append = false)
157+
def visitExt(tag: Byte, bytes: Array[Byte], offset: Int, len: Int, index: Int) =
158+
wrap(index, r.visitExt(tag, bytes, offset, len, index), append = false)
159+
}
49160
}

‎core/exec/src/mill/exec/GroupExecution.scala‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package mill.exec
22

33
import mill.api.ExecResult.{OuterStack, Success}
44
import mill.api.*
5-
import mill.api.internal.{AppendLocated, Cached, Located}
5+
import mill.api.internal.{Cached, Located}
66
import mill.internal.MultiLogger
77
import mill.internal.FileLogger
88

@@ -44,14 +44,14 @@ trait GroupExecution {
4444

4545
/** Evaluate a build override YAML value and deserialize it */
4646
private def evaluateBuildOverride(
47-
appendLocated: AppendLocated[BufferedValue],
47+
located: Located[BufferedValue],
4848
labelled: Task.Named[_]
4949
): Either[upickle.core.TraceVisitor.TraceException, Any] = {
5050
try {
5151
Right(PathRef.currentOverrideModulePath.withValue(
5252
labelled.ctx.enclosingModule.moduleCtx.millSourcePath
5353
) {
54-
upickle.read[Any](interpolateEnvVarsInJson(appendLocated.value))(
54+
upickle.read[Any](interpolateEnvVarsInJson(located.value))(
5555
using labelled.readWriterOpt.get.asInstanceOf[upickle.Reader[Any]]
5656
)
5757
})
@@ -60,15 +60,15 @@ trait GroupExecution {
6060
}
6161
}
6262

63-
val staticBuildOverrides: Map[String, AppendLocated[BufferedValue]] = staticBuildOverrideFiles
63+
val staticBuildOverrides: Map[String, Located[BufferedValue]] = staticBuildOverrideFiles
6464
.flatMap { case (path0, rawText) =>
6565
val path = os.Path(path0)
6666
val headerDataReader = mill.api.internal.HeaderData.headerDataReader(path)
6767

6868
def rec(
6969
segments: Seq[String],
7070
bufValue: upickle.core.BufferedValue
71-
): Seq[(String, AppendLocated[BufferedValue])] = {
71+
): Seq[(String, Located[BufferedValue])] = {
7272
val upickle.core.BufferedValue.Obj(kvs, _, _) = bufValue
7373
val (rawKvs, nested) = kvs.partitionMap { case (upickle.core.BufferedValue.Str(k, i), v) =>
7474
k.toString.split(" +") match {
@@ -77,7 +77,7 @@ trait GroupExecution {
7777
}
7878
}
7979

80-
val currentResults: Seq[(String, AppendLocated[BufferedValue])] =
80+
val currentResults: Seq[(String, Located[BufferedValue])] =
8181
BufferedValue.transform(
8282
BufferedValue.Obj(
8383
rawKvs.map { case (k, i, v) => (BufferedValue.Str(k, i), v) }.to(mutable.ArrayBuffer),
@@ -88,15 +88,12 @@ trait GroupExecution {
8888
)
8989
.rest
9090
.map { case (k, v) =>
91-
val (actualValue, append) = AppendLocated.unwrapAppendMarker(v)
92-
(segments ++ Seq(k.value)).mkString(".") -> AppendLocated(
93-
Located(path, k.index, actualValue),
94-
append
95-
)
91+
val (actualValue, append) = Located.unwrapAppendMarker(v)
92+
(segments ++ Seq(k.value)).mkString(".") -> Located(path, k.index, actualValue, append)
9693
}
9794
.toSeq
9895

99-
val nestedResults: Seq[(String, AppendLocated[BufferedValue])] = nested.flatten.toSeq
96+
val nestedResults: Seq[(String, Located[BufferedValue])] = nested.flatten.toSeq
10097

10198
currentResults ++ nestedResults
10299
}
@@ -315,13 +312,13 @@ trait GroupExecution {
315312
}
316313

317314
// Helper to evaluate build override only (no task evaluation)
318-
def evaluateBuildOverrideOnly(appendLocated: AppendLocated[BufferedValue])
315+
def evaluateBuildOverrideOnly(located: Located[BufferedValue])
319316
: GroupExecution.Results = {
320-
lazy val originalText = os.read(appendLocated.path)
317+
lazy val originalText = os.read(located.path)
321318
lazy val strippedText = originalText.replace("\n//|", "\n")
322319
lazy val lookupLineSuffix = fastparse
323320
.IndexedParserInput(strippedText)
324-
.prettyIndex(appendLocated.index)
321+
.prettyIndex(located.index)
325322
.takeWhile(_ != ':')
326323

327324
val (execRes, serializedPaths) =

‎core/internal/src/mill/internal/Util.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ object Util {
245245
// Check for !append tag - if present, wrap in marker object
246246
val hasAppendTag = sequence.getTag.getValue == "!append"
247247
if (hasAppendTag) {
248-
import mill.api.internal.AppendLocated.{AppendMarkerKey, ValuesMarkerKey}
248+
import mill.api.internal.Located.{AppendMarkerKey, ValuesMarkerKey}
249249
// Wrap array in object with marker keys
250250
val objVisitor = v.visitObject(2, jsonableKeys = true, index)
251251
.asInstanceOf[upickle.core.ObjVisitor[Any, J]]

‎runner/meta/src/mill/meta/CodeGen.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ object CodeGen {
170170

171171
def renderModuleDepsSnippet(
172172
name: String,
173-
deps: mill.api.internal.AppendLocated[Seq[mill.api.internal.Located[String]]]
173+
deps: mill.api.internal.Located[Seq[mill.api.internal.Located[String]]]
174174
): String = {
175175
if (deps.value.isEmpty && !deps.append) ""
176176
else {

0 commit comments

Comments
 (0)