A list of `Core.Box` in Base and stdlibs

[KristofferC]

This uses #60478 to get a list of methods that get variables boxed due to #15276 (or other reasons). Generated with ./julia contrib/scan-closure-boxes.jl --module=Base,stdlibs --format=markdown. The lowered code can be get via e.g. code_warntype(sig) where the sig is as given in the table.

Base

var	func	sig	location
all_effects	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
argtypesi	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
callfuture	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
ctfuture	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
ctypes	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
ctypes´	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
exctype	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
i	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
infos	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
infos´	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
j	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
nextstate	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
res	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
ti	abstract_apply	Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1711
argtypes	abstract_call_known	Tuple{typeof(Compiler.abstract_call_known), Compiler.AbstractInterpreter, Any, Compiler.ArgInfo, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:2664
effects	abstract_call_known	Tuple{typeof(Compiler.abstract_call_known), Compiler.AbstractInterpreter, Any, Compiler.ArgInfo, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:2664
exct	abstract_call_known	Tuple{typeof(Compiler.abstract_call_known), Compiler.AbstractInterpreter, Any, Compiler.ArgInfo, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:2664
exct	abstract_invoke	Tuple{typeof(Compiler.abstract_invoke), Compiler.AbstractInterpreter, Compiler.ArgInfo, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:2237
RT	abstract_modifyop!	Tuple{typeof(Compiler.abstract_modifyop!), Compiler.AbstractInterpreter, Any, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}	../usr/share/julia/Compiler/src/tfuncs.jl:1367
TF	abstract_modifyop!	Tuple{typeof(Compiler.abstract_modifyop!), Compiler.AbstractInterpreter, Any, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}	../usr/share/julia/Compiler/src/tfuncs.jl:1367
info	abstract_modifyop!	Tuple{typeof(Compiler.abstract_modifyop!), Compiler.AbstractInterpreter, Any, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}	../usr/share/julia/Compiler/src/tfuncs.jl:1367
call2future	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
may_have_terminated	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
nextstate	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
stateordonet	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
stateordonet_widened	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
statetype	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
valtype	inferiterate	Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}	../usr/share/julia/Compiler/src/abstractinterpretation.jl:1583
old_restrict	return_type_tfunc	Tuple{typeof(Compiler.return_type_tfunc), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}	../usr/share/julia/Compiler/src/tfuncs.jl:3084
tt	return_type_tfunc	Tuple{typeof(Compiler.return_type_tfunc), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}	../usr/share/julia/Compiler/src/tfuncs.jl:3084

Distributed

var	func	sig	location
x	#remotecall_pool#130	Tuple{Distributed.var"##remotecall_pool#130", Base.Pairs{Symbol, V, Nothing, NT} where {V, NT<:NamedTuple}, typeof(Distributed.remotecall_pool), typeof(Distributed.remotecall), Any, Distributed.AbstractWorkerPool, Vararg{Any}}	/home/kc/julia/usr/share/julia/stdlib/v1.14/Distributed/src/workerpool.jl:140
x	#remotecall_pool#154	Tuple{Distributed.var"##remotecall_pool#154", Base.Pairs{Symbol, V, Nothing, NT} where {V, NT<:NamedTuple}, typeof(Distributed.remotecall_pool), typeof(Distributed.remotecall), Any, Distributed.CachingPool, Vararg{Any}}	/home/kc/julia/usr/share/julia/stdlib/v1.14/Distributed/src/workerpool.jl:407
sock	#start_worker#24	Tuple{Distributed.var"##start_worker#24", Bool, Bool, typeof(Distributed.start_worker), IO, AbstractString}	/home/kc/julia/usr/share/julia/stdlib/v1.14/Distributed/src/cluster.jl:237
w	create_worker	Tuple{typeof(Distributed.create_worker), Any, Any}	/home/kc/julia/usr/share/julia/stdlib/v1.14/Distributed/src/cluster.jl:601

[bbrehm]

@KristofferC Would you mind writing a few words on your experience fixing these boxes once you're done?

I think there is a lesson on language design here.

I see mostly 2 classes:

First, genuine shared state where you replace Core.Box by Base.RefValue.

Second, cases where lowering fails to understand that the captured variable is SSA; you mostly introduce a new temp x_capture_me_assigned_only_once = x, instead of let x = x. I assume you do this in order to keep diffs clean? Because a let-block would ask for change in indent-level and hence modify all lines of the closure?

I think your "fix Core.Box"-spree could be a strong argument to have a "capture-by-value" strict-mode a la #60180 where closures can either capture-by-value if the captured variable is obviously SSA, or is an error instead of a Core.Box.

[KristofferC]

Either a new variable or let depending on what is most convenient (minimizes the diff).

Also changing

if condition
   x = val1
else
   x = val2
end

into

x = if condition
    val 1
else
    val2
end

It does kind of feel like the compiler (or rather lowering) should be able to spot these type of trivial rewrites...

[bbrehm]

I very much agree that these trivial rewrites should be done by the compiler. There should be clear rules on IR level for capture-boxing, like e.g. "if the closure only reads, and no parent write is CFG-reachable from any instantiation of the closure, then there is no box", and "if no parent read or write is CFG-reachable from closure instantiation then there is no box".

But then, if one looks at the process, it is not so surprising: The boxing decision is made in the LISP world, on the basis of syntax tree, without any of the nice machinery like a control flow graph. Until we can defer it to something that operates on SSAIR pre-inference, this will continue to suck :(

[KristofferC]

The Distributed ones are fixed via JuliaLang/Distributed.jl#161. Other than that, the remaining ones are in the following Compiler signatures:

Tuple{typeof(Compiler.abstract_apply), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}

Tuple{typeof(Compiler.abstract_call_known), Compiler.AbstractInterpreter, Any, Compiler.ArgInfo, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}, Int64}

Tuple{typeof(Compiler.abstract_invoke), Compiler.AbstractInterpreter, Compiler.ArgInfo, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}

Tuple{typeof(Compiler.abstract_modifyop!), Compiler.AbstractInterpreter, Any, Vector{Any}, Compiler.StmtInfo, Union{Nothing, Vector{Compiler.VarState}}, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}

Tuple{Compiler.var"#inferiterate#abstract_iteration##0", Any, Any}

Tuple{typeof(Compiler.return_type_tfunc), Compiler.AbstractInterpreter, Vector{Any}, Compiler.StmtInfo, Union{Compiler.IRInterpretationState, Compiler.InferenceState}}

but they are kind of complicated so I don't really want to touch thouse.