Merge pull request #3646 from JuliaLang/backports-release-1.10

.github/workflows/test.yml

@@ -36,7 +36,7 @@ jobs:
           - "pkg.julialang.org"
         julia-version:
           # - '1.6'
-          - 'nightly'
+          - '1.10-nightly'
         exclude:
           - os: macOS-latest
             julia-arch: x86

src/API.jl

@@ -1104,6 +1104,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
     # make a flat map of each dep and its direct deps
     depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}()
     pkg_specs = PackageSpec[]
+    pkg_exts_map = Dict{Base.PkgId, Vector{Base.PkgId}}()
     for dep in ctx.env.manifest
         pkg = Base.PkgId(first(dep), last(dep).name)
         Base.in_sysimage(pkg) && continue
@@ -1112,6 +1113,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
         # add any extensions
         weakdeps = last(dep).weakdeps
         pkg_exts = Dict{Base.PkgId, Vector{Base.PkgId}}()
+        prev_ext = nothing
         for (ext_name, extdep_names) in last(dep).exts
             ext_deps = Base.PkgId[]
             push!(ext_deps, pkg) # depends on parent package
@@ -1127,22 +1129,31 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
                 end
             end
             all_extdeps_available || continue
+            if prev_ext isa Base.PkgId
+                # also make the exts depend on eachother sequentially to avoid race
+                push!(ext_deps, prev_ext)
+            end
             ext_uuid = Base.uuid5(pkg.uuid, ext_name)
             ext = Base.PkgId(ext_uuid, ext_name)
+            prev_ext = ext
             push!(pkg_specs, PackageSpec(uuid = ext_uuid, name = ext_name)) # create this here as the name cannot be looked up easily later via the uuid
             filter!(!Base.in_sysimage, ext_deps)
             depsmap[ext] = ext_deps
             exts[ext] = pkg.name
             pkg_exts[ext] = ext_deps
         end
         if !isempty(pkg_exts)
-            # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s),
-            # basically injecting the extension into the precompile order in the graph, to avoid race to precompile extensions
-            for (_pkg, deps) in depsmap # for each manifest dep
-                if !in(_pkg, keys(exts)) && pkg in deps # if not an extension and depends on pkg
-                    append!(deps, keys(pkg_exts)) # add the package extensions to deps
-                    filter!(!isequal(pkg), deps) # remove the pkg from deps
-                end
+            pkg_exts_map[pkg] = collect(keys(pkg_exts))
+        end
+    end
+    # this loop must be run after the full depsmap has been populated
+    for (pkg, pkg_exts) in pkg_exts_map
+        # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s),
+        # basically injecting the extension into the precompile order in the graph, to avoid race to precompile extensions
+        for (_pkg, deps) in depsmap # for each manifest dep
+            if !in(_pkg, keys(exts)) && pkg in deps # if not an extension and depends on pkg
+                append!(deps, pkg_exts) # add the package extensions to deps
+                filter!(!isequal(pkg), deps) # remove the pkg from deps
             end
         end
     end

src/Operations.jl

@@ -47,11 +47,10 @@ tracking_registered_version(pkg::Union{PackageSpec, PackageEntry}, julia_version
     !is_stdlib(pkg.uuid, julia_version) && pkg.path === nothing && pkg.repo.source === nothing
 
 function source_path(manifest_file::String, pkg::Union{PackageSpec, PackageEntry}, julia_version = VERSION)
-    return is_stdlib(pkg.uuid, julia_version) ? Types.stdlib_path(pkg.name) :
-        pkg.path        !== nothing ? joinpath(dirname(manifest_file), pkg.path) :
-        pkg.repo.source !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) :
-        pkg.tree_hash   !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) :
-        nothing
+    pkg.tree_hash   !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) :
+    pkg.path        !== nothing ? joinpath(dirname(manifest_file), pkg.path) :
+    is_or_was_stdlib(pkg.uuid, julia_version) ? Types.stdlib_path(pkg.name) :
+    nothing
 end
 
 #TODO rename
@@ -1066,12 +1065,9 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false)
                 error("could not find entry with uuid $uuid in manifest $(ctx.env.manifest_file)")
             end
             name = entry.name
-            if entry.tree_hash !== nothing
-                path = find_installed(name, uuid, entry.tree_hash)
-            elseif entry.path !== nothing
-                path = project_rel_path(ctx.env, entry.path)
-            else
-                pkgerror("Could not find either `git-tree-sha1` or `path` for package $name")
+            path = source_path(ctx.env.manifest_file, entry)
+            if path === nothing
+                pkgerror("Failed to find path for package $name")
             end
             version = something(entry.version, v"0.0")
         end
@@ -1819,13 +1815,6 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
             end
 
             reset_all_compat!(temp_ctx.env.project)
-
-            # Absolutify stdlibs paths
-            for (uuid, entry) in temp_ctx.env.manifest
-                if is_stdlib(uuid)
-                    entry.path = Types.stdlib_path(entry.name)
-                end
-            end
             write_env(temp_ctx.env, update_undo = false)
 
             # Run sandboxed code

src/Types.jl

@@ -23,7 +23,7 @@ using SHA
 export UUID, SHA1, VersionRange, VersionSpec,
     PackageSpec, PackageEntry, EnvCache, Context, GitRepo, Context!, Manifest, Project, err_rep,
     PkgError, pkgerror, PkgPrecompileError,
-    has_name, has_uuid, is_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml,
+    has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml,
     project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
     registered_name,
     manifest_info,
@@ -430,20 +430,23 @@ is_project_uuid(env::EnvCache, uuid::UUID) = project_uuid(env) == uuid
 # Context #
 ###########
 
+const FORMER_STDLIBS = ["DelimitedFiles"]
+const FORMER_STDLIBS_UUIDS = Set{UUID}()
 const STDLIB = Ref{DictStdLibs}()
 function load_stdlib()
     stdlib = DictStdLibs()
     for name in readdir(stdlib_dir())
-        # DelimitedFiles is an upgradable stdlib
-        # TODO: Store this information of upgradable stdlibs somewhere else
-        name == "DelimitedFiles" && continue
         projfile = projectfile_path(stdlib_path(name); strict=true)
         nothing === projfile && continue
         project = parse_toml(projfile)
         uuid = get(project, "uuid", nothing)::Union{String, Nothing}
         v_str = get(project, "version", nothing)::Union{String, Nothing}
         version = isnothing(v_str) ? nothing : VersionNumber(v_str)
         nothing === uuid && continue
+        if name in FORMER_STDLIBS
+            push!(FORMER_STDLIBS_UUIDS, UUID(uuid))
+            continue
+        end
         stdlib[UUID(uuid)] = (name, version)
     end
     return stdlib
@@ -456,6 +459,11 @@ function stdlibs()
     return STDLIB[]
 end
 is_stdlib(uuid::UUID) = uuid in keys(stdlibs())
+# Includes former stdlibs
+function is_or_was_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
+    return is_stdlib(uuid, julia_version) || uuid in FORMER_STDLIBS_UUIDS
+end
+
 
 # Find the entry in `STDLIBS_BY_VERSION`
 # that corresponds to the requested version, and use that.

test/manifest/old/Manifest.toml

@@ -0,0 +1,9 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[DelimitedFiles]]
+deps = ["Mmap"]
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+version = "1.9.1"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"

test/manifest/old/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"

test/new.jl

@@ -1456,6 +1456,15 @@ end
             end
         end
     end
+    # instantiate old manifest
+    isolate(loaded_depot=true) do
+        manifest_dir = joinpath(@__DIR__, "manifest", "old")
+        cd(manifest_dir) do
+            Pkg.activate(".")
+            Pkg.instantiate()
+            @test isinstalled("DelimitedFiles")
+        end
+    end
     # `instantiate` on a lonely manifest should detect duplicate names
     isolate(loaded_depot=true) do; mktempdir() do tempdir
         simple_package_path = copy_test_package(tempdir, "SimplePackage")