Make Pkg more resilient to reading older manifests where ex-stdlibs a…

…re listed (#3634)

* Make Pkg more resilient to reading older manifests where ex-stdlibs are listed

This should prevent errors like:

```
ERROR: Could not locate the source code for the DelimitedFiles package. Are you trying to use a manifest generated by a different version of Julia?
```

When using older manifests that list DelimitedFiles in the stdlib format even though DelimitedFiles now is upgradable and has e.g. a `git-tree-sha1` entry when generated with current Pkg

(cherry picked from commit ddcc7b2)

src/Operations.jl

@@ -47,11 +47,10 @@ tracking_registered_version(pkg::Union{PackageSpec, PackageEntry}, julia_version
     !is_stdlib(pkg.uuid, julia_version) && pkg.path === nothing && pkg.repo.source === nothing
 
 function source_path(manifest_file::String, pkg::Union{PackageSpec, PackageEntry}, julia_version = VERSION)
-    return is_stdlib(pkg.uuid, julia_version) ? Types.stdlib_path(pkg.name) :
-        pkg.path        !== nothing ? joinpath(dirname(manifest_file), pkg.path) :
-        pkg.repo.source !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) :
-        pkg.tree_hash   !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) :
-        nothing
+    pkg.tree_hash   !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) :
+    pkg.path        !== nothing ? joinpath(dirname(manifest_file), pkg.path) :
+    is_or_was_stdlib(pkg.uuid, julia_version) ? Types.stdlib_path(pkg.name) :
+    nothing
 end
 
 #TODO rename
@@ -1066,12 +1065,9 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false)
                 error("could not find entry with uuid $uuid in manifest $(ctx.env.manifest_file)")
             end
             name = entry.name
-            if entry.tree_hash !== nothing
-                path = find_installed(name, uuid, entry.tree_hash)
-            elseif entry.path !== nothing
-                path = project_rel_path(ctx.env, entry.path)
-            else
-                pkgerror("Could not find either `git-tree-sha1` or `path` for package $name")
+            path = source_path(ctx.env.manifest_file, entry)
+            if path === nothing
+                pkgerror("Failed to find path for package $name")
             end
             version = something(entry.version, v"0.0")
         end

src/Types.jl

@@ -23,7 +23,7 @@ using SHA
 export UUID, SHA1, VersionRange, VersionSpec,
     PackageSpec, PackageEntry, EnvCache, Context, GitRepo, Context!, Manifest, Project, err_rep,
     PkgError, pkgerror, PkgPrecompileError,
-    has_name, has_uuid, is_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml,
+    has_name, has_uuid, is_stdlib, is_or_was_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml,
     project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
     registered_name,
     manifest_info,
@@ -430,20 +430,23 @@ is_project_uuid(env::EnvCache, uuid::UUID) = project_uuid(env) == uuid
 # Context #
 ###########
 
+const FORMER_STDLIBS = ["DelimitedFiles", "Statistics"]
+const FORMER_STDLIBS_UUIDS = Set{UUID}()
 const STDLIB = Ref{DictStdLibs}()
 function load_stdlib()
     stdlib = DictStdLibs()
     for name in readdir(stdlib_dir())
-        # DelimitedFiles is an upgradable stdlib
-        # TODO: Store this information of upgradable stdlibs somewhere else
-        name == "DelimitedFiles" && continue
         projfile = projectfile_path(stdlib_path(name); strict=true)
         nothing === projfile && continue
         project = parse_toml(projfile)
         uuid = get(project, "uuid", nothing)::Union{String, Nothing}
         v_str = get(project, "version", nothing)::Union{String, Nothing}
         version = isnothing(v_str) ? nothing : VersionNumber(v_str)
         nothing === uuid && continue
+        if name in FORMER_STDLIBS
+            push!(FORMER_STDLIBS_UUIDS, UUID(uuid))
+            continue
+        end
         stdlib[UUID(uuid)] = (name, version)
     end
     return stdlib
@@ -456,6 +459,11 @@ function stdlibs()
     return STDLIB[]
 end
 is_stdlib(uuid::UUID) = uuid in keys(stdlibs())
+# Includes former stdlibs
+function is_or_was_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
+    return is_stdlib(uuid, julia_version) || uuid in FORMER_STDLIBS_UUIDS
+end
+
 
 # Find the entry in `STDLIBS_BY_VERSION`
 # that corresponds to the requested version, and use that.

test/manifest/old/Manifest.toml

@@ -0,0 +1,9 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[DelimitedFiles]]
+deps = ["Mmap"]
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+version = "1.9.1"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"

test/manifest/old/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"

test/new.jl

@@ -1456,6 +1456,15 @@ end
             end
         end
     end
+    # instantiate old manifest
+    isolate(loaded_depot=true) do
+        manifest_dir = joinpath(@__DIR__, "manifest", "old")
+        cd(manifest_dir) do
+            Pkg.activate(".")
+            Pkg.instantiate()
+            @test isinstalled("DelimitedFiles")
+        end
+    end
     # `instantiate` on a lonely manifest should detect duplicate names
     isolate(loaded_depot=true) do; mktempdir() do tempdir
         simple_package_path = copy_test_package(tempdir, "SimplePackage")