From 09eb76c794c9d1e15d7f896bca85110a6ba6e76e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 9 Oct 2023 15:54:10 +0200 Subject: [PATCH 1/7] remove writing out explicit stdlib paths for the temporary testing manifest (#3640) this should not be needed after https://github.com/JuliaLang/julia/pull/39572 (cherry picked from commit debcade0f2a767bd772ac5673123fa32591ebe26) --- src/Operations.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index e7d80f4a03..21aa29244b 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1818,13 +1818,6 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S end reset_all_compat!(temp_ctx.env.project) - - # Absolutify stdlibs paths - for (uuid, entry) in temp_ctx.env.manifest - if is_stdlib(uuid) - entry.path = Types.stdlib_path(entry.name) - end - end write_env(temp_ctx.env, update_undo = false) # Run sandboxed code From c16a5238ee03ff7f88d3340f8ee04b17eed061d5 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 13 Oct 2023 10:15:20 -0400 Subject: [PATCH 2/7] Switch datastructure Vector -> Set for algorithmic complexity (#3652) In the same project as #3651, this code was also having trouble because of the rapidly growing alldeps, which would end up with many repeated entries. Switch the data structure to fix the algorithmic complexity here also. (cherry picked from commit 0e0cf4514f8ee32d8d53c60aa7c1d9276bfa1bde) --- src/API.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/API.jl b/src/API.jl index ff92efaa34..e1b8a2bc5d 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1248,19 +1248,21 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool # if a list of packages is given, restrict to dependencies of given packages if !isempty(pkgs) pkgs_names = [p.name for p in pkgs] - function collect_all_deps(depsmap, dep, alldeps=Base.PkgId[]) - append!(alldeps, depsmap[dep]) + function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) for _dep in depsmap[dep] - collect_all_deps(depsmap, _dep, alldeps) + if !(_dep in alldeps) + push!(alldeps, _dep) + collect_all_deps(depsmap, _dep, alldeps) + end end return alldeps end - keep = Base.PkgId[] + keep = Set{Base.PkgId}() for dep in depsmap dep_pkgid = first(dep) if dep_pkgid.name in pkgs_names push!(keep, dep_pkgid) - append!(keep, collect_all_deps(depsmap, dep_pkgid)) + collect_all_deps(depsmap, dep_pkgid, keep) end end for ext in keys(exts) From ee5109f5935bc825c50e7bc20d04b4b233babf2c Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 15 Oct 2023 17:05:35 -0400 Subject: [PATCH 3/7] Precompile: Fix algorithmic complexity of cycle detection (#3651) Co-authored-by: Jameson Nash (cherry picked from commit a8648f7c83bf561e0c3c6d33d23476067f382d0f) --- src/API.jl | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/API.jl b/src/API.jl index e1b8a2bc5d..093b1dadea 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1230,13 +1230,32 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool # find and guard against circular deps circular_deps = Base.PkgId[] - function in_deps(_pkgs, deps, dmap) - isempty(deps) && return false - !isempty(intersect(_pkgs, deps)) && return true - return any(dep->in_deps(vcat(_pkgs, dep), dmap[dep], dmap), deps) + # Three states + # !haskey -> never visited + # true -> cannot be compiled due to a cycle (or not yet determined) + # false -> not depending on a cycle + could_be_cycle = Dict{Base.PkgId, Bool}() + function scan_pkg!(pkg, dmap) + did_visit_dep = true + inpath = get!(could_be_cycle, pkg) do + did_visit_dep = false + return true + end + if did_visit_dep ? inpath : scan_deps!(pkg, dmap) + # Found a cycle. Delete this and all parents + return true + end + return false end - for (pkg, deps) in depsmap - if in_deps([pkg], deps, depsmap) + function scan_deps!(pkg, dmap) + for dep in dmap[pkg] + scan_pkg!(dep, dmap) && return true + end + could_be_cycle[pkg] = false + return false + end + for pkg in keys(depsmap) + if scan_pkg!(pkg, depsmap) push!(circular_deps, pkg) notify(was_processed[pkg]) end From 1c5fcf9ba24c16786b5153261abc52986ac9842a Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 16 Oct 2023 16:11:45 -0700 Subject: [PATCH 4/7] Pin registry for MetaGraph tests (#3666) * pin registry for MetaGraph tests * fix * fix (cherry picked from commit 550eadd7e46bd54c0f6a9c2786d43d07d2446ec7) --- test/new.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/new.jl b/test/new.jl index da4f81ec7e..1b120ce883 100644 --- a/test/new.jl +++ b/test/new.jl @@ -904,7 +904,13 @@ end # ## Resolve tiers # @testset "add: resolve tiers" begin - isolate(loaded_depot=true) do; mktempdir() do tmp + # The MetaGraphs version tested below relied on a JLD2 version + # that couldn't actually be loaded on julia 1.9+ so General + # will be patched. This checks out a commit before then to maintain + # these tests. + registry_url = "https://github.com/JuliaRegistries/General.git" + registry_commit = "030d6dae0df2ad6c3b2f90d41749df3eedb8d1b1" + Utils.isolate_and_pin_registry(; registry_url, registry_commit) do; mktempdir() do tmp # All copy_test_package(tmp, "ShouldPreserveAll"; use_pkg=false) Pkg.activate(joinpath(tmp, "ShouldPreserveAll")) From 3ff4de8f5f6473a1920b17c2c368003e3dd5d287 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 31 Oct 2023 09:03:43 -0400 Subject: [PATCH 5/7] `@debug` the command being run when `Pkg.test` runs the tests in a subprocess (#3671) (cherry picked from commit e43501bbb0de43f4e1370691df683d3982bea27c) --- src/Operations.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Operations.jl b/src/Operations.jl index 21aa29244b..1daa1a00ab 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2026,6 +2026,7 @@ end # Handles the interrupting of a subprocess gracefully to avoid orphaning function subprocess_handler(cmd::Cmd, ctx, sandbox_ctx, error_msg::String) + @debug "Running command" cmd p = run(pipeline(ignorestatus(cmd), stdout = sandbox_ctx.io, stderr = stderr_f()), wait = false) interrupted = false try From 39bd01107c88fbcac3aec211647c62f34183f08f Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 31 Oct 2023 09:48:20 -0400 Subject: [PATCH 6/7] update CI --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21dcba730e..a71bd8427e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,8 +46,8 @@ jobs: run: | git config --global core.autocrlf false git config --global core.eol lf - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@latest + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia-version }} arch: ${{ matrix.julia-arch }} @@ -72,8 +72,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v1.0.0 - - uses: julia-actions/setup-julia@latest + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v1 with: # version: '1.6' version: '1.9-nightly' From 2e99ee631918d347d49b853ce8e9b5ba6abc294d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 31 Oct 2023 15:57:08 +0100 Subject: [PATCH 7/7] try use 1.9 instead of 1.9-nightly on CI --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a71bd8427e..a79fa5abdc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: - "pkg.julialang.org" julia-version: # - '1.6' - - '1.9-nightly' + - '1.9' exclude: - os: macOS-latest julia-arch: x86