Projects with Manifest.tomls may encounter failures with registry caching #110
Description
It appears that my assessment that enabling cache-registries: true by default was safe was incorrect. For projects that commit a Manifest.toml the registry may not get updated in Pkg.instantiate. Specifically, while performing some interactive CI debugging I found that this line fails only throws an exception when a package is missing from the registry and not when a package version is missing.
The reason for filing this issue in julia-actions/cache is that we may want to reconsider having cache-registries default to true.
As for the original problem here's the a portion of the workflow YAML:
- uses: julia-actions/setup-julia@v1
with:
version: "1.9.0"
arch: x64
- uses: julia-actions/cache@v1
- uses: julia-actions/add-julia-registry@v1
with:
key: ...
registry: ...
- uses: julia-actions/julia-runtest@v1
with:
annotate: trueAnd here's the logs from julia-actions/julia-runtest with the private package names replaced with placeholders:
Testing X
┌ Warning: Could not use exact versions of packages in manifest, re-resolving
└ @ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1800
ERROR: LoadError: Unsatisfiable requirements detected for package Y [0541aee2]:
Y [0541aee2] log:
├─possible versions are: 0.0.1-0.6.1 or uninstalled
└─restricted to versions 0.6.2-0.6 by X [d036d29a] — no versions left
└─X [d036d29a] log:
├─possible versions are: 0.2.0 or uninstalled
└─X [d036d29a] is fixed to version 0.2.0
Stacktrace:
[1] check_constraints(graph::Pkg.Resolve.Graph)
@ Pkg.Resolve /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Resolve/graphtype.jl:998
[2] Pkg.Resolve.Graph(compat::Dict{Base.UUID, Dict{VersionNumber, Dict{Base.UUID, Pkg.Versions.VersionSpec}}}, compat_weak::Dict{Base.UUID, Dict{VersionNumber, Set{Base.UUID}}}, uuid_to_name::Dict{Base.UUID, String}, reqs::Dict{Base.UUID, Pkg.Versions.VersionSpec}, fixed::Dict{Base.UUID, Pkg.Resolve.Fixed}, verbose::Bool, julia_version::VersionNumber)
@ Pkg.Resolve /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Resolve/graphtype.jl:345
[3] deps_graph(env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, uuid_to_name::Dict{Base.UUID, String}, reqs::Dict{Base.UUID, Pkg.Versions.VersionSpec}, fixed::Dict{Base.UUID, Pkg.Resolve.Fixed}, julia_version::VersionNumber, installed_only::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:587
[4] resolve_versions!(env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, pkgs::Vector{Pkg.Types.PackageSpec}, julia_version::VersionNumber, installed_only::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:407
[5] up(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}, level::Pkg.Types.UpgradeLevel; skip_writing_project::Bool, preserve::Nothing)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1533
[6] up(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; level::Pkg.Types.UpgradeLevel, mode::Pkg.Types.PackageMode, preserve::Nothing, update_registry::Bool, skip_writing_project::Bool, kwargs::Base.Pairs{Symbol, Base.DevNull, Tuple{Symbol}, NamedTuple{(:io,), Tuple{Base.DevNull}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:348
[7] up
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:323 [inlined]
[8] #up#48
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:161 [inlined]
[9] up
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:161 [inlined]
[10] #resolve#143
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:354 [inlined]
[11] resolve
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:353 [inlined]
[12] (::Pkg.Operations.var"#117#122"{String, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.PackageSpec})()
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1802
[13] with_temp_env(fn::Pkg.Operations.var"#117#122"{String, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.PackageSpec}, temp_env::String)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1692
[14] (::Pkg.Operations.var"#115#120"{Dict{String, Any}, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.Context, Pkg.Types.PackageSpec, String, Pkg.Types.Project, String})(tmp::String)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1781
[15] mktempdir(fn::Pkg.Operations.var"#115#120"{Dict{String, Any}, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.Context, Pkg.Types.PackageSpec, String, Pkg.Types.Project, String}, parent::String; prefix::String)
@ Base.Filesystem ./file.jl:760
[16] mktempdir(fn::Function, parent::String)
@ Base.Filesystem ./file.jl:756
[17] mktempdir
@ ./file.jl:756 [inlined]
[18] sandbox(fn::Function, ctx::Pkg.Types.Context, target::Pkg.Types.PackageSpec, target_path::String, sandbox_path::String, sandbox_project_override::Pkg.Types.Project; preferences::Dict{String, Any}, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1739
[19] test(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; coverage::Bool, julia_args::Cmd, test_args::Cmd, test_fn::Nothing, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1949
[20] test
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1893 [inlined]
[21] test(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; coverage::Bool, test_fn::Nothing, julia_args::Vector{String}, test_args::Cmd, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool, kwargs::Base.Pairs{Symbol, IOContext{Base.PipeEndpoint}, Tuple{Symbol}, NamedTuple{(:io,), Tuple{IOContext{Base.PipeEndpoint}}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:441
[22] test(pkgs::Vector{Pkg.Types.PackageSpec}; io::IOContext{Base.PipeEndpoint}, kwargs::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:coverage, :julia_args, :force_latest_compatible_version), Tuple{Bool, Vector{String}, Bool}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:156
[23] test(; name::Nothing, uuid::Nothing, version::Nothing, url::Nothing, rev::Nothing, path::Nothing, mode::Pkg.Types.PackageMode, subdir::Nothing, kwargs::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:coverage, :julia_args, :force_latest_compatible_version), Tuple{Bool, Vector{String}, Bool}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:171
[24] top-level scope
@ ~/work/_actions/julia-actions/julia-runtest/v1/test_harness.jl:15
[25] include(fname::String)
@ Base.MainInclude ./client.jl:478
[26] top-level scope
@ none:1
in expression starting at /home/runner/work/_actions/julia-actions/julia-runtest/v1/test_harness.jl:7
caused by: Unsatisfiable requirements detected for package Y [0541aee2]:
Y [0541aee2] log:
├─possible versions are: 0.0.1-0.6.1 or uninstalled
└─restricted to versions 0.6.2-0.6 by X [d036d29a] — no versions left
└─X [d036d29a] log:
├─possible versions are: 0.2.0 or uninstalled
└─X [d036d29a] is fixed to version 0.2.0
Stacktrace:
[1] check_constraints(graph::Pkg.Resolve.Graph)
@ Pkg.Resolve /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Resolve/graphtype.jl:998
[2] Pkg.Resolve.Graph(compat::Dict{Base.UUID, Dict{VersionNumber, Dict{Base.UUID, Pkg.Versions.VersionSpec}}}, compat_weak::Dict{Base.UUID, Dict{VersionNumber, Set{Base.UUID}}}, uuid_to_name::Dict{Base.UUID, String}, reqs::Dict{Base.UUID, Pkg.Versions.VersionSpec}, fixed::Dict{Base.UUID, Pkg.Resolve.Fixed}, verbose::Bool, julia_version::VersionNumber)
@ Pkg.Resolve /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Resolve/graphtype.jl:345
[3] deps_graph(env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, uuid_to_name::Dict{Base.UUID, String}, reqs::Dict{Base.UUID, Pkg.Versions.VersionSpec}, fixed::Dict{Base.UUID, Pkg.Resolve.Fixed}, julia_version::VersionNumber, installed_only::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:587
[4] resolve_versions!(env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, pkgs::Vector{Pkg.Types.PackageSpec}, julia_version::VersionNumber, installed_only::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:407
[5] up(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}, level::Pkg.Types.UpgradeLevel; skip_writing_project::Bool, preserve::Nothing)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1533
[6] up(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; level::Pkg.Types.UpgradeLevel, mode::Pkg.Types.PackageMode, preserve::Nothing, update_registry::Bool, skip_writing_project::Bool, kwargs::Base.Pairs{Symbol, Base.DevNull, Tuple{Symbol}, NamedTuple{(:io,), Tuple{Base.DevNull}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:348
[7] up
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:323 [inlined]
[8] #up#48
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:161 [inlined]
[9] up
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:161 [inlined]
[10] #resolve#143
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:354 [inlined]
[11] resolve
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:353 [inlined]
[12] (::Pkg.Operations.var"#117#122"{String, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.PackageSpec})()
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1794
[13] with_temp_env(fn::Pkg.Operations.var"#117#122"{String, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.PackageSpec}, temp_env::String)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1692
[14] (::Pkg.Operations.var"#115#120"{Dict{String, Any}, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.Context, Pkg.Types.PackageSpec, String, Pkg.Types.Project, String})(tmp::String)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1781
[15] mktempdir(fn::Pkg.Operations.var"#115#120"{Dict{String, Any}, Bool, Bool, Bool, Pkg.Operations.var"#130#134"{Bool, Cmd, Cmd, Nothing, Pkg.Types.Context, Vector{Tuple{String, Base.Process}}, String, Pkg.Types.PackageSpec}, Pkg.Types.Context, Pkg.Types.PackageSpec, String, Pkg.Types.Project, String}, parent::String; prefix::String)
@ Base.Filesystem ./file.jl:760
[16] mktempdir(fn::Function, parent::String)
@ Base.Filesystem ./file.jl:756
[17] mktempdir
@ ./file.jl:756 [inlined]
[18] sandbox(fn::Function, ctx::Pkg.Types.Context, target::Pkg.Types.PackageSpec, target_path::String, sandbox_path::String, sandbox_project_override::Pkg.Types.Project; preferences::Dict{String, Any}, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1739
[19] test(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; coverage::Bool, julia_args::Cmd, test_args::Cmd, test_fn::Nothing, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool)
@ Pkg.Operations /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1949
[20] test
@ /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1893 [inlined]
[21] test(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; coverage::Bool, test_fn::Nothing, julia_args::Vector{String}, test_args::Cmd, force_latest_compatible_version::Bool, allow_earlier_backwards_compatible_versions::Bool, allow_reresolve::Bool, kwargs::Base.Pairs{Symbol, IOContext{Base.PipeEndpoint}, Tuple{Symbol}, NamedTuple{(:io,), Tuple{IOContext{Base.PipeEndpoint}}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:441
[22] test(pkgs::Vector{Pkg.Types.PackageSpec}; io::IOContext{Base.PipeEndpoint}, kwargs::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:coverage, :julia_args, :force_latest_compatible_version), Tuple{Bool, Vector{String}, Bool}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:156
[23] test(; name::Nothing, uuid::Nothing, version::Nothing, url::Nothing, rev::Nothing, path::Nothing, mode::Pkg.Types.PackageMode, subdir::Nothing, kwargs::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:coverage, :julia_args, :force_latest_compatible_version), Tuple{Bool, Vector{String}, Bool}}})
@ Pkg.API /opt/hostedtoolcache/julia/1.9.0/x64/share/julia/stdlib/v1.9/Pkg/src/API.jl:171
[24] top-level scope
@ ~/work/_actions/julia-actions/julia-runtest/v1/test_harness.jl:15
[25] include(fname::String)
@ Base.MainInclude ./client.jl:478
[26] top-level scope
@ none:1
Here's an example of the behaviour inside the CI environment with package names replaced with placeholders (emulating some of what Pkg.test does):
julia> using Pkg
julia> Pkg.instantiate(; allow_autoprecomp=false)
Cloning [0541aee2-...] Y from https://github.com/...
Installed Y ─ v0.6.2
false
julia> Pkg.resolve()
ERROR: Unsatisfiable requirements detected for package X [0541aee2]:
Y [0541aee2] log:
├─possible versions are: 0.0.1-0.6.1 or uninstalled
└─restricted to versions 0.6.2-0.6 by X [7d1a9472] — no versions left
└─X [7d1a9472] log:
├─possible versions are: 0.2.0 or uninstalled
└─X [7d1a9472] is fixed to version 0.2.0
julia> using Y
julia> pkgversion(Y)
v"0.6.2"From what I've gathered it seems that Pkg.instantiate is actually finding and installing the version required but when Pkg.test attempts a re-resolve it fails as the registry doesn't contain an entry for that item. Probably Pkg.test should update the registries when attempting a re-resolve.