Skip to content

Move max_default_jobs to default_njobs #212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
christiangnrd wants to merge 1 commit into from
Closed

Move max_default_jobs to default_njobs #212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions src/ParallelTestRunner.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ include("execute.jl")
stderr = Base.stderr,
max_worker_rss = get_max_worker_rss(),
memory_per_worker = 2 * 2^30,
max_default_jobs = nothing,
max_default_jobs = typemax(Int64),
serial = String[],
serial_position::Symbol = :before,
recycle_on_failure::Bool = false,
Expand Down Expand Up @@ -238,7 +238,7 @@ function runtests(mod::Module, args::ParsedArgs;
stderr = Base.stderr,
max_worker_rss = get_max_worker_rss(),
memory_per_worker = DEFAULT_MEMORY_PER_WORKER,
max_default_jobs = nothing,
max_default_jobs = DEFAULT_MAX_DEFAULT_JOBS,
recycle_on_failure::Bool = false,
retries::Integer = 0,
history_flush_every::Integer = 20,
Expand Down Expand Up @@ -341,7 +341,7 @@ function _runtests(mod::Module, args::ParsedArgs;
stderr = Base.stderr,
max_worker_rss = get_max_worker_rss(),
memory_per_worker = DEFAULT_MEMORY_PER_WORKER,
max_default_jobs = nothing,
max_default_jobs = DEFAULT_MAX_DEFAULT_JOBS,
recycle_on_failure::Bool = false,
retries::Integer = 0,
history_flush_every::Integer = 20,
Expand All @@ -352,14 +352,8 @@ function _runtests(mod::Module, args::ParsedArgs;

# determine parallelism
env_jobs = tryparse(Int, get(ENV, "PTR_NUM_JOBS", ""))
if max_default_jobs !== nothing
max_default_jobs isa Integer && !(max_default_jobs isa Bool) && max_default_jobs >= 1 ||
throw(ArgumentError("`max_default_jobs` must be a positive integer, got $(repr(max_default_jobs))"))
end
_jobs = @something args.jobs env_jobs begin
default_jobs = default_njobs(; memory_per_worker)
max_default_jobs === nothing ? default_jobs : min(default_jobs, max_default_jobs)
end

_jobs = _jobs = @something args.jobs env_jobs default_njobs(; memory_per_worker, max_default_jobs)
jobs = clamp(_jobs, 1, max(1, length(parallel_tests)))
worker_pool = Channel{Union{Nothing, PTRWorker}}(jobs)
for _ in 1:jobs
Expand Down
7 changes: 6 additions & 1 deletion src/resources.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ end
# memory). Packages whose tests are heavier can pass a larger
# `memory_per_worker` to `runtests`.
const DEFAULT_MEMORY_PER_WORKER = 2 * Int64(2)^30
const DEFAULT_MAX_DEFAULT_JOBS = typemax(Int64)

"""
default_njobs(; memory_per_worker = 2 * 2^30)
Expand All @@ -45,10 +46,14 @@ bytes of the available system memory.
"""
function default_njobs(;
memory_per_worker = DEFAULT_MEMORY_PER_WORKER,
max_default_jobs::Integer = DEFAULT_MAX_DEFAULT_JOBS,
# Just use Sys.EFFECTIVE_CPU_THREADS when min VERSION >= v"1.13"
_cpu_threads = (@static isdefined(Sys, :EFFECTIVE_CPU_THREADS) ? Sys.EFFECTIVE_CPU_THREADS : Sys.CPU_THREADS),
_free_memory = available_memory(),
)
!(max_default_jobs isa Bool) && max_default_jobs >= 1 ||
throw(ArgumentError("`max_default_jobs` must be a positive integer, got $(repr(max_default_jobs))"))

memory_jobs = Int64(_free_memory) ÷ memory_per_worker
return max(1, min(_cpu_threads, memory_jobs))
return max(1, min(_cpu_threads, memory_jobs, max_default_jobs))
end
6 changes: 5 additions & 1 deletion test/core.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,12 @@ end
end
@test contains(String(take!(io)), "using 2 parallel jobs")

for bad in (0, -1, true, 1.5, "2")
for badint in (0, -1, true)
@test_throws ArgumentError runtests(ParallelTestRunner, String[]; testsuite,
stdout=io, stderr=io, max_default_jobs=badint)
end
for bad in (1.5, "2")
@test_throws TypeError runtests(ParallelTestRunner, String[]; testsuite,
stdout=io, stderr=io, max_default_jobs=bad)
end
end
Expand Down
Loading