Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,24 @@ The main entry point of `ParallelTestRunner` is the exported function `runtests`
* the module for which you want to run the tests
* the list of arguments passed to the test script, as a `Vector{String}`, this will typically be [`Base.ARGS`](https://docs.julialang.org/en/v1/base/constants/#Base.ARGS).

With the `--help` flag you can print a help message:
With the `--help` (or `-h`) flag you can print a help message:

```julia
julia> using ParallelTestRunner

julia> ParallelTestRunner.runtests(ParallelTestRunner, ["--help"])
Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...]

--help Show this text.
--list List available tests alphabetically.
--verbose Print more information during testing.
--quickfail Fail the entire run as soon as a single test errored.
--jobs=N Launch `N` processes to perform tests. Can also be set
-h, --help Show this text.
-l, --list List available tests alphabetically.
-V, --verbose Print more information during testing.
-q, --quickfail Fail the entire run as soon as a single test errored.
-j N, --jobs=N Launch `N` processes to perform tests. Can also be set
with the PTR_NUM_JOBS environment
variable, with `--jobs=N` taking precedence.

Remaining arguments filter the tests that will be executed.
Remaining arguments filter the tests that will be executed.
Prefix your argument with '!' to instead exclude those tests
```

## Setup
Expand Down
13 changes: 8 additions & 5 deletions docs/src/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ julia --project test/runtests.jl [OPTIONS] [TESTS...]

### Available Options

- `--help`: Show usage information and exit
- `--list`: List all available tests alphabetically and exit. Each entry shows the
- `-h`, `--help`: Show usage information and exit
- `-l`, `--list`: List all available tests alphabetically and exit. Each entry shows the
test's historical duration, if known, and is marked with `×` and printed in red if its last run failed.
- `--verbose`: Print more detailed information during test execution (including start times for each test)
- `--quickfail`: Stop the entire test run as soon as any test fails
- `--jobs=N`: Use `N` worker processes (default: based on CPU threads and available memory)
- `-V`, `--verbose`: Print more detailed information during test execution (including start times for each test)
- `-q`, `--quickfail`: Stop the entire test run as soon as any test fails
- `-j N`, `--jobs=N`: Use `N` worker processes (default: based on CPU threads and available memory)
- `TESTS...`: Filter test files by name, matched using `startswith`. Arguments starting with '!' will instead be excluded from the test selection.

### Examples
Expand All @@ -84,6 +84,9 @@ julia --project test/runtests.jl integration
# Run with verbose output and 4 workers
julia --project test/runtests.jl --verbose --jobs=4

# Same as above, using the short forms
julia --project test/runtests.jl -V -j 4

# Run with quick-fail enabled
julia --project test/runtests.jl --quickfail
```
Expand Down
10 changes: 5 additions & 5 deletions src/ParallelTestRunner.jl
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,12 @@ Several keyword arguments are also supported:

## Command Line Options

- `--help`: Show usage information and exit
- `--list`: List all available tests alphabetically and exit. Each entry shows the
- `-h`, `--help`: Show usage information and exit
- `-l`, `--list`: List all available tests alphabetically and exit. Each entry shows the
test's historical duration, if known, and is marked with `×` and printed in red if its last run failed.
- `--verbose`: Print more detailed information during test execution
- `--quickfail`: Stop the entire test run as soon as any test fails
- `--jobs=N`: Use N worker processes (default: based on CPU threads and available memory;
- `-V`, `--verbose`: Print more detailed information during test execution
- `-q`, `--quickfail`: Stop the entire test run as soon as any test fails
- `-j N`, `--jobs=N`: Use N worker processes (default: based on CPU threads and available memory;
can also be set with the `PTR_NUM_JOBS` environment variable, with
`--jobs=N` taking precedence)
- `TESTS...`: Filter test files by name, matched using `startswith`. Arguments starting with '!' will instead be excluded from the test selection.
Expand Down
87 changes: 51 additions & 36 deletions src/args.jl
Original file line number Diff line number Diff line change
Expand Up @@ -70,34 +70,49 @@ struct ParsedArgs
end

# parse some command-line arguments
function extract_flag!(args, flag; typ = Nothing)
for f in args
# only accept the exact flag or `--flag=value`, so that flags sharing a
# prefix (e.g. `--list` and `--listing`) don't capture each other
if f == flag || startswith(f, flag * "=")
# Check if it's just `--flag` or if it's `--flag=foo`
val = if f == flag
typ === Nothing ||
error("Option `$flag` requires a value (use `$flag=<value>`)")
nothing
else
_, value = split(f, '='; limit = 2)
if typ === Nothing || typ <: AbstractString
value
else
parsed = tryparse(typ, value)
parsed === nothing &&
error("Invalid value `$value` for option `$flag` (expected a value of type $typ)")
parsed
end
end

# Drop this value from our args
filter!(x -> x != f, args)
return Some(val)
#
# `flag` is the long form (`--flag` or `--flag=value`). If `short` is given, the short
# form is also accepted: `-f` for boolean flags, and `-f value` or `-fvalue` for flags
# that take a value. If the flag is passed multiple times, the first occurrence wins.
function extract_flag!(args, flag; short = nothing, typ = Nothing)
# only accept the exact flag or `--flag=value`, so that flags sharing a
# prefix (e.g. `--list` and `--listing`) don't capture each other
is_long(f) = f == flag || startswith(f, flag * "=")
is_short(f) = short !== nothing && (f == short || (typ !== Nothing && startswith(f, short)))

i = findfirst(f -> is_long(f) || is_short(f), args)
i === nothing && return nothing
f = popat!(args, i)

val = if f == flag || f == short
# just `--flag` or `-f`, which for typed flags means `-f value`
if typ === Nothing
nothing
elseif f == short && i <= length(args)
popat!(args, i)
else
hint = short === nothing ? "`$flag=<value>`" : "`$flag=<value>` or `$short <value>`"
error("Option `$f` requires a value (use $hint)")
end
elseif is_long(f)
# `--flag=foo`
last(split(f, '='; limit = 2))
else
# `-ffoo`
f[(length(short) + 1):end]
end

if !(val === nothing || typ === Nothing || typ <: AbstractString)
parsed = tryparse(typ, val)
parsed === nothing &&
error("Invalid value `$val` for option `$(is_long(f) ? flag : short)` (expected a value of type $typ)")
val = parsed
end
return nothing

# drop any further occurrences of this flag
extract_flag!(args, flag; short, typ)

return Some(val)
end

"""
Expand All @@ -115,17 +130,17 @@ of these flags will be recorded in the `custom` field of the returned [`ParsedAr
function parse_args(args; custom::Array{String} = String[])
args = copy(args)

help = extract_flag!(args, "--help")
help = extract_flag!(args, "--help"; short = "-h")
if help !== nothing
usage =
"""
Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...]

--help Show this text.
--list List available tests alphabetically.
--verbose Print more information during testing.
--quickfail Fail the entire run as soon as a single test errored.
--jobs=N Launch `N` processes to perform tests. Can also be set
-h, --help Show this text.
-l, --list List available tests alphabetically.
-V, --verbose Print more information during testing.
-q, --quickfail Fail the entire run as soon as a single test errored.
-j N, --jobs=N Launch `N` processes to perform tests. Can also be set
with the PTR_NUM_JOBS environment
variable, with `--jobs=N` taking precedence."""

Expand All @@ -141,10 +156,10 @@ function parse_args(args; custom::Array{String} = String[])
exit(0)
end

jobs = extract_flag!(args, "--jobs"; typ = Int)
verbose = extract_flag!(args, "--verbose")
quickfail = extract_flag!(args, "--quickfail")
list = extract_flag!(args, "--list")
jobs = extract_flag!(args, "--jobs"; short = "-j", typ = Int)
verbose = extract_flag!(args, "--verbose"; short = "-V")
quickfail = extract_flag!(args, "--quickfail"; short = "-q")
list = extract_flag!(args, "--list"; short = "-l")

# boolean flags don't take values
for (flag, val) in (("--verbose", verbose), ("--quickfail", quickfail), ("--list", list))
Expand Down
54 changes: 54 additions & 0 deletions test/unit.jl
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,41 @@
# a typed flag with an unparseable value is an error
args = ["--jobs=abc"]
@test_throws ErrorException ParallelTestRunner.extract_flag!(args, "--jobs"; typ=Int)

# short forms
args = ["-V", "test1"]
result = ParallelTestRunner.extract_flag!(args, "--verbose"; short="-V")
@test result === Some(nothing)
@test args == ["test1"]

args = ["-j", "4", "test1"]
result = ParallelTestRunner.extract_flag!(args, "--jobs"; short="-j", typ=Int)
@test something(result) == 4
@test args == ["test1"]

args = ["-j4", "test1"]
result = ParallelTestRunner.extract_flag!(args, "--jobs"; short="-j", typ=Int)
@test something(result) == 4
@test args == ["test1"]

# boolean short flags only match exactly
args = ["-Vx"]
result = ParallelTestRunner.extract_flag!(args, "--verbose"; short="-V")
@test result === nothing
@test args == ["-Vx"]

# repeated flags, in either form, are all consumed
args = ["--verbose", "-V", "--verbose"]
result = ParallelTestRunner.extract_flag!(args, "--verbose"; short="-V")
@test result === Some(nothing)
@test isempty(args)

# a typed short flag without a value is an error
args = ["-j"]
@test_throws ErrorException ParallelTestRunner.extract_flag!(args, "--jobs"; short="-j", typ=Int)

args = ["-j", "abc"]
@test_throws ErrorException ParallelTestRunner.extract_flag!(args, "--jobs"; short="-j", typ=Int)
end

@testset "parse_args" begin
Expand Down Expand Up @@ -68,6 +103,25 @@ end
@test something(args.jobs) == 2
end

@testset "short flags" begin
args = parse_args(["-V", "-q", "-l", "-j", "2", "basic"])
@test args.verbose !== nothing
@test args.quickfail !== nothing
@test args.list !== nothing
@test something(args.jobs) == 2
@test args.positionals == ["basic"]

args = parse_args(["-j3"])
@test something(args.jobs) == 3

# mixing long and short forms of the same flag is fine
args = parse_args(["--verbose", "-V"])
@test args.verbose !== nothing

@test_throws ErrorException parse_args(["-x"])
@test_throws ErrorException parse_args(["-j"])
end

@testset "positional arguments" begin
args = parse_args(["--verbose", "basic", "subdir"])
@test args.verbose !== nothing
Expand Down
Loading