Mirrors/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-05-21 17:52:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #46450 - Gilnaa:libtest_json_output, r=nrc

Browse Source 
Libtest json output

A revisit to my [last PR](https://github.com/rust-lang/rust/pull/45923).

Events are now more atomic, printed in a flat hierarchy.

For the normal test output:
```
running 1 test
test f ... FAILED

failures:

---- f stdout ----
	thread 'f' panicked at 'assertion failed: `(left == right)`
  left: `3`,
 right: `4`', f.rs:3:1
note: Run with `RUST_BACKTRACE=1` for a backtrace.

failures:
    f

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out
```

The JSON equivalent is:
```
{ "type": "suite", "event": "started", "test_count": "1" }
{ "type": "test", "event": "started", "name": "f" }
{ "type": "test", "event": "failed", "name": "f" }
{ "type": "suite", "event": "failed", "passed": 0, "failed": 1, "allowed_fail": 0, "ignored": 0,  "measured": 0, "filtered_out": "0" }
{ "type": "test_output", "name": "f", "output": "thread 'f' panicked at 'assertion failed: `(left == right)`
  left: `3`,
 right: `4`', f.rs:3:1
note: Run with `RUST_BACKTRACE=1` for a backtrace.
" }
```
This commit is contained in:
bors
2018-01-27 10:56:56 +00:00
parent 6272b60dca 8b7f1d0cec
commit 6b99adeb11
12 changed files with 1711 additions and 754 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libtest/formatters/json.rs
+229
View File
@@ -0,0 +1,229 @@
// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use super::*;
pub(crate) struct JsonFormatter<T> {
out: OutputLocation<T>,
}
impl<T: Write> JsonFormatter<T> {
pub fn new(out: OutputLocation<T>) -> Self {
Self { out }
}
fn write_message(&mut self, s: &str) -> io::Result<()> {
assert!(!s.contains('\n'));
self.out.write_all(s.as_ref())?;
self.out.write_all(b"\n")
}
fn write_event(
&mut self,
ty: &str,
name: &str,
evt: &str,
extra: Option<String>,
) -> io::Result<()> {
if let Some(extras) = extra {
self.write_message(&*format!(
r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#,
ty,
name,
evt,
extras
))
} else {
self.write_message(&*format!(
r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#,
ty,
name,
evt
))
}
}
}
impl<T: Write> OutputFormatter for JsonFormatter<T> {
fn write_run_start(&mut self, test_count: usize) -> io::Result<()> {
self.write_message(&*format!(
r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#,
test_count
))
}
fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> {
self.write_message(&*format!(
r#"{{ "type": "test", "event": "started", "name": "{}" }}"#,
desc.name
))
}
fn write_result(
&mut self,
desc: &TestDesc,
result: &TestResult,
stdout: &[u8],
) -> io::Result<()> {
match *result {
TrOk => self.write_event("test", desc.name.as_slice(), "ok", None),
TrFailed => {
let extra_data = if stdout.len() > 0 {
Some(format!(
r#""stdout": "{}""#,
EscapedString(String::from_utf8_lossy(stdout))
))
} else {
None
};
self.write_event("test", desc.name.as_slice(), "failed", extra_data)
}
TrFailedMsg(ref m) => {
self.write_event(
"test",
desc.name.as_slice(),
"failed",
Some(format!(r#""message": "{}""#, EscapedString(m))),
)
}
TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None),
TrAllowedFail => {
self.write_event("test", desc.name.as_slice(), "allowed_failure", None)
}
TrBench(ref bs) => {
let median = bs.ns_iter_summ.median as usize;
let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
let mbps = if bs.mb_s == 0 {
"".into()
} else {
format!(r#", "mib_per_second": {}"#, bs.mb_s)
};
let line = format!(
"{{ \"type\": \"bench\", \
\"name\": \"{}\", \
\"median\": {}, \
\"deviation\": {}{} }}",
desc.name,
median,
deviation,
mbps
);
self.write_message(&*line)
}
}
}
fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
self.write_message(&*format!(
r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#,
desc.name
))
}
fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
self.write_message(&*format!(
"{{ \"type\": \"suite\", \
\"event\": \"{}\", \
\"passed\": {}, \
\"failed\": {}, \
\"allowed_fail\": {}, \
\"ignored\": {}, \
\"measured\": {}, \
\"filtered_out\": \"{}\" }}",
if state.failed == 0 { "ok" } else { "failed" },
state.passed,
state.failed + state.allowed_fail,
state.allowed_fail,
state.ignored,
state.measured,
state.filtered_out
))?;
Ok(state.failed == 0)
}
}
/// A formatting utility used to print strings with characters in need of escaping.
/// Base code taken form `libserialize::json::escape_str`
struct EscapedString<S: AsRef<str>>(S);
impl<S: AsRef<str>> ::std::fmt::Display for EscapedString<S> {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
let mut start = 0;
for (i, byte) in self.0.as_ref().bytes().enumerate() {
let escaped = match byte {
b'"' => "\\\"",
b'\\' => "\\\\",
b'\x00' => "\\u0000",
b'\x01' => "\\u0001",
b'\x02' => "\\u0002",
b'\x03' => "\\u0003",
b'\x04' => "\\u0004",
b'\x05' => "\\u0005",
b'\x06' => "\\u0006",
b'\x07' => "\\u0007",
b'\x08' => "\\b",
b'\t' => "\\t",
b'\n' => "\\n",
b'\x0b' => "\\u000b",
b'\x0c' => "\\f",
b'\r' => "\\r",
b'\x0e' => "\\u000e",
b'\x0f' => "\\u000f",
b'\x10' => "\\u0010",
b'\x11' => "\\u0011",
b'\x12' => "\\u0012",
b'\x13' => "\\u0013",
b'\x14' => "\\u0014",
b'\x15' => "\\u0015",
b'\x16' => "\\u0016",
b'\x17' => "\\u0017",
b'\x18' => "\\u0018",
b'\x19' => "\\u0019",
b'\x1a' => "\\u001a",
b'\x1b' => "\\u001b",
b'\x1c' => "\\u001c",
b'\x1d' => "\\u001d",
b'\x1e' => "\\u001e",
b'\x1f' => "\\u001f",
b'\x7f' => "\\u007f",
_ => {
continue;
}
};
if start < i {
f.write_str(&self.0.as_ref()[start..i])?;
}
f.write_str(escaped)?;
start = i + 1;
}
if start != self.0.as_ref().len() {
f.write_str(&self.0.as_ref()[start..])?;
}
Ok(())
}
}
src/libtest/formatters/mod.rs
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use super::*;
mod pretty;
mod json;
mod terse;
pub(crate) use self::pretty::PrettyFormatter;
pub(crate) use self::json::JsonFormatter;
pub(crate) use self::terse::TerseFormatter;
pub(crate) trait OutputFormatter {
fn write_run_start(&mut self, test_count: usize) -> io::Result<()>;
fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>;
fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>;
fn write_result(
&mut self,
desc: &TestDesc,
result: &TestResult,
stdout: &[u8],
) -> io::Result<()>;
fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool>;
}
src/libtest/formatters/pretty.rs
+247
View File
@@ -0,0 +1,247 @@
// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use super::*;
pub(crate) struct PrettyFormatter<T> {
out: OutputLocation<T>,
use_color: bool,
/// Number of columns to fill when aligning names
max_name_len: usize,
is_multithreaded: bool,
}
impl<T: Write> PrettyFormatter<T> {
pub fn new(
out: OutputLocation<T>,
use_color: bool,
max_name_len: usize,
is_multithreaded: bool,
) -> Self {
PrettyFormatter {
out,
use_color,
max_name_len,
is_multithreaded,
}
}
#[cfg(test)]
pub fn output_location(&self) -> &OutputLocation<T> {
&self.out
}
pub fn write_ok(&mut self) -> io::Result<()> {
self.write_short_result("ok", term::color::GREEN)
}
pub fn write_failed(&mut self) -> io::Result<()> {
self.write_short_result("FAILED", term::color::RED)
}
pub fn write_ignored(&mut self) -> io::Result<()> {
self.write_short_result("ignored", term::color::YELLOW)
}
pub fn write_allowed_fail(&mut self) -> io::Result<()> {
self.write_short_result("FAILED (allowed)", term::color::YELLOW)
}
pub fn write_bench(&mut self) -> io::Result<()> {
self.write_pretty("bench", term::color::CYAN)
}
pub fn write_short_result(
&mut self,
result: &str,
color: term::color::Color,
) -> io::Result<()> {
self.write_pretty(result, color)?;
self.write_plain("\n")
}
pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
match self.out {
Pretty(ref mut term) => {
if self.use_color {
term.fg(color)?;
}
term.write_all(word.as_bytes())?;
if self.use_color {
term.reset()?;
}
term.flush()
}
Raw(ref mut stdout) => {
stdout.write_all(word.as_bytes())?;
stdout.flush()
}
}
}
pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> {
let s = s.as_ref();
self.out.write_all(s.as_bytes())?;
self.out.flush()
}
pub fn write_successes(&mut self, state: &ConsoleTestState) -> io::Result<()> {
self.write_plain("\nsuccesses:\n")?;
let mut successes = Vec::new();
let mut stdouts = String::new();
for &(ref f, ref stdout) in &state.not_failures {
successes.push(f.name.to_string());
if !stdout.is_empty() {
stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
stdouts.push_str(&output);
stdouts.push_str("\n");
}
}
if !stdouts.is_empty() {
self.write_plain("\n")?;
self.write_plain(&stdouts)?;
}
self.write_plain("\nsuccesses:\n")?;
successes.sort();
for name in &successes {
self.write_plain(&format!(" {}\n", name))?;
}
Ok(())
}
pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> {
self.write_plain("\nfailures:\n")?;
let mut failures = Vec::new();
let mut fail_out = String::new();
for &(ref f, ref stdout) in &state.failures {
failures.push(f.name.to_string());
if !stdout.is_empty() {
fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
fail_out.push_str(&output);
fail_out.push_str("\n");
}
}
if !fail_out.is_empty() {
self.write_plain("\n")?;
self.write_plain(&fail_out)?;
}
self.write_plain("\nfailures:\n")?;
failures.sort();
for name in &failures {
self.write_plain(&format!(" {}\n", name))?;
}
Ok(())
}
fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> {
let name = desc.padded_name(self.max_name_len, desc.name.padding());
self.write_plain(&format!("test {} ... ", name))?;
Ok(())
}
}
impl<T: Write> OutputFormatter for PrettyFormatter<T> {
fn write_run_start(&mut self, test_count: usize) -> io::Result<()> {
let noun = if test_count != 1 { "tests" } else { "test" };
self.write_plain(&format!("\nrunning {} {}\n", test_count, noun))
}
fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> {
// When running tests concurrently, we should not print
// the test's name as the result will be mis-aligned.
// When running the tests serially, we print the name here so
// that the user can see which test hangs.
if !self.is_multithreaded {
self.write_test_name(desc)?;
}
Ok(())
}
fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> {
if self.is_multithreaded {
self.write_test_name(desc)?;
}
match *result {
TrOk => self.write_ok(),
TrFailed | TrFailedMsg(_) => self.write_failed(),
TrIgnored => self.write_ignored(),
TrAllowedFail => self.write_allowed_fail(),
TrBench(ref bs) => {
self.write_bench()?;
self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
}
}
}
fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
if self.is_multithreaded {
self.write_test_name(desc)?;
}
self.write_plain(&format!(
"test {} has been running for over {} seconds\n",
desc.name,
TEST_WARN_TIMEOUT_S
))
}
fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
if state.options.display_output {
self.write_successes(state)?;
}
let success = state.failed == 0;
if !success {
self.write_failures(state)?;
}
self.write_plain("\ntest result: ")?;
if success {
// There's no parallelism at this point so it's safe to use color
self.write_pretty("ok", term::color::GREEN)?;
} else {
self.write_pretty("FAILED", term::color::RED)?;
}
let s = if state.allowed_fail > 0 {
format!(
". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n",
state.passed,
state.failed + state.allowed_fail,
state.allowed_fail,
state.ignored,
state.measured,
state.filtered_out
)
} else {
format!(
". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
state.passed,
state.failed,
state.ignored,
state.measured,
state.filtered_out
)
};
self.write_plain(&s)?;
Ok(success)
}
}
src/libtest/formatters/terse.rs
+246
View File
@@ -0,0 +1,246 @@
// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use super::*;
pub(crate) struct TerseFormatter<T> {
out: OutputLocation<T>,
use_color: bool,
is_multithreaded: bool,
/// Number of columns to fill when aligning names
max_name_len: usize,
test_count: usize,
}
impl<T: Write> TerseFormatter<T> {
pub fn new(
out: OutputLocation<T>,
use_color: bool,
max_name_len: usize,
is_multithreaded: bool,
) -> Self {
TerseFormatter {
out,
use_color,
max_name_len,
is_multithreaded,
test_count: 0,
}
}
pub fn write_ok(&mut self) -> io::Result<()> {
self.write_short_result(".", term::color::GREEN)
}
pub fn write_failed(&mut self) -> io::Result<()> {
self.write_short_result("F", term::color::RED)
}
pub fn write_ignored(&mut self) -> io::Result<()> {
self.write_short_result("i", term::color::YELLOW)
}
pub fn write_allowed_fail(&mut self) -> io::Result<()> {
self.write_short_result("a", term::color::YELLOW)
}
pub fn write_bench(&mut self) -> io::Result<()> {
self.write_pretty("bench", term::color::CYAN)
}
pub fn write_short_result(
&mut self,
result: &str,
color: term::color::Color,
) -> io::Result<()> {
self.write_pretty(result, color)?;
if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 {
// we insert a new line every 100 dots in order to flush the
// screen when dealing with line-buffered output (e.g. piping to
// `stamp` in the rust CI).
self.write_plain("\n")?;
}
self.test_count += 1;
Ok(())
}
pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
match self.out {
Pretty(ref mut term) => {
if self.use_color {
term.fg(color)?;
}
term.write_all(word.as_bytes())?;
if self.use_color {
term.reset()?;
}
term.flush()
}
Raw(ref mut stdout) => {
stdout.write_all(word.as_bytes())?;
stdout.flush()
}
}
}
pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> {
let s = s.as_ref();
self.out.write_all(s.as_bytes())?;
self.out.flush()
}
pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> {
self.write_plain("\nsuccesses:\n")?;
let mut successes = Vec::new();
let mut stdouts = String::new();
for &(ref f, ref stdout) in &state.not_failures {
successes.push(f.name.to_string());
if !stdout.is_empty() {
stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
stdouts.push_str(&output);
stdouts.push_str("\n");
}
}
if !stdouts.is_empty() {
self.write_plain("\n")?;
self.write_plain(&stdouts)?;
}
self.write_plain("\nsuccesses:\n")?;
successes.sort();
for name in &successes {
self.write_plain(&format!(" {}\n", name))?;
}
Ok(())
}
pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> {
self.write_plain("\nfailures:\n")?;
let mut failures = Vec::new();
let mut fail_out = String::new();
for &(ref f, ref stdout) in &state.failures {
failures.push(f.name.to_string());
if !stdout.is_empty() {
fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
fail_out.push_str(&output);
fail_out.push_str("\n");
}
}
if !fail_out.is_empty() {
self.write_plain("\n")?;
self.write_plain(&fail_out)?;
}
self.write_plain("\nfailures:\n")?;
failures.sort();
for name in &failures {
self.write_plain(&format!(" {}\n", name))?;
}
Ok(())
}
fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> {
let name = desc.padded_name(self.max_name_len, desc.name.padding());
self.write_plain(&format!("test {} ... ", name))?;
Ok(())
}
}
impl<T: Write> OutputFormatter for TerseFormatter<T> {
fn write_run_start(&mut self, test_count: usize) -> io::Result<()> {
let noun = if test_count != 1 { "tests" } else { "test" };
self.write_plain(&format!("\nrunning {} {}\n", test_count, noun))
}
fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> {
// Remnants from old libtest code that used the padding value
// in order to indicate benchmarks.
// When running benchmarks, terse-mode should still print their name as if
// it is the Pretty formatter.
if !self.is_multithreaded && desc.name.padding() == PadOnRight {
self.write_test_name(desc)?;
}
Ok(())
}
fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> {
match *result {
TrOk => self.write_ok(),
TrFailed | TrFailedMsg(_) => self.write_failed(),
TrIgnored => self.write_ignored(),
TrAllowedFail => self.write_allowed_fail(),
TrBench(ref bs) => {
if self.is_multithreaded {
self.write_test_name(desc)?;
}
self.write_bench()?;
self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
}
}
}
fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
self.write_plain(&format!(
"test {} has been running for over {} seconds\n",
desc.name,
TEST_WARN_TIMEOUT_S
))
}
fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
if state.options.display_output {
self.write_outputs(state)?;
}
let success = state.failed == 0;
if !success {
self.write_failures(state)?;
}
self.write_plain("\ntest result: ")?;
if success {
// There's no parallelism at this point so it's safe to use color
self.write_pretty("ok", term::color::GREEN)?;
} else {
self.write_pretty("FAILED", term::color::RED)?;
}
let s = if state.allowed_fail > 0 {
format!(
". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n",
state.passed,
state.failed + state.allowed_fail,
state.allowed_fail,
state.ignored,
state.measured,
state.filtered_out
)
} else {
format!(
". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
state.passed,
state.failed,
state.ignored,
state.measured,
state.filtered_out
)
};
self.write_plain(&s)?;
Ok(success)
}
}
src/libtest/lib.rs
+618 -511
View File
@@ -1,4 +1,4 @@
// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
@@ -71,6 +71,7 @@
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Instant, Duration};
use std::borrow::Cow;
const TEST_WARN_TIMEOUT_S: u64 = 60;
const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode
@@ -84,6 +85,9 @@ pub mod test {
}
pub mod stats;
mod formatters;
use formatters::{OutputFormatter, PrettyFormatter, TerseFormatter, JsonFormatter};
// The name of a test. By convention this follows the rules for rust
// paths; i.e. it should be a series of identifiers separated by double
@@ -94,14 +98,33 @@ pub mod test {
pub enum TestName {
StaticTestName(&'static str),
DynTestName(String),
AlignedTestName(Cow<'static, str>, NamePadding),
}
impl TestName {
fn as_slice(&self) -> &str {
match *self {
StaticTestName(s) => s,
DynTestName(ref s) => s,
AlignedTestName(ref s, _) => &*s,
}
}
fn padding(&self) -> NamePadding {
match self {
&AlignedTestName(_, p) => p,
_ => PadNone,
}
}
fn with_padding(&self, padding: NamePadding) -> TestName {
let name = match self {
&TestName::StaticTestName(name) => Cow::Borrowed(name),
&TestName::DynTestName(ref name) => Cow::Owned(name.clone()),
&TestName::AlignedTestName(ref name, _) => name.clone(),
};
TestName::AlignedTestName(name, padding)
}
}
impl fmt::Display for TestName {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -109,7 +132,7 @@ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub enum NamePadding {
PadNone,
PadOnRight,
@@ -217,10 +240,7 @@ pub struct Metric {
impl Metric {
pub fn new(value: f64, noise: f64) -> Metric {
Metric {
value,
noise,
}
Metric { value, noise }
}
}
@@ -232,9 +252,7 @@ pub struct Options {
impl Options {
pub fn new() -> Options {
Options {
display_output: false,
}
Options { display_output: false }
}
pub fn display_output(mut self, display_output: bool) -> Options {
@@ -274,25 +292,24 @@ pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) {
// rather than a &[].
pub fn test_main_static(tests: &[TestDescAndFn]) {
let args = env::args().collect::<Vec<_>>();
let owned_tests = tests.iter()
.map(|t| {
match t.testfn {
StaticTestFn(f) => {
TestDescAndFn {
testfn: StaticTestFn(f),
desc: t.desc.clone(),
}
}
StaticBenchFn(f) => {
TestDescAndFn {
testfn: StaticBenchFn(f),
desc: t.desc.clone(),
}
}
_ => panic!("non-static tests passed to test::test_main_static"),
}
})
.collect();
let owned_tests = tests
.iter()
.map(|t| match t.testfn {
StaticTestFn(f) => {
TestDescAndFn {
testfn: StaticTestFn(f),
desc: t.desc.clone(),
}
}
StaticBenchFn(f) => {
TestDescAndFn {
testfn: StaticBenchFn(f),
desc: t.desc.clone(),
}
}
_ => panic!("non-static tests passed to test::test_main_static"),
})
.collect();
test_main(&args, owned_tests, Options::new())
}
@@ -303,6 +320,13 @@ pub enum ColorConfig {
NeverColor,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum OutputFormat {
Pretty,
Terse,
Json,
}
#[derive(Debug)]
pub struct TestOpts {
pub list: bool,
@@ -314,7 +338,7 @@ pub struct TestOpts {
pub logfile: Option<PathBuf>,
pub nocapture: bool,
pub color: ColorConfig,
pub quiet: bool,
pub format: OutputFormat,
pub test_threads: Option<usize>,
pub skip: Vec<String>,
pub options: Options,
@@ -333,7 +357,7 @@ fn new() -> TestOpts {
logfile: None,
nocapture: false,
color: AutoColor,
quiet: false,
format: OutputFormat::Pretty,
test_threads: None,
skip: vec![],
options: Options::new(),
@@ -351,26 +375,76 @@ fn optgroups() -> getopts::Options {
.optflag("", "bench", "Run benchmarks instead of tests")
.optflag("", "list", "List all tests and benchmarks")
.optflag("h", "help", "Display this message (longer with --help)")
.optopt("", "logfile", "Write logs to the specified file instead \
of stdout", "PATH")
.optflag("", "nocapture", "don't capture stdout/stderr of each \
task, allow printing directly")
.optopt("", "test-threads", "Number of threads used for running tests \
in parallel", "n_threads")
.optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \
be used multiple times)","FILTER")
.optflag("q", "quiet", "Display one character per test instead of one line")
.optflag("", "exact", "Exactly match filters rather than by substring")
.optopt("", "color", "Configure coloring of output:
.optopt(
"",
"logfile",
"Write logs to the specified file instead \
of stdout",
"PATH",
)
.optflag(
"",
"nocapture",
"don't capture stdout/stderr of each \
task, allow printing directly",
)
.optopt(
"",
"test-threads",
"Number of threads used for running tests \
in parallel",
"n_threads",
)
.optmulti(
"",
"skip",
"Skip tests whose names contain FILTER (this flag can \
be used multiple times)",
"FILTER",
)
.optflag(
"q",
"quiet",
"Display one character per test instead of one line. \
Alias to --format=terse",
)
.optflag(
"",
"exact",
"Exactly match filters rather than by substring",
)
.optopt(
"",
"color",
"Configure coloring of output:
auto = colorize if stdout is a tty and tests are run on serially (default);
always = always colorize output;
never = never colorize output;", "auto|always|never");
return opts
never = never colorize output;",
"auto|always|never",
)
.optopt(
"",
"format",
"Configure formatting of output:
pretty = Print verbose output;
terse = Display one character per test;
json = Output a json document",
"pretty|terse|json",
)
.optopt(
"Z",
"",
"Enable nightly-only flags:
unstable-options = Allow use of experimental features",
"unstable-options",
);
return opts;
}
fn usage(binary: &str, options: &getopts::Options) {
let message = format!("Usage: {} [OPTIONS] [FILTER]", binary);
println!(r#"{usage}
println!(
r#"{usage}
The FILTER string is tested against the name of all tests, and only those
tests whose names contain the filter are run.
@@ -397,11 +471,23 @@ fn usage(binary: &str, options: &getopts::Options) {
test, then the test runner will ignore these tests during
normal test runs. Running with --ignored will run these
tests."#,
usage = options.usage(&message));
usage = options.usage(&message)
);
}
// FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566
fn is_nightly() -> bool {
// Whether this is a feature-staged build, i.e. on the beta or stable channel
let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some();
// Whether we should enable unstable features for bootstrapping
let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
bootstrap || !disable_unstable_features
}
// Parses command line arguments into test options
pub fn parse_opts(args: &[String]) -> Option<OptRes> {
let mut allow_unstable = false;
let opts = optgroups();
let args = args.get(1..).unwrap_or(args);
let matches = match opts.parse(args) {
@@ -409,6 +495,24 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
Err(f) => return Some(Err(f.to_string())),
};
if let Some(opt) = matches.opt_str("Z") {
if !is_nightly() {
return Some(Err(
"the option `Z` is only accepted on the nightly compiler"
.into(),
));
}
match &*opt {
"unstable-options" => {
allow_unstable = true;
}
_ => {
return Some(Err("Unrecognized option to `Z`".into()));
}
}
};
if matches.opt_present("h") {
usage(&args[0], &opts);
return None;
@@ -435,22 +539,25 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
if !nocapture {
nocapture = match env::var("RUST_TEST_NOCAPTURE") {
Ok(val) => &val != "0",
Err(_) => false
Err(_) => false,
};
}
let test_threads = match matches.opt_str("test-threads") {
Some(n_str) =>
Some(n_str) => {
match n_str.parse::<usize>() {
Ok(0) =>
return Some(Err(format!("argument for --test-threads must not be 0"))),
Ok(0) => return Some(Err(format!("argument for --test-threads must not be 0"))),
Ok(n) => Some(n),
Err(e) =>
return Some(Err(format!("argument for --test-threads must be a number > 0 \
(error: {})", e)))
},
None =>
None,
Err(e) => {
return Some(Err(format!(
"argument for --test-threads must be a number > 0 \
(error: {})",
e
)))
}
}
}
None => None,
};
let color = match matches.opt_str("color").as_ref().map(|s| &**s) {
@@ -459,9 +566,34 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
Some("never") => NeverColor,
Some(v) => {
return Some(Err(format!("argument for --color must be auto, always, or never (was \
return Some(Err(format!(
"argument for --color must be auto, always, or never (was \
{})",
v)))
v
)))
}
};
let format = match matches.opt_str("format").as_ref().map(|s| &**s) {
None if quiet => OutputFormat::Terse,
Some("pretty") | None => OutputFormat::Pretty,
Some("terse") => OutputFormat::Terse,
Some("json") => {
if !allow_unstable {
return Some(Err(
"The \"json\" format is only accepted on the nightly compiler"
.into(),
));
}
OutputFormat::Json
}
Some(v) => {
return Some(Err(format!(
"argument for --format must be pretty, terse, or json (was \
{})",
v
)))
}
};
@@ -475,7 +607,7 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
logfile,
nocapture,
color,
quiet,
format,
test_threads,
skip: matches.opt_strs("skip"),
options: Options::new(),
@@ -507,11 +639,24 @@ enum OutputLocation<T> {
Raw(T),
}
struct ConsoleTestState<T> {
impl<T: Write> Write for OutputLocation<T> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
match *self {
Pretty(ref mut term) => term.write(buf),
Raw(ref mut stdout) => stdout.write(buf),
}
}
fn flush(&mut self) -> io::Result<()> {
match *self {
Pretty(ref mut term) => term.flush(),
Raw(ref mut stdout) => stdout.flush(),
}
}
}
struct ConsoleTestState {
log_out: Option<File>,
out: OutputLocation<T>,
use_color: bool,
quiet: bool,
total: usize,
passed: usize,
failed: usize,
@@ -522,26 +667,18 @@ struct ConsoleTestState<T> {
metrics: MetricMap,
failures: Vec<(TestDesc, Vec<u8>)>,
not_failures: Vec<(TestDesc, Vec<u8>)>,
max_name_len: usize, // number of columns to fill when aligning names
options: Options,
}
impl<T: Write> ConsoleTestState<T> {
pub fn new(opts: &TestOpts, _: Option<T>) -> io::Result<ConsoleTestState<io::Stdout>> {
impl ConsoleTestState {
pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState> {
let log_out = match opts.logfile {
Some(ref path) => Some(File::create(path)?),
None => None,
};
let out = match term::stdout() {
None => Raw(io::stdout()),
Some(t) => Pretty(t),
};
Ok(ConsoleTestState {
out,
log_out,
use_color: use_color(opts),
quiet: opts.quiet,
total: 0,
passed: 0,
failed: 0,
@@ -552,119 +689,10 @@ pub fn new(opts: &TestOpts, _: Option<T>) -> io::Result<ConsoleTestState<io::Std
metrics: MetricMap::new(),
failures: Vec::new(),
not_failures: Vec::new(),
max_name_len: 0,
options: opts.options,
})
}
pub fn write_ok(&mut self) -> io::Result<()> {
self.write_short_result("ok", ".", term::color::GREEN)
}
pub fn write_failed(&mut self) -> io::Result<()> {
self.write_short_result("FAILED", "F", term::color::RED)
}
pub fn write_ignored(&mut self) -> io::Result<()> {
self.write_short_result("ignored", "i", term::color::YELLOW)
}
pub fn write_allowed_fail(&mut self) -> io::Result<()> {
self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW)
}
pub fn write_bench(&mut self) -> io::Result<()> {
self.write_pretty("bench", term::color::CYAN)
}
pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color)
-> io::Result<()> {
if self.quiet {
self.write_pretty(quiet, color)?;
if self.current_test_count() % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 {
// we insert a new line every 100 dots in order to flush the
// screen when dealing with line-buffered output (e.g. piping to
// `stamp` in the rust CI).
self.write_plain("\n")?;
}
Ok(())
} else {
self.write_pretty(verbose, color)?;
self.write_plain("\n")
}
}
pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
match self.out {
Pretty(ref mut term) => {
if self.use_color {
term.fg(color)?;
}
term.write_all(word.as_bytes())?;
if self.use_color {
term.reset()?;
}
term.flush()
}
Raw(ref mut stdout) => {
stdout.write_all(word.as_bytes())?;
stdout.flush()
}
}
}
pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> {
let s = s.as_ref();
match self.out {
Pretty(ref mut term) => {
term.write_all(s.as_bytes())?;
term.flush()
}
Raw(ref mut stdout) => {
stdout.write_all(s.as_bytes())?;
stdout.flush()
}
}
}
pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
self.total = len;
let noun = if len != 1 {
"tests"
} else {
"test"
};
self.write_plain(&format!("\nrunning {} {}\n", len, noun))
}
pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
if self.quiet && align != PadOnRight {
Ok(())
} else {
let name = test.padded_name(self.max_name_len, align);
self.write_plain(&format!("test {} ... ", name))
}
}
pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
match *result {
TrOk => self.write_ok(),
TrFailed | TrFailedMsg(_) => self.write_failed(),
TrIgnored => self.write_ignored(),
TrAllowedFail => self.write_allowed_fail(),
TrBench(ref bs) => {
self.write_bench()?;
self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
}
}
}
pub fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
self.write_plain(&format!("test {} has been running for over {} seconds\n",
desc.name,
TEST_WARN_TIMEOUT_S))
}
pub fn write_log<S: AsRef<str>>(&mut self, msg: S) -> io::Result<()> {
let msg = msg.as_ref();
match self.log_out {
@@ -674,114 +702,23 @@ pub fn write_log<S: AsRef<str>>(&mut self, msg: S) -> io::Result<()> {
}
pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
self.write_log(
format!("{} {}\n",
match *result {
TrOk => "ok".to_owned(),
TrFailed => "failed".to_owned(),
TrFailedMsg(ref msg) => format!("failed: {}", msg),
TrIgnored => "ignored".to_owned(),
TrAllowedFail => "failed (allowed)".to_owned(),
TrBench(ref bs) => fmt_bench_samples(bs),
},
test.name))
}
pub fn write_failures(&mut self) -> io::Result<()> {
self.write_plain("\nfailures:\n")?;
let mut failures = Vec::new();
let mut fail_out = String::new();
for &(ref f, ref stdout) in &self.failures {
failures.push(f.name.to_string());
if !stdout.is_empty() {
fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
fail_out.push_str(&output);
fail_out.push_str("\n");
}
}
if !fail_out.is_empty() {
self.write_plain("\n")?;
self.write_plain(&fail_out)?;
}
self.write_plain("\nfailures:\n")?;
failures.sort();
for name in &failures {
self.write_plain(&format!(" {}\n", name))?;
}
Ok(())
}
pub fn write_outputs(&mut self) -> io::Result<()> {
self.write_plain("\nsuccesses:\n")?;
let mut successes = Vec::new();
let mut stdouts = String::new();
for &(ref f, ref stdout) in &self.not_failures {
successes.push(f.name.to_string());
if !stdout.is_empty() {
stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name));
let output = String::from_utf8_lossy(stdout);
stdouts.push_str(&output);
stdouts.push_str("\n");
}
}
if !stdouts.is_empty() {
self.write_plain("\n")?;
self.write_plain(&stdouts)?;
}
self.write_plain("\nsuccesses:\n")?;
successes.sort();
for name in &successes {
self.write_plain(&format!(" {}\n", name))?;
}
Ok(())
self.write_log(format!(
"{} {}\n",
match *result {
TrOk => "ok".to_owned(),
TrFailed => "failed".to_owned(),
TrFailedMsg(ref msg) => format!("failed: {}", msg),
TrIgnored => "ignored".to_owned(),
TrAllowedFail => "failed (allowed)".to_owned(),
TrBench(ref bs) => fmt_bench_samples(bs),
},
test.name
))
}
fn current_test_count(&self) -> usize {
self.passed + self.failed + self.ignored + self.measured + self.allowed_fail
}
pub fn write_run_finish(&mut self) -> io::Result<bool> {
assert!(self.current_test_count() == self.total);
if self.options.display_output {
self.write_outputs()?;
}
let success = self.failed == 0;
if !success {
self.write_failures()?;
}
self.write_plain("\ntest result: ")?;
if success {
// There's no parallelism at this point so it's safe to use color
self.write_pretty("ok", term::color::GREEN)?;
} else {
self.write_pretty("FAILED", term::color::RED)?;
}
let s = if self.allowed_fail > 0 {
format!(
". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n",
self.passed,
self.failed + self.allowed_fail,
self.allowed_fail,
self.ignored,
self.measured,
self.filtered_out)
} else {
format!(
". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
self.passed,
self.failed,
self.ignored,
self.measured,
self.filtered_out)
};
self.write_plain(&s)?;
return Ok(success);
}
}
// Format a number with thousands separators
@@ -815,19 +752,30 @@ pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
let median = bs.ns_iter_summ.median as usize;
let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
fmt_thousands_sep(median, ','),
fmt_thousands_sep(deviation, ',')))
.unwrap();
output
.write_fmt(format_args!(
"{:>11} ns/iter (+/- {})",
fmt_thousands_sep(median, ','),
fmt_thousands_sep(deviation, ',')
))
.unwrap();
if bs.mb_s != 0 {
output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
output
.write_fmt(format_args!(" = {} MB/s", bs.mb_s))
.unwrap();
}
output
}
// List the tests to console, and optionally to logfile. Filters are honored.
pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
let mut st = ConsoleTestState::new(opts, None::<io::Stdout>)?;
let mut output = match term::stdout() {
None => Raw(io::stdout()),
Some(t) => Pretty(t),
};
let quiet = opts.format == OutputFormat::Terse;
let mut st = ConsoleTestState::new(opts)?;
let mut ntest = 0;
let mut nbench = 0;
@@ -835,14 +783,24 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Res
for test in filter_tests(&opts, tests) {
use TestFn::*;
let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test;
let TestDescAndFn {
desc: TestDesc { name, .. },
testfn,
} = test;
let fntype = match testfn {
StaticTestFn(..) | DynTestFn(..) => { ntest += 1; "test" },
StaticBenchFn(..) | DynBenchFn(..) => { nbench += 1; "benchmark" },
StaticTestFn(..) | DynTestFn(..) => {
ntest += 1;
"test"
}
StaticBenchFn(..) |
DynBenchFn(..) => {
nbench += 1;
"benchmark"
}
};
st.write_plain(format!("{}: {}\n", name, fntype))?;
writeln!(output, "{}: {}", name, fntype)?;
st.write_log(format!("{} {}\n", fntype, name))?;
}
@@ -853,13 +811,16 @@ fn plural(count: u32, s: &str) -> String {
}
}
if !opts.quiet {
if !quiet {
if ntest != 0 || nbench != 0 {
st.write_plain("\n")?;
writeln!(output, "")?;
}
st.write_plain(format!("{}, {}\n",
writeln!(output,
"{}, {}",
plural(ntest, "test"),
plural(nbench, "benchmark")))?;
plural(nbench, "benchmark")
)?;
}
Ok(())
@@ -867,16 +828,23 @@ fn plural(count: u32, s: &str) -> String {
// A simple console test runner
pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
fn callback(
event: &TestEvent,
st: &mut ConsoleTestState,
out: &mut OutputFormatter,
) -> io::Result<()> {
fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
match (*event).clone() {
TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
TeFiltered(ref filtered_tests) => {
st.total = filtered_tests.len();
out.write_run_start(filtered_tests.len())
}
TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out),
TeWait(ref test, padding) => st.write_test_start(test, padding),
TeTimeout(ref test) => st.write_timeout(test),
TeWait(ref test) => out.write_test_start(test),
TeTimeout(ref test) => out.write_timeout(test),
TeResult(test, result, stdout) => {
st.write_log_result(&test, &result)?;
st.write_result(&result)?;
out.write_result(&test, &result, &*stdout)?;
match result {
TrOk => {
st.passed += 1;
@@ -885,9 +853,11 @@ fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Re
TrIgnored => st.ignored += 1,
TrAllowedFail => st.allowed_fail += 1,
TrBench(bs) => {
st.metrics.insert_metric(test.name.as_slice(),
bs.ns_iter_summ.median,
bs.ns_iter_summ.max - bs.ns_iter_summ.min);
st.metrics.insert_metric(
test.name.as_slice(),
bs.ns_iter_summ.median,
bs.ns_iter_summ.max - bs.ns_iter_summ.min,
);
st.measured += 1
}
TrFailed => {
@@ -897,9 +867,7 @@ fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Re
TrFailedMsg(msg) => {
st.failed += 1;
let mut stdout = stdout;
stdout.extend_from_slice(
format!("note: {}", msg).as_bytes()
);
stdout.extend_from_slice(format!("note: {}", msg).as_bytes());
st.failures.push((test, stdout));
}
}
@@ -908,19 +876,47 @@ fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Re
}
}
let mut st = ConsoleTestState::new(opts, None::<io::Stdout>)?;
let output = match term::stdout() {
None => Raw(io::stdout()),
Some(t) => Pretty(t),
};
let max_name_len = tests
.iter()
.max_by_key(|t| len_if_padded(*t))
.map(|t| t.desc.name.as_slice().len())
.unwrap_or(0);
let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1;
let mut out: Box<OutputFormatter> = match opts.format {
OutputFormat::Pretty => Box::new(PrettyFormatter::new(
output,
use_color(opts),
max_name_len,
is_multithreaded,
)),
OutputFormat::Terse => Box::new(TerseFormatter::new(
output,
use_color(opts),
max_name_len,
is_multithreaded,
)),
OutputFormat::Json => Box::new(JsonFormatter::new(output)),
};
let mut st = ConsoleTestState::new(opts)?;
fn len_if_padded(t: &TestDescAndFn) -> usize {
match t.testfn.padding() {
PadNone => 0,
PadOnRight => t.desc.name.as_slice().len(),
}
}
if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) {
let n = t.desc.name.as_slice();
st.max_name_len = n.len();
}
run_tests(opts, tests, |x| callback(&x, &mut st))?;
return st.write_run_finish();
run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?;
assert!(st.current_test_count() == st.total);
return out.write_run_finish(&st);
}
#[test]
@@ -939,11 +935,10 @@ fn should_sort_failures_before_printing_them() {
allow_fail: false,
};
let mut st = ConsoleTestState {
let mut out = PrettyFormatter::new(Raw(Vec::new()), false, 10, false);
let st = ConsoleTestState {
log_out: None,
out: Raw(Vec::new()),
use_color: false,
quiet: false,
total: 0,
passed: 0,
failed: 0,
@@ -951,17 +946,16 @@ fn should_sort_failures_before_printing_them() {
allowed_fail: 0,
filtered_out: 0,
measured: 0,
max_name_len: 10,
metrics: MetricMap::new(),
failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
options: Options::new(),
not_failures: Vec::new(),
};
st.write_failures().unwrap();
let s = match st.out {
Raw(ref m) => String::from_utf8_lossy(&m[..]),
Pretty(_) => unreachable!(),
out.write_failures(&st).unwrap();
let s = match out.output_location() {
&Raw(ref m) => String::from_utf8_lossy(&m[..]),
&Pretty(_) => unreachable!(),
};
let apos = s.find("a").unwrap();
@@ -1009,7 +1003,7 @@ fn stdout_isatty() -> bool {
#[derive(Clone)]
pub enum TestEvent {
TeFiltered(Vec<TestDesc>),
TeWait(TestDesc, NamePadding),
TeWait(TestDesc),
TeResult(TestDesc, TestResult, Vec<u8>),
TeTimeout(TestDesc),
TeFilteredOut(usize),
@@ -1017,9 +1011,19 @@ pub enum TestEvent {
pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
struct Sink(Arc<Mutex<Vec<u8>>>);
impl Write for Sink {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
Write::write(&mut *self.0.lock().unwrap(), data)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
where F: FnMut(TestEvent) -> io::Result<()>
where
F: FnMut(TestEvent) -> io::Result<()>,
{
use std::collections::HashMap;
use std::sync::mpsc::RecvTimeoutError;
@@ -1031,27 +1035,29 @@ pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F)
filtered_tests = convert_benchmarks_to_tests(filtered_tests);
}
let filtered_tests = {
let mut filtered_tests = filtered_tests;
for test in filtered_tests.iter_mut() {
test.desc.name = test.desc.name.with_padding(test.testfn.padding());
}
filtered_tests
};
let filtered_out = tests_len - filtered_tests.len();
callback(TeFilteredOut(filtered_out))?;
let filtered_descs = filtered_tests.iter()
.map(|t| t.desc.clone())
.collect();
let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
callback(TeFiltered(filtered_descs))?;
let (filtered_tests, filtered_benchs): (Vec<_>, _) =
filtered_tests.into_iter().partition(|e| {
match e.testfn {
StaticTestFn(_) | DynTestFn(_) => true,
_ => false,
}
filtered_tests.into_iter().partition(|e| match e.testfn {
StaticTestFn(_) | DynTestFn(_) => true,
_ => false,
});
let concurrency = match opts.test_threads {
Some(n) => n,
None => get_concurrency(),
};
let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
let mut remaining = filtered_tests;
remaining.reverse();
@@ -1063,8 +1069,13 @@ pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F)
fn get_timed_out_tests(running_tests: &mut HashMap<TestDesc, Instant>) -> Vec<TestDesc> {
let now = Instant::now();
let timed_out = running_tests.iter()
.filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone())} else { None })
let timed_out = running_tests
.iter()
.filter_map(|(desc, timeout)| if &now >= timeout {
Some(desc.clone())
} else {
None
})
.collect();
for test in &timed_out {
running_tests.remove(test);
@@ -1079,13 +1090,14 @@ fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration>
*next_timeout - now
} else {
Duration::new(0, 0)
}})
}
})
};
if concurrency == 1 {
while !remaining.is_empty() {
let test = remaining.pop().unwrap();
callback(TeWait(test.desc.clone(), test.testfn.padding()))?;
callback(TeWait(test.desc.clone()))?;
run_test(opts, !opts.run_tests, test, tx.clone());
let (test, result, stdout) = rx.recv().unwrap();
callback(TeResult(test, result, stdout))?;
@@ -1096,6 +1108,7 @@ fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration>
let test = remaining.pop().unwrap();
let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S);
running_tests.insert(test.desc.clone(), timeout);
callback(TeWait(test.desc.clone()))?; //here no pad
run_test(opts, !opts.run_tests, test, tx.clone());
pending += 1;
}
@@ -1119,7 +1132,6 @@ fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration>
let (desc, result, stdout) = res.unwrap();
running_tests.remove(&desc);
callback(TeWait(desc.clone(), PadNone))?;
callback(TeResult(desc, result, stdout))?;
pending -= 1;
}
@@ -1128,7 +1140,7 @@ fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration>
if opts.bench_benchmarks {
// All benchmarks run at the end, in serial.
for b in filtered_benchs {
callback(TeWait(b.desc.clone(), b.testfn.padding()))?;
callback(TeWait(b.desc.clone()))?;
run_test(opts, false, b, tx.clone());
let (test, result, stdout) = rx.recv().unwrap();
callback(TeResult(test, result, stdout))?;
@@ -1145,8 +1157,10 @@ fn get_concurrency() -> usize {
match opt_n {
Some(n) if n > 0 => n,
_ => {
panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.",
s)
panic!(
"RUST_TEST_THREADS is `{}`, should be a positive integer.",
s
)
}
}
}
@@ -1203,10 +1217,8 @@ fn num_cpus() -> usize {
unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
}
#[cfg(any(target_os = "freebsd",
target_os = "dragonfly",
target_os = "bitrig",
target_os = "netbsd"))]
#[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "bitrig",
target_os = "netbsd"))]
fn num_cpus() -> usize {
use std::ptr;
@@ -1219,12 +1231,14 @@ fn num_cpus() -> usize {
if cpus < 1 {
let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
unsafe {
libc::sysctl(mib.as_mut_ptr(),
2,
&mut cpus as *mut _ as *mut _,
&mut cpus_size as *mut _ as *mut _,
ptr::null_mut(),
0);
libc::sysctl(
mib.as_mut_ptr(),
2,
&mut cpus as *mut _ as *mut _,
&mut cpus_size as *mut _ as *mut _,
ptr::null_mut(),
0,
);
}
if cpus < 1 {
cpus = 1;
@@ -1242,12 +1256,14 @@ fn num_cpus() -> usize {
let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
unsafe {
libc::sysctl(mib.as_mut_ptr(),
2,
&mut cpus as *mut _ as *mut _,
&mut cpus_size as *mut _ as *mut _,
ptr::null_mut(),
0);
libc::sysctl(
mib.as_mut_ptr(),
2,
&mut cpus as *mut _ as *mut _,
&mut cpus_size as *mut _ as *mut _,
ptr::null_mut(),
0,
);
}
if cpus < 1 {
cpus = 1;
@@ -1269,27 +1285,27 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescA
filtered = match opts.filter {
None => filtered,
Some(ref filter) => {
filtered.into_iter()
.filter(|test| {
if opts.filter_exact {
test.desc.name.as_slice() == &filter[..]
} else {
test.desc.name.as_slice().contains(&filter[..])
}
})
.collect()
filtered
.into_iter()
.filter(|test| if opts.filter_exact {
test.desc.name.as_slice() == &filter[..]
} else {
test.desc.name.as_slice().contains(&filter[..])
})
.collect()
}
};
// Skip tests that match any of the skip filters
filtered = filtered.into_iter()
.filter(|t| !opts.skip.iter().any(|sf| {
if opts.filter_exact {
t.desc.name.as_slice() == &sf[..]
} else {
t.desc.name.as_slice().contains(&sf[..])
}
}))
filtered = filtered
.into_iter()
.filter(|t| {
!opts.skip.iter().any(|sf| if opts.filter_exact {
t.desc.name.as_slice() == &sf[..]
} else {
t.desc.name.as_slice().contains(&sf[..])
})
})
.collect();
// Maybe pull out the ignored test and unignore them
@@ -1298,9 +1314,12 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescA
} else {
fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
if test.desc.ignore {
let TestDescAndFn {desc, testfn} = test;
let TestDescAndFn { desc, testfn } = test;
Some(TestDescAndFn {
desc: TestDesc { ignore: false, ..desc },
desc: TestDesc {
ignore: false,
..desc
},
testfn,
})
} else {
@@ -1311,7 +1330,9 @@ fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
};
// Sort the tests alphabetically
filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
filtered.sort_by(|t1, t2| {
t1.desc.name.as_slice().cmp(t2.desc.name.as_slice())
});
filtered
}
@@ -1334,24 +1355,26 @@ pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAnd
})
}))
}
f => f,
};
TestDescAndFn {
desc: x.desc,
testfn,
}
}).collect()
f => f,
};
TestDescAndFn {
desc: x.desc,
testfn,
}
})
.collect()
}
pub fn run_test(opts: &TestOpts,
force_ignore: bool,
test: TestDescAndFn,
monitor_ch: Sender<MonitorMsg>) {
pub fn run_test(
opts: &TestOpts,
force_ignore: bool,
test: TestDescAndFn,
monitor_ch: Sender<MonitorMsg>,
) {
let TestDescAndFn {desc, testfn} = test;
let TestDescAndFn { desc, testfn } = test;
let ignore_because_panic_abort =
cfg!(target_arch = "wasm32") &&
let ignore_because_panic_abort = cfg!(target_arch = "wasm32") &&
!cfg!(target_os = "emscripten") &&
desc.should_panic != ShouldPanic::No;
@@ -1364,16 +1387,6 @@ fn run_test_inner(desc: TestDesc,
monitor_ch: Sender<MonitorMsg>,
nocapture: bool,
testfn: Box<FnBox() + Send>) {
struct Sink(Arc<Mutex<Vec<u8>>>);
impl Write for Sink {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
Write::write(&mut *self.0.lock().unwrap(), data)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
// Buffer for capturing standard I/O
let data = Arc::new(Mutex::new(Vec::new()));
let data2 = data.clone();
@@ -1383,7 +1396,7 @@ fn flush(&mut self) -> io::Result<()> {
let oldio = if !nocapture {
Some((
io::set_print(Some(Box::new(Sink(data2.clone())))),
io::set_panic(Some(Box::new(Sink(data2))))
io::set_panic(Some(Box::new(Sink(data2)))),
))
} else {
None
@@ -1398,21 +1411,18 @@ fn flush(&mut self) -> io::Result<()> {
let test_result = calc_result(&desc, result);
let stdout = data.lock().unwrap().to_vec();
monitor_ch.send((desc.clone(), test_result, stdout)).unwrap();
monitor_ch
.send((desc.clone(), test_result, stdout))
.unwrap();
};
// If the platform is single-threaded we're just going to run
// the test synchronously, regardless of the concurrency
// level.
let supports_threads =
!cfg!(target_os = "emscripten") &&
!cfg!(target_arch = "wasm32");
let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
if supports_threads {
let cfg = thread::Builder::new().name(match name {
DynTestName(ref name) => name.clone(),
StaticTestName(name) => name.to_owned(),
});
let cfg = thread::Builder::new().name(name.as_slice().to_owned());
cfg.spawn(runtest).unwrap();
} else {
runtest();
@@ -1421,14 +1431,16 @@ fn flush(&mut self) -> io::Result<()> {
match testfn {
DynBenchFn(bencher) => {
let bs = ::bench::benchmark(|harness| bencher.run(harness));
monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
return;
::bench::benchmark(desc,
monitor_ch,
opts.nocapture,
|harness| bencher.run(harness));
}
StaticBenchFn(benchfn) => {
let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness));
monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
return;
::bench::benchmark(desc,
monitor_ch,
opts.nocapture,
|harness| (benchfn.clone())(harness));
}
DynTestFn(f) => {
let cb = move || {
@@ -1436,9 +1448,10 @@ fn flush(&mut self) -> io::Result<()> {
};
run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb))
}
StaticTestFn(f) =>
StaticTestFn(f) => {
run_test_inner(desc, monitor_ch, opts.nocapture,
Box::new(move || __rust_begin_short_backtrace(f))),
Box::new(move || __rust_begin_short_backtrace(f)))
}
}
}
@@ -1452,12 +1465,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> Tes
match (&desc.should_panic, task_result) {
(&ShouldPanic::No, Ok(())) |
(&ShouldPanic::Yes, Err(_)) => TrOk,
(&ShouldPanic::YesWithMessage(msg), Err(ref err)) =>
(&ShouldPanic::YesWithMessage(msg), Err(ref err)) => {
if err.downcast_ref::<String>()
.map(|e| &**e)
.or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
.map(|e| e.contains(msg))
.unwrap_or(false) {
.map(|e| &**e)
.or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
.map(|e| e.contains(msg))
.unwrap_or(false)
{
TrOk
} else {
if desc.allow_fail {
@@ -1465,7 +1479,8 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> Tes
} else {
TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
}
},
}
}
_ if desc.allow_fail => TrAllowedFail,
_ => TrFailed,
}
@@ -1493,18 +1508,15 @@ pub fn new() -> MetricMap {
/// you want to see grow larger, so a change larger than `noise` in the
/// negative direction represents a regression.
pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) {
let m = Metric {
value,
noise,
};
let m = Metric { value, noise };
self.0.insert(name.to_owned(), m);
}
pub fn fmt_metrics(&self) -> String {
let v = self.0
.iter()
.map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
.collect::<Vec<_>>();
.iter()
.map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
.collect::<Vec<_>>();
v.join(", ")
}
}
@@ -1534,7 +1546,8 @@ pub fn black_box<T>(dummy: T) -> T {
impl Bencher {
/// Callback for benchmark functions to run in their body.
pub fn iter<T, F>(&mut self, mut inner: F)
where F: FnMut() -> T
where
F: FnMut() -> T,
{
if self.mode == BenchMode::Single {
ns_iter_inner(&mut inner, 1);
@@ -1545,7 +1558,8 @@ pub fn iter<T, F>(&mut self, mut inner: F)
}
pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary>
where F: FnMut(&mut Bencher)
where
F: FnMut(&mut Bencher),
{
f(self);
return self.summary;
@@ -1557,7 +1571,8 @@ fn ns_from_dur(dur: Duration) -> u64 {
}
fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64
where F: FnMut() -> T
where
F: FnMut() -> T,
{
let start = Instant::now();
for _ in 0..k {
@@ -1568,7 +1583,8 @@ fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64
pub fn iter<T, F>(inner: &mut F) -> stats::Summary
where F: FnMut() -> T
where
F: FnMut() -> T,
{
// Initial bench run to get ballpark figure.
let ns_single = ns_iter_inner(inner, 1);
@@ -1610,7 +1626,8 @@ pub fn iter<T, F>(inner: &mut F) -> stats::Summary
// If we've run for 100ms and seem to have converged to a
// stable median.
if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
summ.median - summ5.median < summ5.median_abs_dev {
summ.median - summ5.median < summ5.median_abs_dev
{
return summ5;
}
@@ -1634,12 +1651,16 @@ pub fn iter<T, F>(inner: &mut F) -> stats::Summary
}
pub mod bench {
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::cmp;
use std::io;
use std::sync::{Arc, Mutex};
use stats;
use super::{Bencher, BenchSamples, BenchMode};
use super::{Bencher, BenchSamples, BenchMode, Sink, MonitorMsg, TestDesc, Sender, TestResult};
pub fn benchmark<F>(f: F) -> BenchSamples
where F: FnMut(&mut Bencher)
pub fn benchmark<F>(desc: TestDesc, monitor_ch: Sender<MonitorMsg>, nocapture: bool, f: F)
where
F: FnMut(&mut Bencher),
{
let mut bs = Bencher {
mode: BenchMode::Auto,
@@ -1647,30 +1668,58 @@ pub fn benchmark<F>(f: F) -> BenchSamples
bytes: 0,
};
return match bs.bench(f) {
Some(ns_iter_summ) => {
let data = Arc::new(Mutex::new(Vec::new()));
let data2 = data.clone();
let oldio = if !nocapture {
Some((
io::set_print(Some(Box::new(Sink(data2.clone())))),
io::set_panic(Some(Box::new(Sink(data2)))),
))
} else {
None
};
let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f)));
if let Some((printio, panicio)) = oldio {
io::set_print(printio);
io::set_panic(panicio);
};
let test_result = match result { //bs.bench(f) {
Ok(Some(ns_iter_summ)) => {
let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
let mb_s = bs.bytes * 1000 / ns_iter;
BenchSamples {
let bs = BenchSamples {
ns_iter_summ,
mb_s: mb_s as usize,
}
};
TestResult::TrBench(bs)
}
None => {
Ok(None) => {
// iter not called, so no data.
// FIXME: error in this case?
let samples: &mut [f64] = &mut [0.0_f64; 1];
BenchSamples {
let bs = BenchSamples {
ns_iter_summ: stats::Summary::new(samples),
mb_s: 0,
}
};
TestResult::TrBench(bs)
}
Err(_) => {
TestResult::TrFailed
}
};
let stdout = data.lock().unwrap().to_vec();
monitor_ch.send((desc, test_result, stdout)).unwrap();
}
pub fn run_once<F>(f: F)
where F: FnMut(&mut Bencher)
where
F: FnMut(&mut Bencher),
{
let mut bs = Bencher {
mode: BenchMode::Single,
@@ -1810,7 +1859,11 @@ fn f() {}
#[test]
fn parse_ignored_flag() {
let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
let args = vec![
"progname".to_string(),
"filter".to_string(),
"--ignored".to_string(),
];
let opts = match parse_opts(&args) {
Some(Ok(o)) => o,
_ => panic!("Malformed arg in parse_ignored_flag"),
@@ -1827,7 +1880,8 @@ pub fn filter_for_ignored_option() {
opts.run_tests = true;
opts.run_ignored = true;
let tests = vec![TestDescAndFn {
let tests =
vec![TestDescAndFn {
desc: TestDesc {
name: StaticTestName("1"),
ignore: true,
@@ -1855,72 +1909,95 @@ pub fn filter_for_ignored_option() {
#[test]
pub fn exact_filter_match() {
fn tests() -> Vec<TestDescAndFn> {
vec!["base",
"base::test",
"base::test1",
"base::test2",
].into_iter()
.map(|name| TestDescAndFn {
desc: TestDesc {
name: StaticTestName(name),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
},
testfn: DynTestFn(Box::new(move || {}))
})
.collect()
vec!["base", "base::test", "base::test1", "base::test2"]
.into_iter()
.map(|name| {
TestDescAndFn {
desc: TestDesc {
name: StaticTestName(name),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
},
testfn: DynTestFn(Box::new(move || {}))
}
}).collect()
}
let substr = filter_tests(&TestOpts {
let substr = filter_tests(
&TestOpts {
filter: Some("base".into()),
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(substr.len(), 4);
let substr = filter_tests(&TestOpts {
let substr = filter_tests(
&TestOpts {
filter: Some("bas".into()),
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(substr.len(), 4);
let substr = filter_tests(&TestOpts {
let substr = filter_tests(
&TestOpts {
filter: Some("::test".into()),
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(substr.len(), 3);
let substr = filter_tests(&TestOpts {
let substr = filter_tests(
&TestOpts {
filter: Some("base::test".into()),
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(substr.len(), 3);
let exact = filter_tests(&TestOpts {
let exact = filter_tests(
&TestOpts {
filter: Some("base".into()),
filter_exact: true, ..TestOpts::new()
}, tests());
filter_exact: true,
..TestOpts::new()
},
tests(),
);
assert_eq!(exact.len(), 1);
let exact = filter_tests(&TestOpts {
let exact = filter_tests(
&TestOpts {
filter: Some("bas".into()),
filter_exact: true,
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(exact.len(), 0);
let exact = filter_tests(&TestOpts {
let exact = filter_tests(
&TestOpts {
filter: Some("::test".into()),
filter_exact: true,
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(exact.len(), 0);
let exact = filter_tests(&TestOpts {
let exact = filter_tests(
&TestOpts {
filter: Some("base::test".into()),
filter_exact: true,
..TestOpts::new()
}, tests());
},
tests(),
);
assert_eq!(exact.len(), 1);
}
@@ -1929,15 +2006,17 @@ pub fn sort_tests() {
let mut opts = TestOpts::new();
opts.run_tests = true;
let names = vec!["sha1::test".to_string(),
"isize::test_to_str".to_string(),
"isize::test_pow".to_string(),
"test::do_not_run_ignored_tests".to_string(),
"test::ignored_tests_result_in_ignored".to_string(),
"test::first_free_arg_should_be_a_filter".to_string(),
"test::parse_ignored_flag".to_string(),
"test::filter_for_ignored_option".to_string(),
"test::sort_tests".to_string()];
let names = vec![
"sha1::test".to_string(),
"isize::test_to_str".to_string(),
"isize::test_pow".to_string(),
"test::do_not_run_ignored_tests".to_string(),
"test::ignored_tests_result_in_ignored".to_string(),
"test::first_free_arg_should_be_a_filter".to_string(),
"test::parse_ignored_flag".to_string(),
"test::filter_for_ignored_option".to_string(),
"test::sort_tests".to_string(),
];
let tests = {
fn testfn() {}
let mut tests = Vec::new();
@@ -1957,15 +2036,17 @@ fn testfn() {}
};
let filtered = filter_tests(&opts, tests);
let expected = vec!["isize::test_pow".to_string(),
"isize::test_to_str".to_string(),
"sha1::test".to_string(),
"test::do_not_run_ignored_tests".to_string(),
"test::filter_for_ignored_option".to_string(),
"test::first_free_arg_should_be_a_filter".to_string(),
"test::ignored_tests_result_in_ignored".to_string(),
"test::parse_ignored_flag".to_string(),
"test::sort_tests".to_string()];
let expected = vec![
"isize::test_pow".to_string(),
"isize::test_to_str".to_string(),
"sha1::test".to_string(),
"test::do_not_run_ignored_tests".to_string(),
"test::filter_for_ignored_option".to_string(),
"test::first_free_arg_should_be_a_filter".to_string(),
"test::ignored_tests_result_in_ignored".to_string(),
"test::parse_ignored_flag".to_string(),
"test::sort_tests".to_string(),
];
for (a, b) in expected.iter().zip(filtered) {
assert!(*a == b.desc.name.to_string());
@@ -2004,8 +2085,7 @@ fn f(_: &mut Bencher) {}
#[test]
pub fn test_bench_once_iter() {
fn f(b: &mut Bencher) {
b.iter(|| {
})
b.iter(|| {})
}
bench::run_once(f);
}
@@ -2013,15 +2093,42 @@ fn f(b: &mut Bencher) {
#[test]
pub fn test_bench_no_iter() {
fn f(_: &mut Bencher) {}
bench::benchmark(f);
let (tx, rx) = channel();
let desc = TestDesc {
name: StaticTestName("f"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
};
::bench::benchmark(desc,
tx,
true,
f);
rx.recv().unwrap();
}
#[test]
pub fn test_bench_iter() {
fn f(b: &mut Bencher) {
b.iter(|| {
})
b.iter(|| {})
}
bench::benchmark(f);
let (tx, rx) = channel();
let desc = TestDesc {
name: StaticTestName("f"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
};
::bench::benchmark(desc,
tx,
true,
f);
rx.recv().unwrap();
}
}
src/libtest/stats.rs
+263 -241
View File
@@ -400,16 +400,18 @@ fn test_norm2() {
}
#[test]
fn test_norm10narrow() {
let val = &[966.0000000000,
985.0000000000,
1110.0000000000,
848.0000000000,
821.0000000000,
975.0000000000,
962.0000000000,
1157.0000000000,
1217.0000000000,
955.0000000000];
let val = &[
966.0000000000,
985.0000000000,
1110.0000000000,
848.0000000000,
821.0000000000,
975.0000000000,
962.0000000000,
1157.0000000000,
1217.0000000000,
955.0000000000,
];
let summ = &Summary {
sum: 9996.0000000000,
min: 821.0000000000,
@@ -428,16 +430,18 @@ fn test_norm10narrow() {
}
#[test]
fn test_norm10medium() {
let val = &[954.0000000000,
1064.0000000000,
855.0000000000,
1000.0000000000,
743.0000000000,
1084.0000000000,
704.0000000000,
1023.0000000000,
357.0000000000,
869.0000000000];
let val = &[
954.0000000000,
1064.0000000000,
855.0000000000,
1000.0000000000,
743.0000000000,
1084.0000000000,
704.0000000000,
1023.0000000000,
357.0000000000,
869.0000000000,
];
let summ = &Summary {
sum: 8653.0000000000,
min: 357.0000000000,
@@ -456,16 +460,18 @@ fn test_norm10medium() {
}
#[test]
fn test_norm10wide() {
let val = &[505.0000000000,
497.0000000000,
1591.0000000000,
887.0000000000,
1026.0000000000,
136.0000000000,
1580.0000000000,
940.0000000000,
754.0000000000,
1433.0000000000];
let val = &[
505.0000000000,
497.0000000000,
1591.0000000000,
887.0000000000,
1026.0000000000,
136.0000000000,
1580.0000000000,
940.0000000000,
754.0000000000,
1433.0000000000,
];
let summ = &Summary {
sum: 9349.0000000000,
min: 136.0000000000,
@@ -484,31 +490,33 @@ fn test_norm10wide() {
}
#[test]
fn test_norm25verynarrow() {
let val = &[991.0000000000,
1018.0000000000,
998.0000000000,
1013.0000000000,
974.0000000000,
1007.0000000000,
1014.0000000000,
999.0000000000,
1011.0000000000,
978.0000000000,
985.0000000000,
999.0000000000,
983.0000000000,
982.0000000000,
1015.0000000000,
1002.0000000000,
977.0000000000,
948.0000000000,
1040.0000000000,
974.0000000000,
996.0000000000,
989.0000000000,
1015.0000000000,
994.0000000000,
1024.0000000000];
let val = &[
991.0000000000,
1018.0000000000,
998.0000000000,
1013.0000000000,
974.0000000000,
1007.0000000000,
1014.0000000000,
999.0000000000,
1011.0000000000,
978.0000000000,
985.0000000000,
999.0000000000,
983.0000000000,
982.0000000000,
1015.0000000000,
1002.0000000000,
977.0000000000,
948.0000000000,
1040.0000000000,
974.0000000000,
996.0000000000,
989.0000000000,
1015.0000000000,
994.0000000000,
1024.0000000000,
];
let summ = &Summary {
sum: 24926.0000000000,
min: 948.0000000000,
@@ -527,16 +535,18 @@ fn test_norm25verynarrow() {
}
#[test]
fn test_exp10a() {
let val = &[23.0000000000,
11.0000000000,
2.0000000000,
57.0000000000,
4.0000000000,
12.0000000000,
5.0000000000,
29.0000000000,
3.0000000000,
21.0000000000];
let val = &[
23.0000000000,
11.0000000000,
2.0000000000,
57.0000000000,
4.0000000000,
12.0000000000,
5.0000000000,
29.0000000000,
3.0000000000,
21.0000000000,
];
let summ = &Summary {
sum: 167.0000000000,
min: 2.0000000000,
@@ -555,16 +565,18 @@ fn test_exp10a() {
}
#[test]
fn test_exp10b() {
let val = &[24.0000000000,
17.0000000000,
6.0000000000,
38.0000000000,
25.0000000000,
7.0000000000,
51.0000000000,
2.0000000000,
61.0000000000,
32.0000000000];
let val = &[
24.0000000000,
17.0000000000,
6.0000000000,
38.0000000000,
25.0000000000,
7.0000000000,
51.0000000000,
2.0000000000,
61.0000000000,
32.0000000000,
];
let summ = &Summary {
sum: 263.0000000000,
min: 2.0000000000,
@@ -583,16 +595,18 @@ fn test_exp10b() {
}
#[test]
fn test_exp10c() {
let val = &[71.0000000000,
2.0000000000,
32.0000000000,
1.0000000000,
6.0000000000,
28.0000000000,
13.0000000000,
37.0000000000,
16.0000000000,
36.0000000000];
let val = &[
71.0000000000,
2.0000000000,
32.0000000000,
1.0000000000,
6.0000000000,
28.0000000000,
13.0000000000,
37.0000000000,
16.0000000000,
36.0000000000,
];
let summ = &Summary {
sum: 242.0000000000,
min: 1.0000000000,
@@ -611,31 +625,33 @@ fn test_exp10c() {
}
#[test]
fn test_exp25() {
let val = &[3.0000000000,
24.0000000000,
1.0000000000,
19.0000000000,
7.0000000000,
5.0000000000,
30.0000000000,
39.0000000000,
31.0000000000,
13.0000000000,
25.0000000000,
48.0000000000,
1.0000000000,
6.0000000000,
42.0000000000,
63.0000000000,
2.0000000000,
12.0000000000,
108.0000000000,
26.0000000000,
1.0000000000,
7.0000000000,
44.0000000000,
25.0000000000,
11.0000000000];
let val = &[
3.0000000000,
24.0000000000,
1.0000000000,
19.0000000000,
7.0000000000,
5.0000000000,
30.0000000000,
39.0000000000,
31.0000000000,
13.0000000000,
25.0000000000,
48.0000000000,
1.0000000000,
6.0000000000,
42.0000000000,
63.0000000000,
2.0000000000,
12.0000000000,
108.0000000000,
26.0000000000,
1.0000000000,
7.0000000000,
44.0000000000,
25.0000000000,
11.0000000000,
];
let summ = &Summary {
sum: 593.0000000000,
min: 1.0000000000,
@@ -654,31 +670,33 @@ fn test_exp25() {
}
#[test]
fn test_binom25() {
let val = &[18.0000000000,
17.0000000000,
27.0000000000,
15.0000000000,
21.0000000000,
25.0000000000,
17.0000000000,
24.0000000000,
25.0000000000,
24.0000000000,
26.0000000000,
26.0000000000,
23.0000000000,
15.0000000000,
23.0000000000,
17.0000000000,
18.0000000000,
18.0000000000,
21.0000000000,
16.0000000000,
15.0000000000,
31.0000000000,
20.0000000000,
17.0000000000,
15.0000000000];
let val = &[
18.0000000000,
17.0000000000,
27.0000000000,
15.0000000000,
21.0000000000,
25.0000000000,
17.0000000000,
24.0000000000,
25.0000000000,
24.0000000000,
26.0000000000,
26.0000000000,
23.0000000000,
15.0000000000,
23.0000000000,
17.0000000000,
18.0000000000,
18.0000000000,
21.0000000000,
16.0000000000,
15.0000000000,
31.0000000000,
20.0000000000,
17.0000000000,
15.0000000000,
];
let summ = &Summary {
sum: 514.0000000000,
min: 15.0000000000,
@@ -697,31 +715,33 @@ fn test_binom25() {
}
#[test]
fn test_pois25lambda30() {
let val = &[27.0000000000,
33.0000000000,
34.0000000000,
34.0000000000,
24.0000000000,
39.0000000000,
28.0000000000,
27.0000000000,
31.0000000000,
28.0000000000,
38.0000000000,
21.0000000000,
33.0000000000,
36.0000000000,
29.0000000000,
37.0000000000,
32.0000000000,
34.0000000000,
31.0000000000,
39.0000000000,
25.0000000000,
31.0000000000,
32.0000000000,
40.0000000000,
24.0000000000];
let val = &[
27.0000000000,
33.0000000000,
34.0000000000,
34.0000000000,
24.0000000000,
39.0000000000,
28.0000000000,
27.0000000000,
31.0000000000,
28.0000000000,
38.0000000000,
21.0000000000,
33.0000000000,
36.0000000000,
29.0000000000,
37.0000000000,
32.0000000000,
34.0000000000,
31.0000000000,
39.0000000000,
25.0000000000,
31.0000000000,
32.0000000000,
40.0000000000,
24.0000000000,
];
let summ = &Summary {
sum: 787.0000000000,
min: 21.0000000000,
@@ -740,31 +760,33 @@ fn test_pois25lambda30() {
}
#[test]
fn test_pois25lambda40() {
let val = &[42.0000000000,
50.0000000000,
42.0000000000,
46.0000000000,
34.0000000000,
45.0000000000,
34.0000000000,
49.0000000000,
39.0000000000,
28.0000000000,
40.0000000000,
35.0000000000,
37.0000000000,
39.0000000000,
46.0000000000,
44.0000000000,
32.0000000000,
45.0000000000,
42.0000000000,
37.0000000000,
48.0000000000,
42.0000000000,
33.0000000000,
42.0000000000,
48.0000000000];
let val = &[
42.0000000000,
50.0000000000,
42.0000000000,
46.0000000000,
34.0000000000,
45.0000000000,
34.0000000000,
49.0000000000,
39.0000000000,
28.0000000000,
40.0000000000,
35.0000000000,
37.0000000000,
39.0000000000,
46.0000000000,
44.0000000000,
32.0000000000,
45.0000000000,
42.0000000000,
37.0000000000,
48.0000000000,
42.0000000000,
33.0000000000,
42.0000000000,
48.0000000000,
];
let summ = &Summary {
sum: 1019.0000000000,
min: 28.0000000000,
@@ -783,31 +805,33 @@ fn test_pois25lambda40() {
}
#[test]
fn test_pois25lambda50() {
let val = &[45.0000000000,
43.0000000000,
44.0000000000,
61.0000000000,
51.0000000000,
53.0000000000,
59.0000000000,
52.0000000000,
49.0000000000,
51.0000000000,
51.0000000000,
50.0000000000,
49.0000000000,
56.0000000000,
42.0000000000,
52.0000000000,
51.0000000000,
43.0000000000,
48.0000000000,
48.0000000000,
50.0000000000,
42.0000000000,
43.0000000000,
42.0000000000,
60.0000000000];
let val = &[
45.0000000000,
43.0000000000,
44.0000000000,
61.0000000000,
51.0000000000,
53.0000000000,
59.0000000000,
52.0000000000,
49.0000000000,
51.0000000000,
51.0000000000,
50.0000000000,
49.0000000000,
56.0000000000,
42.0000000000,
52.0000000000,
51.0000000000,
43.0000000000,
48.0000000000,
48.0000000000,
50.0000000000,
42.0000000000,
43.0000000000,
42.0000000000,
60.0000000000,
];
let summ = &Summary {
sum: 1235.0000000000,
min: 42.0000000000,
@@ -826,31 +850,33 @@ fn test_pois25lambda50() {
}
#[test]
fn test_unif25() {
let val = &[99.0000000000,
55.0000000000,
92.0000000000,
79.0000000000,
14.0000000000,
2.0000000000,
33.0000000000,
49.0000000000,
3.0000000000,
32.0000000000,
84.0000000000,
59.0000000000,
22.0000000000,
86.0000000000,
76.0000000000,
31.0000000000,
29.0000000000,
11.0000000000,
41.0000000000,
53.0000000000,
45.0000000000,
44.0000000000,
98.0000000000,
98.0000000000,
7.0000000000];
let val = &[
99.0000000000,
55.0000000000,
92.0000000000,
79.0000000000,
14.0000000000,
2.0000000000,
33.0000000000,
49.0000000000,
3.0000000000,
32.0000000000,
84.0000000000,
59.0000000000,
22.0000000000,
86.0000000000,
76.0000000000,
31.0000000000,
29.0000000000,
11.0000000000,
41.0000000000,
53.0000000000,
45.0000000000,
44.0000000000,
98.0000000000,
98.0000000000,
7.0000000000,
];
let summ = &Summary {
sum: 1242.0000000000,
min: 2.0000000000,
@@ -885,18 +911,14 @@ mod bench {
#[bench]
pub fn sum_three_items(b: &mut Bencher) {
b.iter(|| {
[1e20f64, 1.5f64, -1e20f64].sum();
})
b.iter(|| { [1e20f64, 1.5f64, -1e20f64].sum(); })
}
#[bench]
pub fn sum_many_f64(b: &mut Bencher) {
let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60];
let v = (0..500).map(|i| nums[i % 5]).collect::<Vec<_>>();
b.iter(|| {
v.sum();
})
b.iter(|| { v.sum(); })
}
#[bench]
src/test/run-make/libtest-json/Makefile
+14
View File
@@ -0,0 +1,14 @@
-include ../tools.mk
# Test expected libtest's JSON output
OUTPUT_FILE := $(TMPDIR)/libtest-json-output.json
all:
$(RUSTC) --test f.rs
$(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE) || true
cat $(OUTPUT_FILE) | "$(PYTHON)" validate_json.py
# Compare to output file
diff output.json $(OUTPUT_FILE)
src/test/run-make/libtest-json/f.rs
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#[test]
fn a() {
// Should pass
}
#[test]
fn b() {
assert!(false)
}
#[test]
#[should_panic]
fn c() {
assert!(false);
}
#[test]
#[ignore]
fn d() {
assert!(false);
}
src/test/run-make/libtest-json/output.json
+10
View File
@@ -0,0 +1,10 @@
{ "type": "suite", "event": "started", "test_count": "4" }
{ "type": "test", "event": "started", "name": "a" }
{ "type": "test", "name": "a", "event": "ok" }
{ "type": "test", "event": "started", "name": "b" }
{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:5\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" }
{ "type": "test", "event": "started", "name": "c" }
{ "type": "test", "name": "c", "event": "ok" }
{ "type": "test", "event": "started", "name": "d" }
{ "type": "test", "name": "d", "event": "ignored" }
{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": "0" }
src/test/run-make/libtest-json/validate_json.py
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env python
# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import sys
import json
# Try to decode line in order to ensure it is a valid JSON document
for line in sys.stdin:
json.loads(line)
src/tools/cargo
+1 -1
Submodule src/tools/cargo updated: 91e36aa86c...1d6dfea44f
src/tools/compiletest/src/main.rs
+1 -1
View File
@@ -485,7 +485,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
filter: config.filter.clone(),
filter_exact: config.filter_exact,
run_ignored: config.run_ignored,
quiet: config.quiet,
format: if config.quiet { test::OutputFormat::Terse } else { test::OutputFormat::Pretty },
logfile: config.logfile.clone(),
run_tests: true,
bench_benchmarks: true,