libtest_mimic/
lib.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
//! Write your own tests and benchmarks that look and behave like built-in tests!
//!
//! This is a simple and small test harness that mimics the original `libtest`
//! (used by `cargo test`/`rustc --test`). That means: all output looks pretty
//! much like `cargo test` and most CLI arguments are understood and used. With
//! that plumbing work out of the way, your test runner can focus on the actual
//! testing.
//!
//! For a small real world example, see [`examples/tidy.rs`][1].
//!
//! [1]: https://github.com/LukasKalbertodt/libtest-mimic/blob/master/examples/tidy.rs
//!
//! # Usage
//!
//! To use this, you most likely want to add a manual `[[test]]` section to
//! `Cargo.toml` and set `harness = false`. For example:
//!
//! ```toml
//! [[test]]
//! name = "mytest"
//! path = "tests/mytest.rs"
//! harness = false
//! ```
//!
//! And in `tests/mytest.rs` you would call [`run`] in the `main` function:
//!
//! ```no_run
//! use libtest_mimic::{Arguments, Trial};
//!
//!
//! // Parse command line arguments
//! let args = Arguments::from_args();
//!
//! // Create a list of tests and/or benchmarks (in this case: two dummy tests).
//! let tests = vec![
//!     Trial::test("succeeding_test", move || Ok(())),
//!     Trial::test("failing_test", move || Err("Woops".into())),
//! ];
//!
//! // Run all tests and exit the application appropriatly.
//! libtest_mimic::run(&args, tests).exit();
//! ```
//!
//! Instead of returning `Ok` or `Err` directly, you want to actually perform
//! your tests, of course. See [`Trial::test`] for more information on how to
//! define a test. You can of course list all your tests manually. But in many
//! cases it is useful to generate one test per file in a directory, for
//! example.
//!
//! You can then run `cargo test --test mytest` to run it. To see the CLI
//! arguments supported by this crate, run `cargo test --test mytest -- -h`.
//!
//!
//! # Known limitations and differences to the official test harness
//!
//! `libtest-mimic` works on a best-effort basis: it tries to be as close to
//! `libtest` as possible, but there are differences for a variety of reasons.
//! For example, some rarely used features might not be implemented, some
//! features are extremely difficult to implement, and removing minor,
//! unimportant differences is just not worth the hassle.
//!
//! Some of the notable differences:
//!
//! - Output capture and `--nocapture`: simply not supported. The official
//!   `libtest` uses internal `std` functions to temporarily redirect output.
//!   `libtest-mimic` cannot use those. See [this issue][capture] for more
//!   information.
//! - `--format=junit`
//! - Also see [#13](https://github.com/LukasKalbertodt/libtest-mimic/issues/13)
//!
//! [capture]: https://github.com/LukasKalbertodt/libtest-mimic/issues/9

#![forbid(unsafe_code)]

use std::{
    borrow::Cow,
    fmt,
    process::{self, ExitCode},
    sync::{mpsc, Mutex},
    thread,
    time::Instant,
};

mod args;
mod printer;

use printer::Printer;

pub use crate::args::{Arguments, ColorSetting, FormatSetting};



/// A single test or benchmark.
///
/// The original `libtest` often calls benchmarks "tests", which is a bit
/// confusing. So in this library, it is called "trial".
///
/// A trial is created via [`Trial::test`] or [`Trial::bench`]. The trial's
/// `name` is printed and used for filtering. The `runner` is called when the
/// test/benchmark is executed to determine its outcome. If `runner` panics,
/// the trial is considered "failed". If you need the behavior of
/// `#[should_panic]` you need to catch the panic yourself. You likely want to
/// compare the panic payload to an expected value anyway.
pub struct Trial {
    runner: Box<dyn FnOnce(bool) -> Outcome + Send>,
    info: TestInfo,
}

impl Trial {
    /// Creates a (non-benchmark) test with the given name and runner.
    ///
    /// The runner returning `Ok(())` is interpreted as the test passing. If the
    /// runner returns `Err(_)`, the test is considered failed.
    pub fn test<R>(name: impl Into<String>, runner: R) -> Self
    where
        R: FnOnce() -> Result<(), Failed> + Send + 'static,
    {
        Self {
            runner: Box::new(move |_test_mode| match runner() {
                Ok(()) => Outcome::Passed,
                Err(failed) => Outcome::Failed(failed),
            }),
            info: TestInfo {
                name: name.into(),
                kind: String::new(),
                is_ignored: false,
                is_bench: false,
            },
        }
    }

    /// Creates a benchmark with the given name and runner.
    ///
    /// If the runner's parameter `test_mode` is `true`, the runner function
    /// should run all code just once, without measuring, just to make sure it
    /// does not panic. If the parameter is `false`, it should perform the
    /// actual benchmark. If `test_mode` is `true` you may return `Ok(None)`,
    /// but if it's `false`, you have to return a `Measurement`, or else the
    /// benchmark is considered a failure.
    ///
    /// `test_mode` is `true` if neither `--bench` nor `--test` are set, and
    /// `false` when `--bench` is set. If `--test` is set, benchmarks are not
    /// ran at all, and both flags cannot be set at the same time.
    pub fn bench<R>(name: impl Into<String>, runner: R) -> Self
    where
        R: FnOnce(bool) -> Result<Option<Measurement>, Failed> + Send + 'static,
    {
        Self {
            runner: Box::new(move |test_mode| match runner(test_mode) {
                Err(failed) => Outcome::Failed(failed),
                Ok(_) if test_mode => Outcome::Passed,
                Ok(Some(measurement)) => Outcome::Measured(measurement),
                Ok(None)
                    => Outcome::Failed("bench runner returned `Ok(None)` in bench mode".into()),
            }),
            info: TestInfo {
                name: name.into(),
                kind: String::new(),
                is_ignored: false,
                is_bench: true,
            },
        }
    }

    /// Sets the "kind" of this test/benchmark. If this string is not
    /// empty, it is printed in brackets before the test name (e.g.
    /// `test [my-kind] test_name`). (Default: *empty*)
    ///
    /// This is the only extension to the original libtest.
    pub fn with_kind(self, kind: impl Into<String>) -> Self {
        Self {
            info: TestInfo {
                kind: kind.into(),
                ..self.info
            },
            ..self
        }
    }

    /// Sets whether or not this test is considered "ignored". (Default: `false`)
    ///
    /// With the built-in test suite, you can annotate `#[ignore]` on tests to
    /// not execute them by default (for example because they take a long time
    /// or require a special environment). If the `--ignored` flag is set,
    /// ignored tests are executed, too.
    pub fn with_ignored_flag(self, is_ignored: bool) -> Self {
        Self {
            info: TestInfo {
                is_ignored,
                ..self.info
            },
            ..self
        }
    }

    /// Returns the name of this trial.
    pub fn name(&self) -> &str {
        &self.info.name
    }

    /// Returns the kind of this trial. If you have not set a kind, this is an
    /// empty string.
    pub fn kind(&self) -> &str {
        &self.info.kind
    }

    /// Returns whether this trial has been marked as *ignored*.
    pub fn has_ignored_flag(&self) -> bool {
        self.info.is_ignored
    }

    /// Returns `true` iff this trial is a test (as opposed to a benchmark).
    pub fn is_test(&self) -> bool {
        !self.info.is_bench
    }

    /// Returns `true` iff this trial is a benchmark (as opposed to a test).
    pub fn is_bench(&self) -> bool {
        self.info.is_bench
    }
}

impl fmt::Debug for Trial {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        struct OpaqueRunner;
        impl fmt::Debug for OpaqueRunner {
            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                f.write_str("<runner>")
            }
        }

        f.debug_struct("Test")
            .field("runner", &OpaqueRunner)
            .field("name", &self.info.name)
            .field("kind", &self.info.kind)
            .field("is_ignored", &self.info.is_ignored)
            .field("is_bench", &self.info.is_bench)
            .finish()
    }
}

#[derive(Debug)]
struct TestInfo {
    name: String,
    kind: String,
    is_ignored: bool,
    is_bench: bool,
}

impl TestInfo {
    fn test_name_with_kind(&self) -> Cow<'_, str> {
        if self.kind.is_empty() {
            Cow::Borrowed(&self.name)
        } else {
            Cow::Owned(format!("[{}] {}", self.kind, self.name))
        }
    }
}

/// Output of a benchmark.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Measurement {
    /// Average time in ns.
    pub avg: u64,

    /// Variance in ns.
    pub variance: u64,
}

/// Indicates that a test/benchmark has failed. Optionally carries a message.
///
/// You usually want to use the `From` impl of this type, which allows you to
/// convert any `T: fmt::Display` (e.g. `String`, `&str`, ...) into `Failed`.
#[derive(Debug, Clone)]
pub struct Failed {
    msg: Option<String>,
}

impl Failed {
    /// Creates an instance without message.
    pub fn without_message() -> Self {
        Self { msg: None }
    }

    /// Returns the message of this instance.
    pub fn message(&self) -> Option<&str> {
        self.msg.as_deref()
    }
}

impl<M: std::fmt::Display> From<M> for Failed {
    fn from(msg: M) -> Self {
        Self {
            msg: Some(msg.to_string())
        }
    }
}



/// The outcome of performing a test/benchmark.
#[derive(Debug, Clone)]
enum Outcome {
    /// The test passed.
    Passed,

    /// The test or benchmark failed.
    Failed(Failed),

    /// The test or benchmark was ignored.
    Ignored,

    /// The benchmark was successfully run.
    Measured(Measurement),
}

/// Contains information about the entire test run. Is returned by[`run`].
///
/// This type is marked as `#[must_use]`. Usually, you just call
/// [`exit()`][Conclusion::exit] on the result of `run` to exit the application
/// with the correct exit code. But you can also store this value and inspect
/// its data.
#[derive(Clone, Debug, PartialEq, Eq)]
#[must_use = "Call `exit()` or `exit_if_failed()` to set the correct return code"]
pub struct Conclusion {
    /// Number of tests and benchmarks that were filtered out (either by the
    /// filter-in pattern or by `--skip` arguments).
    pub num_filtered_out: u64,

    /// Number of passed tests.
    pub num_passed: u64,

    /// Number of failed tests and benchmarks.
    pub num_failed: u64,

    /// Number of ignored tests and benchmarks.
    pub num_ignored: u64,

    /// Number of benchmarks that successfully ran.
    pub num_measured: u64,
}

impl Conclusion {
    /// Returns an exit code that can be returned from `main` to signal
    /// success/failure to the calling process.
    pub fn exit_code(&self) -> ExitCode {
        if self.has_failed() {
            ExitCode::from(101)
        } else {
            ExitCode::SUCCESS
        }
    }

    /// Returns whether there have been any failures.
    pub fn has_failed(&self) -> bool {
        self.num_failed > 0
    }

    /// Exits the application with an appropriate error code (0 if all tests
    /// have passed, 101 if there have been failures). This uses
    /// [`process::exit`], meaning that destructors are not ran. Consider
    /// using [`Self::exit_code`] instead for a proper program cleanup.
    pub fn exit(&self) -> ! {
        self.exit_if_failed();
        process::exit(0);
    }

    /// Exits the application with error code 101 if there were any failures.
    /// Otherwise, returns normally. This uses [`process::exit`], meaning that
    /// destructors are not ran. Consider using [`Self::exit_code`] instead for
    /// a proper program cleanup.
    pub fn exit_if_failed(&self) {
        if self.has_failed() {
            process::exit(101)
        }
    }

    fn empty() -> Self {
        Self {
            num_filtered_out: 0,
            num_passed: 0,
            num_failed: 0,
            num_ignored: 0,
            num_measured: 0,
        }
    }
}

impl Arguments {
    /// Returns `true` if the given test should be ignored.
    fn is_ignored(&self, test: &Trial) -> bool {
        (test.info.is_ignored && !self.ignored && !self.include_ignored)
            || (test.info.is_bench && self.test)
            || (!test.info.is_bench && self.bench)
    }

    fn is_filtered_out(&self, test: &Trial) -> bool {
        let test_name = test.name();
        // Match against the full test name, including the kind. This upholds the invariant that if
        // --list prints out:
        //
        // <some string>: test
        //
        // then "--exact <some string>" runs exactly that test.
        let test_name_with_kind = test.info.test_name_with_kind();

        // If a filter was specified, apply this
        if let Some(filter) = &self.filter {
            match self.exact {
                // For exact matches, we want to match against either the test name (to maintain
                // backwards compatibility with older versions of libtest-mimic), or the test kind
                // (technically more correct with respect to matching against the output of --list.)
                true if test_name != filter && &test_name_with_kind != filter => return true,
                false if !test_name_with_kind.contains(filter) => return true,
                _ => {}
            };
        }

        // If any skip pattern were specified, test for all patterns.
        for skip_filter in &self.skip {
            match self.exact {
                // For exact matches, we want to match against either the test name (to maintain
                // backwards compatibility with older versions of libtest-mimic), or the test kind
                // (technically more correct with respect to matching against the output of --list.)
                true if test_name == skip_filter || &test_name_with_kind == skip_filter => {
                    return true
                }
                false if test_name_with_kind.contains(skip_filter) => return true,
                _ => {}
            }
        }

        if self.ignored && !test.info.is_ignored {
            return true;
        }

        false
    }
}

/// Runs all given trials (tests & benchmarks).
///
/// This is the central function of this crate. It provides the framework for
/// the testing harness. It does all the printing and house keeping.
///
/// The returned value contains a couple of useful information. See
/// [`Conclusion`] for more information. If `--list` was specified, a list is
/// printed and a dummy `Conclusion` is returned.
pub fn run(args: &Arguments, mut tests: Vec<Trial>) -> Conclusion {
    let start_instant = Instant::now();
    let mut conclusion = Conclusion::empty();

    // Apply filtering
    if args.filter.is_some() || !args.skip.is_empty() || args.ignored {
        let len_before = tests.len() as u64;
        tests.retain(|test| !args.is_filtered_out(test));
        conclusion.num_filtered_out = len_before - tests.len() as u64;
    }
    let tests = tests;

    // Create printer which is used for all output.
    let mut printer = printer::Printer::new(args, &tests);

    // If `--list` is specified, just print the list and return.
    if args.list {
        printer.print_list(&tests, args.ignored);
        return Conclusion::empty();
    }

    // Print number of tests
    printer.print_title(tests.len() as u64);

    let mut failed_tests = Vec::new();
    let mut handle_outcome = |outcome: Outcome, test: TestInfo, printer: &mut Printer| {
        printer.print_single_outcome(&test, &outcome);

        // Handle outcome
        match outcome {
            Outcome::Passed => conclusion.num_passed += 1,
            Outcome::Failed(failed) => {
                failed_tests.push((test, failed.msg));
                conclusion.num_failed += 1;
            },
            Outcome::Ignored => conclusion.num_ignored += 1,
            Outcome::Measured(_) => conclusion.num_measured += 1,
        }
    };

    // Execute all tests.
    let test_mode = !args.bench;

    let num_threads = platform_defaults_to_one_thread()
        .then_some(1)
        .or(args.test_threads)
        .or_else(|| std::thread::available_parallelism().ok().map(Into::into))
        .unwrap_or(1);

    if num_threads == 1 {
        // Run test sequentially in main thread
        for test in tests {
            // Print `test foo    ...`, run the test, then print the outcome in
            // the same line.
            printer.print_test(&test.info);
            let outcome = if args.is_ignored(&test) {
                Outcome::Ignored
            } else {
                run_single(test.runner, test_mode)
            };
            handle_outcome(outcome, test.info, &mut printer);
        }
    } else {
        // Run test in thread pool.
        let (sender, receiver) = mpsc::channel();

        let num_tests = tests.len();
        // TODO: this should use a mpmc channel, once that's stabilized in std.
        let iter = Mutex::new(tests.into_iter());
        thread::scope(|scope| {
            // Start worker threads
            for _ in 0..num_threads {
                scope.spawn(|| {
                    loop {
                        // Get next test to process from the iterator.
                        let Some(trial) = iter.lock().unwrap().next() else {
                            break;
                        };

                        let payload = if args.is_ignored(&trial) {
                            (Outcome::Ignored, trial.info)
                        } else {
                            let outcome = run_single(trial.runner, test_mode);
                            (outcome, trial.info)
                        };

                        // It's fine to ignore the result of sending. If the
                        // receiver has hung up, everything will wind down soon
                        // anyway.
                        let _ = sender.send(payload);
                    }
                });
            }

            // Print results of tests that already dinished
            for (outcome, test_info) in receiver.iter().take(num_tests) {
                // In multithreaded mode, we do only print the start of the line
                // after the test ran, as otherwise it would lead to terribly
                // interleaved output.
                printer.print_test(&test_info);
                handle_outcome(outcome, test_info, &mut printer);
            }
        });

    }

    // Print failures if there were any, and the final summary.
    if !failed_tests.is_empty() {
        printer.print_failures(&failed_tests);
    }

    printer.print_summary(&conclusion, start_instant.elapsed());

    conclusion
}

/// Returns whether the current host platform should use a single thread by
/// default rather than a thread pool by default. Some platforms, such as
/// WebAssembly, don't have native support for threading at this time.
fn platform_defaults_to_one_thread() -> bool {
    cfg!(target_family = "wasm")
}

/// Runs the given runner, catching any panics and treating them as a failed test.
fn run_single(runner: Box<dyn FnOnce(bool) -> Outcome + Send>, test_mode: bool) -> Outcome {
    use std::panic::{catch_unwind, AssertUnwindSafe};

    catch_unwind(AssertUnwindSafe(move || runner(test_mode))).unwrap_or_else(|e| {
        // The `panic` information is just an `Any` object representing the
        // value the panic was invoked with. For most panics (which use
        // `panic!` like `println!`), this is either `&str` or `String`.
        let payload = e.downcast_ref::<String>()
            .map(|s| s.as_str())
            .or(e.downcast_ref::<&str>().map(|s| *s));

        let msg = match payload {
            Some(payload) => format!("test panicked: {payload}"),
            None => format!("test panicked"),
        };
        Outcome::Failed(msg.into())
    })
}