tiny_bench/output/analysis/
criterion.rs

1//! Everything in this module is more or less copied from [criterion.rs](https://github.com/bheisler/criterion.rs)
2//! with some rewrites to make it fit, the license is included in this file's directory
3use crate::output::analysis::random::Rng;
4use crate::output::wrap_yellow;
5use std::time::Duration;
6
7/// Struct containing all of the configuration options for a benchmark.
8pub struct BenchmarkConfig {
9    /// How long the bench 'should' run, `num_samples` is prioritized so benching will take
10    /// longer to be able to collect `num_samples` if the code to be benched is slower
11    /// than this time limit allowed.
12    pub measurement_time: Duration,
13    /// How many resamples should be done
14    pub num_resamples: usize,
15    /// Recommended at least 50, above 100 <https://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Recommendations>
16    /// doesn't seem to yield a significantly different result
17    pub num_samples: usize,
18    /// How long the bench should warm up
19    pub warm_up_time: Duration,
20    /// Puts results in target/tiny-bench/label/.. if target can be found.
21    /// used for comparing previous runs
22    pub dump_results_to_disk: bool,
23
24    /// Sets a hard ceiling on max iterations, overriding the heuristic calculations for iteration
25    /// count. A rule of thumb; if this is used, the results are unlikely to be statistically
26    /// significant.
27    pub max_iterations: Option<u64>,
28}
29
30impl Default for BenchmarkConfig {
31    fn default() -> Self {
32        BenchmarkConfig {
33            measurement_time: Duration::from_secs(5),
34            num_resamples: 100_000,
35            num_samples: 100,
36            warm_up_time: Duration::from_secs(3),
37            dump_results_to_disk: true,
38            max_iterations: None,
39        }
40    }
41}
42
43pub(crate) fn calculate_iterations(
44    warmup_mean_execution_time: f64,
45    num_samples: u64,
46    target_time: Duration,
47) -> Vec<u64> {
48    let met = warmup_mean_execution_time;
49    let m_ns = target_time.as_nanos();
50    // Solve: [d + 2*d + 3*d + ... + n*d] * met = m_ns
51
52    let total_runs = num_samples * (num_samples + 1) / 2;
53    let d = ((m_ns as f64 / met / total_runs as f64).ceil() as u64).max(1);
54    let expected_nanoseconds = total_runs as f64 * d as f64 * met;
55    if d == 1 {
56        let actual_time = Duration::from_nanos(expected_nanoseconds as u64);
57        println!(
58            "{} You may wish to increase target time to {:.1?} or lower the requested number of samples",
59            wrap_yellow(&format!(
60                "Unable to complete {num_samples} samples in {target_time:.1?}"
61            )),
62            actual_time
63        );
64    }
65
66    (1..=num_samples).map(|a| a * d).collect()
67}
68
69pub(crate) fn calculate_t_value(sample_a: &[f64], sample_b: &[f64]) -> f64 {
70    let a_mean = calculate_mean(sample_a);
71    let b_mean = calculate_mean(sample_b);
72    let a_var = calculate_variance(sample_a, a_mean);
73    let b_var = calculate_variance(sample_b, b_mean);
74    let a_len = sample_a.len() as f64;
75    let b_len = sample_b.len() as f64;
76    let mean_diff = a_mean - b_mean;
77    let d = (a_var / a_len + b_var / b_len).sqrt();
78    mean_diff / d
79}
80
81pub(crate) fn calculate_mean(a: &[f64]) -> f64 {
82    a.iter().sum::<f64>() / a.len() as f64
83}
84
85pub(crate) fn calculate_variance(sample: &[f64], mean: f64) -> f64 {
86    let sum = sample
87        .iter()
88        .copied()
89        .map(|val| (val - mean).powi(2))
90        .sum::<f64>();
91    sum / (sample.len() as f64 - 1f64) // use n - 1 when measuring variance from a sample
92}
93
94pub(crate) fn resample(sample_a: &[f64], sample_b: &[f64], times: usize) -> Vec<f64> {
95    let a_len = sample_a.len();
96    let mut combined = Vec::with_capacity(a_len + sample_b.len());
97    combined.extend_from_slice(sample_a);
98    combined.extend_from_slice(sample_b);
99    let mut rng = Rng::new();
100    let combined_len = combined.len();
101    let mut distributions = Vec::new();
102    for _ in 0..times {
103        let mut sample = Vec::with_capacity(combined_len);
104        for _ in 0..combined_len {
105            let index = (rng.next() % combined.len() as u64) as usize;
106            sample.push(combined[index]);
107        }
108        let sample_a = Vec::from(&sample[..a_len]);
109        let sample_b = Vec::from(&sample[a_len..]);
110        let t = calculate_t_value(&sample_a, &sample_b);
111        distributions.push(t);
112    }
113    distributions
114}
115
116pub(crate) fn calculate_p_value(total_t: f64, distribution: &[f64]) -> f64 {
117    let hits = distribution.iter().filter(|x| x < &&total_t).count();
118    let tails = 2; // I don't know what this is, Two-tailed significance testing something something
119    let min = std::cmp::min(hits, distribution.len() - hits);
120    (min * tails) as f64 / distribution.len() as f64
121}
122
123#[inline]
124pub(crate) fn calculate_median(sample: &mut [f64]) -> f64 {
125    sample.sort_by(f64::total_cmp);
126    sample.get(sample.len() / 2).copied().unwrap_or_default()
127}
128
129pub(crate) struct SamplingDataSimpleAnalysis {
130    pub(crate) elapsed: u128,
131    pub(crate) min: f64,
132    pub(crate) max: f64,
133    pub(crate) average: f64,
134    pub(crate) median: f64,
135    pub(crate) variance: f64,
136    pub(crate) stddev: f64,
137    pub(crate) per_sample_average: Vec<f64>,
138}
139
140#[cfg(test)]
141mod tests {
142    use crate::output::analysis::criterion::{
143        calculate_mean, calculate_t_value, calculate_variance,
144    };
145
146    #[test]
147    fn calculates_mean() {
148        let data = vec![46.0, 69.0, 32.0, 60.0, 52.0, 41.0];
149        assert!(calculate_mean(&data) - 50.0 < 0.0000_001);
150    }
151
152    #[test]
153    fn calculates_variance() {
154        let data = vec![46.0, 69.0, 32.0, 60.0, 52.0, 41.0];
155        assert!(calculate_variance(&data, 50.0) - 177.2 < 0.00001);
156    }
157
158    #[test]
159    fn calculate_t() {
160        let sample_a = vec![19.7, 20.4, 19.6, 17.8, 18.5, 18.9, 18.3, 18.9, 19.5, 21.95];
161        let sample_b = vec![
162            28.3, 26.7, 20.1, 23.3, 25.2, 22.1, 17.7, 27.6, 20.6, 13.7, 23.2, 17.5, 20.6, 18.0,
163            23.9, 21.6, 24.3, 20.4, 23.9, 13.3,
164        ];
165        assert!(calculate_t_value(&sample_a, &sample_b).abs() - 2.24787 < 0.0001);
166    }
167}
tiny_bench/output/analysis/criterion.rs

tiny_bench/output/analysis/
criterion.rs
