1
use std::{
2
    collections::{BTreeMap, HashMap},
3
    ops::{Range, RangeInclusive},
4
    path::Path,
5
    sync::Arc,
6
    time::{Duration, Instant},
7
};
8

            
9
use bonsaidb::core::async_trait::async_trait;
10
use cli_table::{Cell, Table};
11
use futures::{stream::FuturesUnordered, StreamExt};
12
use plotters::{
13
    coord::ranged1d::{NoDefaultFormatting, ValueFormatter},
14
    element::{BackendCoordOnly, CoordMapper, Drawable, PointCollection},
15
    prelude::*,
16
};
17
use plotters_backend::DrawingErrorKind;
18
use rand::{rngs::SmallRng, SeedableRng};
19
use serde::{Deserialize, Serialize};
20
use tera::Tera;
21
use tokio::runtime::Runtime;
22

            
23
use crate::{
24
    bonsai::{Bonsai, BonsaiBackend},
25
    model::{InitialDataSet, InitialDataSetConfig},
26
    plan::{
27
        AddProductToCart, Checkout, CreateCart, FindProduct, Load, LookupProduct, Operation,
28
        OperationResult, Plan, ReviewProduct, ShopperPlanConfig,
29
    },
30
    plot::{label_to_color, BACKGROUND_COLOR, TEXT_COLOR},
31
    utils::{current_timestamp_string, format_nanoseconds, local_git_rev},
32
};
33

            
34
4
pub fn execute_plans_for_all_backends(
35
4
    name_filter: &str,
36
4
    plans: &[Arc<Plan>],
37
4
    initial_data: &Arc<InitialDataSet>,
38
4
    number_of_agents: usize,
39
4
    measurements: &Measurements,
40
4
) {
41
4
    if name_filter.is_empty()
42
        || name_filter == "bonsaidb"
43
        || name_filter.starts_with("bonsaidb-local")
44
4
    {
45
4
        println!("Executing bonsaidb-local");
46
4
        BonsaiBackend::execute_async(
47
4
            Bonsai::Local,
48
4
            plans,
49
4
            initial_data,
50
4
            number_of_agents,
51
4
            measurements,
52
4
        );
53
4
    }
54
4
    if name_filter.is_empty()
55
        || name_filter == "bonsaidb"
56
        || name_filter.starts_with("bonsaidb-local+lz4")
57
4
    {
58
4
        println!("Executing bonsaidb-local+lz4");
59
4
        BonsaiBackend::execute_async(
60
4
            Bonsai::LocalLz4,
61
4
            plans,
62
4
            initial_data,
63
4
            number_of_agents,
64
4
            measurements,
65
4
        );
66
4
    }
67
4
    if name_filter.is_empty()
68
        || name_filter == "bonsaidb"
69
        || name_filter.starts_with("bonsaidb-quic")
70
4
    {
71
4
        println!("Executing bonsaidb-quic");
72
4
        BonsaiBackend::execute_async(
73
4
            Bonsai::Quic,
74
4
            plans,
75
4
            initial_data,
76
4
            number_of_agents,
77
4
            measurements,
78
4
        );
79
4
    }
80
4
    if name_filter.is_empty() || name_filter == "bonsaidb" || name_filter.starts_with("bonsaidb-ws")
81
4
    {
82
4
        println!("Executing bonsaidb-ws");
83
4
        BonsaiBackend::execute_async(
84
4
            Bonsai::WebSockets,
85
4
            plans,
86
4
            initial_data,
87
4
            number_of_agents,
88
4
            measurements,
89
4
        );
90
4
    }
91
    #[cfg(feature = "postgresql")]
92
4
    if name_filter.is_empty() || name_filter.starts_with("postgresql") {
93
4
        if let Ok(url) = std::env::var("COMMERCE_POSTGRESQL_URL") {
94
4
            println!("Executing postgresql");
95
4
            crate::postgres::Postgres::execute_async(
96
4
                url,
97
4
                plans,
98
4
                initial_data,
99
4
                number_of_agents,
100
4
                measurements,
101
4
            );
102
4
        } else {
103
            eprintln!("postgresql feature is enabled, but environment variable COMMERCE_POSTGRESQL_URL is missing.");
104
        }
105
    }
106
4
}
107

            
108
#[async_trait]
109
pub trait Backend: Sized + Send + Sync + 'static {
110
    type Operator: BackendOperator;
111
    type Config: Send + Sync;
112

            
113
    async fn new(config: Self::Config) -> Self;
114

            
115
    fn label(&self) -> &'static str;
116

            
117
20
    fn execute_async(
118
20
        config: Self::Config,
119
20
        plans: &[Arc<Plan>],
120
20
        initial_data: &Arc<InitialDataSet>,
121
20
        concurrent_agents: usize,
122
20
        measurements: &Measurements,
123
20
    ) {
124
20
        let (plan_sender, plan_receiver) = flume::bounded(concurrent_agents * 2);
125
20
        let runtime = Runtime::new().unwrap();
126
20
        let backend = runtime.block_on(Self::new(config));
127
20
        // Load the initial data
128
20
        println!("Loading data");
129
20
        runtime.block_on(async {
130
20
            let _ = backend
131
20
                .new_operator_async()
132
16
                .await
133
20
                .operate(
134
20
                    &Load {
135
20
                        initial_data: initial_data.clone(),
136
20
                    },
137
20
                    &[],
138
20
                    measurements,
139
12287
                )
140
12287
                .await;
141
20
        });
142
20
        println!("Executing plans");
143
20
        let agent_handles = FuturesUnordered::new();
144
40
        for _ in 0..concurrent_agents {
145
40
            let operator = runtime.block_on(backend.new_operator_async());
146
40
            agent_handles.push(runtime.spawn(agent::<Self>(
147
40
                operator,
148
40
                plan_receiver.clone(),
149
40
                measurements.clone(),
150
40
            )));
151
40
        }
152
20
        runtime.block_on(async {
153
            // Send the plans to the channel that the agents are waiting for
154
            // them on.
155
2800
            for plan in plans {
156
2780
                plan_sender.send_async(plan.clone()).await.unwrap();
157
            }
158
            // Disconnect the receivers, allowing the agents to exit once there
159
            // are no more plans in queue.
160
20
            drop(plan_sender);
161
            // Wait for each of the agents to return.
162
60
            for result in agent_handles.collect::<Vec<_>>().await {
163
40
                result.unwrap();
164
40
            }
165
20
        })
166
20
    }
167

            
168
    async fn new_operator_async(&self) -> Self::Operator;
169
}
170

            
171
#[async_trait]
172
pub trait Operator<T> {
173
    async fn operate(
174
        &mut self,
175
        operation: &T,
176
        results: &[OperationResult],
177
        measurements: &Measurements,
178
    ) -> OperationResult;
179
}
180

            
181
40
async fn agent<B: Backend>(
182
40
    mut operator: B::Operator,
183
40
    plan_receiver: flume::Receiver<Arc<Plan>>,
184
40
    measurements: Measurements,
185
40
) {
186
2820
    while let Ok(plan) = plan_receiver.recv_async().await {
187
2780
        let mut results = Vec::with_capacity(plan.operations.len());
188
30360
        for step in &plan.operations {
189
77402
            results.push(operator.operate(step, &results, &measurements).await)
190
        }
191
    }
192
40
}
193

            
194
pub trait BackendOperator:
195
    Operator<Load>
196
    + Operator<LookupProduct>
197
    + Operator<FindProduct>
198
    + Operator<CreateCart>
199
    + Operator<AddProductToCart>
200
    + Operator<ReviewProduct>
201
    + Operator<Checkout>
202
    + Send
203
    + Sync
204
{
205
}
206

            
207
#[async_trait]
208
impl<T> Operator<Operation> for T
209
where
210
    T: BackendOperator,
211
{
212
30360
    async fn operate(
213
30360
        &mut self,
214
30360
        operation: &Operation,
215
30360
        results: &[OperationResult],
216
30360
        measurements: &Measurements,
217
30360
    ) -> OperationResult {
218
30360
        match operation {
219
31163
            Operation::FindProduct(op) => self.operate(op, results, measurements).await,
220
22262
            Operation::LookupProduct(op) => self.operate(op, results, measurements).await,
221
4660
            Operation::CreateCart(op) => self.operate(op, results, measurements).await,
222
15702
            Operation::AddProductToCart(op) => self.operate(op, results, measurements).await,
223
1452
            Operation::RateProduct(op) => self.operate(op, results, measurements).await,
224
2165
            Operation::Checkout(op) => self.operate(op, results, measurements).await,
225
        }
226
60720
    }
227
}
228

            
229
4
#[derive(Serialize, Deserialize, Debug)]
230
pub struct BenchmarkSummary {
231
    label: String,
232
    timestamp: String,
233
    revision: String,
234
    plan_count: usize,
235
    agent_count: usize,
236
    product_count: usize,
237
    category_count: usize,
238
    customer_count: usize,
239
    order_count: usize,
240
    summaries: Vec<BackendSummary>,
241
    operations: Vec<MetricSummary>,
242
}
243

            
244
impl BenchmarkSummary {
245
4
    pub fn render_to(&self, location: &Path, tera: &Tera) {
246
4
        std::fs::write(
247
4
            location,
248
4
            tera.render("run.html", &tera::Context::from_serialize(self).unwrap())
249
4
                .unwrap()
250
4
                .as_bytes(),
251
4
        )
252
4
        .unwrap()
253
4
    }
254
}
255

            
256
20
#[derive(Serialize, Deserialize, Debug)]
257
pub struct BackendSummary {
258
    backend: String,
259
    transport: String,
260
    total_time: String,
261
    wall_time: String,
262
}
263

            
264
28
#[derive(Serialize, Deserialize, Debug)]
265
pub struct MetricSummary {
266
    metric: Metric,
267
    description: String,
268
    invocations: usize,
269
    summaries: Vec<OperationSummary>,
270
}
271

            
272
140
#[derive(Serialize, Deserialize, Debug)]
273
pub struct OperationSummary {
274
    backend: String,
275
    avg: String,
276
    min: String,
277
    max: String,
278
    stddev: String,
279
    outliers: String,
280
}
281

            
282
pub struct Benchmark<'a> {
283
    pub label: String,
284
    pub seed: Option<u64>,
285
    pub agents: Option<usize>,
286
    pub shoppers: Option<usize>,
287
    pub data_config: &'a InitialDataSetConfig,
288
    pub shopper_config: &'a ShopperPlanConfig,
289
}
290

            
291
impl<'a> Benchmark<'a> {
292
4
    pub fn execute(
293
4
        self,
294
4
        name_filter: &str,
295
4
        plot_dir: impl AsRef<Path>,
296
4
        tera: Arc<Tera>,
297
4
    ) -> BTreeMap<&'static str, Duration> {
298
4
        let plot_dir = plot_dir.as_ref().to_path_buf();
299
4
        std::fs::create_dir_all(&plot_dir).unwrap();
300

            
301
4
        let mut rng = if let Some(seed) = self.seed {
302
3
            SmallRng::seed_from_u64(seed)
303
        } else {
304
1
            SmallRng::from_entropy()
305
        };
306
4
        let initial_data = Arc::new(self.data_config.fake(&mut rng));
307
4
        let number_of_agents = self.agents.unwrap_or_else(num_cpus::get);
308
4
        let shoppers = self.shoppers.unwrap_or(number_of_agents * 100);
309
4

            
310
4
        println!(
311
4
            "Running {} plans across {} agents",
312
4
            shoppers, number_of_agents
313
4
        );
314
4
        // Generate plans to execute.
315
4
        let mut plans = Vec::with_capacity(shoppers as usize);
316
556
        for _ in 0..shoppers {
317
556
            plans.push(Arc::new(
318
556
                self.shopper_config.random_plan(&mut rng, &initial_data),
319
556
            ));
320
556
        }
321
        // Set up our statistics gathering thread
322
4
        let (metric_sender, metric_receiver) = flume::unbounded();
323
4
        let measurements = Measurements {
324
4
            sender: metric_sender,
325
4
        };
326
4
        let thread_initial_data = initial_data.clone();
327
4
        let stats_thread = std::thread::spawn(move || {
328
4
            stats_thread(
329
4
                self.label,
330
4
                metric_receiver,
331
4
                shoppers,
332
4
                number_of_agents,
333
4
                &thread_initial_data,
334
4
                &plot_dir,
335
4
                &tera,
336
4
            )
337
4
        });
338
4
        // Perform all benchmarks
339
4
        execute_plans_for_all_backends(
340
4
            name_filter,
341
4
            &plans,
342
4
            &initial_data,
343
4
            number_of_agents,
344
4
            &measurements,
345
4
        );
346
4
        // Drop the measurements instance to allow the stats thread to know
347
4
        // there are no more metrics coming.
348
4
        drop(measurements);
349
4
        // Wait for the statistics thread to report all the results.
350
4
        stats_thread.join().unwrap()
351
4
    }
352
}
353

            
354
40
#[derive(Clone)]
355
pub struct Measurements {
356
    sender: flume::Sender<(&'static str, Metric, Duration)>,
357
}
358

            
359
impl Measurements {
360
30380
    pub fn begin(&self, label: &'static str, metric: Metric) -> Measurement<'_> {
361
30380
        Measurement {
362
30380
            target: &self.sender,
363
30380
            label,
364
30380
            metric,
365
30380
            start: Instant::now(),
366
30380
        }
367
30380
    }
368
}
369

            
370
pub struct Measurement<'a> {
371
    target: &'a flume::Sender<(&'static str, Metric, Duration)>,
372
    label: &'static str,
373
    metric: Metric,
374
    start: Instant,
375
}
376

            
377
impl<'a> Measurement<'a> {
378
30380
    pub fn finish(self) {
379
30380
        let duration = Instant::now()
380
30380
            .checked_duration_since(self.start)
381
30380
            .expect("time went backwards. Restart benchmarks.");
382
30380
        self.target
383
30380
            .send((self.label, self.metric, duration))
384
30380
            .unwrap();
385
30380
    }
386
}
387

            
388
97899
#[derive(Serialize, Deserialize, Clone, Copy, Hash, Eq, PartialEq, Debug, Ord, PartialOrd)]
389
pub enum Metric {
390
    Load,
391
    LookupProduct,
392
    FindProduct,
393
    CreateCart,
394
    AddProductToCart,
395
    Checkout,
396
    RateProduct,
397
}
398

            
399
impl Metric {
400
28
    pub fn description(&self) -> &'static str {
401
28
        match self {
402
4
            Metric::Load => "Measures the time spent loading the initial data set and performing any pre-cache operations that most database administrators would perform on their databases periodically to ensure good performance.",
403
4
            Metric::LookupProduct => "Meaures the time spent looking up a product by its id. This operation is meant to simulate the basic needs of the database to provide a product details page after a user clicked a direct link that contians the product's unique id, including the product's current rating.",
404
4
            Metric::FindProduct => "Measures the time spent looking up a product by its name (exact match, indexed). This operation is meant to simulate the basic needs of the database to provide a product details after finding a product by its name, including the product's current rating.",
405
4
            Metric::CreateCart => "Measures the time spent creating a shopping cart.",
406
4
            Metric::AddProductToCart => "Measures the time spent adding a product to a shopping cart.",
407
4
            Metric::RateProduct => "Measures the time spent adding or updating a review of a product by a customer. Each customer can only have one review per product. When this operation is complete, all subsequent calls to LookupProduct and FindProduct should reflect the new rating. This simulates an 'upsert' (insert or update) operation using a unique index.",
408
4
            Metric::Checkout => "Measures the time spent converting a shopping cart into an order for a customer."
409
        }
410
28
    }
411
}
412

            
413
#[derive(Clone, Debug)]
414
struct NanosRange(RangeInclusive<Nanos>);
415
3092
#[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq)]
416
struct Nanos(u64);
417

            
418
impl ValueFormatter<Nanos> for NanosRange {
419
1680
    fn format(value: &Nanos) -> String {
420
1680
        format_nanoseconds(value.0 as f64)
421
1680
    }
422
}
423

            
424
impl Ranged for NanosRange {
425
    type ValueType = Nanos;
426
    type FormatOption = NoDefaultFormatting;
427

            
428
51896
    fn map(&self, value: &Self::ValueType, limit: (i32, i32)) -> i32 {
429
51896
        let limited_size = limit.1 - limit.0;
430
51896
        let full_size = self.0.end().0 + 1 - self.0.start().0;
431
51896
        let normalized_offset = value.0.saturating_sub(self.0.start().0) as f64 / full_size as f64;
432
51896
        limit.0 + (normalized_offset * limited_size as f64) as i32
433
51896
    }
434

            
435
336
    fn key_points<Hint: plotters::coord::ranged1d::KeyPointHint>(
436
336
        &self,
437
336
        hint: Hint,
438
336
    ) -> Vec<Self::ValueType> {
439
336
        let total_range = self.0.end().0 - self.0.start().0;
440
336
        let num_points = hint.max_num_points();
441
336
        let mut important_points = Vec::with_capacity(num_points);
442
336
        important_points.push(*self.0.start());
443
336
        if num_points > 2 {
444
336
            let steps = num_points - 2;
445
336
            let step_size = total_range as f64 / steps as f64;
446
336
            important_points.extend(
447
336
                (1..num_points - 1)
448
17808
                    .map(|step| Nanos(self.0.start().0 + (step as f64 * step_size) as u64)),
449
336
            );
450
336
        }
451
336
        important_points.push(*self.0.end());
452
336

            
453
336
        important_points
454
336
    }
455

            
456
    fn range(&self) -> std::ops::Range<Self::ValueType> {
457
        Nanos(self.0.start().0)..Nanos(self.0.end().0 + 1)
458
    }
459
}
460

            
461
impl DiscreteRanged for NanosRange {
462
    fn size(&self) -> usize {
463
        (self.0.end().0 - self.0.start().0) as usize
464
    }
465

            
466
    fn index_of(&self, value: &Self::ValueType) -> Option<usize> {
467
        if value.0 <= self.0.end().0 {
468
            if let Some(index) = value.0.checked_sub(self.0.start().0) {
469
                return Some(index as usize);
470
            }
471
        }
472
        None
473
    }
474

            
475
    fn from_index(&self, index: usize) -> Option<Self::ValueType> {
476
        Some(Nanos(self.0.start().0 + index as u64))
477
    }
478
}
479

            
480
4
fn stats_thread(
481
4
    label: String,
482
4
    metric_receiver: flume::Receiver<(&'static str, Metric, Duration)>,
483
4
    number_of_plans: usize,
484
4
    number_of_agents: usize,
485
4
    initial_data: &InitialDataSet,
486
4
    plot_dir: &Path,
487
4
    tera: &Tera,
488
4
) -> BTreeMap<&'static str, Duration> {
489
4
    let mut all_results: BTreeMap<Metric, BTreeMap<&'static str, Vec<u64>>> = BTreeMap::new();
490
4
    let mut accumulated_label_stats: BTreeMap<&'static str, Duration> = BTreeMap::new();
491
4
    let mut longest_by_metric = HashMap::new();
492
30384
    while let Ok((label, metric, duration)) = metric_receiver.recv() {
493
30380
        let metric_results = all_results.entry(metric).or_default();
494
30380
        let label_results = metric_results.entry(label).or_default();
495
30380
        let nanos = u64::try_from(duration.as_nanos()).unwrap();
496
30380
        label_results.push(nanos);
497
30380
        let label_duration = accumulated_label_stats.entry(label).or_default();
498
30380
        longest_by_metric
499
30380
            .entry(metric)
500
30380
            .and_modify(|existing: &mut Duration| {
501
30352
                *existing = (*existing).max(duration);
502
30380
            })
503
30380
            .or_insert(duration);
504
30380
        *label_duration += duration;
505
30380
    }
506

            
507
4
    let mut operations = BTreeMap::new();
508
4
    let mut metric_ranges = HashMap::new();
509
32
    for (metric, label_metrics) in all_results {
510
28
        let label_stats = label_metrics
511
28
            .iter()
512
140
            .map(|(label, stats)| {
513
140
                let mut sum = 0;
514
140
                let mut min = u64::MAX;
515
140
                let mut max = 0;
516
30520
                for &nanos in stats {
517
30380
                    sum += nanos;
518
30380
                    min = min.min(nanos);
519
30380
                    max = max.max(nanos);
520
30380
                }
521
140
                let average = sum as f64 / stats.len() as f64;
522
140
                let stddev = stddev(stats, average);
523
140

            
524
140
                let mut outliers = Vec::new();
525
140
                let mut plottable_stats = Vec::new();
526
140
                let mut min_plottable = u64::MAX;
527
140
                let mut max_plottable = 0;
528
30520
                for &nanos in stats {
529
30380
                    let diff = (nanos as f64 - average).abs();
530
30380
                    let diff_magnitude = diff / stddev;
531
30380
                    if stats.len() == 1 || diff_magnitude < 3. {
532
29937
                        plottable_stats.push(nanos);
533
29937
                        min_plottable = min_plottable.min(nanos);
534
29937
                        max_plottable = max_plottable.max(nanos);
535
29937
                    } else {
536
443
                        // Outlier
537
443
                        outliers.push(diff_magnitude);
538
443
                    }
539
                }
540

            
541
140
                if !outliers.is_empty() {
542
101
                    eprintln!("Not plotting {} outliers for {}", outliers.len(), label);
543
101
                }
544

            
545
140
                metric_ranges
546
140
                    .entry(metric)
547
140
                    .and_modify(|range: &mut Range<u64>| {
548
112
                        range.start = range.start.min(min_plottable);
549
112
                        range.end = range.end.max(max_plottable + 1);
550
140
                    })
551
140
                    .or_insert(min_plottable..max_plottable + 1);
552
140

            
553
140
                (
554
140
                    label,
555
140
                    MetricStats {
556
140
                        average,
557
140
                        min,
558
140
                        max,
559
140
                        stddev,
560
140
                        outliers,
561
140
                        plottable_stats,
562
140
                    },
563
140
                )
564
140
            })
565
28
            .collect::<BTreeMap<_, _>>();
566
28
        println!(
567
28
            "{:?}: {} operations",
568
28
            metric,
569
28
            label_metrics.values().next().unwrap().len()
570
28
        );
571
28
        cli_table::print_stdout(
572
28
            label_metrics
573
28
                .iter()
574
140
                .map(|(label, stats)| {
575
140
                    let average = stats.iter().sum::<u64>() as f64 / stats.len() as f64;
576
140
                    let min = *stats.iter().min().unwrap() as f64;
577
140
                    let max = *stats.iter().max().unwrap() as f64;
578
140
                    let stddev = stddev(stats, average);
579
140

            
580
140
                    vec![
581
140
                        label.cell(),
582
140
                        format_nanoseconds(average).cell(),
583
140
                        format_nanoseconds(min).cell(),
584
140
                        format_nanoseconds(max).cell(),
585
140
                        format_nanoseconds(stddev).cell(),
586
140
                    ]
587
140
                })
588
28
                .table()
589
28
                .title(vec![
590
28
                    "Backend".cell(),
591
28
                    "Avg".cell(),
592
28
                    "Min".cell(),
593
28
                    "Max".cell(),
594
28
                    "StdDev".cell(),
595
28
                ]),
596
28
        )
597
28
        .unwrap();
598
28

            
599
28
        let mut label_chart_data = BTreeMap::new();
600
140
        for (label, metrics) in label_stats.iter() {
601
140
            let report = operations.entry(metric).or_insert_with(|| MetricSummary {
602
28
                metric,
603
28
                invocations: label_metrics.values().next().unwrap().len(),
604
28
                description: metric.description().to_string(),
605
28
                summaries: Vec::new(),
606
140
            });
607
140
            report.summaries.push(OperationSummary {
608
140
                backend: label.to_string(),
609
140
                avg: format_nanoseconds(metrics.average),
610
140
                min: format_nanoseconds(metrics.min as f64),
611
140
                max: format_nanoseconds(metrics.max as f64),
612
140
                stddev: format_nanoseconds(metrics.stddev),
613
140
                outliers: metrics.outliers.len().to_string(),
614
140
            });
615
140
            let mut histogram = vec![0; 63];
616
140
            let range = &metric_ranges[&metric];
617
140
            let bucket_width = range.end / (histogram.len() as u64 - 1);
618
30077
            for &nanos in &metrics.plottable_stats {
619
29937
                let bucket = (nanos / bucket_width) as usize;
620
29937
                histogram[bucket] += 1;
621
29937
            }
622
140
            let chart_data = HistogramBars {
623
140
                bars: histogram
624
140
                    .iter()
625
140
                    .enumerate()
626
8820
                    .filter_map(|(bucket, &count)| {
627
8820
                        if count > 0 {
628
1518
                            let bucket_value = bucket as u64 * bucket_width;
629
1518

            
630
1518
                            Some(HistogramBar::new(
631
1518
                                (Nanos(bucket_value), count),
632
1518
                                bucket_width,
633
1518
                                label_to_color(label),
634
1518
                            ))
635
                        } else {
636
7302
                            None
637
                        }
638
8820
                    })
639
140
                    .collect::<Vec<_>>(),
640
140
            };
641
140
            label_chart_data.insert(label, chart_data);
642
        }
643
28
        let highest_count = Iterator::max(
644
28
            label_chart_data
645
28
                .values()
646
1518
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.upper_left.1)),
647
28
        )
648
28
        .unwrap();
649
28
        let highest_nanos = Iterator::max(
650
28
            label_chart_data
651
28
                .values()
652
1518
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.lower_right.0)),
653
28
        )
654
28
        .unwrap();
655
28
        let lowest_nanos = Iterator::min(
656
28
            label_chart_data
657
28
                .values()
658
1518
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.upper_left.0)),
659
28
        )
660
28
        .unwrap();
661
168
        for (label, chart_data) in label_chart_data {
662
140
            println!("Plotting {}: {:?}", label, metric);
663
140
            let chart_path = plot_dir.join(format!("{}-{:?}.png", label, metric));
664
140
            let chart_root = BitMapBackend::new(&chart_path, (800, 240)).into_drawing_area();
665
140
            chart_root.fill(&BACKGROUND_COLOR).unwrap();
666
140
            let mut chart = ChartBuilder::on(&chart_root)
667
140
                .caption(
668
140
                    format!("{}: {:?}", label, metric),
669
140
                    ("sans-serif", 30., &TEXT_COLOR),
670
140
                )
671
140
                .margin_left(10)
672
140
                .margin_right(50)
673
140
                .margin_bottom(10)
674
140
                .x_label_area_size(50)
675
140
                .y_label_area_size(80)
676
140
                .build_cartesian_2d(
677
140
                    NanosRange(lowest_nanos..=highest_nanos),
678
140
                    0..highest_count + 1,
679
140
                )
680
140
                .unwrap();
681
140

            
682
140
            chart
683
140
                .configure_mesh()
684
140
                .disable_x_mesh()
685
140
                .y_desc("Count")
686
140
                .x_desc("Execution Time")
687
140
                .axis_desc_style(("sans-serif", 15, &TEXT_COLOR))
688
140
                .x_label_style(&TEXT_COLOR)
689
140
                .y_label_style(&TEXT_COLOR)
690
140
                .light_line_style(&TEXT_COLOR.mix(0.1))
691
140
                .bold_line_style(&TEXT_COLOR.mix(0.3))
692
140
                .draw()
693
140
                .unwrap();
694
140

            
695
140
            chart.draw_series(chart_data).unwrap();
696
140
            chart_root.present().unwrap();
697
140
        }
698
28
        let label_lines = label_metrics
699
28
            .iter()
700
140
            .map(|(label, stats)| {
701
140
                let mut running_data = Vec::new();
702
140
                let mut elapsed = 0;
703
30380
                for (index, &nanos) in stats.iter().enumerate() {
704
30380
                    elapsed += nanos;
705
30380
                    running_data.push((index, Nanos(elapsed)));
706
30380
                }
707
140
                (label, running_data)
708
140
            })
709
28
            .collect::<BTreeMap<_, _>>();
710
28
        let metric_chart_path = plot_dir.join(format!("{:?}.png", metric));
711
28
        let metric_chart_root =
712
28
            BitMapBackend::new(&metric_chart_path, (800, 480)).into_drawing_area();
713
28
        metric_chart_root.fill(&BACKGROUND_COLOR).unwrap();
714
28
        let mut metric_chart = ChartBuilder::on(&metric_chart_root)
715
28
            .caption(format!("{:?}", metric), ("sans-serif", 30., &TEXT_COLOR))
716
28
            .margin_left(10)
717
28
            .margin_right(50)
718
28
            .margin_bottom(10)
719
28
            .x_label_area_size(50)
720
28
            .y_label_area_size(80)
721
28
            .build_cartesian_2d(
722
28
                0..label_lines
723
28
                    .iter()
724
140
                    .map(|(_, data)| data.len())
725
28
                    .max()
726
28
                    .unwrap(),
727
28
                NanosRange(
728
28
                    Nanos(0)
729
28
                        ..=label_lines
730
28
                            .iter()
731
140
                            .map(|(_, stats)| stats.last().unwrap().1)
732
28
                            .max()
733
28
                            .unwrap(),
734
28
                ),
735
28
            )
736
28
            .unwrap();
737
28

            
738
28
        metric_chart
739
28
            .configure_mesh()
740
28
            .disable_x_mesh()
741
28
            .x_desc("Invocations")
742
28
            .y_desc("Accumulated Execution Time")
743
28
            .axis_desc_style(("sans-serif", 15, &TEXT_COLOR))
744
28
            .x_label_style(&TEXT_COLOR)
745
28
            .y_label_style(&TEXT_COLOR)
746
28
            .light_line_style(&TEXT_COLOR.mix(0.1))
747
28
            .bold_line_style(&TEXT_COLOR.mix(0.3))
748
28
            .draw()
749
28
            .unwrap();
750

            
751
168
        for (label, data) in label_lines {
752
140
            metric_chart
753
140
                .draw_series(LineSeries::new(data.into_iter(), &label_to_color(label)))
754
140
                .unwrap()
755
140
                .label(label.to_string())
756
140
                .legend(|(x, y)| {
757
140
                    PathElement::new(vec![(x, y), (x + 20, y)], &label_to_color(label))
758
140
                });
759
140
        }
760
28
        metric_chart
761
28
            .configure_series_labels()
762
28
            .border_style(&TEXT_COLOR)
763
28
            .background_style(&BACKGROUND_COLOR)
764
28
            .label_font(&TEXT_COLOR)
765
28
            .position(SeriesLabelPosition::UpperLeft)
766
28
            .draw()
767
28
            .unwrap();
768
28
        metric_chart_root.present().unwrap();
769
    }
770
4
    cli_table::print_stdout(
771
4
        accumulated_label_stats
772
4
            .iter()
773
20
            .map(|(label, duration)| {
774
20
                vec![
775
20
                    label.cell(),
776
20
                    format_nanoseconds(duration.as_nanos() as f64).cell(),
777
20
                ]
778
20
            })
779
4
            .table()
780
4
            .title(vec![
781
4
                "Backend".cell(),
782
4
                format!("Total Execution Time across {} agents", number_of_agents).cell(),
783
4
            ]),
784
4
    )
785
4
    .unwrap();
786
4
    BenchmarkSummary {
787
4
        label,
788
4
        timestamp: current_timestamp_string(),
789
4
        revision: local_git_rev(),
790
4
        plan_count: number_of_plans,
791
4
        agent_count: number_of_agents,
792
4
        product_count: initial_data.products.len(),
793
4
        category_count: initial_data.categories.len(),
794
4
        customer_count: initial_data.customers.len(),
795
4
        order_count: initial_data.orders.len(),
796
4
        summaries: accumulated_label_stats
797
4
            .iter()
798
20
            .map(|(&backend, duration)| BackendSummary {
799
20
                backend: backend.to_string(),
800
20
                transport: match backend {
801
20
                    "bonsaidb-local" => String::from("None"),
802
16
                    "bonsaidb-quic" => String::from("UDP with TLS"),
803
12
                    _ => String::from("TCP"),
804
                },
805
20
                total_time: format_nanoseconds(duration.as_nanos() as f64),
806
20
                wall_time: format_nanoseconds(duration.as_nanos() as f64 / number_of_agents as f64),
807
20
            })
808
4
            .collect(),
809
28
        operations: operations.into_iter().map(|(_k, v)| v).collect(),
810
4
    }
811
4
    .render_to(&plot_dir.join("index.html"), tera);
812
4
    accumulated_label_stats
813
4
}
814

            
815
struct MetricStats {
816
    average: f64,
817
    min: u64,
818
    max: u64,
819
    stddev: f64,
820
    plottable_stats: Vec<u64>,
821
    outliers: Vec<f64>,
822
}
823

            
824
struct HistogramBars {
825
    bars: Vec<HistogramBar>,
826
}
827

            
828
struct HistogramBar {
829
    upper_left: (Nanos, u64),
830
    lower_right: (Nanos, u64),
831

            
832
    color: RGBColor,
833
}
834

            
835
impl HistogramBar {
836
1518
    pub fn new(coord: (Nanos, u64), width: u64, color: RGBColor) -> Self {
837
1518
        Self {
838
1518
            upper_left: (Nanos(coord.0 .0), coord.1),
839
1518
            lower_right: (Nanos(coord.0 .0 + width), 0),
840
1518
            color,
841
1518
        }
842
1518
    }
843
}
844

            
845
impl<'a> Drawable<BitMapBackend<'a>> for HistogramBar {
846
1518
    fn draw<I: Iterator<Item = <BackendCoordOnly as CoordMapper>::Output>>(
847
1518
        &self,
848
1518
        mut pos: I,
849
1518
        backend: &mut BitMapBackend,
850
1518
        _parent_dim: (u32, u32),
851
1518
    ) -> Result<(), DrawingErrorKind<<BitMapBackend as DrawingBackend>::ErrorType>> {
852
1518
        let upper_left = pos.next().unwrap();
853
1518
        let lower_right = pos.next().unwrap();
854
1518
        backend.draw_rect(upper_left, lower_right, &self.color, true)?;
855

            
856
1518
        Ok(())
857
1518
    }
858
}
859

            
860
impl<'a> PointCollection<'a, (Nanos, u64)> for &'a HistogramBar {
861
    type Point = &'a (Nanos, u64);
862

            
863
    type IntoIter = HistogramBarIter<'a>;
864

            
865
1518
    fn point_iter(self) -> Self::IntoIter {
866
1518
        HistogramBarIter::UpperLeft(self)
867
1518
    }
868
}
869

            
870
enum HistogramBarIter<'a> {
871
    UpperLeft(&'a HistogramBar),
872
    LowerRight(&'a HistogramBar),
873
    Done,
874
}
875

            
876
impl<'a> Iterator for HistogramBarIter<'a> {
877
    type Item = &'a (Nanos, u64);
878

            
879
3036
    fn next(&mut self) -> Option<Self::Item> {
880
3036
        let (next, result) = match self {
881
1518
            HistogramBarIter::UpperLeft(bar) => {
882
1518
                (HistogramBarIter::LowerRight(bar), Some(&bar.upper_left))
883
            }
884
1518
            HistogramBarIter::LowerRight(bar) => (HistogramBarIter::Done, Some(&bar.lower_right)),
885
            HistogramBarIter::Done => (HistogramBarIter::Done, None),
886
        };
887
3036
        *self = next;
888
3036
        result
889
3036
    }
890
}
891

            
892
impl IntoIterator for HistogramBars {
893
    type Item = HistogramBar;
894

            
895
    type IntoIter = std::vec::IntoIter<HistogramBar>;
896

            
897
140
    fn into_iter(self) -> Self::IntoIter {
898
140
        self.bars.into_iter()
899
140
    }
900
}
901

            
902
280
fn stddev(data: &[u64], average: f64) -> f64 {
903
280
    if data.is_empty() {
904
        0.
905
    } else {
906
280
        let variance = data
907
280
            .iter()
908
60760
            .map(|value| {
909
60760
                let diff = average - (*value as f64);
910
60760

            
911
60760
                diff * diff
912
60760
            })
913
280
            .sum::<f64>()
914
280
            / data.len() as f64;
915
280

            
916
280
        variance.sqrt()
917
    }
918
280
}