1
use std::collections::{BTreeMap, HashMap};
2
use std::ops::{Range, RangeInclusive};
3
use std::path::Path;
4
use std::sync::Arc;
5
use std::time::{Duration, Instant};
6

            
7
use bonsaidb::core::async_trait::async_trait;
8
use cli_table::{Cell, Table};
9
use futures::stream::FuturesUnordered;
10
use futures::StreamExt;
11
use plotters::coord::ranged1d::{NoDefaultFormatting, ValueFormatter};
12
use plotters::element::{BackendCoordOnly, CoordMapper, Drawable, PointCollection};
13
use plotters::prelude::*;
14
use plotters_backend::DrawingErrorKind;
15
use rand::rngs::SmallRng;
16
use rand::SeedableRng;
17
use serde::{Deserialize, Serialize};
18
use tera::Tera;
19
use tokio::runtime::Runtime;
20

            
21
use crate::bonsai::{Bonsai, BonsaiBackend};
22
use crate::model::{InitialDataSet, InitialDataSetConfig};
23
use crate::plan::{
24
    AddProductToCart, Checkout, CreateCart, FindProduct, Load, LookupProduct, Operation,
25
    OperationResult, Plan, ReviewProduct, ShopperPlanConfig,
26
};
27
use crate::plot::{label_to_color, BACKGROUND_COLOR, TEXT_COLOR};
28
use crate::utils::{current_timestamp_string, format_nanoseconds, local_git_rev};
29

            
30
4
pub fn execute_plans_for_all_backends(
31
4
    name_filter: &str,
32
4
    plans: &[Arc<Plan>],
33
4
    initial_data: &Arc<InitialDataSet>,
34
4
    number_of_agents: usize,
35
4
    measurements: &Measurements,
36
4
) {
37
4
    if name_filter.is_empty()
38
        || name_filter == "bonsaidb"
39
        || name_filter.starts_with("bonsaidb-local")
40
4
    {
41
4
        println!("Executing bonsaidb-local");
42
4
        BonsaiBackend::execute_async(
43
4
            Bonsai::Local,
44
4
            plans,
45
4
            initial_data,
46
4
            number_of_agents,
47
4
            measurements,
48
4
        );
49
4
    }
50
4
    if name_filter.is_empty()
51
        || name_filter == "bonsaidb"
52
        || name_filter.starts_with("bonsaidb-local+lz4")
53
4
    {
54
4
        println!("Executing bonsaidb-local+lz4");
55
4
        BonsaiBackend::execute_async(
56
4
            Bonsai::LocalLz4,
57
4
            plans,
58
4
            initial_data,
59
4
            number_of_agents,
60
4
            measurements,
61
4
        );
62
4
    }
63
4
    if name_filter.is_empty()
64
        || name_filter == "bonsaidb"
65
        || name_filter.starts_with("bonsaidb-quic")
66
4
    {
67
4
        println!("Executing bonsaidb-quic");
68
4
        BonsaiBackend::execute_async(
69
4
            Bonsai::Quic,
70
4
            plans,
71
4
            initial_data,
72
4
            number_of_agents,
73
4
            measurements,
74
4
        );
75
4
    }
76
4
    if name_filter.is_empty() || name_filter == "bonsaidb" || name_filter.starts_with("bonsaidb-ws")
77
4
    {
78
4
        println!("Executing bonsaidb-ws");
79
4
        BonsaiBackend::execute_async(
80
4
            Bonsai::WebSockets,
81
4
            plans,
82
4
            initial_data,
83
4
            number_of_agents,
84
4
            measurements,
85
4
        );
86
4
    }
87
    #[cfg(feature = "postgresql")]
88
4
    if name_filter.is_empty() || name_filter.starts_with("postgresql") {
89
4
        if let Ok(url) = std::env::var("COMMERCE_POSTGRESQL_URL") {
90
4
            println!("Executing postgresql");
91
4
            crate::postgres::Postgres::execute_async(
92
4
                url,
93
4
                plans,
94
4
                initial_data,
95
4
                number_of_agents,
96
4
                measurements,
97
4
            );
98
4
        } else {
99
            eprintln!("postgresql feature is enabled, but environment variable COMMERCE_POSTGRESQL_URL is missing.");
100
        }
101
    }
102
    #[cfg(feature = "mongo")]
103
4
    if name_filter.is_empty() || name_filter.starts_with("mongo") {
104
4
        if let Ok(url) = std::env::var("COMMERCE_MONGODB_URL") {
105
            println!("Executing mongodb");
106
            crate::mongo::MongoBackend::execute_async(
107
                url,
108
                plans,
109
                initial_data,
110
                number_of_agents,
111
                measurements,
112
            );
113
4
        } else {
114
4
            eprintln!("mongo feature is enabled, but environment variable COMMERCE_MONGODB_URL is missing.");
115
4
        }
116
    }
117
4
}
118

            
119
#[async_trait]
120
pub trait Backend: Sized + Send + Sync + 'static {
121
    type Operator: BackendOperator;
122
    type Config: Send + Sync;
123

            
124
    async fn new(config: Self::Config) -> Self;
125

            
126
    fn label(&self) -> &'static str;
127

            
128
20
    fn execute_async(
129
20
        config: Self::Config,
130
20
        plans: &[Arc<Plan>],
131
20
        initial_data: &Arc<InitialDataSet>,
132
20
        concurrent_agents: usize,
133
20
        measurements: &Measurements,
134
20
    ) {
135
20
        let (plan_sender, plan_receiver) = flume::bounded(concurrent_agents * 2);
136
20
        let runtime = Runtime::new().unwrap();
137
20
        let backend = runtime.block_on(Self::new(config));
138
20
        // Load the initial data
139
20
        println!("Loading data");
140
20
        runtime.block_on(async {
141
20
            let _ = backend
142
20
                .new_operator_async()
143
16
                .await
144
20
                .operate(
145
20
                    &Load {
146
20
                        initial_data: initial_data.clone(),
147
20
                    },
148
20
                    &[],
149
20
                    measurements,
150
20
                )
151
17103
                .await;
152
20
        });
153
20
        println!("Executing plans");
154
20
        let agent_handles = FuturesUnordered::new();
155
40
        for _ in 0..concurrent_agents {
156
40
            let operator = runtime.block_on(backend.new_operator_async());
157
40
            agent_handles.push(runtime.spawn(agent::<Self>(
158
40
                operator,
159
40
                plan_receiver.clone(),
160
40
                measurements.clone(),
161
40
            )));
162
40
        }
163
20
        runtime.block_on(async {
164
            // Send the plans to the channel that the agents are waiting for
165
            // them on.
166
2800
            for plan in plans {
167
2780
                plan_sender.send_async(plan.clone()).await.unwrap();
168
            }
169
            // Disconnect the receivers, allowing the agents to exit once there
170
            // are no more plans in queue.
171
20
            drop(plan_sender);
172
            // Wait for each of the agents to return.
173
60
            for result in agent_handles.collect::<Vec<_>>().await {
174
40
                result.unwrap();
175
40
            }
176
20
        })
177
20
    }
178

            
179
    async fn new_operator_async(&self) -> Self::Operator;
180
}
181

            
182
#[async_trait]
183
pub trait Operator<T, R> {
184
    async fn operate(
185
        &mut self,
186
        operation: &T,
187
        results: &[OperationResult<R>],
188
        measurements: &Measurements,
189
    ) -> OperationResult<R>;
190
}
191

            
192
40
async fn agent<B: Backend>(
193
40
    mut operator: B::Operator,
194
40
    plan_receiver: flume::Receiver<Arc<Plan>>,
195
40
    measurements: Measurements,
196
40
) {
197
2820
    while let Ok(plan) = plan_receiver.recv_async().await {
198
2780
        let mut results = Vec::with_capacity(plan.operations.len());
199
31990
        for step in &plan.operations {
200
77773
            results.push(operator.operate(step, &results, &measurements).await)
201
        }
202
    }
203
40
}
204

            
205
pub trait BackendOperator:
206
    Operator<Load, Self::Id>
207
    + Operator<LookupProduct, Self::Id>
208
    + Operator<FindProduct, Self::Id>
209
    + Operator<CreateCart, Self::Id>
210
    + Operator<AddProductToCart, Self::Id>
211
    + Operator<ReviewProduct, Self::Id>
212
    + Operator<Checkout, Self::Id>
213
    + Send
214
    + Sync
215
{
216
    type Id: Send + Sync;
217
}
218

            
219
#[async_trait]
220
impl<T> Operator<Operation, T::Id> for T
221
where
222
    T: BackendOperator,
223
{
224
31990
    async fn operate(
225
31990
        &mut self,
226
31990
        operation: &Operation,
227
31990
        results: &[OperationResult<T::Id>],
228
31990
        measurements: &Measurements,
229
31990
    ) -> OperationResult<T::Id> {
230
31990
        match operation {
231
31599
            Operation::FindProduct(op) => self.operate(op, results, measurements).await,
232
21042
            Operation::LookupProduct(op) => self.operate(op, results, measurements).await,
233
4585
            Operation::CreateCart(op) => self.operate(op, results, measurements).await,
234
16097
            Operation::AddProductToCart(op) => self.operate(op, results, measurements).await,
235
1745
            Operation::RateProduct(op) => self.operate(op, results, measurements).await,
236
2705
            Operation::Checkout(op) => self.operate(op, results, measurements).await,
237
        }
238
63980
    }
239
}
240

            
241
4
#[derive(Serialize, Deserialize, Debug)]
242
pub struct BenchmarkSummary {
243
    label: String,
244
    timestamp: String,
245
    revision: String,
246
    plan_count: usize,
247
    agent_count: usize,
248
    product_count: usize,
249
    category_count: usize,
250
    customer_count: usize,
251
    order_count: usize,
252
    summaries: Vec<BackendSummary>,
253
    operations: Vec<MetricSummary>,
254
}
255

            
256
impl BenchmarkSummary {
257
4
    pub fn render_to(&self, location: &Path, tera: &Tera) {
258
4
        std::fs::write(
259
4
            location,
260
4
            tera.render("run.html", &tera::Context::from_serialize(self).unwrap())
261
4
                .unwrap()
262
4
                .as_bytes(),
263
4
        )
264
4
        .unwrap()
265
4
    }
266
}
267

            
268
20
#[derive(Serialize, Deserialize, Debug)]
269
pub struct BackendSummary {
270
    backend: String,
271
    transport: String,
272
    total_time: String,
273
    wall_time: String,
274
}
275

            
276
28
#[derive(Serialize, Deserialize, Debug)]
277
pub struct MetricSummary {
278
    metric: Metric,
279
    description: String,
280
    invocations: usize,
281
    summaries: Vec<OperationSummary>,
282
}
283

            
284
140
#[derive(Serialize, Deserialize, Debug)]
285
pub struct OperationSummary {
286
    backend: String,
287
    avg: String,
288
    min: String,
289
    max: String,
290
    stddev: String,
291
    outliers: String,
292
}
293

            
294
pub struct Benchmark<'a> {
295
    pub label: String,
296
    pub seed: Option<u64>,
297
    pub agents: Option<usize>,
298
    pub shoppers: Option<usize>,
299
    pub data_config: &'a InitialDataSetConfig,
300
    pub shopper_config: &'a ShopperPlanConfig,
301
}
302

            
303
impl<'a> Benchmark<'a> {
304
4
    pub fn execute(
305
4
        self,
306
4
        name_filter: &str,
307
4
        plot_dir: impl AsRef<Path>,
308
4
        tera: Arc<Tera>,
309
4
    ) -> BTreeMap<&'static str, Duration> {
310
4
        let plot_dir = plot_dir.as_ref().to_path_buf();
311
4
        std::fs::create_dir_all(&plot_dir).unwrap();
312

            
313
4
        let mut rng = if let Some(seed) = self.seed {
314
3
            SmallRng::seed_from_u64(seed)
315
        } else {
316
1
            SmallRng::from_entropy()
317
        };
318
4
        let initial_data = Arc::new(self.data_config.fake(&mut rng));
319
4
        let number_of_agents = self.agents.unwrap_or_else(num_cpus::get);
320
4
        let shoppers = self.shoppers.unwrap_or(number_of_agents * 100);
321
4

            
322
4
        println!("Running {shoppers} plans across {number_of_agents} agents");
323
4
        // Generate plans to execute.
324
4
        let mut plans = Vec::with_capacity(shoppers);
325
556
        for _ in 0..shoppers {
326
556
            plans.push(Arc::new(
327
556
                self.shopper_config.random_plan(&mut rng, &initial_data),
328
556
            ));
329
556
        }
330
        // Set up our statistics gathering thread
331
4
        let (metric_sender, metric_receiver) = flume::unbounded();
332
4
        let measurements = Measurements {
333
4
            sender: metric_sender,
334
4
        };
335
4
        let thread_initial_data = initial_data.clone();
336
4
        let stats_thread = std::thread::spawn(move || {
337
4
            stats_thread(
338
4
                self.label,
339
4
                metric_receiver,
340
4
                shoppers,
341
4
                number_of_agents,
342
4
                &thread_initial_data,
343
4
                &plot_dir,
344
4
                &tera,
345
4
            )
346
4
        });
347
4
        // Perform all benchmarks
348
4
        execute_plans_for_all_backends(
349
4
            name_filter,
350
4
            &plans,
351
4
            &initial_data,
352
4
            number_of_agents,
353
4
            &measurements,
354
4
        );
355
4
        // Drop the measurements instance to allow the stats thread to know
356
4
        // there are no more metrics coming.
357
4
        drop(measurements);
358
4
        // Wait for the statistics thread to report all the results.
359
4
        stats_thread.join().unwrap()
360
4
    }
361
}
362

            
363
40
#[derive(Clone)]
364
pub struct Measurements {
365
    sender: flume::Sender<(&'static str, Metric, Duration)>,
366
}
367

            
368
impl Measurements {
369
32010
    pub fn begin(&self, label: &'static str, metric: Metric) -> Measurement<'_> {
370
32010
        Measurement {
371
32010
            target: &self.sender,
372
32010
            label,
373
32010
            metric,
374
32010
            start: Instant::now(),
375
32010
        }
376
32010
    }
377
}
378

            
379
pub struct Measurement<'a> {
380
    target: &'a flume::Sender<(&'static str, Metric, Duration)>,
381
    label: &'static str,
382
    metric: Metric,
383
    start: Instant,
384
}
385

            
386
impl<'a> Measurement<'a> {
387
32010
    pub fn finish(self) {
388
32010
        let duration = Instant::now()
389
32010
            .checked_duration_since(self.start)
390
32010
            .expect("time went backwards. Restart benchmarks.");
391
32010
        self.target
392
32010
            .send((self.label, self.metric, duration))
393
32010
            .unwrap();
394
32010
    }
395
}
396

            
397
103879
#[derive(Serialize, Deserialize, Clone, Copy, Hash, Eq, PartialEq, Debug, Ord, PartialOrd)]
398
pub enum Metric {
399
    Load,
400
    LookupProduct,
401
    FindProduct,
402
    CreateCart,
403
    AddProductToCart,
404
    Checkout,
405
    RateProduct,
406
}
407

            
408
impl Metric {
409
28
    pub fn description(&self) -> &'static str {
410
28
        match self {
411
4
            Metric::Load => "Measures the time spent loading the initial data set and performing any pre-cache operations that most database administrators would perform on their databases periodically to ensure good performance.",
412
4
            Metric::LookupProduct => "Meaures the time spent looking up a product by its id. This operation is meant to simulate the basic needs of the database to provide a product details page after a user clicked a direct link that contians the product's unique id, including the product's current rating.",
413
4
            Metric::FindProduct => "Measures the time spent looking up a product by its name (exact match, indexed). This operation is meant to simulate the basic needs of the database to provide a product details after finding a product by its name, including the product's current rating.",
414
4
            Metric::CreateCart => "Measures the time spent creating a shopping cart.",
415
4
            Metric::AddProductToCart => "Measures the time spent adding a product to a shopping cart.",
416
4
            Metric::RateProduct => "Measures the time spent adding or updating a review of a product by a customer. Each customer can only have one review per product. When this operation is complete, all subsequent calls to LookupProduct and FindProduct should reflect the new rating. This simulates an 'upsert' (insert or update) operation using a unique index.",
417
4
            Metric::Checkout => "Measures the time spent converting a shopping cart into an order for a customer."
418
        }
419
28
    }
420
}
421

            
422
#[derive(Clone, Debug)]
423
struct NanosRange(RangeInclusive<Nanos>);
424
3340
#[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq)]
425
struct Nanos(u64);
426

            
427
impl ValueFormatter<Nanos> for NanosRange {
428
1848
    fn format(value: &Nanos) -> String {
429
1848
        format_nanoseconds(value.0 as f64)
430
1848
    }
431
}
432

            
433
impl Ranged for NanosRange {
434
    type FormatOption = NoDefaultFormatting;
435
    type ValueType = Nanos;
436

            
437
55622
    fn map(&self, value: &Self::ValueType, limit: (i32, i32)) -> i32 {
438
55622
        let limited_size = limit.1 - limit.0;
439
55622
        let full_size = self.0.end().0 + 1 - self.0.start().0;
440
55622
        let normalized_offset = value.0.saturating_sub(self.0.start().0) as f64 / full_size as f64;
441
55622
        limit.0 + (normalized_offset * limited_size as f64) as i32
442
55622
    }
443

            
444
336
    fn key_points<Hint: plotters::coord::ranged1d::KeyPointHint>(
445
336
        &self,
446
336
        hint: Hint,
447
336
    ) -> Vec<Self::ValueType> {
448
336
        let total_range = self.0.end().0 - self.0.start().0;
449
336
        let num_points = hint.max_num_points();
450
336
        let mut important_points = Vec::with_capacity(num_points);
451
336
        important_points.push(*self.0.start());
452
336
        if num_points > 2 {
453
336
            let steps = num_points - 2;
454
336
            let step_size = total_range as f64 / steps as f64;
455
336
            important_points.extend(
456
336
                (1..num_points - 1)
457
19656
                    .map(|step| Nanos(self.0.start().0 + (step as f64 * step_size) as u64)),
458
336
            );
459
336
        }
460
336
        important_points.push(*self.0.end());
461
336

            
462
336
        important_points
463
336
    }
464

            
465
    fn range(&self) -> std::ops::Range<Self::ValueType> {
466
        Nanos(self.0.start().0)..Nanos(self.0.end().0 + 1)
467
    }
468
}
469

            
470
impl DiscreteRanged for NanosRange {
471
    fn size(&self) -> usize {
472
        (self.0.end().0 - self.0.start().0) as usize
473
    }
474

            
475
    fn index_of(&self, value: &Self::ValueType) -> Option<usize> {
476
        if value.0 <= self.0.end().0 {
477
            if let Some(index) = value.0.checked_sub(self.0.start().0) {
478
                return Some(index as usize);
479
            }
480
        }
481
        None
482
    }
483

            
484
    fn from_index(&self, index: usize) -> Option<Self::ValueType> {
485
        Some(Nanos(self.0.start().0 + index as u64))
486
    }
487
}
488

            
489
4
fn stats_thread(
490
4
    label: String,
491
4
    metric_receiver: flume::Receiver<(&'static str, Metric, Duration)>,
492
4
    number_of_plans: usize,
493
4
    number_of_agents: usize,
494
4
    initial_data: &InitialDataSet,
495
4
    plot_dir: &Path,
496
4
    tera: &Tera,
497
4
) -> BTreeMap<&'static str, Duration> {
498
4
    let mut all_results: BTreeMap<Metric, BTreeMap<&'static str, Vec<u64>>> = BTreeMap::new();
499
4
    let mut accumulated_label_stats: BTreeMap<&'static str, Duration> = BTreeMap::new();
500
4
    let mut longest_by_metric = HashMap::new();
501
32014
    while let Ok((label, metric, duration)) = metric_receiver.recv() {
502
32010
        let metric_results = all_results.entry(metric).or_default();
503
32010
        let label_results = metric_results.entry(label).or_default();
504
32010
        let nanos = u64::try_from(duration.as_nanos()).unwrap();
505
32010
        label_results.push(nanos);
506
32010
        let label_duration = accumulated_label_stats.entry(label).or_default();
507
32010
        longest_by_metric
508
32010
            .entry(metric)
509
32010
            .and_modify(|existing: &mut Duration| {
510
31982
                *existing = (*existing).max(duration);
511
32010
            })
512
32010
            .or_insert(duration);
513
32010
        *label_duration += duration;
514
32010
    }
515

            
516
4
    let mut operations = BTreeMap::new();
517
4
    let mut metric_ranges = HashMap::new();
518
32
    for (metric, label_metrics) in all_results {
519
28
        let label_stats = label_metrics
520
28
            .iter()
521
140
            .map(|(label, stats)| {
522
140
                let mut sum = 0;
523
140
                let mut min = u64::MAX;
524
140
                let mut max = 0;
525
32150
                for &nanos in stats {
526
32010
                    sum += nanos;
527
32010
                    min = min.min(nanos);
528
32010
                    max = max.max(nanos);
529
32010
                }
530
140
                let average = sum as f64 / stats.len() as f64;
531
140
                let stddev = stddev(stats, average);
532
140

            
533
140
                let mut outliers = Vec::new();
534
140
                let mut plottable_stats = Vec::new();
535
140
                let mut min_plottable = u64::MAX;
536
140
                let mut max_plottable = 0;
537
32150
                for &nanos in stats {
538
32010
                    let diff = (nanos as f64 - average).abs();
539
32010
                    let diff_magnitude = diff / stddev;
540
32010
                    if stats.len() == 1 || diff_magnitude < 3. {
541
31642
                        plottable_stats.push(nanos);
542
31642
                        min_plottable = min_plottable.min(nanos);
543
31642
                        max_plottable = max_plottable.max(nanos);
544
31642
                    } else {
545
368
                        // Outlier
546
368
                        outliers.push(diff_magnitude);
547
368
                    }
548
                }
549

            
550
140
                if !outliers.is_empty() {
551
100
                    eprintln!("Not plotting {} outliers for {}", outliers.len(), label);
552
100
                }
553

            
554
140
                metric_ranges
555
140
                    .entry(metric)
556
140
                    .and_modify(|range: &mut Range<u64>| {
557
112
                        range.start = range.start.min(min_plottable);
558
112
                        range.end = range.end.max(max_plottable + 1);
559
140
                    })
560
140
                    .or_insert(min_plottable..max_plottable + 1);
561
140

            
562
140
                (
563
140
                    label,
564
140
                    MetricStats {
565
140
                        average,
566
140
                        min,
567
140
                        max,
568
140
                        stddev,
569
140
                        outliers,
570
140
                        plottable_stats,
571
140
                    },
572
140
                )
573
140
            })
574
28
            .collect::<BTreeMap<_, _>>();
575
28
        println!(
576
28
            "{:?}: {} operations",
577
28
            metric,
578
28
            label_metrics.values().next().unwrap().len()
579
28
        );
580
28
        cli_table::print_stdout(
581
28
            label_metrics
582
28
                .iter()
583
140
                .map(|(label, stats)| {
584
140
                    let average = stats.iter().sum::<u64>() as f64 / stats.len() as f64;
585
140
                    let min = *stats.iter().min().unwrap() as f64;
586
140
                    let max = *stats.iter().max().unwrap() as f64;
587
140
                    let stddev = stddev(stats, average);
588
140

            
589
140
                    vec![
590
140
                        label.cell(),
591
140
                        format_nanoseconds(average).cell(),
592
140
                        format_nanoseconds(min).cell(),
593
140
                        format_nanoseconds(max).cell(),
594
140
                        format_nanoseconds(stddev).cell(),
595
140
                    ]
596
140
                })
597
28
                .table()
598
28
                .title(vec![
599
28
                    "Backend".cell(),
600
28
                    "Avg".cell(),
601
28
                    "Min".cell(),
602
28
                    "Max".cell(),
603
28
                    "StdDev".cell(),
604
28
                ]),
605
28
        )
606
28
        .unwrap();
607
28

            
608
28
        let mut label_chart_data = BTreeMap::new();
609
140
        for (label, metrics) in label_stats.iter() {
610
140
            let report = operations.entry(metric).or_insert_with(|| MetricSummary {
611
28
                metric,
612
28
                invocations: label_metrics.values().next().unwrap().len(),
613
28
                description: metric.description().to_string(),
614
28
                summaries: Vec::new(),
615
140
            });
616
140
            report.summaries.push(OperationSummary {
617
140
                backend: label.to_string(),
618
140
                avg: format_nanoseconds(metrics.average),
619
140
                min: format_nanoseconds(metrics.min as f64),
620
140
                max: format_nanoseconds(metrics.max as f64),
621
140
                stddev: format_nanoseconds(metrics.stddev),
622
140
                outliers: metrics.outliers.len().to_string(),
623
140
            });
624
140
            let mut histogram = vec![0; 63];
625
140
            let range = &metric_ranges[&metric];
626
140
            let bucket_width = range.end / (histogram.len() as u64 - 1);
627
31782
            for &nanos in &metrics.plottable_stats {
628
31642
                let bucket = (nanos / bucket_width) as usize;
629
31642
                histogram[bucket] += 1;
630
31642
            }
631
140
            let chart_data = HistogramBars {
632
140
                bars: histogram
633
140
                    .iter()
634
140
                    .enumerate()
635
8820
                    .filter_map(|(bucket, &count)| {
636
8820
                        if count > 0 {
637
1642
                            let bucket_value = bucket as u64 * bucket_width;
638
1642

            
639
1642
                            Some(HistogramBar::new(
640
1642
                                (Nanos(bucket_value), count),
641
1642
                                bucket_width,
642
1642
                                label_to_color(label),
643
1642
                            ))
644
                        } else {
645
7178
                            None
646
                        }
647
8820
                    })
648
140
                    .collect::<Vec<_>>(),
649
140
            };
650
140
            label_chart_data.insert(label, chart_data);
651
        }
652
28
        let highest_count = Iterator::max(
653
28
            label_chart_data
654
28
                .values()
655
1642
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.upper_left.1)),
656
28
        )
657
28
        .unwrap();
658
28
        let highest_nanos = Iterator::max(
659
28
            label_chart_data
660
28
                .values()
661
1642
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.lower_right.0)),
662
28
        )
663
28
        .unwrap();
664
28
        let lowest_nanos = Iterator::min(
665
28
            label_chart_data
666
28
                .values()
667
1642
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.upper_left.0)),
668
28
        )
669
28
        .unwrap();
670
168
        for (label, chart_data) in label_chart_data {
671
140
            println!("Plotting {label}: {metric:?}");
672
140
            let chart_path = plot_dir.join(format!("{label}-{metric:?}.png"));
673
140
            let chart_root = BitMapBackend::new(&chart_path, (800, 240)).into_drawing_area();
674
140
            chart_root.fill(&BACKGROUND_COLOR).unwrap();
675
140
            let mut chart = ChartBuilder::on(&chart_root)
676
140
                .caption(
677
140
                    format!("{label}: {metric:?}"),
678
140
                    ("sans-serif", 30., &TEXT_COLOR),
679
140
                )
680
140
                .margin_left(10)
681
140
                .margin_right(50)
682
140
                .margin_bottom(10)
683
140
                .x_label_area_size(50)
684
140
                .y_label_area_size(80)
685
140
                .build_cartesian_2d(
686
140
                    NanosRange(lowest_nanos..=highest_nanos),
687
140
                    0..highest_count + 1,
688
140
                )
689
140
                .unwrap();
690
140

            
691
140
            chart
692
140
                .configure_mesh()
693
140
                .disable_x_mesh()
694
140
                .y_desc("Count")
695
140
                .x_desc("Execution Time")
696
140
                .axis_desc_style(("sans-serif", 15, &TEXT_COLOR))
697
140
                .x_label_style(&TEXT_COLOR)
698
140
                .y_label_style(&TEXT_COLOR)
699
140
                .light_line_style(TEXT_COLOR.mix(0.1))
700
140
                .bold_line_style(TEXT_COLOR.mix(0.3))
701
140
                .draw()
702
140
                .unwrap();
703
140

            
704
140
            chart.draw_series(chart_data).unwrap();
705
140
            chart_root.present().unwrap();
706
140
        }
707
28
        let label_lines = label_metrics
708
28
            .iter()
709
140
            .map(|(label, stats)| {
710
140
                let mut running_data = Vec::new();
711
140
                let mut elapsed = 0;
712
32010
                for (index, &nanos) in stats.iter().enumerate() {
713
32010
                    elapsed += nanos;
714
32010
                    running_data.push((index, Nanos(elapsed)));
715
32010
                }
716
140
                (label, running_data)
717
140
            })
718
28
            .collect::<BTreeMap<_, _>>();
719
28
        let metric_chart_path = plot_dir.join(format!("{metric:?}.png"));
720
28
        let metric_chart_root =
721
28
            BitMapBackend::new(&metric_chart_path, (800, 480)).into_drawing_area();
722
28
        metric_chart_root.fill(&BACKGROUND_COLOR).unwrap();
723
28
        let mut metric_chart = ChartBuilder::on(&metric_chart_root)
724
28
            .caption(format!("{metric:?}"), ("sans-serif", 30., &TEXT_COLOR))
725
28
            .margin_left(10)
726
28
            .margin_right(50)
727
28
            .margin_bottom(10)
728
28
            .x_label_area_size(50)
729
28
            .y_label_area_size(80)
730
28
            .build_cartesian_2d(
731
140
                0..label_lines.values().map(|data| data.len()).max().unwrap(),
732
28
                NanosRange(
733
28
                    Nanos(0)
734
28
                        ..=label_lines
735
28
                            .values()
736
140
                            .map(|stats| stats.last().unwrap().1)
737
28
                            .max()
738
28
                            .unwrap(),
739
28
                ),
740
28
            )
741
28
            .unwrap();
742
28

            
743
28
        metric_chart
744
28
            .configure_mesh()
745
28
            .disable_x_mesh()
746
28
            .x_desc("Invocations")
747
28
            .y_desc("Accumulated Execution Time")
748
28
            .axis_desc_style(("sans-serif", 15, &TEXT_COLOR))
749
28
            .x_label_style(&TEXT_COLOR)
750
28
            .y_label_style(&TEXT_COLOR)
751
28
            .light_line_style(TEXT_COLOR.mix(0.1))
752
28
            .bold_line_style(TEXT_COLOR.mix(0.3))
753
28
            .draw()
754
28
            .unwrap();
755

            
756
168
        for (label, data) in label_lines {
757
140
            metric_chart
758
140
                .draw_series(LineSeries::new(data.into_iter(), label_to_color(label)))
759
140
                .unwrap()
760
140
                .label(label.to_string())
761
140
                .legend(|(x, y)| {
762
140
                    PathElement::new(vec![(x, y), (x + 20, y)], label_to_color(label))
763
140
                });
764
140
        }
765
28
        metric_chart
766
28
            .configure_series_labels()
767
28
            .border_style(TEXT_COLOR)
768
28
            .background_style(BACKGROUND_COLOR)
769
28
            .label_font(&TEXT_COLOR)
770
28
            .position(SeriesLabelPosition::UpperLeft)
771
28
            .draw()
772
28
            .unwrap();
773
28
        metric_chart_root.present().unwrap();
774
    }
775
4
    cli_table::print_stdout(
776
4
        accumulated_label_stats
777
4
            .iter()
778
20
            .map(|(label, duration)| {
779
20
                vec![
780
20
                    label.cell(),
781
20
                    format_nanoseconds(duration.as_nanos() as f64).cell(),
782
20
                ]
783
20
            })
784
4
            .table()
785
4
            .title(vec![
786
4
                "Backend".cell(),
787
4
                format!("Total Execution Time across {number_of_agents} agents").cell(),
788
4
            ]),
789
4
    )
790
4
    .unwrap();
791
4
    BenchmarkSummary {
792
4
        label,
793
4
        timestamp: current_timestamp_string(),
794
4
        revision: local_git_rev(),
795
4
        plan_count: number_of_plans,
796
4
        agent_count: number_of_agents,
797
4
        product_count: initial_data.products.len(),
798
4
        category_count: initial_data.categories.len(),
799
4
        customer_count: initial_data.customers.len(),
800
4
        order_count: initial_data.orders.len(),
801
4
        summaries: accumulated_label_stats
802
4
            .iter()
803
20
            .map(|(&backend, duration)| BackendSummary {
804
20
                backend: backend.to_string(),
805
20
                transport: match backend {
806
20
                    "bonsaidb-local" => String::from("None"),
807
16
                    "bonsaidb-quic" => String::from("UDP with TLS"),
808
12
                    _ => String::from("TCP"),
809
                },
810
20
                total_time: format_nanoseconds(duration.as_nanos() as f64),
811
20
                wall_time: format_nanoseconds(duration.as_nanos() as f64 / number_of_agents as f64),
812
20
            })
813
4
            .collect(),
814
4
        operations: operations.into_values().collect(),
815
4
    }
816
4
    .render_to(&plot_dir.join("index.html"), tera);
817
4
    accumulated_label_stats
818
4
}
819

            
820
struct MetricStats {
821
    average: f64,
822
    min: u64,
823
    max: u64,
824
    stddev: f64,
825
    plottable_stats: Vec<u64>,
826
    outliers: Vec<f64>,
827
}
828

            
829
struct HistogramBars {
830
    bars: Vec<HistogramBar>,
831
}
832

            
833
struct HistogramBar {
834
    upper_left: (Nanos, u64),
835
    lower_right: (Nanos, u64),
836

            
837
    color: RGBColor,
838
}
839

            
840
impl HistogramBar {
841
1642
    pub fn new(coord: (Nanos, u64), width: u64, color: RGBColor) -> Self {
842
1642
        Self {
843
1642
            upper_left: (Nanos(coord.0 .0), coord.1),
844
1642
            lower_right: (Nanos(coord.0 .0 + width), 0),
845
1642
            color,
846
1642
        }
847
1642
    }
848
}
849

            
850
impl<'a> Drawable<BitMapBackend<'a>> for HistogramBar {
851
1642
    fn draw<I: Iterator<Item = <BackendCoordOnly as CoordMapper>::Output>>(
852
1642
        &self,
853
1642
        mut pos: I,
854
1642
        backend: &mut BitMapBackend,
855
1642
        _parent_dim: (u32, u32),
856
1642
    ) -> Result<(), DrawingErrorKind<<BitMapBackend as DrawingBackend>::ErrorType>> {
857
1642
        let upper_left = pos.next().unwrap();
858
1642
        let lower_right = pos.next().unwrap();
859
1642
        backend.draw_rect(upper_left, lower_right, &self.color, true)?;
860

            
861
1642
        Ok(())
862
1642
    }
863
}
864

            
865
impl<'a> PointCollection<'a, (Nanos, u64)> for &'a HistogramBar {
866
    type IntoIter = HistogramBarIter<'a>;
867
    type Point = &'a (Nanos, u64);
868

            
869
1642
    fn point_iter(self) -> Self::IntoIter {
870
1642
        HistogramBarIter::UpperLeft(self)
871
1642
    }
872
}
873

            
874
enum HistogramBarIter<'a> {
875
    UpperLeft(&'a HistogramBar),
876
    LowerRight(&'a HistogramBar),
877
    Done,
878
}
879

            
880
impl<'a> Iterator for HistogramBarIter<'a> {
881
    type Item = &'a (Nanos, u64);
882

            
883
3284
    fn next(&mut self) -> Option<Self::Item> {
884
3284
        let (next, result) = match self {
885
1642
            HistogramBarIter::UpperLeft(bar) => {
886
1642
                (HistogramBarIter::LowerRight(bar), Some(&bar.upper_left))
887
            }
888
1642
            HistogramBarIter::LowerRight(bar) => (HistogramBarIter::Done, Some(&bar.lower_right)),
889
            HistogramBarIter::Done => (HistogramBarIter::Done, None),
890
        };
891
3284
        *self = next;
892
3284
        result
893
3284
    }
894
}
895

            
896
impl IntoIterator for HistogramBars {
897
    type IntoIter = std::vec::IntoIter<HistogramBar>;
898
    type Item = HistogramBar;
899

            
900
140
    fn into_iter(self) -> Self::IntoIter {
901
140
        self.bars.into_iter()
902
140
    }
903
}
904

            
905
280
fn stddev(data: &[u64], average: f64) -> f64 {
906
280
    if data.is_empty() {
907
        0.
908
    } else {
909
280
        let variance = data
910
280
            .iter()
911
64020
            .map(|value| {
912
64020
                let diff = average - (*value as f64);
913
64020

            
914
64020
                diff * diff
915
64020
            })
916
280
            .sum::<f64>()
917
280
            / data.len() as f64;
918
280

            
919
280
        variance.sqrt()
920
    }
921
280
}