1
use std::collections::{BTreeMap, HashMap};
2
use std::ops::{Range, RangeInclusive};
3
use std::path::Path;
4
use std::sync::Arc;
5
use std::time::{Duration, Instant};
6

            
7
use bonsaidb::core::async_trait::async_trait;
8
use cli_table::{Cell, Table};
9
use futures::stream::FuturesUnordered;
10
use futures::StreamExt;
11
use plotters::coord::ranged1d::{NoDefaultFormatting, ValueFormatter};
12
use plotters::element::{BackendCoordOnly, CoordMapper, Drawable, PointCollection};
13
use plotters::prelude::*;
14
use plotters_backend::DrawingErrorKind;
15
use rand::rngs::SmallRng;
16
use rand::SeedableRng;
17
use serde::{Deserialize, Serialize};
18
use tera::Tera;
19
use tokio::runtime::Runtime;
20

            
21
use crate::bonsai::{Bonsai, BonsaiBackend};
22
use crate::model::{InitialDataSet, InitialDataSetConfig};
23
use crate::plan::{
24
    AddProductToCart, Checkout, CreateCart, FindProduct, Load, LookupProduct, Operation,
25
    OperationResult, Plan, ReviewProduct, ShopperPlanConfig,
26
};
27
use crate::plot::{label_to_color, BACKGROUND_COLOR, TEXT_COLOR};
28
use crate::utils::{current_timestamp_string, format_nanoseconds, local_git_rev};
29

            
30
4
pub fn execute_plans_for_all_backends(
31
4
    name_filter: &str,
32
4
    plans: &[Arc<Plan>],
33
4
    initial_data: &Arc<InitialDataSet>,
34
4
    number_of_agents: usize,
35
4
    measurements: &Measurements,
36
4
) {
37
4
    if name_filter.is_empty()
38
        || name_filter == "bonsaidb"
39
        || name_filter.starts_with("bonsaidb-local")
40
4
    {
41
4
        println!("Executing bonsaidb-local");
42
4
        BonsaiBackend::execute_async(
43
4
            Bonsai::Local,
44
4
            plans,
45
4
            initial_data,
46
4
            number_of_agents,
47
4
            measurements,
48
4
        );
49
4
    }
50
4
    if name_filter.is_empty()
51
        || name_filter == "bonsaidb"
52
        || name_filter.starts_with("bonsaidb-local+lz4")
53
4
    {
54
4
        println!("Executing bonsaidb-local+lz4");
55
4
        BonsaiBackend::execute_async(
56
4
            Bonsai::LocalLz4,
57
4
            plans,
58
4
            initial_data,
59
4
            number_of_agents,
60
4
            measurements,
61
4
        );
62
4
    }
63
4
    if name_filter.is_empty()
64
        || name_filter == "bonsaidb"
65
        || name_filter.starts_with("bonsaidb-quic")
66
4
    {
67
4
        println!("Executing bonsaidb-quic");
68
4
        BonsaiBackend::execute_async(
69
4
            Bonsai::Quic,
70
4
            plans,
71
4
            initial_data,
72
4
            number_of_agents,
73
4
            measurements,
74
4
        );
75
4
    }
76
4
    if name_filter.is_empty() || name_filter == "bonsaidb" || name_filter.starts_with("bonsaidb-ws")
77
4
    {
78
4
        println!("Executing bonsaidb-ws");
79
4
        BonsaiBackend::execute_async(
80
4
            Bonsai::WebSockets,
81
4
            plans,
82
4
            initial_data,
83
4
            number_of_agents,
84
4
            measurements,
85
4
        );
86
4
    }
87
    #[cfg(feature = "postgresql")]
88
4
    if name_filter.is_empty() || name_filter.starts_with("postgresql") {
89
4
        if let Ok(url) = std::env::var("COMMERCE_POSTGRESQL_URL") {
90
4
            println!("Executing postgresql");
91
4
            crate::postgres::Postgres::execute_async(
92
4
                url,
93
4
                plans,
94
4
                initial_data,
95
4
                number_of_agents,
96
4
                measurements,
97
4
            );
98
4
        } else {
99
            eprintln!("postgresql feature is enabled, but environment variable COMMERCE_POSTGRESQL_URL is missing.");
100
        }
101
    }
102
    #[cfg(feature = "mongo")]
103
4
    if name_filter.is_empty() || name_filter.starts_with("mongo") {
104
4
        if let Ok(url) = std::env::var("COMMERCE_MONGODB_URL") {
105
            println!("Executing mongodb");
106
            crate::mongo::MongoBackend::execute_async(
107
                url,
108
                plans,
109
                initial_data,
110
                number_of_agents,
111
                measurements,
112
            );
113
4
        } else {
114
4
            eprintln!("mongo feature is enabled, but environment variable COMMERCE_MONGODB_URL is missing.");
115
4
        }
116
    }
117
4
}
118

            
119
#[async_trait]
120
pub trait Backend: Sized + Send + Sync + 'static {
121
    type Operator: BackendOperator;
122
    type Config: Send + Sync;
123

            
124
    async fn new(config: Self::Config) -> Self;
125

            
126
    fn label(&self) -> &'static str;
127

            
128
20
    fn execute_async(
129
20
        config: Self::Config,
130
20
        plans: &[Arc<Plan>],
131
20
        initial_data: &Arc<InitialDataSet>,
132
20
        concurrent_agents: usize,
133
20
        measurements: &Measurements,
134
20
    ) {
135
20
        let (plan_sender, plan_receiver) = flume::bounded(concurrent_agents * 2);
136
20
        let runtime = Runtime::new().unwrap();
137
20
        let backend = runtime.block_on(Self::new(config));
138
20
        // Load the initial data
139
20
        println!("Loading data");
140
20
        runtime.block_on(async {
141
20
            let _ = backend
142
20
                .new_operator_async()
143
16
                .await
144
20
                .operate(
145
20
                    &Load {
146
20
                        initial_data: initial_data.clone(),
147
20
                    },
148
20
                    &[],
149
20
                    measurements,
150
20
                )
151
26258
                .await;
152
20
        });
153
20
        println!("Executing plans");
154
20
        let agent_handles = FuturesUnordered::new();
155
70
        for _ in 0..concurrent_agents {
156
70
            let operator = runtime.block_on(backend.new_operator_async());
157
70
            agent_handles.push(runtime.spawn(agent::<Self>(
158
70
                operator,
159
70
                plan_receiver.clone(),
160
70
                measurements.clone(),
161
70
            )));
162
70
        }
163
20
        runtime.block_on(async {
164
            // Send the plans to the channel that the agents are waiting for
165
            // them on.
166
2800
            for plan in plans {
167
2780
                plan_sender.send_async(plan.clone()).await.unwrap();
168
            }
169
            // Disconnect the receivers, allowing the agents to exit once there
170
            // are no more plans in queue.
171
20
            drop(plan_sender);
172
            // Wait for each of the agents to return.
173
89
            for result in agent_handles.collect::<Vec<_>>().await {
174
70
                result.unwrap();
175
70
            }
176
20
        })
177
20
    }
178

            
179
    async fn new_operator_async(&self) -> Self::Operator;
180
}
181

            
182
#[async_trait]
183
pub trait Operator<T, R> {
184
    async fn operate(
185
        &mut self,
186
        operation: &T,
187
        results: &[OperationResult<R>],
188
        measurements: &Measurements,
189
    ) -> OperationResult<R>;
190
}
191

            
192
70
async fn agent<B: Backend>(
193
70
    mut operator: B::Operator,
194
70
    plan_receiver: flume::Receiver<Arc<Plan>>,
195
70
    measurements: Measurements,
196
70
) {
197
2850
    while let Ok(plan) = plan_receiver.recv_async().await {
198
2780
        let mut results = Vec::with_capacity(plan.operations.len());
199
31660
        for step in &plan.operations {
200
85096
            results.push(operator.operate(step, &results, &measurements).await)
201
        }
202
    }
203
70
}
204

            
205
pub trait BackendOperator:
206
    Operator<Load, Self::Id>
207
    + Operator<LookupProduct, Self::Id>
208
    + Operator<FindProduct, Self::Id>
209
    + Operator<CreateCart, Self::Id>
210
    + Operator<AddProductToCart, Self::Id>
211
    + Operator<ReviewProduct, Self::Id>
212
    + Operator<Checkout, Self::Id>
213
    + Send
214
    + Sync
215
{
216
    type Id: Send + Sync;
217
}
218

            
219
#[async_trait]
220
impl<T> Operator<Operation, T::Id> for T
221
where
222
    T: BackendOperator,
223
{
224
31660
    async fn operate(
225
31660
        &mut self,
226
31660
        operation: &Operation,
227
31660
        results: &[OperationResult<T::Id>],
228
31660
        measurements: &Measurements,
229
31660
    ) -> OperationResult<T::Id> {
230
31660
        match operation {
231
34413
            Operation::FindProduct(op) => self.operate(op, results, measurements).await,
232
24056
            Operation::LookupProduct(op) => self.operate(op, results, measurements).await,
233
5017
            Operation::CreateCart(op) => self.operate(op, results, measurements).await,
234
17043
            Operation::AddProductToCart(op) => self.operate(op, results, measurements).await,
235
1570
            Operation::RateProduct(op) => self.operate(op, results, measurements).await,
236
2997
            Operation::Checkout(op) => self.operate(op, results, measurements).await,
237
        }
238
94980
    }
239
}
240

            
241
4
#[derive(Serialize, Deserialize, Debug)]
242
pub struct BenchmarkSummary {
243
    label: String,
244
    timestamp: String,
245
    revision: String,
246
    plan_count: usize,
247
    agent_count: usize,
248
    product_count: usize,
249
    category_count: usize,
250
    customer_count: usize,
251
    order_count: usize,
252
    summaries: Vec<BackendSummary>,
253
    operations: Vec<MetricSummary>,
254
}
255

            
256
impl BenchmarkSummary {
257
4
    pub fn render_to(&self, location: &Path, tera: &Tera) {
258
4
        std::fs::write(
259
4
            location,
260
4
            tera.render("run.html", &tera::Context::from_serialize(self).unwrap())
261
4
                .unwrap()
262
4
                .as_bytes(),
263
4
        )
264
4
        .unwrap()
265
4
    }
266
}
267

            
268
20
#[derive(Serialize, Deserialize, Debug)]
269
pub struct BackendSummary {
270
    backend: String,
271
    transport: String,
272
    total_time: String,
273
    wall_time: String,
274
}
275

            
276
28
#[derive(Serialize, Deserialize, Debug)]
277
pub struct MetricSummary {
278
    metric: Metric,
279
    description: String,
280
    invocations: usize,
281
    summaries: Vec<OperationSummary>,
282
}
283

            
284
140
#[derive(Serialize, Deserialize, Debug)]
285
pub struct OperationSummary {
286
    backend: String,
287
    avg: String,
288
    min: String,
289
    max: String,
290
    stddev: String,
291
    outliers: String,
292
}
293

            
294
pub struct Benchmark<'a> {
295
    pub label: String,
296
    pub seed: Option<u64>,
297
    pub agents: Option<usize>,
298
    pub shoppers: Option<usize>,
299
    pub data_config: &'a InitialDataSetConfig,
300
    pub shopper_config: &'a ShopperPlanConfig,
301
}
302

            
303
impl<'a> Benchmark<'a> {
304
4
    pub fn execute(
305
4
        self,
306
4
        name_filter: &str,
307
4
        plot_dir: impl AsRef<Path>,
308
4
        tera: Arc<Tera>,
309
4
    ) -> BTreeMap<&'static str, Duration> {
310
4
        let plot_dir = plot_dir.as_ref().to_path_buf();
311
4
        std::fs::create_dir_all(&plot_dir).unwrap();
312

            
313
4
        let mut rng = if let Some(seed) = self.seed {
314
3
            SmallRng::seed_from_u64(seed)
315
        } else {
316
1
            SmallRng::from_entropy()
317
        };
318
4
        let initial_data = Arc::new(self.data_config.fake(&mut rng));
319
4
        let number_of_agents = self.agents.unwrap_or_else(num_cpus::get);
320
4
        let shoppers = self.shoppers.unwrap_or(number_of_agents * 100);
321
4

            
322
4
        println!("Running {shoppers} plans across {number_of_agents} agents");
323
4
        // Generate plans to execute.
324
4
        let mut plans = Vec::with_capacity(shoppers);
325
556
        for _ in 0..shoppers {
326
556
            plans.push(Arc::new(
327
556
                self.shopper_config.random_plan(&mut rng, &initial_data),
328
556
            ));
329
556
        }
330
        // Set up our statistics gathering thread
331
4
        let (metric_sender, metric_receiver) = flume::unbounded();
332
4
        let measurements = Measurements {
333
4
            sender: metric_sender,
334
4
        };
335
4
        let thread_initial_data = initial_data.clone();
336
4
        let stats_thread = std::thread::spawn(move || {
337
4
            stats_thread(
338
4
                self.label,
339
4
                metric_receiver,
340
4
                shoppers,
341
4
                number_of_agents,
342
4
                &thread_initial_data,
343
4
                &plot_dir,
344
4
                &tera,
345
4
            )
346
4
        });
347
4
        // Perform all benchmarks
348
4
        execute_plans_for_all_backends(
349
4
            name_filter,
350
4
            &plans,
351
4
            &initial_data,
352
4
            number_of_agents,
353
4
            &measurements,
354
4
        );
355
4
        // Drop the measurements instance to allow the stats thread to know
356
4
        // there are no more metrics coming.
357
4
        drop(measurements);
358
4
        // Wait for the statistics thread to report all the results.
359
4
        stats_thread.join().unwrap()
360
4
    }
361
}
362

            
363
70
#[derive(Clone)]
364
pub struct Measurements {
365
    sender: flume::Sender<(&'static str, Metric, Duration)>,
366
}
367

            
368
impl Measurements {
369
31680
    pub fn begin(&self, label: &'static str, metric: Metric) -> Measurement<'_> {
370
31680
        Measurement {
371
31680
            target: &self.sender,
372
31680
            label,
373
31680
            metric,
374
31680
            start: Instant::now(),
375
31680
        }
376
31680
    }
377
}
378

            
379
pub struct Measurement<'a> {
380
    target: &'a flume::Sender<(&'static str, Metric, Duration)>,
381
    label: &'static str,
382
    metric: Metric,
383
    start: Instant,
384
}
385

            
386
impl<'a> Measurement<'a> {
387
31680
    pub fn finish(self) {
388
31680
        let duration = Instant::now()
389
31680
            .checked_duration_since(self.start)
390
31680
            .expect("time went backwards. Restart benchmarks.");
391
31680
        self.target
392
31680
            .send((self.label, self.metric, duration))
393
31680
            .unwrap();
394
31680
    }
395
}
396

            
397
102076
#[derive(Serialize, Deserialize, Clone, Copy, Hash, Eq, PartialEq, Debug, Ord, PartialOrd)]
398
pub enum Metric {
399
    Load,
400
    LookupProduct,
401
    FindProduct,
402
    CreateCart,
403
    AddProductToCart,
404
    Checkout,
405
    RateProduct,
406
}
407

            
408
impl Metric {
409
28
    pub fn description(&self) -> &'static str {
410
28
        match self {
411
4
            Metric::Load => "Measures the time spent loading the initial data set and performing any pre-cache operations that most database administrators would perform on their databases periodically to ensure good performance.",
412
4
            Metric::LookupProduct => "Meaures the time spent looking up a product by its id. This operation is meant to simulate the basic needs of the database to provide a product details page after a user clicked a direct link that contians the product's unique id, including the product's current rating.",
413
4
            Metric::FindProduct => "Measures the time spent looking up a product by its name (exact match, indexed). This operation is meant to simulate the basic needs of the database to provide a product details after finding a product by its name, including the product's current rating.",
414
4
            Metric::CreateCart => "Measures the time spent creating a shopping cart.",
415
4
            Metric::AddProductToCart => "Measures the time spent adding a product to a shopping cart.",
416
4
            Metric::RateProduct => "Measures the time spent adding or updating a review of a product by a customer. Each customer can only have one review per product. When this operation is complete, all subsequent calls to LookupProduct and FindProduct should reflect the new rating. This simulates an 'upsert' (insert or update) operation using a unique index.",
417
4
            Metric::Checkout => "Measures the time spent converting a shopping cart into an order for a customer."
418
        }
419
28
    }
420
}
421

            
422
#[derive(Clone, Debug)]
423
struct NanosRange(RangeInclusive<Nanos>);
424
3560
#[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq)]
425
struct Nanos(u64);
426

            
427
impl ValueFormatter<Nanos> for NanosRange {
428
1848
    fn format(value: &Nanos) -> String {
429
1848
        format_nanoseconds(value.0 as f64)
430
1848
    }
431
}
432

            
433
impl Ranged for NanosRange {
434
    type FormatOption = NoDefaultFormatting;
435
    type ValueType = Nanos;
436

            
437
55512
    fn map(&self, value: &Self::ValueType, limit: (i32, i32)) -> i32 {
438
55512
        let limited_size = limit.1 - limit.0;
439
55512
        let full_size = self.0.end().0 + 1 - self.0.start().0;
440
55512
        let normalized_offset = value.0.saturating_sub(self.0.start().0) as f64 / full_size as f64;
441
55512
        limit.0 + (normalized_offset * limited_size as f64) as i32
442
55512
    }
443

            
444
336
    fn key_points<Hint: plotters::coord::ranged1d::KeyPointHint>(
445
336
        &self,
446
336
        hint: Hint,
447
336
    ) -> Vec<Self::ValueType> {
448
336
        let total_range = self.0.end().0 - self.0.start().0;
449
336
        let num_points = hint.max_num_points();
450
336
        let mut important_points = Vec::with_capacity(num_points);
451
336
        important_points.push(*self.0.start());
452
336
        if num_points > 2 {
453
336
            let steps = num_points - 2;
454
336
            let step_size = total_range as f64 / steps as f64;
455
336
            important_points.extend(
456
336
                (1..num_points - 1)
457
19656
                    .map(|step| Nanos(self.0.start().0 + (step as f64 * step_size) as u64)),
458
336
            );
459
336
        }
460
336
        important_points.push(*self.0.end());
461
336

            
462
336
        important_points
463
336
    }
464

            
465
    fn range(&self) -> std::ops::Range<Self::ValueType> {
466
        Nanos(self.0.start().0)..Nanos(self.0.end().0 + 1)
467
    }
468
}
469

            
470
impl DiscreteRanged for NanosRange {
471
    fn size(&self) -> usize {
472
        (self.0.end().0 - self.0.start().0) as usize
473
    }
474

            
475
    fn index_of(&self, value: &Self::ValueType) -> Option<usize> {
476
        if value.0 <= self.0.end().0 {
477
            if let Some(index) = value.0.checked_sub(self.0.start().0) {
478
                return Some(index as usize);
479
            }
480
        }
481
        None
482
    }
483

            
484
    fn from_index(&self, index: usize) -> Option<Self::ValueType> {
485
        Some(Nanos(self.0.start().0 + index as u64))
486
    }
487
}
488

            
489
4
fn stats_thread(
490
4
    label: String,
491
4
    metric_receiver: flume::Receiver<(&'static str, Metric, Duration)>,
492
4
    number_of_plans: usize,
493
4
    number_of_agents: usize,
494
4
    initial_data: &InitialDataSet,
495
4
    plot_dir: &Path,
496
4
    tera: &Tera,
497
4
) -> BTreeMap<&'static str, Duration> {
498
4
    let mut all_results: BTreeMap<Metric, BTreeMap<&'static str, Vec<u64>>> = BTreeMap::new();
499
4
    let mut accumulated_label_stats: BTreeMap<&'static str, Duration> = BTreeMap::new();
500
4
    let mut longest_by_metric = HashMap::new();
501
31684
    while let Ok((label, metric, duration)) = metric_receiver.recv() {
502
31680
        let metric_results = all_results.entry(metric).or_default();
503
31680
        let label_results = metric_results.entry(label).or_default();
504
31680
        let nanos = u64::try_from(duration.as_nanos()).unwrap();
505
31680
        label_results.push(nanos);
506
31680
        let label_duration = accumulated_label_stats.entry(label).or_default();
507
31680
        longest_by_metric
508
31680
            .entry(metric)
509
31680
            .and_modify(|existing: &mut Duration| {
510
31652
                *existing = (*existing).max(duration);
511
31680
            })
512
31680
            .or_insert(duration);
513
31680
        *label_duration += duration;
514
31680
    }
515

            
516
4
    let mut operations = BTreeMap::new();
517
4
    let mut metric_ranges = HashMap::new();
518
32
    for (metric, label_metrics) in all_results {
519
28
        let label_stats = label_metrics
520
28
            .iter()
521
140
            .map(|(label, stats)| {
522
140
                let mut sum = 0;
523
140
                let mut min = u64::MAX;
524
140
                let mut max = 0;
525
31820
                for &nanos in stats {
526
31680
                    sum += nanos;
527
31680
                    min = min.min(nanos);
528
31680
                    max = max.max(nanos);
529
31680
                }
530
140
                let average = sum as f64 / stats.len() as f64;
531
140
                let stddev = stddev(stats, average);
532
140

            
533
140
                let mut outliers = Vec::new();
534
140
                let mut plottable_stats = Vec::new();
535
140
                let mut min_plottable = u64::MAX;
536
140
                let mut max_plottable = 0;
537
31820
                for &nanos in stats {
538
31680
                    let diff = (nanos as f64 - average).abs();
539
31680
                    let diff_magnitude = diff / stddev;
540
31680
                    if stats.len() == 1 || diff_magnitude < 3. {
541
31296
                        plottable_stats.push(nanos);
542
31296
                        min_plottable = min_plottable.min(nanos);
543
31296
                        max_plottable = max_plottable.max(nanos);
544
31296
                    } else {
545
384
                        // Outlier
546
384
                        outliers.push(diff_magnitude);
547
384
                    }
548
                }
549

            
550
140
                if !outliers.is_empty() {
551
97
                    eprintln!("Not plotting {} outliers for {}", outliers.len(), label);
552
97
                }
553

            
554
140
                metric_ranges
555
140
                    .entry(metric)
556
140
                    .and_modify(|range: &mut Range<u64>| {
557
112
                        range.start = range.start.min(min_plottable);
558
112
                        range.end = range.end.max(max_plottable + 1);
559
140
                    })
560
140
                    .or_insert(min_plottable..max_plottable + 1);
561
140

            
562
140
                (
563
140
                    label,
564
140
                    MetricStats {
565
140
                        average,
566
140
                        min,
567
140
                        max,
568
140
                        stddev,
569
140
                        outliers,
570
140
                        plottable_stats,
571
140
                    },
572
140
                )
573
140
            })
574
28
            .collect::<BTreeMap<_, _>>();
575
28
        println!(
576
28
            "{:?}: {} operations",
577
28
            metric,
578
28
            label_metrics.values().next().unwrap().len()
579
28
        );
580
28
        cli_table::print_stdout(
581
28
            label_metrics
582
28
                .iter()
583
140
                .map(|(label, stats)| {
584
140
                    let average = stats.iter().sum::<u64>() as f64 / stats.len() as f64;
585
140
                    let min = *stats.iter().min().unwrap() as f64;
586
140
                    let max = *stats.iter().max().unwrap() as f64;
587
140
                    let stddev = stddev(stats, average);
588
140

            
589
140
                    vec![
590
140
                        label.cell(),
591
140
                        format_nanoseconds(average).cell(),
592
140
                        format_nanoseconds(min).cell(),
593
140
                        format_nanoseconds(max).cell(),
594
140
                        format_nanoseconds(stddev).cell(),
595
140
                    ]
596
140
                })
597
28
                .table()
598
28
                .title(vec![
599
28
                    "Backend".cell(),
600
28
                    "Avg".cell(),
601
28
                    "Min".cell(),
602
28
                    "Max".cell(),
603
28
                    "StdDev".cell(),
604
28
                ]),
605
28
        )
606
28
        .unwrap();
607
28

            
608
28
        let mut label_chart_data = BTreeMap::new();
609
140
        for (label, metrics) in label_stats.iter() {
610
140
            let report = operations.entry(metric).or_insert_with(|| MetricSummary {
611
28
                metric,
612
28
                invocations: label_metrics.values().next().unwrap().len(),
613
28
                description: metric.description().to_string(),
614
28
                summaries: Vec::new(),
615
140
            });
616
140
            report.summaries.push(OperationSummary {
617
140
                backend: label.to_string(),
618
140
                avg: format_nanoseconds(metrics.average),
619
140
                min: format_nanoseconds(metrics.min as f64),
620
140
                max: format_nanoseconds(metrics.max as f64),
621
140
                stddev: format_nanoseconds(metrics.stddev),
622
140
                outliers: metrics.outliers.len().to_string(),
623
140
            });
624
140
            let mut histogram = vec![0; 63];
625
140
            let range = &metric_ranges[&metric];
626
140
            let bucket_width = range.end / (histogram.len() as u64 - 1);
627
31436
            for &nanos in &metrics.plottable_stats {
628
31296
                let bucket = (nanos / bucket_width) as usize;
629
31296
                histogram[bucket] += 1;
630
31296
            }
631
140
            let chart_data = HistogramBars {
632
140
                bars: histogram
633
140
                    .iter()
634
140
                    .enumerate()
635
8820
                    .filter_map(|(bucket, &count)| {
636
8820
                        if count > 0 {
637
1752
                            let bucket_value = bucket as u64 * bucket_width;
638
1752

            
639
1752
                            Some(HistogramBar::new(
640
1752
                                (Nanos(bucket_value), count),
641
1752
                                bucket_width,
642
1752
                                label_to_color(label),
643
1752
                            ))
644
                        } else {
645
7068
                            None
646
                        }
647
8820
                    })
648
140
                    .collect::<Vec<_>>(),
649
140
            };
650
140
            label_chart_data.insert(label, chart_data);
651
        }
652
28
        let highest_count = Iterator::max(
653
28
            label_chart_data
654
28
                .values()
655
1752
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.upper_left.1)),
656
28
        )
657
28
        .unwrap();
658
28
        let highest_nanos = Iterator::max(
659
28
            label_chart_data
660
28
                .values()
661
1752
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.lower_right.0)),
662
28
        )
663
28
        .unwrap();
664
28
        let lowest_nanos = Iterator::min(
665
28
            label_chart_data
666
28
                .values()
667
1752
                .flat_map(|chart_data| chart_data.bars.iter().map(|bar| bar.upper_left.0)),
668
28
        )
669
28
        .unwrap();
670
168
        for (label, chart_data) in label_chart_data {
671
140
            println!("Plotting {label}: {metric:?}");
672
140
            let chart_path = plot_dir.join(format!("{label}-{metric:?}.png"));
673
140
            let chart_root = BitMapBackend::new(&chart_path, (800, 240)).into_drawing_area();
674
140
            chart_root.fill(&BACKGROUND_COLOR).unwrap();
675
140
            let mut chart = ChartBuilder::on(&chart_root)
676
140
                .caption(
677
140
                    format!("{label}: {metric:?}"),
678
140
                    ("sans-serif", 30., &TEXT_COLOR),
679
140
                )
680
140
                .margin_left(10)
681
140
                .margin_right(50)
682
140
                .margin_bottom(10)
683
140
                .x_label_area_size(50)
684
140
                .y_label_area_size(80)
685
140
                .build_cartesian_2d(
686
140
                    NanosRange(lowest_nanos..=highest_nanos),
687
140
                    0..highest_count + 1,
688
140
                )
689
140
                .unwrap();
690
140

            
691
140
            chart
692
140
                .configure_mesh()
693
140
                .disable_x_mesh()
694
140
                .y_desc("Count")
695
140
                .x_desc("Execution Time")
696
140
                .axis_desc_style(("sans-serif", 15, &TEXT_COLOR))
697
140
                .x_label_style(&TEXT_COLOR)
698
140
                .y_label_style(&TEXT_COLOR)
699
140
                .light_line_style(TEXT_COLOR.mix(0.1))
700
140
                .bold_line_style(TEXT_COLOR.mix(0.3))
701
140
                .draw()
702
140
                .unwrap();
703
140

            
704
140
            chart.draw_series(chart_data).unwrap();
705
140
            chart_root.present().unwrap();
706
140
        }
707
28
        let label_lines = label_metrics
708
28
            .iter()
709
140
            .map(|(label, stats)| {
710
140
                let mut running_data = Vec::new();
711
140
                let mut elapsed = 0;
712
31680
                for (index, &nanos) in stats.iter().enumerate() {
713
31680
                    elapsed += nanos;
714
31680
                    running_data.push((index, Nanos(elapsed)));
715
31680
                }
716
140
                (label, running_data)
717
140
            })
718
28
            .collect::<BTreeMap<_, _>>();
719
28
        let metric_chart_path = plot_dir.join(format!("{metric:?}.png"));
720
28
        let metric_chart_root =
721
28
            BitMapBackend::new(&metric_chart_path, (800, 480)).into_drawing_area();
722
28
        metric_chart_root.fill(&BACKGROUND_COLOR).unwrap();
723
28
        let mut metric_chart = ChartBuilder::on(&metric_chart_root)
724
28
            .caption(format!("{metric:?}"), ("sans-serif", 30., &TEXT_COLOR))
725
28
            .margin_left(10)
726
28
            .margin_right(50)
727
28
            .margin_bottom(10)
728
28
            .x_label_area_size(50)
729
28
            .y_label_area_size(80)
730
28
            .build_cartesian_2d(
731
140
                0..label_lines.values().map(|data| data.len()).max().unwrap(),
732
28
                NanosRange(
733
28
                    Nanos(0)
734
28
                        ..=label_lines
735
28
                            .values()
736
140
                            .map(|stats| stats.last().unwrap().1)
737
28
                            .max()
738
28
                            .unwrap(),
739
28
                ),
740
28
            )
741
28
            .unwrap();
742
28

            
743
28
        metric_chart
744
28
            .configure_mesh()
745
28
            .disable_x_mesh()
746
28
            .x_desc("Invocations")
747
28
            .y_desc("Accumulated Execution Time")
748
28
            .axis_desc_style(("sans-serif", 15, &TEXT_COLOR))
749
28
            .x_label_style(&TEXT_COLOR)
750
28
            .y_label_style(&TEXT_COLOR)
751
28
            .light_line_style(TEXT_COLOR.mix(0.1))
752
28
            .bold_line_style(TEXT_COLOR.mix(0.3))
753
28
            .draw()
754
28
            .unwrap();
755

            
756
168
        for (label, data) in label_lines {
757
140
            metric_chart
758
140
                .draw_series(LineSeries::new(data.into_iter(), label_to_color(label)))
759
140
                .unwrap()
760
140
                .label(label.to_string())
761
140
                .legend(|(x, y)| {
762
140
                    PathElement::new(vec![(x, y), (x + 20, y)], label_to_color(label))
763
140
                });
764
140
        }
765
28
        metric_chart
766
28
            .configure_series_labels()
767
28
            .border_style(TEXT_COLOR)
768
28
            .background_style(BACKGROUND_COLOR)
769
28
            .label_font(&TEXT_COLOR)
770
28
            .position(SeriesLabelPosition::UpperLeft)
771
28
            .draw()
772
28
            .unwrap();
773
28
        metric_chart_root.present().unwrap();
774
    }
775
4
    cli_table::print_stdout(
776
4
        accumulated_label_stats
777
4
            .iter()
778
20
            .map(|(label, duration)| {
779
20
                vec![
780
20
                    label.cell(),
781
20
                    format_nanoseconds(duration.as_nanos() as f64).cell(),
782
20
                ]
783
20
            })
784
4
            .table()
785
4
            .title(vec![
786
4
                "Backend".cell(),
787
4
                format!("Total Execution Time across {number_of_agents} agents").cell(),
788
4
            ]),
789
4
    )
790
4
    .unwrap();
791
4
    BenchmarkSummary {
792
4
        label,
793
4
        timestamp: current_timestamp_string(),
794
4
        revision: local_git_rev(),
795
4
        plan_count: number_of_plans,
796
4
        agent_count: number_of_agents,
797
4
        product_count: initial_data.products.len(),
798
4
        category_count: initial_data.categories.len(),
799
4
        customer_count: initial_data.customers.len(),
800
4
        order_count: initial_data.orders.len(),
801
4
        summaries: accumulated_label_stats
802
4
            .iter()
803
20
            .map(|(&backend, duration)| BackendSummary {
804
20
                backend: backend.to_string(),
805
20
                transport: match backend {
806
20
                    "bonsaidb-local" => String::from("None"),
807
16
                    "bonsaidb-quic" => String::from("UDP with TLS"),
808
12
                    _ => String::from("TCP"),
809
                },
810
20
                total_time: format_nanoseconds(duration.as_nanos() as f64),
811
20
                wall_time: format_nanoseconds(duration.as_nanos() as f64 / number_of_agents as f64),
812
20
            })
813
4
            .collect(),
814
4
        operations: operations.into_values().collect(),
815
4
    }
816
4
    .render_to(&plot_dir.join("index.html"), tera);
817
4
    accumulated_label_stats
818
4
}
819

            
820
struct MetricStats {
821
    average: f64,
822
    min: u64,
823
    max: u64,
824
    stddev: f64,
825
    plottable_stats: Vec<u64>,
826
    outliers: Vec<f64>,
827
}
828

            
829
struct HistogramBars {
830
    bars: Vec<HistogramBar>,
831
}
832

            
833
struct HistogramBar {
834
    upper_left: (Nanos, u64),
835
    lower_right: (Nanos, u64),
836

            
837
    color: RGBColor,
838
}
839

            
840
impl HistogramBar {
841
1752
    pub fn new(coord: (Nanos, u64), width: u64, color: RGBColor) -> Self {
842
1752
        Self {
843
1752
            upper_left: (Nanos(coord.0 .0), coord.1),
844
1752
            lower_right: (Nanos(coord.0 .0 + width), 0),
845
1752
            color,
846
1752
        }
847
1752
    }
848
}
849

            
850
impl<'a> Drawable<BitMapBackend<'a>> for HistogramBar {
851
1752
    fn draw<I: Iterator<Item = <BackendCoordOnly as CoordMapper>::Output>>(
852
1752
        &self,
853
1752
        mut pos: I,
854
1752
        backend: &mut BitMapBackend,
855
1752
        _parent_dim: (u32, u32),
856
1752
    ) -> Result<(), DrawingErrorKind<<BitMapBackend as DrawingBackend>::ErrorType>> {
857
1752
        let upper_left = pos.next().unwrap();
858
1752
        let lower_right = pos.next().unwrap();
859
1752
        backend.draw_rect(upper_left, lower_right, &self.color, true)?;
860

            
861
1752
        Ok(())
862
1752
    }
863
}
864

            
865
impl<'a> PointCollection<'a, (Nanos, u64)> for &'a HistogramBar {
866
    type IntoIter = HistogramBarIter<'a>;
867
    type Point = &'a (Nanos, u64);
868

            
869
1752
    fn point_iter(self) -> Self::IntoIter {
870
1752
        HistogramBarIter::UpperLeft(self)
871
1752
    }
872
}
873

            
874
enum HistogramBarIter<'a> {
875
    UpperLeft(&'a HistogramBar),
876
    LowerRight(&'a HistogramBar),
877
    Done,
878
}
879

            
880
impl<'a> Iterator for HistogramBarIter<'a> {
881
    type Item = &'a (Nanos, u64);
882

            
883
3504
    fn next(&mut self) -> Option<Self::Item> {
884
3504
        let (next, result) = match self {
885
1752
            HistogramBarIter::UpperLeft(bar) => {
886
1752
                (HistogramBarIter::LowerRight(bar), Some(&bar.upper_left))
887
            }
888
1752
            HistogramBarIter::LowerRight(bar) => (HistogramBarIter::Done, Some(&bar.lower_right)),
889
            HistogramBarIter::Done => (HistogramBarIter::Done, None),
890
        };
891
3504
        *self = next;
892
3504
        result
893
3504
    }
894
}
895

            
896
impl IntoIterator for HistogramBars {
897
    type IntoIter = std::vec::IntoIter<HistogramBar>;
898
    type Item = HistogramBar;
899

            
900
140
    fn into_iter(self) -> Self::IntoIter {
901
140
        self.bars.into_iter()
902
140
    }
903
}
904

            
905
280
fn stddev(data: &[u64], average: f64) -> f64 {
906
280
    if data.is_empty() {
907
        0.
908
    } else {
909
280
        let variance = data
910
280
            .iter()
911
63360
            .map(|value| {
912
63360
                let diff = average - (*value as f64);
913
63360

            
914
63360
                diff * diff
915
63360
            })
916
280
            .sum::<f64>()
917
280
            / data.len() as f64;
918
280

            
919
280
        variance.sqrt()
920
    }
921
280
}