1
//! Commerce Benchmark
2
//!
3
//! For more information about this benchmark, see ./README.md.
4

            
5
use std::collections::BTreeMap;
6
use std::io::ErrorKind;
7
use std::sync::Arc;
8
use std::time::Duration;
9

            
10
use clap::Parser;
11
use serde::{Deserialize, Serialize};
12
use tera::{Context, Tera};
13

            
14
use crate::model::InitialDataSetConfig;
15
use crate::utils::{current_timestamp_string, format_nanoseconds, local_git_rev};
16

            
17
mod bonsai;
18
#[cfg(feature = "mongo")]
19
mod mongo;
20
mod plan;
21
#[cfg(feature = "postgresql")]
22
mod postgres;
23
mod utils;
24
use plan::ShopperPlanConfig;
25
mod execute;
26
mod model;
27
mod plot;
28

            
29
2
#[derive(Debug, Parser)]
30
struct Options {
31
    #[clap(long = "bench")]
32
    run_from_cargo_bench: bool,
33
    #[clap(long = "nocapture")]
34
    _cargo_bench_compat_nocapture: bool,
35

            
36
    benchmark_name: Option<String>,
37

            
38
    #[clap(long)]
39
    suite: bool,
40

            
41
    #[clap(long, short, default_value = "Commerce Benchmark")]
42
    label: String,
43

            
44
    #[clap(long, short)]
45
    seed: Option<u64>,
46

            
47
    #[clap(long, default_value = "1")]
48
    min_customers: u32,
49
    #[clap(long, default_value = "500")]
50
    max_customers: u32,
51

            
52
    #[clap(long, default_value = "1")]
53
    min_products: u32,
54
    #[clap(long, default_value = "200")]
55
    max_products: u32,
56

            
57
    #[clap(long, default_value = "1")]
58
    min_categories: u32,
59
    #[clap(long, default_value = "30")]
60
    max_categories: u32,
61

            
62
    #[clap(long, default_value = "10")]
63
    min_orders: u32,
64
    #[clap(long, default_value = "1000")]
65
    max_orders: u32,
66

            
67
    #[clap(long, default_value = "10")]
68
    min_reviews: u32,
69
    #[clap(long, default_value = "500")]
70
    max_reviews: u32,
71

            
72
    #[clap(long, default_value = "0.25")]
73
    add_to_cart_chance: f64,
74
    #[clap(long, default_value = "0.25")]
75
    purchase_chance: f64,
76
    #[clap(long, default_value = "0.25")]
77
    rating_chance: f64,
78

            
79
    #[clap(long, default_value = "1")]
80
    min_searches: u32,
81
    #[clap(long, default_value = "20")]
82
    max_searches: u32,
83

            
84
    #[clap(long, short = 'j')]
85
    agents: Option<usize>,
86

            
87
    #[clap(long, short = 'n')]
88
    shoppers: Option<usize>,
89
}
90

            
91
1
fn main() {
92
1
    match dotenvy::dotenv() {
93
        Ok(_) => {}
94
1
        Err(dotenvy::Error::Io(err)) if err.kind() == ErrorKind::NotFound => {}
95
        Err(other) => panic!("Error with .env file: {other}"),
96
    }
97
1
    let tera = Arc::new(tera::Tera::new("benches/commerce/templates/**/*").unwrap());
98
1
    let options = Options::parse();
99
1
    let benchmark_name = options.benchmark_name.unwrap_or_default();
100
1

            
101
1
    let is_testing = !options.run_from_cargo_bench;
102
1

            
103
1
    if options.suite || is_testing {
104
1
        run_standard_benchmarks(
105
1
            tera.clone(),
106
1
            &benchmark_name,
107
1
            options.shoppers,
108
1
            options.run_from_cargo_bench,
109
1
        );
110
1
    }
111

            
112
1
    if !options.suite || is_testing {
113
        execute::Benchmark {
114
1
            label: options.label,
115
1
            seed: options.seed,
116
1
            agents: if is_testing { Some(1) } else { options.agents },
117
1
            shoppers: if is_testing {
118
1
                Some(256)
119
            } else {
120
                options.shoppers
121
            },
122
1
            data_config: &InitialDataSetConfig {
123
1
                number_of_customers: options.min_customers..=options.max_customers,
124
1
                number_of_products: options.min_products..=options.max_products,
125
1
                number_of_categories: options.min_categories..=options.max_categories,
126
1
                number_of_orders: options.min_orders..=options.max_orders,
127
1
                number_of_reviews: options.min_reviews..=options.max_reviews,
128
1
            },
129
1
            shopper_config: &ShopperPlanConfig {
130
1
                chance_of_adding_product_to_cart: options.add_to_cart_chance,
131
1
                chance_of_purchasing: options.purchase_chance,
132
1
                chance_of_rating: options.rating_chance,
133
1
                product_search_attempts: options.min_searches..=options.max_searches,
134
1
            },
135
1
        }
136
1
        .execute(&benchmark_name, "commerce-bench", tera);
137
    }
138
1
}
139

            
140
1
fn run_standard_benchmarks(
141
1
    tera: Arc<Tera>,
142
1
    name_filter: &str,
143
1
    shoppers: Option<usize>,
144
1
    run_full: bool,
145
1
) {
146
1
    let shoppers = shoppers.unwrap_or(100);
147
1

            
148
1
    let mut initial_datasets = vec![(
149
1
        "small",
150
1
        InitialDataSetConfig {
151
1
            number_of_customers: 100..=100,
152
1
            number_of_products: 100..=100,
153
1
            number_of_categories: 10..=10,
154
1
            number_of_orders: 125..=125,
155
1
            number_of_reviews: 50..=50,
156
1
        },
157
1
    )];
158
1
    if run_full {
159
        initial_datasets.push((
160
            "medium",
161
            InitialDataSetConfig {
162
                number_of_customers: 1_000..=1_000,
163
                number_of_products: 1_000..=1_000,
164
                number_of_categories: 50..=50,
165
                number_of_orders: 1500..=1500,
166
                number_of_reviews: 500..=500,
167
            },
168
        ));
169
        initial_datasets.push((
170
            "large",
171
            InitialDataSetConfig {
172
                number_of_customers: 5_000..=5_000,
173
                number_of_products: 5_000..=5_000,
174
                number_of_categories: 100..=100,
175
                number_of_orders: 5_000..=5_000,
176
                number_of_reviews: 1_000..=1_000,
177
            },
178
        ));
179
1
    }
180

            
181
1
    let mut shopper_plans = vec![(
182
1
        "balanced",
183
1
        ShopperPlanConfig {
184
1
            chance_of_adding_product_to_cart: 0.25,
185
1
            chance_of_purchasing: 0.25,
186
1
            chance_of_rating: 0.25,
187
1
            product_search_attempts: 1..=10,
188
1
        },
189
1
    )];
190
1
    if run_full {
191
        shopper_plans.push((
192
            "readheavy",
193
            ShopperPlanConfig {
194
                chance_of_adding_product_to_cart: 0.1,
195
                chance_of_purchasing: 0.1,
196
                chance_of_rating: 0.1,
197
                product_search_attempts: 1..=10,
198
            },
199
        ));
200
        shopper_plans.push((
201
            "writeheavy",
202
            ShopperPlanConfig {
203
                chance_of_adding_product_to_cart: 0.9,
204
                chance_of_purchasing: 0.9,
205
                chance_of_rating: 0.9,
206
                product_search_attempts: 1..=10,
207
            },
208
        ));
209
1
    }
210

            
211
1
    let mut number_of_agents = vec![1];
212
1
    let num_cpus = num_cpus::get();
213
1
    if num_cpus > 1 {
214
1
        number_of_agents.push(num_cpus);
215
1
    }
216
1
    number_of_agents.push(num_cpus * 2);
217
1

            
218
1
    let mut datasets = Vec::new();
219
1
    let mut summaries = BTreeMap::<usize, Vec<BTreeMap<&'static str, Duration>>>::new();
220
2
    for (dataset_label, data_config) in &initial_datasets {
221
2
        for (plan_label, shopper_config) in &shopper_plans {
222
1
            println!("Running standard benchmark {dataset_label}-{plan_label}");
223
4
            for &concurrency in &number_of_agents {
224
3
                let summaries = summaries.entry(concurrency).or_default();
225
3
                let measurements = execute::Benchmark {
226
3
                    label: format!("{dataset_label}, {plan_label}, {concurrency} agent(s)"),
227
3
                    seed: Some(0),
228
3
                    agents: Some(concurrency),
229
3
                    shoppers: Some(shoppers),
230
3
                    data_config,
231
3
                    shopper_config,
232
3
                }
233
3
                .execute(
234
3
                    name_filter,
235
3
                    format!("./commerce-bench/{dataset_label}-{plan_label}/{concurrency}/"),
236
3
                    tera.clone(),
237
3
                );
238
3
                datasets.push(DataSet {
239
3
                    size: dataset_label.to_string(),
240
3
                    pattern: plan_label.to_string(),
241
3
                    concurrency: concurrency.to_string(),
242
3
                    path: format!("{dataset_label}-{plan_label}/{concurrency}/index.html"),
243
3
                    results: measurements
244
3
                        .iter()
245
15
                        .map(|(k, v)| (k.to_string(), format_nanoseconds(v.as_nanos() as f64)))
246
3
                        .collect(),
247
3
                });
248
3
                summaries.push(measurements);
249
3
            }
250
        }
251
    }
252

            
253
1
    plot::overview_graph(summaries, "./commerce-bench");
254
1

            
255
1
    std::fs::write(
256
1
        "./commerce-bench/index.html",
257
1
        tera.render(
258
1
            "overview.html",
259
1
            &Context::from_serialize(Overview {
260
1
                datasets,
261
1
                timestamp: current_timestamp_string(),
262
1
                revision: local_git_rev(),
263
1
            })
264
1
            .unwrap(),
265
1
        )
266
1
        .unwrap()
267
1
        .as_bytes(),
268
1
    )
269
1
    .unwrap();
270
1
}
271

            
272
1
#[derive(Debug, Serialize, Deserialize)]
273
struct Overview {
274
    datasets: Vec<DataSet>,
275
    timestamp: String,
276
    revision: String,
277
}
278

            
279
3
#[derive(Debug, Serialize, Deserialize)]
280
struct DataSet {
281
    size: String,
282
    pattern: String,
283
    concurrency: String,
284
    path: String,
285
    results: BTreeMap<String, String>,
286
}