1
//! [`VaultKeyStorage`] using S3-compatible storage.
2
//!
3
//! This is the recommended method for securing your BonsaiDb database. There
4
//! are many ways to acquire secure, inexpensive S3-compatible storage, such as
5
//! Backblaze B2.
6
//!
7
//! Do not configure your bucket with public access. You should only allow
8
//! access from the IP addresses that your BonsaiDb server(s) are hosted on,
9
//! or only allow authenticated access.
10
//!
11
//! To use this, specify the `vault_key_storage` configuration parameter:
12
//!
13
//! ```rust
14
//! # use bonsaidb_keystorage_s3::S3VaultKeyStorage;
15
//! # use aws_config::endpoint::Endpoint;
16
//! # use bonsaidb_core::{document::KeyId, test_util::TestDirectory};
17
//! # use bonsaidb_local::config::{StorageConfiguration, Builder};
18
//! # use http::Uri;
19
//! #
20
//! # async fn test() {
21
//! let directory = TestDirectory::new("bonsaidb-keystorage-s3-basic");
22
//! let configuration = StorageConfiguration::new(&directory)
23
//!     .vault_key_storage(
24
//!         S3VaultKeyStorage::new("bucket_name")
25
//!             .endpoint("https://s3.us-west-001.backblazeb2.com"),
26
//!     )
27
//!     .default_encryption_key(KeyId::Master);
28
//! # }
29
//! ```
30
//!
31
//! The API calls are performed by the [`aws-sdk-s3`](aws_sdk_s3) crate.
32

            
33
#![forbid(unsafe_code)]
34
#![warn(
35
    clippy::cargo,
36
    missing_docs,
37
    // clippy::missing_docs_in_private_items,
38
    clippy::pedantic,
39
    future_incompatible,
40
    rust_2018_idioms,
41
)]
42
#![allow(
43
    clippy::missing_errors_doc, // TODO clippy::missing_errors_doc
44
    clippy::missing_panics_doc, // TODO clippy::missing_panics_doc
45
    clippy::option_if_let_else,
46
    clippy::module_name_repetitions,
47
)]
48

            
49
use std::fmt::Display;
50
use std::future::Future;
51

            
52
use async_trait::async_trait;
53
use aws_config::meta::region::RegionProviderChain;
54
use aws_sdk_s3::config::Region;
55
use aws_sdk_s3::primitives::ByteStream;
56
use aws_sdk_s3::Client;
57
use bonsaidb_local::vault::{KeyPair, VaultKeyStorage};
58
use bonsaidb_local::StorageId;
59
use tokio::runtime::{self, Handle, Runtime};
60
pub use {aws_sdk_s3, http};
61

            
62
/// S3-compatible [`VaultKeyStorage`] implementor.
63
6
#[derive(Default, Debug)]
64
#[must_use]
65
pub struct S3VaultKeyStorage {
66
    runtime: Tokio,
67
    bucket: String,
68
    /// The S3 endpoint to use. If not specified, the endpoint will be
69
    /// determined automatically. This field can be used to support non-AWS S3
70
    /// providers.
71
    pub endpoint: Option<String>,
72
    /// The AWS region to use. If not specified, the region will be determined
73
    /// by the aws sdk.
74
    pub region: Option<Region>,
75
    /// The path prefix for keys to be stored within.
76
    pub path: String,
77
}
78

            
79
#[derive(Debug)]
80
enum Tokio {
81
    Runtime(Runtime),
82
    Handle(Handle),
83
}
84

            
85
impl Default for Tokio {
86
6
    fn default() -> Self {
87
6
        Handle::try_current().map_or_else(
88
6
            |_| {
89
3
                Self::Runtime(
90
3
                    runtime::Builder::new_current_thread()
91
3
                        .enable_all()
92
3
                        .build()
93
3
                        .unwrap(),
94
3
                )
95
6
            },
96
6
            Self::Handle,
97
6
        )
98
6
    }
99
}
100

            
101
impl Tokio {
102
8
    pub fn block_on<F: Future<Output = R>, R>(&self, future: F) -> R {
103
8
        match self {
104
4
            Tokio::Runtime(rt) => rt.block_on(future),
105
4
            Tokio::Handle(rt) => rt.block_on(future),
106
        }
107
8
    }
108
}
109

            
110
impl S3VaultKeyStorage {
111
    /// Creates a new key storage instance for `bucket`. This instance will use
112
    /// the currently available Tokio runtime or create one if none is
113
    /// available.
114
    pub fn new(bucket: impl Display) -> Self {
115
        Self::new_with_runtime(bucket, tokio::runtime::Handle::current())
116
    }
117

            
118
    /// Creates a new key storage instance for `bucket`, which performs its
119
    /// networking operations on `runtime`.
120
    pub fn new_with_runtime(bucket: impl Display, runtime: tokio::runtime::Handle) -> Self {
121
        Self {
122
            bucket: bucket.to_string(),
123
            runtime: Tokio::Handle(runtime),
124
            ..Self::default()
125
        }
126
    }
127

            
128
    /// Sets the path prefix for vault keys to be stored within.
129
2
    pub fn path(mut self, prefix: impl Display) -> Self {
130
2
        self.path = prefix.to_string();
131
2
        self
132
2
    }
133

            
134
    /// Sets the endpoint to use. See [`Self::endpoint`] for more information.
135
    #[allow(clippy::missing_const_for_fn)] // destructors
136
    pub fn endpoint(mut self, endpoint: impl Into<String>) -> Self {
137
        self.endpoint = Some(endpoint.into());
138
        self
139
    }
140

            
141
8
    fn path_for_id(&self, storage_id: StorageId) -> String {
142
8
        let mut path = self.path.clone();
143
8
        if !path.is_empty() && !path.ends_with('/') {
144
2
            path.push('/');
145
6
        }
146
8
        path.push_str(&storage_id.to_string());
147
8
        path
148
8
    }
149

            
150
8
    async fn client(&self) -> aws_sdk_s3::Client {
151
8
        let region_provider = RegionProviderChain::first_try(self.region.clone())
152
8
            .or_default_provider()
153
8
            .or_else(Region::new("us-east-1"));
154
53
        let config = aws_config::from_env().load().await;
155
8
        if let Some(endpoint) = self.endpoint.clone() {
156
            Client::from_conf(
157
8
                aws_sdk_s3::Config::builder()
158
8
                    .endpoint_url(endpoint)
159
26
                    .region(region_provider.region().await)
160
8
                    .credentials_provider(config.credentials_provider().unwrap().clone())
161
8
                    .build(),
162
            )
163
        } else {
164
            Client::new(&config)
165
        }
166
8
    }
167
}
168

            
169
#[async_trait]
170
impl VaultKeyStorage for S3VaultKeyStorage {
171
    type Error = anyhow::Error;
172

            
173
2
    fn set_vault_key_for(&self, storage_id: StorageId, key: KeyPair) -> Result<(), Self::Error> {
174
2
        self.runtime.block_on(async {
175
18
            let client = self.client().await;
176
2
            let key = key.to_bytes()?;
177
2
            client
178
2
                .put_object()
179
2
                .bucket(&self.bucket)
180
2
                .key(self.path_for_id(storage_id))
181
2
                .body(ByteStream::from(key.to_vec()))
182
2
                .send()
183
10
                .await?;
184
2
            Ok(())
185
2
        })
186
2
    }
187

            
188
6
    fn vault_key_for(&self, storage_id: StorageId) -> Result<Option<KeyPair>, Self::Error> {
189
6
        self.runtime.block_on(async {
190
61
            let client = self.client().await;
191
6
            match client
192
6
                .get_object()
193
6
                .bucket(&self.bucket)
194
6
                .key(self.path_for_id(storage_id))
195
6
                .send()
196
33
                .await
197
            {
198
4
                Ok(response) => {
199
4
                    let bytes = response.body.collect().await?.into_bytes();
200
4
                    let key = KeyPair::from_bytes(&bytes)
201
4
                        .map_err(|err| anyhow::anyhow!(err.to_string()))?;
202
4
                    Ok(Some(key))
203
                }
204
2
                Err(aws_smithy_client::SdkError::ServiceError(err))
205
                    if matches!(
206
2
                        err.err(),
207
                        aws_sdk_s3::operation::get_object::GetObjectError::NoSuchKey(_)
208
                    ) =>
209
                {
210
2
                    Ok(None)
211
                }
212
                Err(err) => Err(anyhow::anyhow!(err)),
213
            }
214
6
        })
215
6
    }
216
}
217

            
218
#[cfg(test)]
219
macro_rules! env_var {
220
    ($name:expr) => {{
221
        match std::env::var($name) {
222
            Ok(value) if !value.is_empty() => value,
223
            _ => {
224
                log::error!(
225
                    "Ignoring basic_test because of missing environment variable: {}",
226
                    $name
227
                );
228
                return;
229
            }
230
        }
231
    }};
232
}
233

            
234
#[cfg(test)]
235
1
#[tokio::test]
236
1
async fn basic_test() {
237
1
    use bonsaidb_core::connection::AsyncStorageConnection;
238
1
    use bonsaidb_core::document::KeyId;
239
1
    use bonsaidb_core::schema::SerializedCollection;
240
1
    use bonsaidb_core::test_util::{Basic, BasicSchema, TestDirectory};
241
1
    use bonsaidb_local::config::{Builder, StorageConfiguration};
242
1
    use bonsaidb_local::AsyncStorage;
243
1
    drop(dotenvy::dotenv());
244
1

            
245
1
    let bucket = env_var!("S3_BUCKET");
246
1
    let endpoint = env_var!("S3_ENDPOINT");
247
1

            
248
1
    let directory = TestDirectory::new("bonsaidb-keystorage-s3-basic");
249
1

            
250
3
    let configuration = |prefix| {
251
3
        let mut vault_key_storage = S3VaultKeyStorage {
252
3
            bucket: bucket.clone(),
253
3
            endpoint: Some(endpoint.clone()),
254
3
            ..S3VaultKeyStorage::default()
255
3
        };
256
3
        if let Some(prefix) = prefix {
257
1
            vault_key_storage = vault_key_storage.path(prefix);
258
2
        }
259
1

            
260
3
        StorageConfiguration::new(&directory)
261
3
            .vault_key_storage(vault_key_storage)
262
3
            .default_encryption_key(KeyId::Master)
263
3
            .with_schema::<BasicSchema>()
264
3
            .unwrap()
265
3
    };
266
1
    let document = {
267
1
        let bonsai = AsyncStorage::open(configuration(None)).await.unwrap();
268
1
        let db = bonsai
269
1
            .create_database::<BasicSchema>("test", false)
270
2
            .await
271
1
            .unwrap();
272
1
        Basic::new("test").push_into_async(&db).await.unwrap()
273
1
    };
274
1

            
275
1
    {
276
1
        // Should be able to access the storage again
277
1
        let bonsai = AsyncStorage::open(configuration(None)).await.unwrap();
278
1

            
279
1
        let db = bonsai.database::<BasicSchema>("test").await.unwrap();
280
1
        let retrieved = Basic::get_async(&document.header.id, &db)
281
1
            .await
282
1
            .unwrap()
283
1
            .expect("document not found");
284
1
        assert_eq!(document, retrieved);
285
1
    }
286
1

            
287
1
    // Verify that we can't access the storage again without the vault
288
1
    assert!(
289
1
        AsyncStorage::open(configuration(Some(String::from("path-prefix"))))
290
1
            .await
291
1
            .is_err()
292
1
    );
293
1
}
294

            
295
1
#[test]
296
1
fn blocking_test() {
297
1
    use bonsaidb_core::connection::StorageConnection;
298
1
    use bonsaidb_core::document::KeyId;
299
1
    use bonsaidb_core::schema::SerializedCollection;
300
1
    use bonsaidb_core::test_util::{Basic, BasicSchema, TestDirectory};
301
1
    use bonsaidb_local::config::{Builder, StorageConfiguration};
302
1
    use bonsaidb_local::Storage;
303
1
    drop(dotenvy::dotenv());
304

            
305
1
    let bucket = env_var!("S3_BUCKET");
306
1
    let endpoint = env_var!("S3_ENDPOINT");
307

            
308
1
    let directory = TestDirectory::new("bonsaidb-keystorage-s3-blocking");
309
1

            
310
3
    let configuration = |prefix| {
311
3
        let mut vault_key_storage = S3VaultKeyStorage {
312
3
            bucket: bucket.clone(),
313
3
            endpoint: Some(endpoint.clone()),
314
3
            ..S3VaultKeyStorage::default()
315
3
        };
316
3
        if let Some(prefix) = prefix {
317
1
            vault_key_storage = vault_key_storage.path(prefix);
318
2
        }
319

            
320
3
        StorageConfiguration::new(&directory)
321
3
            .vault_key_storage(vault_key_storage)
322
3
            .default_encryption_key(KeyId::Master)
323
3
            .with_schema::<BasicSchema>()
324
3
            .unwrap()
325
3
    };
326
1
    let document = {
327
1
        let bonsai = Storage::open(configuration(None)).unwrap();
328
1
        let db = bonsai
329
1
            .create_database::<BasicSchema>("test", false)
330
1
            .unwrap();
331
1
        Basic::new("test").push_into(&db).unwrap()
332
1
    };
333
1

            
334
1
    {
335
1
        // Should be able to access the storage again
336
1
        let bonsai = Storage::open(configuration(None)).unwrap();
337
1

            
338
1
        let db = bonsai.database::<BasicSchema>("test").unwrap();
339
1
        let retrieved = Basic::get(&document.header.id, &db)
340
1
            .unwrap()
341
1
            .expect("document not found");
342
1
        assert_eq!(document, retrieved);
343
    }
344

            
345
    // Verify that we can't access the storage again without the vault
346
1
    assert!(Storage::open(configuration(Some(String::from("path-prefix")))).is_err());
347
1
}