1
1
//! [`VaultKeyStorage`] using S3-compatible storage.
2
//!
3
//! This is the recommended method for securing your BonsaiDb database. There
4
//! are many ways to acquire secure, inexpensive S3-compatible storage, such as
5
//! Backblaze B2.
6
//!
7
//! Do not configure your bucket with public access. You should only allow
8
//! access from the IP addresses that your BonsaiDb server(s) are hosted on,
9
//! or only allow authenticated access.
10
//!
11
//! To use this, specify the `vault_key_storage` configuration parameter:
12
//!
13
//! ```rust
14
//! # use bonsaidb_keystorage_s3::S3VaultKeyStorage;
15
//! # use aws_sdk_s3::Endpoint;
16
//! # use bonsaidb_core::{document::KeyId, test_util::TestDirectory};
17
//! # use bonsaidb_local::config::{StorageConfiguration, Builder};
18
//! # use http::Uri;
19
//! #
20
//! # async fn test() {
21
//! let directory = TestDirectory::new("bonsaidb-keystorage-s3-basic");
22
//! let configuration = StorageConfiguration::new(&directory)
23
//!     .vault_key_storage(
24
//!         S3VaultKeyStorage::new("bucket_name").endpoint(Endpoint::immutable(
25
//!             Uri::try_from("https://s3.us-west-001.backblazeb2.com").unwrap(),
26
//!         )),
27
//!     )
28
//!     .default_encryption_key(KeyId::Master);
29
//! # }
30
//! ```
31
//!
32
//! The API calls are performed by the [`aws-sdk-s3`](aws_sdk_s3) crate.
33

            
34
#![forbid(unsafe_code)]
35
#![warn(
36
    clippy::cargo,
37
    missing_docs,
38
    // clippy::missing_docs_in_private_items,
39
    clippy::nursery,
40
    clippy::pedantic,
41
    future_incompatible,
42
    rust_2018_idioms,
43
)]
44
#![allow(
45
    clippy::missing_errors_doc, // TODO clippy::missing_errors_doc
46
    clippy::missing_panics_doc, // TODO clippy::missing_panics_doc
47
    clippy::option_if_let_else,
48
    clippy::module_name_repetitions,
49
)]
50

            
51
use std::{fmt::Display, future::Future};
52

            
53
use async_trait::async_trait;
54
use aws_config::meta::region::RegionProviderChain;
55
pub use aws_sdk_s3;
56
use aws_sdk_s3::{types::ByteStream, Client, Endpoint, Region};
57
use bonsaidb_local::{
58
    vault::{KeyPair, VaultKeyStorage},
59
    StorageId,
60
};
61
pub use http;
62
use tokio::runtime::{Handle, Runtime};
63

            
64
/// S3-compatible [`VaultKeyStorage`] implementor.
65
6
#[derive(Default, Debug)]
66
#[must_use]
67
pub struct S3VaultKeyStorage {
68
    runtime: Tokio,
69
    bucket: String,
70
    /// The S3 endpoint to use. If not specified, the endpoint will be
71
    /// determined automatically. This field can be used to support non-AWS S3
72
    /// providers.
73
    pub endpoint: Option<Endpoint>,
74
    /// The AWS region to use. If not specified, the region will be determined
75
    /// by the aws sdk.
76
    pub region: Option<Region>,
77
    /// The path prefix for keys to be stored within.
78
    pub path: String,
79
}
80

            
81
#[derive(Debug)]
82
enum Tokio {
83
    Runtime(Runtime),
84
    Handle(Handle),
85
}
86

            
87
impl Default for Tokio {
88
6
    fn default() -> Self {
89
6
        Handle::try_current().map_or_else(|_| Self::Runtime(Runtime::new().unwrap()), Self::Handle)
90
6
    }
91
}
92

            
93
impl Tokio {
94
8
    pub fn block_on<F: Future<Output = R>, R>(&self, future: F) -> R {
95
8
        match self {
96
4
            Tokio::Runtime(rt) => rt.block_on(future),
97
4
            Tokio::Handle(rt) => rt.block_on(future),
98
        }
99
8
    }
100
}
101

            
102
impl S3VaultKeyStorage {
103
    /// Creates a new key storage instance for `bucket`. This instance will use
104
    /// the currently available Tokio runtime or create one if none is
105
    /// available.
106
    pub fn new(bucket: impl Display) -> Self {
107
        Self::new_with_runtime(bucket, tokio::runtime::Handle::current())
108
    }
109

            
110
    /// Creates a new key storage instance for `bucket`, which performs its
111
    /// networking operations on `runtime`.
112
    pub fn new_with_runtime(bucket: impl Display, runtime: tokio::runtime::Handle) -> Self {
113
        Self {
114
            bucket: bucket.to_string(),
115
            runtime: Tokio::Handle(runtime),
116
            ..Self::default()
117
        }
118
    }
119

            
120
    /// Sets the path prefix for vault keys to be stored within.
121
2
    pub fn path(mut self, prefix: impl Display) -> Self {
122
2
        self.path = prefix.to_string();
123
2
        self
124
2
    }
125

            
126
    /// Sets the endpoint to use. See [`Self::endpoint`] for more information.
127
    #[allow(clippy::missing_const_for_fn)] // destructors
128
    pub fn endpoint(mut self, endpoint: Endpoint) -> Self {
129
        self.endpoint = Some(endpoint);
130
        self
131
    }
132

            
133
8
    fn path_for_id(&self, storage_id: StorageId) -> String {
134
8
        let mut path = self.path.clone();
135
8
        if !path.is_empty() && !path.ends_with('/') {
136
2
            path.push('/');
137
6
        }
138
8
        path.push_str(&storage_id.to_string());
139
8
        path
140
8
    }
141

            
142
8
    async fn client(&self) -> aws_sdk_s3::Client {
143
8
        let region_provider = RegionProviderChain::first_try(self.region.clone())
144
8
            .or_default_provider()
145
8
            .or_else(Region::new("us-east-1"));
146
68
        let config = aws_config::from_env().load().await;
147
8
        if let Some(endpoint) = self.endpoint.clone() {
148
            Client::from_conf(
149
8
                aws_sdk_s3::Config::builder()
150
8
                    .endpoint_resolver(endpoint)
151
30
                    .region(region_provider.region().await)
152
8
                    .credentials_provider(config.credentials_provider().unwrap().clone())
153
8
                    .build(),
154
            )
155
        } else {
156
            Client::new(&config)
157
        }
158
8
    }
159
}
160

            
161
#[async_trait]
162
impl VaultKeyStorage for S3VaultKeyStorage {
163
    type Error = anyhow::Error;
164

            
165
2
    fn set_vault_key_for(&self, storage_id: StorageId, key: KeyPair) -> Result<(), Self::Error> {
166
2
        self.runtime.block_on(async {
167
21
            let client = self.client().await;
168
2
            let key = key.to_bytes()?;
169
2
            client
170
2
                .put_object()
171
2
                .bucket(&self.bucket)
172
2
                .key(self.path_for_id(storage_id))
173
2
                .body(ByteStream::from(key.to_vec()))
174
2
                .send()
175
12
                .await?;
176
2
            Ok(())
177
2
        })
178
2
    }
179

            
180
6
    fn vault_key_for(&self, storage_id: StorageId) -> Result<Option<KeyPair>, Self::Error> {
181
6
        self.runtime.block_on(async {
182
77
            let client = self.client().await;
183
6
            match client
184
6
                .get_object()
185
6
                .bucket(&self.bucket)
186
6
                .key(self.path_for_id(storage_id))
187
6
                .send()
188
32
                .await
189
            {
190
4
                Ok(response) => {
191
4
                    let bytes = response.body.collect().await?.into_bytes();
192
4
                    let key = KeyPair::from_bytes(&bytes)
193
4
                        .map_err(|err| anyhow::anyhow!(err.to_string()))?;
194
4
                    Ok(Some(key))
195
                }
196
                Err(aws_smithy_client::SdkError::ServiceError {
197
                    err:
198
                        aws_sdk_s3::error::GetObjectError {
199
                            kind: aws_sdk_s3::error::GetObjectErrorKind::NoSuchKey(_),
200
                            ..
201
                        },
202
                    ..
203
2
                }) => Ok(None),
204
                Err(err) => Err(anyhow::anyhow!(err)),
205
            }
206
6
        })
207
6
    }
208
}
209

            
210
#[cfg(test)]
211
macro_rules! env_var {
212
    ($name:expr) => {{
213
        match std::env::var($name) {
214
            Ok(value) if !value.is_empty() => value,
215
            _ => {
216
                log::error!(
217
                    "Ignoring basic_test because of missing environment variable: {}",
218
                    $name
219
                );
220
                return;
221
            }
222
        }
223
    }};
224
}
225

            
226
#[cfg(test)]
227
1
#[tokio::test]
228
1
async fn basic_test() {
229
1
    use bonsaidb_core::{
230
1
        connection::AsyncStorageConnection,
231
1
        document::KeyId,
232
1
        schema::SerializedCollection,
233
1
        test_util::{Basic, BasicSchema, TestDirectory},
234
1
    };
235
1
    use bonsaidb_local::{
236
1
        config::{Builder, StorageConfiguration},
237
1
        AsyncStorage,
238
1
    };
239
1
    use http::Uri;
240
1
    drop(dotenv::dotenv());
241

            
242
1
    let bucket = env_var!("S3_BUCKET");
243
1
    let endpoint = env_var!("S3_ENDPOINT");
244

            
245
1
    let directory = TestDirectory::new("bonsaidb-keystorage-s3-basic");
246
1

            
247
3
    let configuration = |prefix| {
248
3
        let mut vault_key_storage = S3VaultKeyStorage {
249
3
            bucket: bucket.clone(),
250
3
            endpoint: Some(Endpoint::immutable(Uri::try_from(&endpoint).unwrap())),
251
3
            ..S3VaultKeyStorage::default()
252
3
        };
253
3
        if let Some(prefix) = prefix {
254
1
            vault_key_storage = vault_key_storage.path(prefix);
255
2
        }
256

            
257
3
        StorageConfiguration::new(&directory)
258
3
            .vault_key_storage(vault_key_storage)
259
3
            .default_encryption_key(KeyId::Master)
260
3
            .with_schema::<BasicSchema>()
261
3
            .unwrap()
262
3
    };
263
1
    let document = {
264
1
        let bonsai = AsyncStorage::open(configuration(None)).await.unwrap();
265
1
        let db = bonsai
266
1
            .create_database::<BasicSchema>("test", false)
267
2
            .await
268
1
            .unwrap();
269
1
        Basic::new("test").push_into_async(&db).await.unwrap()
270
    };
271

            
272
    {
273
        // Should be able to access the storage again
274
1
        let bonsai = AsyncStorage::open(configuration(None)).await.unwrap();
275

            
276
1
        let db = bonsai.database::<BasicSchema>("test").await.unwrap();
277
1
        let retrieved = Basic::get_async(document.header.id, &db)
278
1
            .await
279
1
            .unwrap()
280
1
            .expect("document not found");
281
1
        assert_eq!(document, retrieved);
282
    }
283

            
284
    // Verify that we can't access the storage again without the vault
285
1
    assert!(
286
1
        AsyncStorage::open(configuration(Some(String::from("path-prefix"))))
287
1
            .await
288
1
            .is_err()
289
    );
290
}
291

            
292
1
#[test]
293
1
fn blocking_test() {
294
1
    use bonsaidb_core::{
295
1
        connection::StorageConnection,
296
1
        document::KeyId,
297
1
        schema::SerializedCollection,
298
1
        test_util::{Basic, BasicSchema, TestDirectory},
299
1
    };
300
1
    use bonsaidb_local::{
301
1
        config::{Builder, StorageConfiguration},
302
1
        Storage,
303
1
    };
304
1
    use http::Uri;
305
1
    drop(dotenv::dotenv());
306

            
307
1
    let bucket = env_var!("S3_BUCKET");
308
1
    let endpoint = env_var!("S3_ENDPOINT");
309

            
310
1
    let directory = TestDirectory::new("bonsaidb-keystorage-s3-blocking");
311
1

            
312
3
    let configuration = |prefix| {
313
3
        let mut vault_key_storage = S3VaultKeyStorage {
314
3
            bucket: bucket.clone(),
315
3
            endpoint: Some(Endpoint::immutable(Uri::try_from(&endpoint).unwrap())),
316
3
            ..S3VaultKeyStorage::default()
317
3
        };
318
3
        if let Some(prefix) = prefix {
319
1
            vault_key_storage = vault_key_storage.path(prefix);
320
2
        }
321

            
322
3
        StorageConfiguration::new(&directory)
323
3
            .vault_key_storage(vault_key_storage)
324
3
            .default_encryption_key(KeyId::Master)
325
3
            .with_schema::<BasicSchema>()
326
3
            .unwrap()
327
3
    };
328
1
    let document = {
329
1
        let bonsai = Storage::open(configuration(None)).unwrap();
330
1
        let db = bonsai
331
1
            .create_database::<BasicSchema>("test", false)
332
1
            .unwrap();
333
1
        Basic::new("test").push_into(&db).unwrap()
334
1
    };
335
1

            
336
1
    {
337
1
        // Should be able to access the storage again
338
1
        let bonsai = Storage::open(configuration(None)).unwrap();
339
1

            
340
1
        let db = bonsai.database::<BasicSchema>("test").unwrap();
341
1
        let retrieved = Basic::get(document.header.id, &db)
342
1
            .unwrap()
343
1
            .expect("document not found");
344
1
        assert_eq!(document, retrieved);
345
    }
346

            
347
    // Verify that we can't access the storage again without the vault
348
1
    assert!(Storage::open(configuration(Some(String::from("path-prefix")))).is_err());
349
1
}