From e7303ed7c6da2dd8840d89a7a1dbe7d090822b29 Mon Sep 17 00:00:00 2001 From: Julien Cretin Date: Tue, 27 Oct 2020 14:17:50 +0100 Subject: [PATCH 1/2] Add model for new store --- libraries/persistent_store/src/lib.rs | 6 +- libraries/persistent_store/src/model.rs | 164 ++++++++++++++++++++++++ libraries/persistent_store/src/store.rs | 53 ++++++++ 3 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 libraries/persistent_store/src/model.rs diff --git a/libraries/persistent_store/src/lib.rs b/libraries/persistent_store/src/lib.rs index f2753db..f1c1653 100644 --- a/libraries/persistent_store/src/lib.rs +++ b/libraries/persistent_store/src/lib.rs @@ -352,9 +352,13 @@ extern crate alloc; mod bitfield; mod buffer; mod format; +#[cfg(feature = "std")] +mod model; mod storage; mod store; pub use self::buffer::{BufferCorruptFunction, BufferOptions, BufferStorage}; +#[cfg(feature = "std")] +pub use self::model::{StoreModel, StoreOperation}; pub use self::storage::{Storage, StorageError, StorageIndex, StorageResult}; -pub use self::store::{StoreError, StoreResult}; +pub use self::store::{StoreError, StoreRatio, StoreResult, StoreUpdate}; diff --git a/libraries/persistent_store/src/model.rs b/libraries/persistent_store/src/model.rs new file mode 100644 index 0000000..74eea69 --- /dev/null +++ b/libraries/persistent_store/src/model.rs @@ -0,0 +1,164 @@ +// Copyright 2019-2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::format::Format; +use crate::{StoreError, StoreRatio, StoreResult, StoreUpdate}; +use std::collections::{HashMap, HashSet}; + +/// Models the mutable operations of a store. +/// +/// The model doesn't model the storage and read-only operations. This is done by the driver. +#[derive(Clone, Debug)] +pub struct StoreModel { + /// Represents the content of the store. + map: HashMap>, + + /// The modeled storage configuration. + format: Format, +} + +/// Mutable operations on a store. +#[derive(Clone, Debug)] +pub enum StoreOperation { + /// Applies a transaction. + Transaction { + /// The list of updates to be applied. + updates: Vec, + }, + + /// Deletes all keys above a threshold. + Clear { + /// The minimum key to be deleted. + min_key: usize, + }, + + /// Compacts the store until a given capacity is immediately available. + Prepare { + /// How much capacity should be immediately available after compaction. + length: usize, + }, +} + +impl StoreModel { + /// Creates an empty model for a given storage configuration. + pub fn new(format: Format) -> StoreModel { + let map = HashMap::new(); + StoreModel { map, format } + } + + /// Returns the modeled content. + pub fn map(&self) -> &HashMap> { + &self.map + } + + /// Returns the storage configuration. + pub fn format(&self) -> &Format { + &self.format + } + + /// Simulates a store operation. + pub fn apply(&mut self, operation: StoreOperation) -> StoreResult<()> { + match operation { + StoreOperation::Transaction { updates } => self.transaction(updates), + StoreOperation::Clear { min_key } => self.clear(min_key), + StoreOperation::Prepare { length } => self.prepare(length), + } + } + + /// Returns the capacity according to the model. + pub fn capacity(&self) -> StoreRatio { + let total = self.format.total_capacity(); + let used: usize = self.map.values().map(|x| self.entry_size(x)).sum(); + StoreRatio { used, total } + } + + /// Applies a transaction. + fn transaction(&mut self, updates: Vec) -> StoreResult<()> { + // Fail if too many updates. + if updates.len() > self.format.max_updates() { + return Err(StoreError::InvalidArgument); + } + // Fail if an update is invalid. + if !updates.iter().all(|x| self.update_valid(x)) { + return Err(StoreError::InvalidArgument); + } + // Fail if updates are not disjoint, i.e. there are duplicate keys. + let keys: HashSet<_> = updates.iter().map(|x| x.key()).collect(); + if keys.len() != updates.len() { + return Err(StoreError::InvalidArgument); + } + // Fail if there is not enough capacity. + let capacity = match updates.len() { + 0 => 0, + 1 => match &updates[0] { + StoreUpdate::Insert { value, .. } => self.entry_size(value), + StoreUpdate::Remove { .. } => 0, + }, + _ => 1 + updates.iter().map(|x| self.update_size(x)).sum::(), + }; + if self.capacity().remaining() < capacity { + return Err(StoreError::NoCapacity); + } + // Apply the updates. + for update in updates { + match update { + StoreUpdate::Insert { key, value } => { + self.map.insert(key, value.into_boxed_slice()); + } + StoreUpdate::Remove { key } => { + self.map.remove(&key); + } + } + } + Ok(()) + } + + /// Applies a clear operation. + fn clear(&mut self, min_key: usize) -> StoreResult<()> { + if min_key > self.format.max_key() { + return Err(StoreError::InvalidArgument); + } + self.map.retain(|&k, _| k < min_key); + Ok(()) + } + + /// Applies a prepare operation. + fn prepare(&self, length: usize) -> StoreResult<()> { + if self.capacity().remaining() < length { + return Err(StoreError::NoCapacity); + } + Ok(()) + } + + /// Returns the word capacity of an update. + fn update_size(&self, update: &StoreUpdate) -> usize { + match update { + StoreUpdate::Insert { value, .. } => self.entry_size(value), + StoreUpdate::Remove { .. } => 1, + } + } + + /// Returns the word capacity of an entry. + fn entry_size(&self, value: &[u8]) -> usize { + 1 + self.format.bytes_to_words(value.len()) + } + + /// Returns whether an update is valid. + fn update_valid(&self, update: &StoreUpdate) -> bool { + update.key() <= self.format.max_key() + && update + .value() + .map_or(true, |x| x.len() <= self.format.max_value_len()) + } +} diff --git a/libraries/persistent_store/src/store.rs b/libraries/persistent_store/src/store.rs index 0c94196..d8d39ae 100644 --- a/libraries/persistent_store/src/store.rs +++ b/libraries/persistent_store/src/store.rs @@ -13,7 +13,9 @@ // limitations under the License. use crate::StorageError; +use alloc::vec::Vec; +/// Errors returned by store operations. #[derive(Debug, PartialEq, Eq)] pub enum StoreError { /// Invalid argument. @@ -60,4 +62,55 @@ impl From for StoreError { } } +/// Result of store operations. pub type StoreResult = Result; + +/// Progression ratio for store metrics. +/// +/// This is used for the [capacity] and [lifetime] metrics. Those metrics are measured in words. +/// +/// [capacity]: struct.Store.html#method.capacity +/// [lifetime]: struct.Store.html#method.lifetime +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct StoreRatio { + /// How much of the metric is used. + pub used: usize, + + /// How much of the metric can be used at most. + pub total: usize, +} + +impl StoreRatio { + /// How much of the metric is remaining. + pub fn remaining(self) -> usize { + self.total - self.used + } +} + +/// Represents an update to the store as part of a transaction. +#[derive(Clone, Debug)] +pub enum StoreUpdate { + /// Inserts or replaces an entry in the store. + Insert { key: usize, value: Vec }, + + /// Removes an entry from the store. + Remove { key: usize }, +} + +impl StoreUpdate { + /// Returns the key affected by the update. + pub fn key(&self) -> usize { + match *self { + StoreUpdate::Insert { key, .. } => key, + StoreUpdate::Remove { key } => key, + } + } + + /// Returns the value written by the update. + pub fn value(&self) -> Option<&[u8]> { + match self { + StoreUpdate::Insert { value, .. } => Some(value), + StoreUpdate::Remove { .. } => None, + } + } +} From 5ce91947b6b458d9b49336ef62e9ef8a3a0d0bad Mon Sep 17 00:00:00 2001 From: Julien Cretin Date: Thu, 29 Oct 2020 12:23:34 +0100 Subject: [PATCH 2/2] Fix StoreRatio fields visibility and improve documentation --- libraries/persistent_store/src/model.rs | 22 +++++++++++++--------- libraries/persistent_store/src/store.rs | 18 ++++++++++++++++-- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/libraries/persistent_store/src/model.rs b/libraries/persistent_store/src/model.rs index 74eea69..2677265 100644 --- a/libraries/persistent_store/src/model.rs +++ b/libraries/persistent_store/src/model.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, HashSet}; #[derive(Clone, Debug)] pub struct StoreModel { /// Represents the content of the store. - map: HashMap>, + content: HashMap>, /// The modeled storage configuration. format: Format, @@ -53,13 +53,13 @@ pub enum StoreOperation { impl StoreModel { /// Creates an empty model for a given storage configuration. pub fn new(format: Format) -> StoreModel { - let map = HashMap::new(); - StoreModel { map, format } + let content = HashMap::new(); + StoreModel { content, format } } /// Returns the modeled content. - pub fn map(&self) -> &HashMap> { - &self.map + pub fn content(&self) -> &HashMap> { + &self.content } /// Returns the storage configuration. @@ -79,7 +79,7 @@ impl StoreModel { /// Returns the capacity according to the model. pub fn capacity(&self) -> StoreRatio { let total = self.format.total_capacity(); - let used: usize = self.map.values().map(|x| self.entry_size(x)).sum(); + let used: usize = self.content.values().map(|x| self.entry_size(x)).sum(); StoreRatio { used, total } } @@ -100,11 +100,15 @@ impl StoreModel { } // Fail if there is not enough capacity. let capacity = match updates.len() { + // An empty transaction doesn't consume anything. 0 => 0, + // Transactions with a single update are optimized by avoiding a marker entry. 1 => match &updates[0] { StoreUpdate::Insert { value, .. } => self.entry_size(value), + // Transactions with a single update which is a removal don't consume anything. StoreUpdate::Remove { .. } => 0, }, + // A transaction consumes one word for the marker entry in addition to its updates. _ => 1 + updates.iter().map(|x| self.update_size(x)).sum::(), }; if self.capacity().remaining() < capacity { @@ -114,10 +118,10 @@ impl StoreModel { for update in updates { match update { StoreUpdate::Insert { key, value } => { - self.map.insert(key, value.into_boxed_slice()); + self.content.insert(key, value.into_boxed_slice()); } StoreUpdate::Remove { key } => { - self.map.remove(&key); + self.content.remove(&key); } } } @@ -129,7 +133,7 @@ impl StoreModel { if min_key > self.format.max_key() { return Err(StoreError::InvalidArgument); } - self.map.retain(|&k, _| k < min_key); + self.content.retain(|&k, _| k < min_key); Ok(()) } diff --git a/libraries/persistent_store/src/store.rs b/libraries/persistent_store/src/store.rs index d8d39ae..473310b 100644 --- a/libraries/persistent_store/src/store.rs +++ b/libraries/persistent_store/src/store.rs @@ -69,18 +69,32 @@ pub type StoreResult = Result; /// /// This is used for the [capacity] and [lifetime] metrics. Those metrics are measured in words. /// +/// # Invariant +/// +/// - The used value does not exceed the total: `used <= total`. +/// /// [capacity]: struct.Store.html#method.capacity /// [lifetime]: struct.Store.html#method.lifetime #[derive(Copy, Clone, PartialEq, Eq)] pub struct StoreRatio { /// How much of the metric is used. - pub used: usize, + pub(crate) used: usize, /// How much of the metric can be used at most. - pub total: usize, + pub(crate) total: usize, } impl StoreRatio { + /// How much of the metric is used. + pub fn used(self) -> usize { + self.used + } + + /// How much of the metric can be used at most. + pub fn total(self) -> usize { + self.total + } + /// How much of the metric is remaining. pub fn remaining(self) -> usize { self.total - self.used