Skip to content
GitLab
Commit 37966866 authored by Szymon's avatar Szymon
Browse files

use btree for deterministic hashes

parent c38ce5df
Hide whitespace changes
Inline Side-by-side
Showing
with 246 additions and 60 deletions
+246 -60
libpod/src/database_pool.rs
+ 0
- 1
View file @ 37966866
......@@ -100,7 +100,6 @@ pub async fn remove_db(owner: &str, init_db: &InitDb, database_key: &DatabaseKey
db.remove(owner);
info!("Removing database at {:?}", owner_database_path(owner));
// TODO: handle errors
let remove_futures = vec![
remove_file(owner_database_path(owner)),
remove_file(owner_database_shm_path(owner)),
......
libpod/src/v5/api_model.rs
+ 167
- 0
View file @ 37966866
pub mod types;
use sha2::{Digest, Sha256};
use std::{
cmp::Ordering,
collections::{HashMap, HashSet, VecDeque},
......@@ -29,6 +30,13 @@ pub struct CreateSchemaReq {
pub meta: SchemaMeta,
}
impl CreateSchemaReq {
/// Get deterministic hash of the request
pub fn compute_hash(&self) -> String {
hex::encode(Sha256::new_with_prefix(format!("{self:?}")).finalize())
}
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
......@@ -356,3 +364,162 @@ impl HasSortProp for SearchEdgeResItem {
&self.node.sort_property_value
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use crate::v5::{
api_model::types::{IndexConstrains, NonEmptyVector},
schema::SchemaInfo,
};
use super::{
types::{EdgeDefinition, ItemPropertiesDefinition, Properties, SchemaPropertyType},
*,
};
#[test]
fn schema_request_returns_deterministic_hash() {
let req1 = CreateSchemaReq {
nodes: ItemsDefinition::from([
(
"item2".try_into().unwrap(),
ItemPropertiesDefinition {
properties: Properties::from([
("prop1".try_into().unwrap(), SchemaPropertyType::Text),
("prop3".try_into().unwrap(), SchemaPropertyType::Integer),
("prop2".try_into().unwrap(), SchemaPropertyType::Bool),
]),
..Default::default()
},
),
(
"item1".try_into().unwrap(),
ItemPropertiesDefinition {
properties: Properties::from([
("prop3".try_into().unwrap(), SchemaPropertyType::Integer),
("prop1".try_into().unwrap(), SchemaPropertyType::Text),
("prop2".try_into().unwrap(), SchemaPropertyType::Bool),
]),
index: vec![
vec!["prop2".try_into().unwrap(), "prop1".try_into().unwrap()]
.try_into()
.unwrap(),
],
not_null: Default::default(),
unique: Default::default(),
},
),
(
"item3".try_into().unwrap(),
ItemPropertiesDefinition {
properties: Properties::from([
("prop3".try_into().unwrap(), SchemaPropertyType::Integer),
("prop2".try_into().unwrap(), SchemaPropertyType::Bool),
("prop1".try_into().unwrap(), SchemaPropertyType::Text),
]),
..Default::default()
},
),
]),
edges: EdgesDefinition::from([(
"edge2".try_into().unwrap(),
BTreeSet::from([
EdgeDefinition {
source: "item2".try_into().unwrap(),
target: "item2".try_into().unwrap(),
// properties: Default::default(),
},
EdgeDefinition {
source: "item1".try_into().unwrap(),
target: "item2".try_into().unwrap(),
// properties: Default::default(),
},
EdgeDefinition {
source: "item3".try_into().unwrap(),
target: "item1".try_into().unwrap(),
// properties: Default::default(),
},
]),
)]),
meta: SchemaMeta {
name: "name".to_string(),
info: SchemaInfo {
version: "0.1".to_string().try_into().unwrap(),
url: "example.com".to_string(),
},
},
};
let req2 = CreateSchemaReq {
nodes: ItemsDefinition::from([
(
"item3".try_into().unwrap(),
ItemPropertiesDefinition {
properties: Properties::from([
("prop2".try_into().unwrap(), SchemaPropertyType::Bool),
("prop1".try_into().unwrap(), SchemaPropertyType::Text),
("prop3".try_into().unwrap(), SchemaPropertyType::Integer),
]),
..Default::default()
},
),
(
"item2".try_into().unwrap(),
ItemPropertiesDefinition {
properties: Properties::from([
("prop1".try_into().unwrap(), SchemaPropertyType::Text),
("prop2".try_into().unwrap(), SchemaPropertyType::Bool),
("prop3".try_into().unwrap(), SchemaPropertyType::Integer),
]),
..Default::default()
},
),
(
"item1".try_into().unwrap(),
ItemPropertiesDefinition {
properties: Properties::from([
("prop2".try_into().unwrap(), SchemaPropertyType::Bool),
("prop3".try_into().unwrap(), SchemaPropertyType::Integer),
("prop1".try_into().unwrap(), SchemaPropertyType::Text),
]),
index: vec![
vec!["prop2".try_into().unwrap(), "prop1".try_into().unwrap()]
.try_into()
.unwrap(),
],
not_null: Default::default(),
unique: Default::default(),
},
),
]),
edges: EdgesDefinition::from([(
"edge2".try_into().unwrap(),
BTreeSet::from([
EdgeDefinition {
target: "item2".try_into().unwrap(),
source: "item2".try_into().unwrap(),
},
EdgeDefinition {
source: "item3".try_into().unwrap(),
target: "item1".try_into().unwrap(),
},
EdgeDefinition {
target: "item2".try_into().unwrap(),
source: "item1".try_into().unwrap(),
},
]),
)]),
meta: SchemaMeta {
name: "name".to_string(),
info: SchemaInfo {
version: "0.1".to_string().try_into().unwrap(),
url: "example.com".to_string(),
},
},
};
assert_eq!(req1.compute_hash(), req2.compute_hash());
}
}
libpod/src/v5/api_model/types.rs
+ 11
- 10
View file @ 37966866
......@@ -11,6 +11,7 @@ use serde::de::value::StringDeserializer;
use serde::de::{self, IntoDeserializer};
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::Value;
use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Display;
use std::ops::{Deref, DerefMut};
use std::str::FromStr;
......@@ -83,7 +84,7 @@ pub type NodeName = ItemName;
pub type EdgeName = ItemName;
/// Newtype pattern used to validate content at the parse stage
#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq, Hash)]
#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq, Hash, Ord, PartialOrd)]
#[serde(try_from = "String")]
pub struct ItemName(String);
......@@ -150,7 +151,7 @@ impl TryFrom<&str> for ItemName {
pub type EdgePropertyName = PropertyName;
pub type NodePropertyName = PropertyName;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, Ord, PartialOrd)]
#[serde(try_from = "String")]
pub struct PropertyName(String);
......@@ -362,10 +363,12 @@ pub struct SchemaEdge {
/// Definition of a property, for example:
/// "age" -> Integer
pub type Properties = HashMap<PropertyName, SchemaPropertyType>;
pub type ItemsDefinition = HashMap<ItemName, ItemPropertiesDefinition>;
/// Use BTreeMap to keep node names and properties ordered - makes request deterministic
/// no matter order of insertion
pub type Properties = BTreeMap<PropertyName, SchemaPropertyType>;
pub type ItemsDefinition = BTreeMap<ItemName, ItemPropertiesDefinition>;
pub type UniqueConstrains = Vec<NonEmptyVector<PropertyName>>;
pub type NotNullConstrains = HashSet<PropertyName>;
pub type NotNullConstrains = BTreeSet<PropertyName>;
pub type IndexConstrains = Vec<NonEmptyVector<PropertyName>>;
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
......@@ -380,15 +383,13 @@ pub struct ItemPropertiesDefinition {
pub index: IndexConstrains,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, Ord, PartialOrd, Eq, PartialEq)]
pub struct EdgeDefinition {
pub source: NodeName,
pub target: NodeName,
#[serde(default)]
pub properties: ItemPropertiesDefinition,
}
// TODO: Vec<Edge> should be a set
pub type EdgesDefinition = HashMap<EdgeName, Vec<EdgeDefinition>>;
pub type EdgesDefinition = BTreeMap<EdgeName, BTreeSet<EdgeDefinition>>;
pub fn validate_name_syntax(name: &str) -> Result<()> {
lazy_static! {
......
libpod/src/v5/database_api/graph_schema.rs
+ 6
- 6
View file @ 37966866
......@@ -242,12 +242,12 @@ async fn create_edge_schema(
edge: &EdgeDefinition,
) -> Result<()> {
debug!("Creating new edge: {edge_name}, {edge:#?}");
if !edge.properties.properties.is_empty() {
// TODO: support properties over edges
return Err(bad_request!(
"Edge {} - {edge_name} -> {} contains properties. Properties over edges are not supported yet",
edge.source, edge.target));
}
// if !edge.properties.properties.is_empty() {
// // TODO: support properties over edges
// return Err(bad_request!(
// "Edge {} - {edge_name} -> {} contains properties. Properties over edges are not supported yet",
// edge.source, edge.target));
// }
if schema.get_node(&edge.source).is_none() {
return Err(bad_request!(
......
libpod/src/v5/migrations_api/mod.rs
+ 1
- 1
View file @ 37966866
......@@ -68,7 +68,7 @@ pub mod tests {
assert_eq!(
schema.info().get("pod").unwrap().version,
SchemaVersion { major: 0, minor: 3 }
SchemaVersion { major: 0, minor: 4 }
);
let acc = schema.nodes().get("PodUserAccount").unwrap();
......
libpod/src/v5/migrations_api/pod_schema.rs
+ 50
- 31
View file @ 37966866
......@@ -6,17 +6,15 @@ use crate::async_db_connection::AsyncConnection;
use crate::error::Result;
use crate::v5::api_model::types::SchemaMeta;
use crate::v5::api_model::CreateSchemaReq;
use crate::v5::database_api::graph_schema;
use crate::v5::internal_api;
use crate::v5::schema::{SchemaInfo, SchemaVersion};
use crate::{async_db_connection::AsyncTx, error::ErrorContext};
pub const NAME: &str = "pod";
pub const URL: &str = "gitlab.memri.io/memri/pod";
pub const URL: &str = "https://gitlab.memri.io/memri/pod";
const MIGRATIONS: [&dyn Migration; 3] = [&V0_1, &V0_2, &V0_3];
const MIGRATIONS: [&dyn Migration; 4] = [&V0_1, &V0_2, &V0_3, &V0_4];
pub async fn update_schema(conn: &mut AsyncConnection) -> Result<()> {
for migration in MIGRATIONS {
......@@ -29,34 +27,34 @@ pub async fn update_schema(conn: &mut AsyncConnection) -> Result<()> {
Ok(())
}
fn get_schema_meta(version: SchemaVersion) -> SchemaMeta {
SchemaMeta {
name: NAME.to_string(),
info: SchemaInfo {
version: version,
url: URL.to_string(),
},
}
}
struct V0_1;
#[async_trait(?Send)]
impl Migration for V0_1 {
async fn do_migrate(&self, conn: &mut AsyncConnection) -> Result<()> {
// TODO: add create_schema_from_raw_sql or something
// TODO: do all those validations that are done normally
conn.in_write_transaction(|tx: AsyncTx| async move {
let sql = include_str!("pod/V1__initial.sql");
tx.execute_batch(sql)?;
graph_schema::store_schema_info(
&tx,
&CreateSchemaReq {
nodes: Default::default(),
edges: Default::default(),
meta: SchemaMeta {
name: NAME.to_string(),
info: SchemaInfo {
version: self.version(),
url: URL.to_string(),
},
},
},
)
.await?;
Ok(())
})
.await?;
// Store schema info using create_schema
internal_api::create_schema(
conn,
&serde_json::from_value(json!({
"meta": get_schema_meta(self.version())
}))?,
)
.await
}
......@@ -173,11 +171,7 @@ impl Migration for V0_2 {
]
},
"meta": {
"name": NAME,
"url": URL,
"version": self.version().to_string()
}
"meta": get_schema_meta(self.version())
}))?,
)
.await
......@@ -202,8 +196,7 @@ impl Migration for V0_3 {
},
},
},
// TODO: creation of schemameta is repeating
"meta": SchemaMeta { name: NAME.to_string() , info: SchemaInfo { version: self.version(), url: URL.to_string() } }
"meta": get_schema_meta(self.version())
}))?,
)
.await?;
......@@ -211,8 +204,34 @@ impl Migration for V0_3 {
Ok(())
}
// TODO: if you forget update version, request will silently return Ok but no changes would apply
fn version(&self) -> SchemaVersion {
SchemaVersion { major: 0, minor: 3 }
}
}
struct V0_4;
#[async_trait(?Send)]
impl Migration for V0_4 {
async fn do_migrate(&self, conn: &mut AsyncConnection) -> Result<()> {
conn.in_write_transaction(|tx: AsyncTx| async move {
let sql = "ALTER TABLE _SchemaInfo
ADD schemaHistory TEXT;";
tx.execute_batch(sql)?;
Ok(())
})
.await?;
// Store schema info using create_schema
internal_api::create_schema(
conn,
&serde_json::from_value(json!({
"meta": get_schema_meta(self.version())
}))?,
)
.await
}
fn version(&self) -> SchemaVersion {
SchemaVersion { major: 0, minor: 4 }
}
}
libpod/src/v5/schema.rs
+ 11
- 11
View file @ 37966866
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, convert::TryInto, fmt::Display};
use std::{collections::{HashMap, BTreeMap}, convert::TryInto, fmt::Display};
use time::OffsetDateTime;
use crate::bad_request;
......@@ -14,9 +14,9 @@ use super::api_model::types::{
/// for example A - quotes -> B, Article - quotes -> [Article, Book], etc.
// TODO: add properties over edges
pub type Connections =
HashMap<ItemName /* source */, HashMap<EdgeName, Vec<ItemName /* Targets */>>>;
BTreeMap<ItemName /* source */, BTreeMap<EdgeName, Vec<ItemName /* Targets */>>>;
pub type SchemasInfo = HashMap<String, SchemaInfo>;
pub type SchemasInfo = BTreeMap<String, SchemaInfo>;
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct SchemaInfo {
pub version: SchemaVersion,
......@@ -76,9 +76,9 @@ pub struct Schema {
impl Schema {
pub fn empty() -> Schema {
Schema {
nodes_types: HashMap::new(),
nodes_connections: HashMap::new(),
edges_types: HashMap::new(),
nodes_types: BTreeMap::new(),
nodes_connections: BTreeMap::new(),
edges_types: BTreeMap::new(),
info: SchemasInfo::new(),
}
}
......@@ -126,10 +126,10 @@ impl Schema {
.iter()
.any(|e| e.source == source_type && e.target == target_type)
{
edge.push(EdgeDefinition {
edge.insert(EdgeDefinition {
source: source_type,
target: target_type,
properties: ItemPropertiesDefinition::default(),
// properties: ItemPropertiesDefinition::default(),
});
}
}
......@@ -150,15 +150,15 @@ impl Schema {
pub fn get_out_edges_for_node(
&self,
source: &ItemName,
) -> Option<&HashMap<EdgeName, Vec<ItemName /* Targets */>>> {
) -> Option<&BTreeMap<EdgeName, Vec<ItemName /* Targets */>>> {
self.nodes_connections.get(source)
}
pub fn get_in_edges_for_node(
&self,
target: &ItemName,
) -> Option<HashMap<EdgeName, Vec<ItemName /* Sources */>>> {
let mut res: HashMap<EdgeName, Vec<ItemName>> = HashMap::new();
) -> Option<BTreeMap<EdgeName, Vec<ItemName /* Sources */>>> {
let mut res: BTreeMap<EdgeName, Vec<ItemName>> = BTreeMap::new();
for (source, edges) in &self.nodes_connections {
for (edge_name, targets) in edges {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment