Skip to content
GitLab
Commit dacb0c30 authored by Koen van der Veen's avatar Koen van der Veen
Browse files

Merge branch 'dev' of gitlab.memri.io:memri/pod into logs

parents 1efe9fae d281c72d
Hide whitespace changes
Inline Side-by-side
Showing
with 280 additions and 134 deletions
+280 -134
docs/HTTP_API.md
+ 14
- 9
View file @ dacb0c30
......@@ -84,18 +84,23 @@ The following properties are expected:
}
```
Note that you cannot both change the Schema and refer to the new Schema
in one `bulk` request, so if you want changes to the Schema to happen first,
split updates to the Schema into a separate request.
(This constraint might be lifted in the future.)
If a Schema with the same `propertyName` but different `valueType` already exists,
the request will fail (even if the `itemType` is different).
(Explanation: this is done to support heterogeneous queries in the Pod.)
If the new Schema item conflicts with already existing Schema, a failure will be returned.
If the new Schema item creates (valid) item properties that have not yet been defined,
the new Schema will be added permanently.
If the new Schema item duplicates already existing Schema, the new item will be silently ignored
and not inserted into the database.
If the new Schema item has an `id` specified, the new item will be created
and may potentially duplicate already existing Schema definition items.
(Note that it is recommended to avoid specifying an `id` for Schema items
except for Memri Webapp/Mobile Clients.)
If the new Schema item creates (valid) properties that have not yet been defined,
the new Schema will be added permanently.
If no `id` has been specified and the new Schema item duplicates an already existing Schema item,
no new items will be created on the server and the `id` of the old item will be returned.
Note that this slightly violates the semantic meaning of item creation, however,
this is the explicit choice due to the special meaning of the Schema items
(and lack of a need for logic duplicates).
⚠️ UNSTABLE: We might require more properties to be defined here in the future,
e.g. to what Plugin does the Schema addition belong to.
......
src/database_api.rs
+ 74
- 32
View file @ dacb0c30
......@@ -506,30 +506,73 @@ pub fn get_incoming_edges(tx: &Tx, target: Rowid) -> Result<Vec<EdgePointer>> {
pub fn get_schema(tx: &Tx) -> Result<Schema> {
let mut stmt = tx
.prepare_cached(
"SELECT thisProperty.value, thisType.value \
"SELECT itemType.value, propertyName.value, valueType.value \
FROM \
items as item, \
strings as thisProperty, \
strings as thisType \
WHERE item.type = 'ItemPropertySchema' \
AND thisProperty.item = item.rowid \
AND thisType.item = item.rowid \
AND thisProperty.name = 'propertyName' \
AND thisType.name = 'valueType';",
strings as itemType, \
strings as propertyName, \
strings as valueType \
WHERE \
item.type = 'ItemPropertySchema' \
AND item.deleted = 0 \
AND itemType.item = item.rowid \
AND propertyName.item = item.rowid \
AND valueType.item = item.rowid \
AND itemType.name = 'itemType' \
AND propertyName.name = 'propertyName' \
AND valueType.name = 'valueType';",
)
.context_str("Failed to prepare SQL get_schema query")?;
let mut rows = stmt.query([])?;
let mut property_types: HashMap<String, SchemaPropertyType> = HashMap::new();
let mut schema = Schema::empty();
while let Some(row) = rows.next()? {
let this_property: String = row.get(0)?;
let this_type: String = row.get(1)?;
let value_type = SchemaPropertyType::from_string(&this_type).map_err(|e| Error {
let item_type: String = row.get(0)?;
let property_name: String = row.get(1)?;
let value_type: String = row.get(2)?;
let value_type = SchemaPropertyType::from_string(&value_type).map_err(|e| Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: e,
})?;
property_types.insert(this_property, value_type);
schema.set_type_unchecked(item_type, property_name, value_type);
}
Ok(schema)
}
pub fn get_schema_item_id_for_property(
tx: &Tx,
item_type: &str,
property_name: &str,
) -> Result<String> {
let sql = "\
SELECT \
item.id \
FROM \
items as item, \
strings as nameEntry, \
strings as typeEntry \
WHERE \
item.type = 'ItemPropertySchema' \
AND item.deleted = 0 \
AND typeEntry.item = item.rowid \
AND nameEntry.item = item.rowid \
AND typeEntry.name = 'itemType' \
AND typeEntry.value = ? \
AND nameEntry.name = 'propertyName' \
AND nameEntry.value = ? \
;";
let mut stmt = tx
.prepare_cached(sql)
.context_str("Failed to prepare SQL query")
.context_str(sql)?;
let mut rows: Rows = stmt.query(params![item_type, property_name])?;
if let Some(row) = rows.next()? {
Ok(row.get(0)?)
} else {
Err(Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: format!("Failed to find property {}.{}", item_type, property_name),
})
}
Ok(Schema { property_types })
}
pub fn delete_schema_items_by_item_type_and_prop(
......@@ -537,14 +580,19 @@ pub fn delete_schema_items_by_item_type_and_prop(
item_type: &str,
property_name: &str,
) -> Result<()> {
let sql = "SELECT rowid FROM items as item, strings as itemTypeStr, strings as propNameStr \
WHERE item.type = 'ItemPropertySchema' \
AND item.rowid = itemTypeStr.item \
AND item.rowid = propNameStr.item \
AND itemTypeStr.name = 'itemType' \
AND itemTypeStr.value = ? \
AND propNameStr.name = 'propertyName' \
AND propNameStr.value = ? \
let sql = "\
SELECT \
item.rowid \
FROM \
items as item, strings as itemTypeStr, strings as propNameStr \
WHERE \
item.type = 'ItemPropertySchema' \
AND item.rowid = itemTypeStr.item \
AND item.rowid = propNameStr.item \
AND itemTypeStr.name = 'itemType' \
AND itemTypeStr.value = ? \
AND propNameStr.name = 'propertyName' \
AND propNameStr.value = ? \
;";
let mut stmt = tx.prepare_cached(sql)?;
let mut rows = stmt.query(params![item_type, property_name])?;
......@@ -639,7 +687,7 @@ pub mod tests {
let mut conn = new_conn();
let tx = conn.transaction()?;
let schema = get_schema(&tx)?;
assert!(schema.property_types.len() >= 3);
assert!(schema.all_properties_len() >= 3);
Ok(())
}
......@@ -675,15 +723,9 @@ pub mod tests {
insert_string(&tx, item, "valueType", "Text")?;
let schema = get_schema(&tx)?;
assert_eq!(
schema.property_types.get("age"),
Some(&SchemaPropertyType::Integer)
);
assert_eq!(
schema.property_types.get("name"),
Some(&SchemaPropertyType::Text)
);
assert!(schema.property_types.len() >= 3);
assert_eq!(schema.property("age"), Some(&SchemaPropertyType::Integer));
assert_eq!(schema.property("name"), Some(&SchemaPropertyType::Text));
assert!(schema.all_properties_len() >= 3);
Ok(())
}
......
src/database_utils.rs
+ 35
- 23
View file @ dacb0c30
......@@ -23,7 +23,7 @@ pub fn get_item_properties(tx: &Tx, rowid: i64, schema: &Schema) -> Result<Map<S
for IntegersNameValue { name, value } in database_api::get_integers_records_for_item(tx, rowid)?
{
match schema.property_types.get(&name) {
match schema.property(&name) {
Some(SchemaPropertyType::Bool) => {
json.insert(name, (value == 1).into());
}
......@@ -42,7 +42,7 @@ pub fn get_item_properties(tx: &Tx, rowid: i64, schema: &Schema) -> Result<Map<S
}
for StringsNameValue { name, value } in database_api::get_strings_records_for_item(tx, rowid)? {
match schema.property_types.get(&name) {
match schema.property(&name) {
Some(SchemaPropertyType::Text) => {
json.insert(name, value.into());
}
......@@ -58,7 +58,7 @@ pub fn get_item_properties(tx: &Tx, rowid: i64, schema: &Schema) -> Result<Map<S
}
for RealsNameValue { name, value } in database_api::get_reals_records_for_item(tx, rowid)? {
match schema.property_types.get(&name) {
match schema.property(&name) {
Some(SchemaPropertyType::Real) => {
json.insert(name, value.into());
}
......@@ -108,7 +108,7 @@ pub fn check_item_has_property(
name: &str,
value: &Value,
) -> Result<bool> {
let dbtype = if let Some(t) = schema.property_types.get(name) {
let dbtype = if let Some(t) = schema.property(name) {
t
} else {
return Err(Error {
......@@ -200,7 +200,7 @@ pub fn insert_property(
name: &str,
json: &Value,
) -> Result<()> {
let dbtype = if let Some(t) = schema.property_types.get(name) {
let dbtype = if let Some(t) = schema.property(name) {
t
} else {
return Err(Error {
......@@ -331,15 +331,21 @@ mod tests {
let mut conn = new_conn();
let tx = conn.transaction()?;
let mut schema = database_api::get_schema(&tx).unwrap();
schema
.property_types
.insert("age".to_string(), SchemaPropertyType::Integer);
schema
.property_types
.insert("strength".to_string(), SchemaPropertyType::Real);
schema
.property_types
.insert("myDescription".to_string(), SchemaPropertyType::Text);
schema.set_type_unchecked(
"Person".to_string(),
"age".to_string(),
SchemaPropertyType::Integer,
);
schema.set_type_unchecked(
"Person".to_string(),
"strength".to_string(),
SchemaPropertyType::Real,
);
schema.set_type_unchecked(
"Person".to_string(),
"myDescription".to_string(),
SchemaPropertyType::Text,
);
let date = schema::utc_millis();
let item: Rowid =
......@@ -392,15 +398,21 @@ mod tests {
let mut conn = new_conn();
let tx = conn.transaction()?;
let mut schema = database_api::get_schema(&tx).unwrap();
schema
.property_types
.insert("age".to_string(), SchemaPropertyType::Integer);
schema
.property_types
.insert("strength".to_string(), SchemaPropertyType::Real);
schema
.property_types
.insert("myDescription".to_string(), SchemaPropertyType::Text);
schema.set_type_unchecked(
"Person".to_string(),
"age".to_string(),
SchemaPropertyType::Integer,
);
schema.set_type_unchecked(
"Person".to_string(),
"strength".to_string(),
SchemaPropertyType::Real,
);
schema.set_type_unchecked(
"Person".to_string(),
"myDescription".to_string(),
SchemaPropertyType::Text,
);
let date = schema::utc_millis();
let item: Rowid =
......
src/internal_api.rs
+ 36
- 37
View file @ dacb0c30
......@@ -23,6 +23,7 @@ use crate::schema;
use crate::schema::validate_property_name;
use crate::schema::Schema;
use crate::triggers;
use crate::triggers::SchemaAdditionChange;
use log::info;
use rand::Rng;
use rusqlite::Transaction as Tx;
......@@ -84,38 +85,40 @@ pub fn create_item_tx(
database_key: &DatabaseKey,
) -> Result<String> {
let id: String = if let Some(id) = &item.id {
schema::validate_create_item_id(id)?;
id.to_string()
} else {
new_random_item_id()
};
if let Err(err) = schema::validate_create_item_id(&id) {
return Err(err);
}
let time_now = schema::utc_millis();
let _is_new_schema = triggers::add_item_as_schema_opt(schema, &item)?;
let rowid = database_api::insert_item_base(
tx,
&id,
&item._type,
item.date_created.unwrap_or(time_now),
item.date_modified.unwrap_or(time_now),
time_now,
item.deleted,
)?;
for (prop_name, prop_value) in &item.fields {
insert_property(tx, schema, rowid, prop_name, prop_value)?;
let schema_addition = triggers::add_item_as_schema_opt(tx, schema, &item)?;
if let SchemaAdditionChange::OldSchema { old_id } = schema_addition {
Ok(old_id)
} else {
let rowid = database_api::insert_item_base(
tx,
&id,
&item._type,
item.date_created.unwrap_or(time_now),
item.date_modified.unwrap_or(time_now),
time_now,
item.deleted,
)?;
for (prop_name, prop_value) in &item.fields {
insert_property(tx, schema, rowid, prop_name, prop_value)?;
}
triggers::trigger_after_item_create(
tx,
schema,
rowid,
&id,
&item,
pod_owner,
cli,
database_key,
)?;
Ok(id)
}
triggers::trigger_after_item_create(
tx,
schema,
rowid,
&id,
&item,
pod_owner,
cli,
database_key,
)?;
Ok(id)
}
pub fn update_item_tx(
......@@ -450,7 +453,6 @@ mod tests {
use crate::plugin_auth_crypto::DatabaseKey;
use crate::schema::Schema;
use serde_json::json;
use std::collections::HashMap;
use warp::hyper::StatusCode;
#[test]
......@@ -546,23 +548,20 @@ mod tests {
"valueType": "Bool",
});
let create_item: CreateItem = serde_json::from_value(json).unwrap();
let expected_error = "Schema for property dateCreated is already defined to type DateTime, cannot override to type Bool";
assert!(internal_api::create_item_tx(
let expected_error = "Schema for property dateCreated is already defined to valueType DateTime, cannot override to Bool";
let result = internal_api::create_item_tx(
&tx,
&mut minimal_schema,
create_item,
"",
&cli,
&database_key
)
.unwrap_err()
.msg
.contains(expected_error));
&database_key,
);
assert!(result.is_err());
assert!(result.unwrap_err().msg.contains(expected_error));
}
let mut bad_empty_schema = Schema {
property_types: HashMap::new(),
};
let mut bad_empty_schema = Schema::empty();
let create_item: CreateItem = serde_json::from_value(json).unwrap();
let result = internal_api::create_item_tx(
&tx,
......
src/schema.rs
+ 45
- 1
View file @ dacb0c30
......@@ -46,7 +46,51 @@ impl SchemaPropertyType {
#[derive(Debug)]
pub struct Schema {
pub property_types: HashMap<String, SchemaPropertyType>,
/// United properties across all types, for example:
/// "age" -> Integer
property_types: HashMap<String, SchemaPropertyType>,
/// All types and their properties, for example:
/// "Person", "age" -> Integer
type_properties: HashMap<String, HashMap<String, SchemaPropertyType>>,
}
impl Schema {
pub fn empty() -> Schema {
Schema {
property_types: HashMap::new(),
type_properties: HashMap::new(),
}
}
pub fn set_type_unchecked(
&mut self,
item_type: String,
property_name: String,
value_type: SchemaPropertyType,
) {
self.property_types
.insert(property_name.to_string(), value_type);
self.type_properties
.entry(item_type)
.or_insert_with(HashMap::new)
.insert(property_name, value_type);
}
pub fn property(&self, property_name: &str) -> Option<&SchemaPropertyType> {
self.property_types.get(property_name)
}
pub fn item_property(
&self,
item_type: &str,
property_name: &str,
) -> Option<&SchemaPropertyType> {
self.type_properties
.get(item_type)
.and_then(|i| i.get(property_name))
}
/// The total number of properties united for all types.
/// (E.g. "Person.age" and "Computer.age" only count once, because that's one property "age".)
pub fn all_properties_len(&self) -> usize {
self.property_types.len()
}
}
/// Validation of _new_ item ids. Note that it is not applied to already existing
......
src/triggers.rs
+ 76
- 32
View file @ dacb0c30
......@@ -4,6 +4,7 @@
use crate::api_model::CreateItem;
use crate::command_line_interface::CliOptions;
use crate::database_api;
use crate::database_api::Rowid;
use crate::database_utils::get_item_from_rowid;
use crate::error::Error;
......@@ -38,37 +39,70 @@ pub struct PluginRunItem {
pub enum SchemaAdditionChange {
NotASchema,
NewSchemaAdded,
OldSchemaIgnored,
OldSchema { old_id: String },
}
/// If an item is a Schema, add it to the schema. Return the change.
/// If an item is a Schema, add it to the schema.
/// Fail if an incompatible schema is attempted to be inserted.
pub fn add_item_as_schema_opt(
tx: &Tx,
schema: &mut Schema,
item: &CreateItem,
) -> Result<SchemaAdditionChange> {
// We'll do something ugly here.
// We'll convert the item into JSON and back into the desired type for type check and parsing.
// This is easier code-wise than to do manual conversions.
// It only triggers for specific, rarely used items. This implementation might change later.
if item._type == "ItemPropertySchema" {
// We'll do something ugly here.
// We'll convert the item into JSON, and then back into the desired type
// for type check and parsing. This is easier code-wise than to do manual conversions.
// It only triggers for specific, rarely used items. This implementation might change later.
let json = serde_json::to_value(item)?;
let parsed: SchemaItem = serde_json::from_value(json)
.context(|| format!("Parsing of Schema item {:?}, {}:{}", item, file!(), line!()))?;
if let Some(old) = schema.property_types.get(&parsed.property_name) {
if old == &parsed.value_type {
Ok(OldSchemaIgnored)
} else {
// The result of the operation depends on:
// * the "heterogeneous" property's type
// * this particular itemType's property type
// * whether an ID is present in the request
match (
schema.property(&parsed.property_name),
schema.item_property(&parsed.item_type, &parsed.property_name),
&item.id,
) {
(Some(old_value_type), _, _) if &parsed.value_type != old_value_type => {
Err(Error {
code: StatusCode::BAD_REQUEST,
msg: format!("Schema for property {} is already defined to type {}, cannot override to type {}", parsed.property_name, old, parsed.value_type)
msg: format!(
"Schema for property {} is already defined to valueType {}, cannot override to {}",
parsed.property_name,
old_value_type,
parsed.value_type,
)
})
}
(_, None, _) | (_, _, Some(_)) => {
schema.set_type_unchecked(parsed.item_type, parsed.property_name, parsed.value_type);
Ok(NewSchemaAdded)
}
(_, Some(old_value_type), None) if &parsed.value_type == old_value_type => {
let old_id = database_api::get_schema_item_id_for_property(
tx,
&parsed.item_type,
&parsed.property_name,
)?;
Ok(OldSchema { old_id })
}
(union_value_type, Some(old_value_type), None) => {
Err(Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: format!(
"Unexpected failure, {}.{} has valueType {}. However, this Schema should not have been allowed as {} is already defined as {:?}",
parsed.item_type,
parsed.property_name,
old_value_type,
parsed.property_name,
union_value_type,
)
})
}
} else {
schema
.property_types
.insert(parsed.property_name, parsed.value_type);
Ok(NewSchemaAdded)
}
} else {
Ok(NotASchema)
......@@ -108,28 +142,32 @@ pub fn trigger_after_item_create(
mod tests {
use super::add_item_as_schema_opt;
use crate::api_model::CreateItem;
use crate::database_api::tests::new_conn;
use crate::error::Error;
use crate::error::Result;
use crate::schema::Schema;
use crate::schema::SchemaPropertyType;
use crate::triggers::SchemaAdditionChange;
use serde_json::json;
use std::collections::HashMap;
use warp::http::StatusCode;
#[test]
fn my_test() -> Result<()> {
// let mut minimal_schema = database_api::get_schema(&tx).unwrap();
let mut schema = Schema {
property_types: HashMap::new(),
};
schema
.property_types
.insert("age".to_string(), SchemaPropertyType::Integer);
let mut conn = new_conn();
let tx = conn.transaction().unwrap();
let mut schema = Schema::empty();
schema.set_type_unchecked(
"Person".to_string(),
"age".to_string(),
SchemaPropertyType::Integer,
);
let json = json!({
"type": "Something"
});
let create_item: CreateItem = serde_json::from_value(json).unwrap();
let result = add_item_as_schema_opt(&mut schema, &create_item);
let result = add_item_as_schema_opt(&tx, &mut schema, &create_item);
assert_eq!(result, Ok(SchemaAdditionChange::NotASchema));
let json = json!({
......@@ -139,8 +177,14 @@ mod tests {
"valueType": "Integer",
});
let create_item: CreateItem = serde_json::from_value(json).unwrap();
let result = add_item_as_schema_opt(&mut schema, &create_item);
assert_eq!(result, Ok(SchemaAdditionChange::OldSchemaIgnored));
let result = add_item_as_schema_opt(&tx, &mut schema, &create_item);
assert_eq!(
result,
Err(Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: "Failed to find property Person.age".to_string()
})
);
let json = json!({
"type": "ItemPropertySchema",
......@@ -149,11 +193,11 @@ mod tests {
"valueType": "Integer",
});
let create_item: CreateItem = serde_json::from_value(json).unwrap();
assert_eq!(schema.property_types.len(), 1);
let result = add_item_as_schema_opt(&mut schema, &create_item);
assert_eq!(schema.property_types.len(), 2);
assert_eq!(schema.all_properties_len(), 1);
let result = add_item_as_schema_opt(&tx, &mut schema, &create_item);
assert_eq!(schema.all_properties_len(), 2);
assert_eq!(
schema.property_types.get("agility"),
schema.property("agility"),
Some(SchemaPropertyType::Integer).as_ref()
);
assert_eq!(result, Ok(SchemaAdditionChange::NewSchemaAdded));
......@@ -165,7 +209,7 @@ mod tests {
"valueType": "Text",
});
let create_item: CreateItem = serde_json::from_value(json).unwrap();
let result = add_item_as_schema_opt(&mut schema, &create_item);
let result = add_item_as_schema_opt(&tx, &mut schema, &create_item);
assert!(result.is_err(), "result should be an error {:?}", result);
Ok(())
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment