Use strict database type

E.g. forbid items with incorrect type to be inserted in the database

README.md

@@ -106,6 +106,11 @@ to the clients, however, and clients should only ever receive/send `true` and `f
 Use this database type to denote DateTime.
 Internally stored as Integer and should be passed as Integer.
 
+All column definitions of the same case-insensitive name MUST have the same type and indexing.
+All column names MUST consist of `a-zA-Z_` characters only, and start with `a-zA-Z`.
+All type names MUST consist of `a-zA-Z_` characters only,
+and start with `a-zA-Z` (same as column names).
+
 ### Changing the schema locally
 If you want to make local changes to the schema while developing
 new functionality, you can edit the schema directly.

src/database_migrate_schema.rs

@@ -11,15 +11,6 @@ use std::collections::HashMap;
 use std::collections::HashSet;
 use std::hash::Hash;
 
-/// Constraints:
-///
-/// * All column definitions of the same case-insensitive name MUST have the same type and indexing
-///
-/// * All column names MUST consist of `a-zA-Z_` characters only,
-/// and start with `a-zA-Z`
-///
-/// * All type names MUST consist of `a-zA-Z_` characters only,
-/// and start with `a-zA-Z` (same as column names)
 #[derive(Serialize, Deserialize)]
 struct DatabaseSchema {
     types: Vec<DatabaseType>,
@@ -41,7 +32,7 @@ struct DatabaseColumn {
 /// See `README.md#understanding-the-schema` to understand possible
 /// property types and their meaning
 #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq)]
-enum DatabaseColumnType {
+pub enum DatabaseColumnType {
     Text,
     Integer,
     Real,
@@ -49,28 +40,75 @@ enum DatabaseColumnType {
     DateTime,
 }
 
+fn get_columns_of_type(dbtype: DatabaseColumnType) -> HashSet<String> {
+    let parsed_schema: DatabaseSchema = serde_json::from_slice(AUTOGENERATED_SCHEMA)
+        .expect("Failed to parse autogenerated_database_schema to JSON");
+    let columns = parsed_schema.types.iter().flat_map(|t| &t.properties);
+    let mut result = HashSet::new();
+    for column in columns {
+        if column.dbtype == dbtype {
+            result.insert(column.name.to_string());
+        }
+    }
+    result
+}
+lazy_static! {
+    pub static ref TEXT_COLUMNS: HashSet<String> = {
+        let mut result = get_columns_of_type(DatabaseColumnType::Text);
+        result.insert("_type".to_string());
+        result
+    };
+}
+lazy_static! {
+    pub static ref INTEGER_COLUMNS: HashSet<String> = {
+        let mut result = get_columns_of_type(DatabaseColumnType::Integer);
+        result.insert("uid".to_string());
+        result.insert("version".to_string());
+        result
+    };
+}
+lazy_static! {
+    pub static ref REAL_COLUMNS: HashSet<String> = get_columns_of_type(DatabaseColumnType::Real);
+}
 lazy_static! {
     pub static ref BOOL_COLUMNS: HashSet<String> = {
+        let mut result = get_columns_of_type(DatabaseColumnType::Bool);
+        result.insert("deleted".to_string());
+        result
+    };
+}
+lazy_static! {
+    pub static ref DATE_TIME_COLUMNS: HashSet<String> = {
+        let mut result = get_columns_of_type(DatabaseColumnType::DateTime);
+        result.insert("dateCreated".to_string());
+        result.insert("dateModified".to_string());
+        result
+    };
+}
+lazy_static! {
+    pub static ref ALL_COLUMN_TYPES: HashMap<String, DatabaseColumnType> = {
         let parsed_schema: DatabaseSchema = serde_json::from_slice(AUTOGENERATED_SCHEMA)
             .expect("Failed to parse autogenerated_database_schema to JSON");
         let columns = parsed_schema.types.iter().flat_map(|t| &t.properties);
-        let mut result = HashSet::new();
-        for column in columns {
-            if column.dbtype == DatabaseColumnType::Bool {
-                result.insert(column.name.to_string());
-            }
-        }
-        result.insert("deleted".to_string());
-        result
+        columns.map(|c| (c.name.to_string(), c.dbtype)).collect()
     };
 }
 
+const MANDATORY_ITEMS_FIELDS: &[&str] = &[
+    "uid",
+    "_type",
+    "dateCreated",
+    "dateModified",
+    "deleted",
+    "version",
+];
+
 pub const AUTOGENERATED_SCHEMA: &[u8] = include_bytes!("../res/autogenerated_database_schema.json");
 
 pub fn migrate(sqlite: &Pool<SqliteConnectionManager>) {
     let conn = sqlite
         .get()
-        .expect("Failed to aquire SQLite connection during db initialization");
+        .expect("Failed to acquire SQLite connection during db initialization");
     info!("Initializing database schema (additional columns)");
     let parsed_schema: DatabaseSchema = serde_json::from_slice(AUTOGENERATED_SCHEMA)
         .expect("Failed to parse autogenerated_database_schema to JSON");
@@ -199,12 +237,3 @@ where
     }
     map
 }
-
-const MANDATORY_ITEMS_FIELDS: &[&str] = &[
-    "uid",
-    "_type",
-    "dateCreated",
-    "dateModified",
-    "deleted",
-    "version",
-];

src/internal_api.rs

@@ -89,7 +89,7 @@ fn write_sql_body(sql: &mut String, keys: &[&String], separator: &str) {
 fn execute_sql(tx: &Transaction, sql: &str, fields: &HashMap<String, Value>) -> Result<()> {
     let mut sql_params = Vec::new();
     for (key, value) in fields {
-        sql_params.push((format!(":{}", key), json_value_to_sqlite(value)?));
+        sql_params.push((format!(":{}", key), json_value_to_sqlite(value, key)?));
     }
     let sql_params: Vec<_> = sql_params
         .iter()

src/sql_converters.rs

 use crate::database_migrate_schema;
 use crate::error::Error;
+use database_migrate_schema::ALL_COLUMN_TYPES;
+use database_migrate_schema::BOOL_COLUMNS;
+use database_migrate_schema::DATE_TIME_COLUMNS;
+use database_migrate_schema::INTEGER_COLUMNS;
+use database_migrate_schema::REAL_COLUMNS;
+use database_migrate_schema::TEXT_COLUMNS;
 use lazy_static::lazy_static;
+use log::warn;
 use regex::Regex;
 use rusqlite::types::ToSqlOutput;
 use rusqlite::types::ValueRef;
@@ -59,14 +66,13 @@ pub fn fields_mapping_to_owned_sql_params(
     fields_map: &Map<String, serde_json::Value>,
 ) -> crate::error::Result<Vec<(String, ToSqlOutput)>> {
     let mut sql_params = Vec::new();
-    for (field, value) in fields_map {
-        match value {
-            Value::Array(_) => continue,
-            Value::Object(_) => continue,
-            _ => (),
+    for (key, value) in fields_map {
+        if value.is_array() || value.is_object() {
+            continue;
         };
-        let field = format!(":{}", field);
-        sql_params.push((field, json_value_to_sqlite(value)?));
+        let value = json_value_to_sqlite(value, key)?;
+        let key = format!(":{}", key);
+        sql_params.push((key, value));
     }
     Ok(sql_params)
 }
@@ -80,15 +86,42 @@ pub fn borrow_sql_params<'a>(
         .collect()
 }
 
-pub fn json_value_to_sqlite(json: &Value) -> crate::error::Result<ToSqlOutput<'_>> {
+pub fn json_value_to_sqlite<'a>(
+    json: &'a Value,
+    column: &str,
+) -> crate::error::Result<ToSqlOutput<'a>> {
     match json {
         Value::Null => Ok(ToSqlOutput::Borrowed(ValueRef::Null)),
-        Value::String(s) => Ok(ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes()))),
-        Value::Number(n) => {
+        Value::String(s) if TEXT_COLUMNS.contains(column) => {
+            Ok(ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())))
+        }
+        Value::Number(n) if INTEGER_COLUMNS.contains(column) => {
+            if let Some(int) = n.as_i64() {
+                Ok(ToSqlOutput::Borrowed(ValueRef::Integer(int)))
+            } else {
+                Err(Error {
+                    code: StatusCode::BAD_REQUEST,
+                    msg: format!("Failed to parse JSON number {} to i64 ({})", n, column),
+                })
+            }
+        }
+        Value::Number(n) if REAL_COLUMNS.contains(column) => {
+            if let Some(int) = n.as_f64() {
+                Ok(ToSqlOutput::Borrowed(ValueRef::Real(int)))
+            } else {
+                Err(Error {
+                    code: StatusCode::BAD_REQUEST,
+                    msg: format!("Failed to parse JSON number {} to f64 ({})", n, column),
+                })
+            }
+        }
+        Value::Bool(b) if BOOL_COLUMNS.contains(column) => Ok((if *b { 1 } else { 0 }).into()),
+        Value::Number(n) if DATE_TIME_COLUMNS.contains(column) => {
             if let Some(int) = n.as_i64() {
                 Ok(ToSqlOutput::Borrowed(ValueRef::Integer(int)))
             } else if let Some(float) = n.as_f64() {
-                Ok(ToSqlOutput::Borrowed(ValueRef::Real(float)))
+                warn!("Using float-to-integer conversion property {}, value {}. This might not be supported in the future, please use a compatible DateTime format https://gitlab.memri.io/memri/pod#understanding-the-schema", float, column);
+                Ok((float.round() as i64).into())
             } else {
                 Err(Error {
                     code: StatusCode::BAD_REQUEST,
@@ -96,15 +129,25 @@ pub fn json_value_to_sqlite(json: &Value) -> crate::error::Result<ToSqlOutput<'_
                 })
             }
         }
-        Value::Bool(b) => Ok((if *b { 1 } else { 0 }).into()),
-        Value::Array(arr) => Err(Error {
-            code: StatusCode::BAD_REQUEST,
-            msg: format!("Cannot convert JSON array to an SQL parameter: {:?}", arr),
-        }),
-        Value::Object(obj) => Err(Error {
-            code: StatusCode::BAD_REQUEST,
-            msg: format!("Cannot convert JSON object to an SQL parameter: {:?}", obj),
-        }),
+        _ => {
+            if let Some(dbtype) = ALL_COLUMN_TYPES.get(column) {
+                Err(Error {
+                    code: StatusCode::BAD_REQUEST,
+                    msg: format!(
+                        "Failed to parse json value {} to {:?} ({})",
+                        json, dbtype, column
+                    ),
+                })
+            } else {
+                Err(Error {
+                    code: StatusCode::BAD_REQUEST,
+                    msg: format!(
+                        "Failed to insert json value {} to property {}, reason: not defined in schema",
+                        json, column
+                    ),
+                })
+            }
+        }
     }
 }