Merge branch 'strict-schema-parsing' into 'dev'

Remove old schema file code, use strict Schema parsing

See merge request memri/pod!223

Cargo.lock

 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "aead"
 version = "0.3.2"
@@ -996,7 +998,7 @@ dependencies = [
 
 [[package]]
 name = "pod"
-version = "0.3.0"
+version = "0.4.1"
 dependencies = [
  "bytes",
  "chacha20poly1305",

Cargo.toml

 [package]
 name = "pod"
-version = "0.3.0"
+version = "0.4.1"
 authors = [
   "Bijun Li <bijun.li@polis.global>",
   "Ruben Seggers <ruben.seggers@polis.global>",

res/example_data.sql

@@ -6,12 +6,12 @@
 INSERT INTO items(rowid, id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(19000, "id-person-age", "ItemPropertySchema", 0, 0, 0, 0);
 INSERT INTO strings(item, name, value) VALUES(19000, "itemType", "Person");
 INSERT INTO strings(item, name, value) VALUES(19000, "propertyName", "age");
-INSERT INTO strings(item, name, value) VALUES(19000, "valueType", "integer");
+INSERT INTO strings(item, name, value) VALUES(19000, "valueType", "Integer");
 
 INSERT INTO items(rowid, id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(19001, "id-person-name", "ItemPropertySchema", 0, 0, 0, 0);
 INSERT INTO strings(item, name, value) VALUES(19001, "itemType", "Person");
 INSERT INTO strings(item, name, value) VALUES(19001, "propertyName", "name");
-INSERT INTO strings(item, name, value) VALUES(19001, "valueType", "text");
+INSERT INTO strings(item, name, value) VALUES(19001, "valueType", "Text");
 
 INSERT INTO items(rowid, id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(19002, "first-id", "Person", 0, 0, 0, 0);
 INSERT INTO integers(item, name, value) VALUES(19002, "age", 20);

res/migrations/V1__initial.sql

@@ -62,16 +62,16 @@ CREATE        INDEX idx_reals_name_item on reals(name, item);
 INSERT INTO items(rowid, id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(1, "ad61e2770eb64d81a3ddfa03fa5f887f", "ItemPropertySchema", 0, 0, 0, 0);
 INSERT INTO strings(item, name, value) VALUES(1, "itemType", "ItemPropertySchema");
 INSERT INTO strings(item, name, value) VALUES(1, "propertyName", "itemType");
-INSERT INTO strings(item, name, value) VALUES(1, "valueType", "text");
+INSERT INTO strings(item, name, value) VALUES(1, "valueType", "Text");
 
 -- Items of type "ItemPropertySchema" have property "propertyName" (text)
 INSERT INTO items(rowid, id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(2, "7bce841921beb3295634b224bc75a990", "ItemPropertySchema", 0, 0, 0, 0);
 INSERT INTO strings(item, name, value) VALUES(2, "itemType", "ItemPropertySchema");
 INSERT INTO strings(item, name, value) VALUES(2, "propertyName", "propertyName");
-INSERT INTO strings(item, name, value) VALUES(2, "valueType", "text");
+INSERT INTO strings(item, name, value) VALUES(2, "valueType", "Text");
 
 -- Items of type "ItemPropertySchema" have property "valueType" (text)
 INSERT INTO items(rowid, id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(3, "791427a0571e394d1f29cb63ee77e5", "ItemPropertySchema", 0, 0, 0, 0);
 INSERT INTO strings(item, name, value) VALUES(3, "itemType", "ItemPropertySchema");
 INSERT INTO strings(item, name, value) VALUES(3, "propertyName", "valueType");
-INSERT INTO strings(item, name, value) VALUES(3, "valueType", "text");
+INSERT INTO strings(item, name, value) VALUES(3, "valueType", "Text");

res/migrations/V2__item_StartPlugin.sql

@@ -13,7 +13,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
-    "valueType", "text"
+    "valueType", "Text"
 );
 
 -- Items of type "PluginRun" have property "targetItemId" (text)
@@ -31,5 +31,5 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"),
-    "valueType", "text"
+    "valueType", "Text"
 );

res/migrations/V3__item_File.sql

@@ -12,7 +12,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "419d1188b61dfa7d0a18e20794c843"),
-    "valueType", "text"
+    "valueType", "Text"
 );
 
 
@@ -30,7 +30,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "16fa737654dc376a20976d8ec89033c2"),
-    "valueType", "text"
+    "valueType", "Text"
 );
 
 
@@ -48,5 +48,5 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "25994759432d4a636599c5e9c987eb95"),
-    "valueType", "text"
+    "valueType", "Text"
 );

res/migrations/V4__base_item_properties.sql

@@ -13,7 +13,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "ccf9f0cba490ed58fa967f6a1136fc8"),
-    "valueType", "text"
+    "valueType", "Text"
 );
 
 
@@ -32,7 +32,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "904ecd61ba00152d9ab9f235c13217"),
-    "valueType", "text"
+    "valueType", "Text"
 );
 
 
@@ -51,7 +51,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "cb8b4455cdcabd25ba9d0eaabcc3786"),
-    "valueType", "datetime"
+    "valueType", "DateTime"
 );
 
 
@@ -70,7 +70,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "d1757e7395932c32cc1dcecc03b2977"),
-    "valueType", "datetime"
+    "valueType", "DateTime"
 );
 
 
@@ -89,7 +89,7 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "8f492d26bd192ad53cf07f3ad36cbae7"),
-    "valueType", "datetime"
+    "valueType", "DateTime"
 );
 
 
@@ -108,5 +108,5 @@ INSERT INTO strings(item, name, value) VALUES(
 );
 INSERT INTO strings(item, name, value) VALUES(
     (SELECT rowid FROM items WHERE id = "8716a96393c48255542ca6d54afe2bc6"),
-    "valueType", "bool"
+    "valueType", "Bool"
 );

src/command_line_interface.rs

 use lazy_static::lazy_static;
 use std::net::IpAddr;
-use std::path::PathBuf;
 use structopt::clap::AppSettings;
 use structopt::StructOpt;
 
@@ -111,21 +110,6 @@ pub struct CliOptions {
     /// for details on what it is, and how it works.
     #[structopt(long)]
     pub shared_server: bool,
-
-    /// Deprecated: schema file to use.
-    /// Note that this Schema is in the process of being re-written to a different format,
-    /// The way it works will change in a breaking manner.
-    #[structopt(
-        long,
-        name = "SCHEMA_FILE",
-        parse(from_os_str),
-        default_value = "res/default_schema.json"
-    )]
-    pub schema_file: PathBuf,
-
-    /// Validate the schema file and exit. Useful in combination with the --schema-file CLI key.
-    #[structopt(long)]
-    pub validate_schema: bool,
 }
 
 lazy_static! {
@@ -157,8 +141,6 @@ pub mod tests {
             insecure_non_tls: Some(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))),
             insecure_http_headers: false,
             shared_server: false,
-            schema_file: Default::default(),
-            validate_schema: false,
         }
     }
 }

src/constants.rs

 // Constants used in the project. These are "convention over configuration" for now.
 
 pub const DATABASE_DIR: &str = "./data/db";
-pub const DATABASE_SUFFIX: &str = ".v4.sqlite";
+pub const DATABASE_SUFFIX: &str = ".v4.1.sqlite";
 
 pub const FILES_DIR: &str = "./data/files";
 /// Directory where fully uploaded and hash-checked files are stored

src/database_api.rs

@@ -577,7 +577,7 @@ pub mod tests {
         )?;
         insert_string(&tx, item, "itemType", "Person")?;
         insert_string(&tx, item, "propertyName", "age")?;
-        insert_string(&tx, item, "valueType", "integer")?;
+        insert_string(&tx, item, "valueType", "Integer")?;
 
         let item = insert_item_base(
             &tx,
@@ -590,7 +590,7 @@ pub mod tests {
         )?;
         insert_string(&tx, item, "itemType", "Person")?;
         insert_string(&tx, item, "propertyName", "name")?;
-        insert_string(&tx, item, "valueType", "text")?;
+        insert_string(&tx, item, "valueType", "Text")?;
 
         let schema = get_schema(&tx)?;
         assert_eq!(
@@ -621,7 +621,7 @@ pub mod tests {
         )?;
         insert_string(&tx, item, "itemType", "Person")?;
         insert_string(&tx, item, "propertyName", "age")?;
-        insert_string(&tx, item, "valueType", "integer")?;
+        insert_string(&tx, item, "valueType", "Integer")?;
 
         assert_eq!(
             search_items_params(&tx, Some(item), None, None, None, None, None)?.len(),
@@ -772,7 +772,7 @@ pub mod tests {
         )?;
         insert_string(&tx, item, "itemType", "Person")?;
         insert_string(&tx, item, "propertyName", "age")?;
-        insert_string(&tx, item, "valueType", "integer")?;
+        insert_string(&tx, item, "valueType", "Integer")?;
 
         assert!(check_string_exists(&tx, item, "itemType", "Person")?);
         assert!(check_string_exists(&tx, item, "itemType", "Person2")?.not());

src/database_migrate_schema.rs

-use crate::schema::validate_property_name;
-use serde::Deserialize;
-use serde::Serialize;
-use std::collections::HashMap;
-use std::path::Path;
-
-#[derive(Serialize, Deserialize)]
-struct DatabaseSchema {
-    types: Vec<DatabaseType>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct DatabaseType {
-    name: String,
-    properties: Vec<SchemaProperty>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct SchemaProperty {
-    name: String,
-    indexed: bool,
-    dbtype: SchemaPropertyType,
-}
-
-/// See `README.md#understanding-the-schema` to understand possible
-/// property types and their meaning
-#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq)]
-pub enum SchemaPropertyType {
-    Text,
-    Integer,
-    Real,
-    Bool,
-    DateTime,
-}
-
-pub fn validate_schema_file(file: &Path) -> Result<(), String> {
-    let file = std::fs::read(file)
-        .map_err(|err| format!("Failed to read data from target file, {}", err))?;
-    let schema: DatabaseSchema = serde_json::from_slice(&file)
-        .map_err(|err| format!("Failed to parse schema file, {}", err))?;
-    validate_schema(&schema)
-}
-
-fn validate_schema(schema: &DatabaseSchema) -> Result<(), String> {
-    for typ in &schema.types {
-        validate_property_name(&typ.name)
-            .map_err(|err| format!("Schema type {} is invalid, {}", typ.name, err))?;
-        if typ.name.starts_with('_') {
-            return Err(format!("Schema type {} starts with underscore", typ.name));
-        }
-    }
-    let mut properties: HashMap<String, SchemaPropertyType> = HashMap::new();
-    for typ in &schema.types {
-        for prop in &typ.properties {
-            validate_property_name(&prop.name).map_err(|err| {
-                format!(
-                    "Schema property {} for type {} is invalid, {}",
-                    prop.name, typ.name, err
-                )
-            })?;
-            if prop.name.starts_with('_') {
-                return Err(format!(
-                    "Schema property {} of type {} starts with underscore",
-                    prop.name, typ.name
-                ));
-            }
-            let prop_name = prop.name.to_lowercase();
-            match properties.get(&prop_name) {
-                None => {
-                    properties.insert(prop_name, prop.dbtype);
-                }
-                Some(old_dbtype) => {
-                    if old_dbtype != &prop.dbtype {
-                        return Err(format!("Schema property {} (lowercase {}) is defined differently for different types", prop.name, prop_name));
-                    };
-                }
-            };
-        }
-    }
-    Ok(())
-}

src/main.rs

@@ -5,7 +5,6 @@ mod command_line_interface;
 mod constants;
 pub mod database_api;
 mod database_migrate_refinery;
-mod database_migrate_schema;
 mod database_utils;
 mod error;
 mod file_api;
@@ -43,15 +42,6 @@ async fn main() {
         })
         .init();
     let cli_options: CliOptions = command_line_interface::PARSED.clone();
-    if cli_options.validate_schema {
-        if let Err(err) = database_migrate_schema::validate_schema_file(&cli_options.schema_file) {
-            log::error!("Schema validation failed: {}", err);
-            std::process::exit(1)
-        } else {
-            log::info!("Schema is valid!");
-            std::process::exit(0)
-        }
-    };
     info!("Starting Pod version {}", get_project_version());
     info!("Running Pod with configuration {:#?}", cli_options);
 

src/schema.rs

@@ -27,13 +27,12 @@ impl std::fmt::Display for SchemaPropertyType {
 
 impl SchemaPropertyType {
     pub fn from_string(str: &str) -> std::result::Result<SchemaPropertyType, String> {
-        let str = str.to_lowercase();
-        match str.as_ref() {
-            "text" => Ok(SchemaPropertyType::Text),
-            "integer" => Ok(SchemaPropertyType::Integer),
-            "real" => Ok(SchemaPropertyType::Real),
-            "bool" => Ok(SchemaPropertyType::Bool),
-            "datetime" => Ok(SchemaPropertyType::DateTime),
+        match str {
+            "Text" => Ok(SchemaPropertyType::Text),
+            "Integer" => Ok(SchemaPropertyType::Integer),
+            "Real" => Ok(SchemaPropertyType::Real),
+            "Bool" => Ok(SchemaPropertyType::Bool),
+            "DateTime" => Ok(SchemaPropertyType::DateTime),
             _ => Err(format!(
                 "Failed to parse {} into {}",
                 str,