Commit 59df391a authored by Vasili Novikov's avatar Vasili Novikov
Browse files

Merge branch 'implement-schema-file-api' into 'dev'

Implement Schema-based File API

See merge request memri/pod!192
parents 49c34d83 93731bcb
Showing with 298 additions and 94 deletions
+298 -94
......@@ -242,9 +242,8 @@ See [Plugins](./Plugins.md) on how plugins are started exactly.
⚠️ UNSTABLE: We might require more properties for Plugins to start in the future,
e.g. permission limitation.
<!--
# File API
# File API
### POST /v3/$owner_key/upload_file/$databaseKey/$sha256hashOfTheFile
```text
......@@ -274,4 +273,3 @@ The properties `nonce` and `key` will be updated for this item.
```
Get a file by its sha256 hash.
If the file does not yet exist in Pod, a 404 NOT FOUND error will be returned.
-->
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"419d1188b61dfa7d0a18e20794c843",
"ItemPropertySchema", 0, 0, 0, 0
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "419d1188b61dfa7d0a18e20794c843"),
"itemType", "File"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "419d1188b61dfa7d0a18e20794c843"),
"propertyName", "sha256"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "419d1188b61dfa7d0a18e20794c843"),
"valueType", "text"
);
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"16fa737654dc376a20976d8ec89033c2",
"ItemPropertySchema", 0, 0, 0, 0
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "16fa737654dc376a20976d8ec89033c2"),
"itemType", "File"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "16fa737654dc376a20976d8ec89033c2"),
"propertyName", "key"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "16fa737654dc376a20976d8ec89033c2"),
"valueType", "text"
);
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"25994759432d4a636599c5e9c987eb95",
"ItemPropertySchema", 0, 0, 0, 0
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "25994759432d4a636599c5e9c987eb95"),
"itemType", "File"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "25994759432d4a636599c5e9c987eb95"),
"propertyName", "nonce"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "25994759432d4a636599c5e9c987eb95"),
"valueType", "text"
);
......@@ -125,3 +125,29 @@ lazy_static! {
lazy_static! {
pub static ref PARSED: CliOptions = CliOptions::from_args();
}
#[cfg(test)]
pub mod tests {
use super::CliOptions;
use std::net::IpAddr;
use std::net::Ipv4Addr;
/// Example test CLI. Purely for convenience,
/// you can instantiate your own / unrelated ones as well.
pub fn test_cli() -> CliOptions {
CliOptions {
port: 3030,
owners: "ANY".to_string(),
plugins_callback_address: None,
plugins_docker_network: None,
tls_pub_crt: "".to_string(),
tls_priv_key: "".to_string(),
non_tls: true,
insecure_non_tls: Some(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))),
insecure_http_headers: false,
shared_server: false,
schema_file: Default::default(),
validate_schema: false,
}
}
}
......@@ -4,3 +4,6 @@ pub const DATABASE_DIR: &str = "./data/db";
pub const DATABASE_SUFFIX: &str = ".db3";
pub const FILES_DIR: &str = "./data/files";
/// Directory where fully uploaded and hash-checked files are stored
/// (in future, the files should also be s3-uploaded).
pub const FILES_FINAL_SUBDIR: &str = "final";
......@@ -7,6 +7,7 @@ use crate::schema::SchemaPropertyType;
use log::debug;
use rusqlite::params;
use rusqlite::types::ToSqlOutput;
use rusqlite::Row;
use rusqlite::Transaction as Tx;
use rusqlite::NO_PARAMS;
use std::collections::HashMap;
......@@ -80,6 +81,18 @@ pub struct DatabaseSearch<'a> {
pub _limit: u64,
}
fn parse_item_base(row: &Row) -> Result<ItemBase> {
Ok(ItemBase {
rowid: row.get(0)?,
id: row.get(1)?,
_type: row.get(2)?,
date_created: row.get(3)?,
date_modified: row.get(4)?,
date_server_modified: row.get(5)?,
deleted: row.get(6)?,
})
}
pub fn search_items(tx: &Tx, query: &DatabaseSearch) -> Result<Vec<ItemBase>> {
let mut sql_query = "\
SELECT \
......@@ -140,14 +153,96 @@ pub fn search_items(tx: &Tx, query: &DatabaseSearch) -> Result<Vec<ItemBase>> {
} else {
num_left -= 1;
}
result.push(ItemBase {
rowid: row.get(0)?,
id: row.get(1)?,
_type: row.get(2)?,
date_created: row.get(3)?,
date_modified: row.get(4)?,
date_server_modified: row.get(5)?,
deleted: row.get(6)?,
result.push(parse_item_base(row)?);
}
Ok(result)
}
/// Search for items that have a certain property equal to certain value
pub fn search_strings(tx: &Tx, property_name: &str, value: &str) -> Result<Vec<Rowid>> {
let mut stmt = tx.prepare_cached("SELECT item FROM strings WHERE name = ? AND value = ?;")?;
let mut rows = stmt.query(params![property_name, value])?;
let mut result = Vec::new();
while let Some(row) = rows.next()? {
let item: Rowid = row.get(0)?;
result.push(item);
}
Ok(result)
}
pub fn get_strings_for_item(tx: &Tx, item_rowid: Rowid) -> Result<HashMap<String, String>> {
let mut stmt = tx.prepare_cached("SELECT name, value FROM strings WHERE item = ?;")?;
let mut rows = stmt.query(params![item_rowid])?;
let mut result = HashMap::new();
while let Some(row) = rows.next()? {
result.insert(row.get(0)?, row.get(1)?);
}
Ok(result)
}
pub struct StringsNameValue {
pub name: String,
pub value: String,
}
pub fn get_strings_records_for_item(tx: &Tx, item_rowid: Rowid) -> Result<Vec<StringsNameValue>> {
let mut stmt = tx.prepare_cached("SELECT name, value FROM strings WHERE item = ?;")?;
let mut rows = stmt.query(params![item_rowid])?;
let mut result = Vec::new();
while let Some(row) = rows.next()? {
result.push(StringsNameValue {
name: row.get(0)?,
value: row.get(1)?,
});
}
Ok(result)
}
pub fn get_integers_for_item(tx: &Tx, item_rowid: Rowid) -> Result<HashMap<String, i64>> {
let mut stmt = tx.prepare_cached("SELECT name, value FROM integers WHERE item = ?;")?;
let mut rows = stmt.query(params![item_rowid])?;
let mut result = HashMap::new();
while let Some(row) = rows.next()? {
result.insert(row.get(0)?, row.get(1)?);
}
Ok(result)
}
pub struct IntegersNameValue {
pub name: String,
pub value: i64,
}
pub fn get_integers_records_for_item(tx: &Tx, item_rowid: Rowid) -> Result<Vec<IntegersNameValue>> {
let mut stmt = tx.prepare_cached("SELECT name, value FROM integers WHERE item = ?;")?;
let mut rows = stmt.query(params![item_rowid])?;
let mut result = Vec::new();
while let Some(row) = rows.next()? {
result.push(IntegersNameValue {
name: row.get(0)?,
value: row.get(1)?,
});
}
Ok(result)
}
pub fn get_reals_for_item(tx: &Tx, item_rowid: Rowid) -> Result<HashMap<String, f64>> {
let mut stmt = tx.prepare_cached("SELECT name, value FROM reals WHERE item = ?;")?;
let mut rows = stmt.query(params![item_rowid])?;
let mut result = HashMap::new();
while let Some(row) = rows.next()? {
result.insert(row.get(0)?, row.get(1)?);
}
Ok(result)
}
pub struct RealsNameValue {
pub name: String,
pub value: f64,
}
pub fn get_reals_records_for_item(tx: &Tx, item_rowid: Rowid) -> Result<Vec<RealsNameValue>> {
let mut stmt = tx.prepare_cached("SELECT name, value FROM reals WHERE item = ?;")?;
let mut rows = stmt.query(params![item_rowid])?;
let mut result = Vec::new();
while let Some(row) = rows.next()? {
result.push(RealsNameValue {
name: row.get(0)?,
value: row.get(1)?,
});
}
Ok(result)
......
use crate::constants;
use crate::database_api;
use crate::error::Error;
use crate::error::Result;
use chacha20poly1305::aead::Aead;
use chacha20poly1305::aead::NewAead;
use chacha20poly1305::Key;
use chacha20poly1305::XChaCha20Poly1305;
// use chacha20poly1305::Nonce;
use chacha20poly1305::XNonce;
use log::warn;
use rand::random;
use rusqlite::named_params;
use rusqlite::OptionalExtension;
use rusqlite::Transaction;
use sha2::Digest;
use sha2::Sha256;
......@@ -24,11 +22,11 @@ use warp::http::status::StatusCode;
pub fn upload_file(
tx: &Transaction,
owner: String,
expected_sha256: String,
owner: &str,
expected_sha256: &str,
body: &[u8],
) -> Result<()> {
if file_exists_on_disk(&owner, &expected_sha256)? {
if file_exists_on_disk(owner, expected_sha256)? {
// Note that checking once for file existence here is not enough.
// To prevent TOCTOU attack, we also need to check file existence below.
// We could avoid doing a check here at all, but we do it to avoid spending CPU power
......@@ -39,7 +37,7 @@ pub fn upload_file(
msg: "File already exists".to_string(),
});
};
validate_hash(&expected_sha256, body)?;
validate_hash(expected_sha256, body)?;
let key: [u8; 32] = random();
let key = Key::from_slice(&key);
......@@ -48,9 +46,9 @@ pub fn upload_file(
let nonce: [u8; 24] = rand::random();
let nonce = XNonce::from_slice(&nonce); // unique per file
let body = cipher.encrypt(nonce, body)?;
update_key_and_nonce(tx, key.deref(), nonce.deref(), &expected_sha256)?;
update_key_and_nonce(tx, key.deref(), nonce.deref(), expected_sha256)?;
let file = final_path(&owner, &expected_sha256)?;
let file = final_path(owner, expected_sha256)?;
let file = OpenOptions::new().write(true).create_new(true).open(file);
let mut file = file.map_err(|err| {
if err.raw_os_error() == Some(libc::EEXIST) {
......@@ -92,7 +90,7 @@ fn file_exists_on_disk(owner: &str, sha256: &str) -> Result<bool> {
fn final_path(owner: &str, sha256: &str) -> Result<PathBuf> {
let result = files_dir()?;
let final_dir = result.join(owner).join(FINAL_DIR);
let final_dir = result.join(owner).join(constants::FILES_FINAL_SUBDIR);
create_dir_all(&final_dir).map_err(|err| Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: format!(
......@@ -128,43 +126,51 @@ fn update_key_and_nonce(
nonce: &[u8],
for_sha256: &str,
) -> Result<()> {
let mut stmt =
tx.prepare("UPDATE items SET key = :key , nonce = :nonce WHERE sha256 = :sha256")?;
let key = hex::encode(key);
let nonce = hex::encode(nonce);
let result = stmt
.execute_named(named_params! { ":key": &key, ":nonce": &nonce, ":sha256": for_sha256 })?;
if result == 0 {
let item_rowids = database_api::search_strings(tx, "sha256", for_sha256)?;
if item_rowids.is_empty() {
Err(Error {
code: StatusCode::NOT_FOUND,
msg: format!("Item with sha256 {} not found in database", for_sha256),
})
} else {
let key = hex::encode(key);
let nonce = hex::encode(nonce);
for item in item_rowids {
database_api::insert_string(tx, item, "key", &key)?;
database_api::insert_string(tx, item, "nonce", &nonce)?;
}
Ok(())
}
}
/// Find `key` and `nonce` in the database for an item with the desired `sha256`
/// Find first `key` and `nonce` pair in the database for an item with the desired `sha256`
fn find_key_and_nonce_by_sha256(tx: &Transaction, sha256: &str) -> Result<(Vec<u8>, Vec<u8>)> {
let key_nonce: Option<(String, String)> = tx
.query_row(
"SELECT key, nonce FROM items WHERE sha256 = ?",
&[sha256],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()?;
let key_nonce = if let Some(key_nonce) = key_nonce {
key_nonce
let item_rowids = database_api::search_strings(tx, "sha256", sha256)?;
if let Some(rowid) = item_rowids.first() {
let mut other_props = database_api::get_strings_for_item(tx, *rowid)?;
let key = other_props.remove("key").ok_or_else(|| Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: format!(
"Item with required hash {} found, but it does not have a 'key' property",
sha256
),
})?;
let nonce = other_props.remove("nonce").ok_or_else(|| Error {
code: StatusCode::INTERNAL_SERVER_ERROR,
msg: format!(
"Item with required hash {} found, but it does not have a 'nonce' property",
sha256
),
})?;
let key = hex::decode(key)?;
let nonce = hex::decode(nonce)?;
Ok((key, nonce))
} else {
return Err(Error {
Err(Error {
code: StatusCode::NOT_FOUND,
msg: format!("Item with sha256={} not found", sha256),
});
};
let (key, nonce) = key_nonce;
let key = hex::decode(key)?;
let nonce = hex::decode(nonce)?;
Ok((key, nonce))
})
}
}
/// Directory where files (e.g. media) are stored
......@@ -178,6 +184,52 @@ fn files_dir() -> Result<PathBuf> {
})
}
/// Directory where fully uploaded and hash-checked files are stored
/// (in future, the files should also be s3-uploaded).
const FINAL_DIR: &str = "final";
#[cfg(test)]
mod tests {
use super::files_dir;
use super::get_file;
use super::upload_file;
use crate::command_line_interface;
use crate::database_api;
use crate::database_migrate_refinery;
use crate::error::Result;
use crate::internal_api;
use crate::plugin_auth_crypto::DatabaseKey;
use rusqlite::Connection;
use serde_json::json;
#[test]
fn test_file_upload_get() -> Result<()> {
let mut conn = new_conn();
let tx = conn.transaction().unwrap();
let schema = database_api::get_schema(&tx)?;
let cli = command_line_interface::tests::test_cli();
let database_key = DatabaseKey::from("".to_string()).unwrap();
let owner = "testOwner".to_string();
let owner_dir = files_dir()?.join(&owner);
std::fs::remove_dir_all(&owner_dir).ok();
let sha = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".to_string();
let json = json!({
"type": "File",
"sha256": &sha,
});
let sha_item = serde_json::from_value(json)?;
internal_api::create_item_tx(&tx, &schema, sha_item, &owner, &cli, &database_key)?;
upload_file(&tx, &owner, &sha, &[])?;
let result = get_file(&tx, &owner, &sha)?;
assert_eq!(result.len(), 0, "{}:{}", file!(), line!());
std::fs::remove_dir_all(owner_dir).ok();
Ok(())
}
fn new_conn() -> Connection {
let mut conn = rusqlite::Connection::open_in_memory().unwrap();
database_migrate_refinery::embedded::migrations::runner()
.run(&mut conn)
.expect("Failed to run refinery migrations");
conn
}
}
......@@ -2,11 +2,15 @@ use crate::api_model::Bulk;
use crate::api_model::CreateItem;
use crate::api_model::Search;
use crate::api_model::SortOrder;
use crate::command_line_interface;
use crate::command_line_interface::CliOptions;
use crate::database_api;
use crate::database_api::DatabaseSearch;
use crate::database_api::IntegersNameValue;
use crate::database_api::ItemBase;
use crate::database_api::RealsNameValue;
use crate::database_api::Rowid;
use crate::database_api::StringsNameValue;
use crate::error::Error;
use crate::error::Result;
use crate::plugin_auth_crypto::DatabaseKey;
......@@ -18,7 +22,6 @@ use crate::triggers;
use chrono::Utc;
use log::info;
use log::warn;
use rusqlite::params;
use rusqlite::Transaction;
use serde_json::Map;
use serde_json::Value;
......@@ -27,7 +30,7 @@ use std::str;
use warp::http::status::StatusCode;
pub fn get_project_version() -> String {
crate::command_line_interface::VERSION.to_string()
command_line_interface::VERSION.to_string()
}
/// Get all properties that the item has, ignoring those
......@@ -39,11 +42,8 @@ pub fn get_item_properties(
) -> Result<Map<String, Value>> {
let mut json = serde_json::Map::new();
let mut stmt = tx.prepare_cached("SELECT name, value FROM integers WHERE item = ? ;")?;
let mut integers = stmt.query(params![rowid])?;
while let Some(row) = integers.next()? {
let name: String = row.get(0)?;
let value: i64 = row.get(1)?;
for IntegersNameValue { name, value } in database_api::get_integers_records_for_item(tx, rowid)?
{
match schema.property_types.get(&name) {
Some(SchemaPropertyType::Bool) => {
json.insert(name, (value == 1).into());
......@@ -62,11 +62,7 @@ pub fn get_item_properties(
};
}
let mut stmt = tx.prepare_cached("SELECT name, value FROM strings WHERE item = ? ;")?;
let mut integers = stmt.query(params![rowid])?;
while let Some(row) = integers.next()? {
let name: String = row.get(0)?;
let value: String = row.get(1)?;
for StringsNameValue { name, value } in database_api::get_strings_records_for_item(tx, rowid)? {
match schema.property_types.get(&name) {
Some(SchemaPropertyType::Text) => {
json.insert(name, value.into());
......@@ -79,14 +75,10 @@ pub fn get_item_properties(
other
);
}
};
}
}
let mut stmt = tx.prepare_cached("SELECT name, value FROM reals WHERE item = ? ;")?;
let mut integers = stmt.query(params![rowid])?;
while let Some(row) = integers.next()? {
let name: String = row.get(0)?;
let value: f64 = row.get(1)?;
for RealsNameValue { name, value } in database_api::get_reals_records_for_item(tx, rowid)? {
match schema.property_types.get(&name) {
Some(SchemaPropertyType::Real) => {
json.insert(name, value.into());
......@@ -405,7 +397,7 @@ pub fn search(tx: &Transaction, schema: &Schema, query: Search) -> Result<Vec<Va
#[cfg(test)]
mod tests {
use crate::api_model::CreateItem;
use crate::command_line_interface::CliOptions;
use crate::command_line_interface;
use crate::database_api;
use crate::database_migrate_refinery;
use crate::error::Result;
......@@ -436,29 +428,12 @@ mod tests {
}
}
fn new_cli() -> CliOptions {
CliOptions {
port: 0,
owners: "".to_string(),
plugins_callback_address: None,
plugins_docker_network: None,
tls_pub_crt: "".to_string(),
tls_priv_key: "".to_string(),
non_tls: false,
insecure_non_tls: None,
insecure_http_headers: false,
shared_server: false,
schema_file: Default::default(),
validate_schema: false,
}
}
#[test]
fn test_schema_checking() -> Result<()> {
let mut conn = new_conn();
let tx = conn.transaction().unwrap();
let database_key = DatabaseKey::from("".to_string()).unwrap();
let cli = new_cli();
let cli = command_line_interface::tests::test_cli();
// first try to insert the Person without Schema
let item_json = json!({
......@@ -511,7 +486,7 @@ mod tests {
fn test_item_insert_schema() {
let mut conn = new_conn();
let minimal_schema = minimal_schema();
let cli = new_cli();
let cli = command_line_interface::tests::test_cli();
let database_key = DatabaseKey::from("".to_string()).unwrap();
let tx = conn.transaction().unwrap();
......
use crate::error::Error;
use crate::error::Result;
use lazy_static::lazy_static;
use regex::Regex;
use serde::Deserialize;
......@@ -41,13 +43,13 @@ pub struct Schema {
/// Validation of _new_ item ids. Note that it is not applied to already existing
/// ids or endpoints that access already existing ids.
pub fn validate_create_item_id(item_id: &str) -> crate::error::Result<()> {
pub fn validate_create_item_id(item_id: &str) -> Result<()> {
lazy_static! {
static ref REGEXP: Regex =
Regex::new(r"^[a-zA-Z0-9_-]{6,36}$").expect("Cannot create regex");
}
if !REGEXP.is_match(item_id) {
Err(crate::error::Error {
Err(Error {
code: StatusCode::BAD_REQUEST,
msg: format!(
"Item id '{}' does not satisfy the format {} (use 32 random hex characters if in doubt on item id creation)",
......@@ -61,13 +63,13 @@ pub fn validate_create_item_id(item_id: &str) -> crate::error::Result<()> {
}
/// All item properties should be of this format
pub fn validate_property_name(property: &str) -> crate::error::Result<()> {
pub fn validate_property_name(property: &str) -> Result<()> {
lazy_static! {
static ref REGEXP: Regex =
Regex::new(r"^[a-zA-Z][_a-zA-Z0-9]{1,30}$").expect("Cannot create regex");
Regex::new(r"^[a-zA-Z][_a-zA-Z0-9]{0,30}[a-zA-Z0-9]$").expect("Cannot create regex");
}
if !REGEXP.is_match(property) {
Err(crate::error::Error {
Err(Error {
code: StatusCode::BAD_REQUEST,
msg: format!(
"Property name {} does not satisfy the format {}",
......@@ -76,7 +78,7 @@ pub fn validate_property_name(property: &str) -> crate::error::Result<()> {
),
})
} else if BLOCKLIST_COLUMN_NAMES.contains(&property.to_lowercase()) {
Err(crate::error::Error {
Err(Error {
code: StatusCode::BAD_REQUEST,
msg: format!("Blocklisted item property {}", property),
})
......
......@@ -6,6 +6,7 @@ use crate::api_model::Search;
use crate::api_model::UpdateItem;
use crate::command_line_interface;
use crate::command_line_interface::CliOptions;
use crate::error::Result;
use crate::internal_api;
use crate::warp_endpoints;
use log::error;
......@@ -257,7 +258,7 @@ pub async fn run_server(cli_options: CliOptions) {
}
}
fn respond_with_result<T: Reply>(result: crate::error::Result<T>) -> Response {
fn respond_with_result<T: Reply>(result: Result<T>) -> Response {
match result {
Err(err) => {
let code = err.code.as_str();
......
......@@ -143,7 +143,7 @@ pub fn upload_file(
let mut conn: Connection = check_owner_and_initialize_db(&owner, &init_db, &database_key)?;
conn.execute_batch("SELECT 1 FROM items;")?; // Check DB access
in_transaction(&mut conn, |tx| {
file_api::upload_file(tx, owner, expected_sha256, body)
file_api::upload_file(tx, &owner, &expected_sha256, body)
})
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment