Commit 49c34d83 authored by Vasili Novikov's avatar Vasili Novikov
Browse files

Merge branch 'implement-sort-limit' into 'dev'

Implement sort order and result limits

See merge request memri/pod!191
parents 02fd4f49 67d605e3
Showing with 163 additions and 56 deletions
+163 -56
......@@ -175,23 +175,25 @@ Mark an item as deleted:
### POST /v3/$owner_key/search
```json
```json5
{
"auth": $auth_json,
"payload": {
"type": "Label",
"dateServerModified>=": 1234567890,
"dateServerModified<": 1234567890,
"deleted": false
"deleted": false,
"_sortOrder": "DESC", // either "ASC" (by default) or "DESC"
"_limit": 100, // exclude this field to have no limit
}
}
```
Search items by their properties.
Search items by their properties, all fields above are optional.
The endpoint will return an array of all items with exactly the same properties.
As a first step of the 2021-03 Pod rewrite, only the above properties are supported.
In the future, any will be available.
In the future, any item properties will be available.
### POST /v3/$owner_key/bulk
......
......@@ -84,14 +84,37 @@ pub struct Search {
pub id: Option<String>,
pub _type: Option<String>,
#[serde(rename = "dateServerModified>=")]
pub _date_server_modified_gte: Option<i64>,
pub date_server_modified_gte: Option<i64>,
#[serde(rename = "dateServerModified<")]
pub _date_server_modified_lt: Option<i64>,
pub date_server_modified_lt: Option<i64>,
pub deleted: Option<bool>,
#[serde(default = "default_api_sort_order", rename = "_sortOrder")]
pub sort_order: SortOrder,
#[serde(default = "default_api_limit", rename = "_limit")]
pub limit: u64,
#[serde(flatten)]
pub other_properties: HashMap<String, Value>,
}
#[derive(Serialize, Deserialize, Debug)]
pub enum SortOrder {
/// Ascending
Asc,
/// Descending
Desc,
}
impl std::fmt::Display for SortOrder {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
std::fmt::Debug::fmt(self, f)
}
}
fn default_api_sort_order() -> SortOrder {
SortOrder::Asc
}
fn default_api_limit() -> u64 {
u64::MAX
}
//
// Files API:
//
......
use crate::api_model::SortOrder;
use crate::error::Error;
use crate::error::ErrorContext;
use crate::error::Result;
......@@ -68,15 +69,18 @@ pub fn get_item_rowid(tx: &Tx, id: &str) -> Result<Option<Rowid>> {
}
}
pub fn search_items(
tx: &Tx,
rowid: Option<Rowid>,
id: Option<&str>,
_type: Option<&str>,
date_server_modified_gte: Option<DbTime>,
date_server_modified_lt: Option<DbTime>,
deleted: Option<bool>,
) -> Result<Vec<ItemBase>> {
pub struct DatabaseSearch<'a> {
pub rowid: Option<Rowid>,
pub id: Option<&'a str>,
pub _type: Option<&'a str>,
pub date_server_modified_gte: Option<DbTime>,
pub date_server_modified_lt: Option<DbTime>,
pub deleted: Option<bool>,
pub sort_order: SortOrder,
pub _limit: u64,
}
pub fn search_items(tx: &Tx, query: &DatabaseSearch) -> Result<Vec<ItemBase>> {
let mut sql_query = "\
SELECT \
rowid, \
......@@ -91,19 +95,19 @@ pub fn search_items(
WHERE "
.to_string();
let mut params_vec: Vec<ToSqlOutput> = Vec::new();
if let Some(r) = rowid {
if let Some(r) = query.rowid {
add_sql_param(&mut sql_query, "rowid", &Comparison::Equals);
params_vec.push(r.into());
}
if let Some(id) = id {
if let Some(id) = &query.id {
add_sql_param(&mut sql_query, "id", &Comparison::Equals);
params_vec.push(id.into());
params_vec.push((*id).into());
}
if let Some(typ) = _type {
if let Some(typ) = &query._type {
add_sql_param(&mut sql_query, "type", &Comparison::Equals);
params_vec.push(typ.into());
params_vec.push((*typ).into());
}
if let Some(dt) = date_server_modified_gte {
if let Some(dt) = query.date_server_modified_gte {
add_sql_param(
&mut sql_query,
"dateServerModified",
......@@ -111,15 +115,17 @@ pub fn search_items(
);
params_vec.push(dt.into());
}
if let Some(dt) = date_server_modified_lt {
if let Some(dt) = query.date_server_modified_lt {
add_sql_param(&mut sql_query, "dateServerModified", &Comparison::LessThan);
params_vec.push(dt.into());
}
if let Some(deleted) = deleted {
if let Some(deleted) = query.deleted {
add_sql_param(&mut sql_query, "deleted", &Comparison::Equals);
params_vec.push(deleted.into());
}
sql_query.push_str("1 ;"); // older sqlite versions do not support `true`
sql_query.push_str("1 "); // older sqlite versions do not support `true`
sql_query.push_str(&format!("ORDER BY dateServerModified {}", query.sort_order));
sql_query.push(';');
debug!("Executing search SQL: {}", sql_query);
let mut stmt = tx
......@@ -127,7 +133,13 @@ pub fn search_items(
.context(|| format!("SQL query: {}", sql_query))?;
let mut rows = stmt.query(params_vec)?;
let mut result = Vec::new();
let mut num_left = query._limit;
while let Some(row) = rows.next()? {
if num_left == 0 {
break;
} else {
num_left -= 1;
}
result.push(ItemBase {
rowid: row.get(0)?,
id: row.get(1)?,
......@@ -422,69 +434,123 @@ mod tests {
insert_string(&tx, item, "valueType", "integer")?;
assert_eq!(
search_items(&tx, Some(item), None, None, None, None, None)?.len(),
search_items_params(&tx, Some(item), None, None, None, None, None)?.len(),
1
);
dangerous_permament_remove_item(&tx, item)?;
assert_eq!(
search_items(&tx, Some(item), None, None, None, None, None)?.len(),
search_items_params(&tx, Some(item), None, None, None, None, None)?.len(),
0
);
Ok(())
}
fn search_items_params(
tx: &Tx,
rowid: Option<Rowid>,
id: Option<&str>,
_type: Option<&str>,
date_server_modified_gte: Option<DbTime>,
date_server_modified_lt: Option<DbTime>,
deleted: Option<bool>,
) -> Result<Vec<ItemBase>> {
let database_search = DatabaseSearch {
rowid,
id,
_type,
date_server_modified_gte,
date_server_modified_lt,
deleted,
sort_order: SortOrder::Asc,
_limit: u64::MAX,
};
search_items(tx, &database_search)
}
#[test]
fn test_search() -> Result<()> {
let mut conn = new_conn();
let tx = conn.transaction()?;
let date = Utc::now().timestamp_millis();
let item1 = insert_item_base(&tx, "one", "Person", date, date, date, false)?;
let _item2 = insert_item_base(&tx, "two", "Book", date, date, date, false)?;
let _item3 = insert_item_base(&tx, "three", "Street", date, date, date, false)?;
let _item2 = insert_item_base(&tx, "two", "Book", date, date, date + 1, false)?;
let _item3 = insert_item_base(&tx, "three", "Street", date, date, date + 2, false)?;
assert_eq!(
search_items(&tx, None, None, Some("Person"), None, None, None)?.len(),
search_items_params(&tx, None, None, Some("Person"), None, None, None)?.len(),
1,
);
assert_eq!(
search_items(&tx, None, None, Some("Void"), None, None, None)?.len(),
search_items_params(&tx, None, None, Some("Void"), None, None, None)?.len(),
0,
);
assert_eq!(
search_items(&tx, Some(item1), None, None, None, None, None)?.len(),
search_items_params(&tx, Some(item1), None, None, None, None, None)?.len(),
1,
);
assert_eq!(
search_items(&tx, None, Some("one"), None, None, None, None)?.len(),
search_items_params(&tx, None, Some("one"), None, None, None, None)?.len(),
1,
);
assert_eq!(
search_items(&tx, None, Some("nothing"), None, None, None, None)?.len(),
search_items_params(&tx, None, Some("nothing"), None, None, None, None)?.len(),
0,
);
assert_eq!(
search_items(&tx, None, None, None, Some(date), None, None)?.len(),
search_items_params(&tx, None, None, None, Some(date), None, None)?.len(),
3,
);
assert_eq!(
search_items(&tx, None, None, None, Some(date), Some(date + 1), None)?.len(),
search_items_params(&tx, None, None, None, Some(date), Some(date + 3), None)?.len(),
3,
);
assert_eq!(
search_items(&tx, None, None, None, Some(date - 1), Some(date), None)?.len(),
search_items_params(&tx, None, None, None, Some(date - 1), Some(date), None)?.len(),
0,
);
assert_eq!(
search_items(&tx, None, None, None, None, None, Some(true))?.len(),
search_items_params(&tx, None, None, None, None, None, Some(true))?.len(),
0,
);
assert_eq!(
search_items(&tx, None, None, None, Some(date - 1), None, Some(false))?.len(),
search_items_params(&tx, None, None, None, Some(date - 1), None, Some(false))?.len(),
3,
);
assert!(search_items(&tx, None, None, None, None, None, None)?.len() >= 3);
assert!(search_items_params(&tx, None, None, None, None, None, None)?.len() >= 3);
{
let mut query = DatabaseSearch {
rowid: None,
id: None,
_type: None,
date_server_modified_gte: Some(date),
date_server_modified_lt: None,
deleted: None,
sort_order: SortOrder::Asc,
_limit: 1,
};
assert_eq!(search_items(&tx, &query)?.len(), 1);
query._limit = 3;
let three_results = search_items(&tx, &query)?;
assert_eq!(three_results.len(), 3);
let first = &three_results[0];
let third = &three_results[2];
assert_ne!(first.rowid, third.rowid);
assert_ne!(first.date_server_modified, third.date_server_modified);
query.sort_order = SortOrder::Asc;
let search_asc = &search_items(&tx, &query)?;
query.sort_order = SortOrder::Desc;
let search_desc = &search_items(&tx, &query)?;
assert_eq!(search_asc[0].rowid, search_desc[2].rowid);
assert_eq!(search_asc[2].rowid, search_desc[0].rowid);
query._limit = 100;
assert!(search_items(&tx, &query)?.len() < 100); // there aren't 100 items there
}
assert_eq!(
search_items(
search_items_params(
&tx,
Some(item1),
Some("one"),
......
......@@ -144,7 +144,7 @@ fn update_key_and_nonce(
}
}
// Find `key` and `nonce` in the database for an item with the desired `sha256`
/// Find `key` and `nonce` in the database for an item with the desired `sha256`
fn find_key_and_nonce_by_sha256(tx: &Transaction, sha256: &str) -> Result<(Vec<u8>, Vec<u8>)> {
let key_nonce: Option<(String, String)> = tx
.query_row(
......
use crate::api_model::Bulk;
use crate::api_model::CreateItem;
use crate::api_model::Search;
use crate::api_model::SortOrder;
use crate::command_line_interface::CliOptions;
use crate::database_api;
use crate::database_api::DatabaseSearch;
use crate::database_api::ItemBase;
use crate::database_api::Rowid;
use crate::error::Error;
......@@ -104,7 +106,17 @@ pub fn get_item_properties(
}
pub fn get_item_from_rowid(tx: &Transaction, schema: &Schema, rowid: Rowid) -> Result<Value> {
let item = database_api::search_items(tx, Some(rowid), None, None, None, None, None)?;
let database_search = DatabaseSearch {
rowid: Some(rowid),
id: None,
_type: None,
date_server_modified_gte: None,
date_server_modified_lt: None,
deleted: None,
sort_order: SortOrder::Asc,
_limit: 1,
};
let item = database_api::search_items(tx, &database_search)?;
let item = if let Some(item) = item.into_iter().next() {
item
} else {
......@@ -123,9 +135,11 @@ pub fn get_item_tx(tx: &Transaction, schema: &Schema, id: &str) -> Result<Vec<Va
let search_query = Search {
id: Some(id.to_string()),
_type: None,
_date_server_modified_gte: None,
_date_server_modified_lt: None,
date_server_modified_gte: None,
date_server_modified_lt: None,
deleted: None,
sort_order: SortOrder::Asc,
limit: 1,
other_properties: Default::default(),
};
let result = search(tx, schema, search_query)?;
......@@ -368,15 +382,17 @@ pub fn search(tx: &Transaction, schema: &Schema, query: Search) -> Result<Vec<Va
),
});
}
let items = database_api::search_items(
tx,
None,
query.id.as_deref(),
query._type.as_deref(),
query._date_server_modified_gte,
query._date_server_modified_lt,
query.deleted,
)?;
let database_search = DatabaseSearch {
rowid: None,
id: query.id.as_deref(),
_type: query._type.as_deref(),
date_server_modified_gte: query.date_server_modified_gte,
date_server_modified_lt: query.date_server_modified_lt,
deleted: query.deleted,
sort_order: query.sort_order,
_limit: query.limit,
};
let items = database_api::search_items(tx, &database_search)?;
let mut result = Vec::new();
for item in items {
let mut properties = get_item_properties(tx, item.rowid, schema)?;
......
......@@ -18,8 +18,8 @@ pub fn generate_xchacha20poly1305_cipher() -> XChaCha20Poly1305 {
XChaCha20Poly1305::new(key)
}
// Given a `database_key` from an already authorized request,
// create a new PluginAuth to be passed to external Plugins.
/// Given a `database_key` from an already authorized request,
/// create a new PluginAuth to be passed to external Plugins.
pub fn create_plugin_auth(database_key: &str) -> Result<PluginAuth> {
let nonce: [u8; 24] = rand::random();
let nonce = XNonce::from_slice(&nonce); // MUST be unique
......
......@@ -107,7 +107,7 @@ const POD_ITEM_MANDATORY_PROPERTIES: &[&str] = &[
"target",
];
// SQLite keywords taken from https://www.sqlite.org/lang_keywords.html
/// SQLite keywords taken from https://www.sqlite.org/lang_keywords.html
const SQLITE_RESERVED_KEYWORDS: &[&str] = &[
"ABORT",
"ACTION",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment