Unverified Commit ab4c3ffa authored by Vasili Novikov's avatar Vasili Novikov
Browse files

Update PluginRun / StartPlugin item

* Rename StartPlugin -> PluginRun
* Rename PluginRun.container -> PluginRun.containerImage
  This terminology is technically better as docker runs
  container _images_ in _containers_.
* Reset database (via name change) due to breaking
  change in migration files content.
  (This could be avoided, but no need for now.)
* Update v3 -> v4 because a rename is a breaking change
Showing with 60 additions and 59 deletions
+60 -59
...@@ -109,7 +109,7 @@ e.g. to what Plugin does the Schema addition belong to. ...@@ -109,7 +109,7 @@ e.g. to what Plugin does the Schema addition belong to.
Get version of the Pod: the git commit and cargo version that it was built from. Get version of the Pod: the git commit and cargo version that it was built from.
### POST /v3/$owner_key/get_item ### POST /v4/$owner_key/get_item
```json ```json
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -122,7 +122,7 @@ Returns an empty array if an item is not found, ...@@ -122,7 +122,7 @@ Returns an empty array if an item is not found,
or an array with 1 item if item exists. or an array with 1 item if item exists.
### POST /v3/$owner_key/create_item ### POST /v4/$owner_key/create_item
```json ```json
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -149,7 +149,7 @@ Returns an error if the new item doesn't conform to the Schema. ...@@ -149,7 +149,7 @@ Returns an error if the new item doesn't conform to the Schema.
Returns `id` of the created item if the operation is successful. Returns `id` of the created item if the operation is successful.
### POST /v3/$owner_key/update_item ### POST /v4/$owner_key/update_item
```json ```json
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -169,7 +169,7 @@ Update a single item. ...@@ -169,7 +169,7 @@ Update a single item.
Returns an empty object if the operation is successful. Returns an empty object if the operation is successful.
### POST /v3/$owner_key/get_edges ### POST /v4/$owner_key/get_edges
```json5 ```json5
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -207,7 +207,7 @@ Returns an array empty array if either the element does not exist or if it has n ...@@ -207,7 +207,7 @@ Returns an array empty array if either the element does not exist or if it has n
⚠ WARNING: this endpoint is unstable, and it might be deprecated and removed in next releases of Pod.️ ⚠ WARNING: this endpoint is unstable, and it might be deprecated and removed in next releases of Pod.️
### POST /v3/$owner_key/create_edge ### POST /v4/$owner_key/create_edge
```json5 ```json5
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -232,7 +232,7 @@ Returns an error if the new item doesn't conform to the Schema. ...@@ -232,7 +232,7 @@ Returns an error if the new item doesn't conform to the Schema.
Returns `id` of the created item if the operation is successful. Returns `id` of the created item if the operation is successful.
### POST /v3/$owner_key/delete_item ### POST /v4/$owner_key/delete_item
```json ```json
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -245,7 +245,7 @@ Mark an item as deleted: ...@@ -245,7 +245,7 @@ Mark an item as deleted:
* Update `dateServerModified` * Update `dateServerModified`
### POST /v3/$owner_key/search ### POST /v4/$owner_key/search
```json5 ```json5
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -277,7 +277,7 @@ For now only literally those two properties are supported, and all edges are ret ...@@ -277,7 +277,7 @@ For now only literally those two properties are supported, and all edges are ret
without filtering. without filtering.
### POST /v3/$owner_key/bulk ### POST /v4/$owner_key/bulk
```json5 ```json5
{ {
"auth": $auth_json, "auth": $auth_json,
...@@ -323,8 +323,8 @@ For now, plugins are started when items of a particular types are inserted into ...@@ -323,8 +323,8 @@ For now, plugins are started when items of a particular types are inserted into
Items of the following structure need to be inserted to Pod to start a plugin: Items of the following structure need to be inserted to Pod to start a plugin:
```json5 ```json5
{ {
"type": "StartPlugin", /* exactly this, and nothing else */ "type": "PluginRun", /* exactly this, and nothing else */
"container": "your_docker_container:optional_version", /* any locally available docker container if you run Pod locally */ "containerImage": "your_docker_image:optional_version", /* any locally available docker container if you run Pod locally */
"targetItemId": "the item id this plugin needs to run against, NOT this item's id", "targetItemId": "the item id this plugin needs to run against, NOT this item's id",
/* Any other optional fields that your plugin might need... */ /* Any other optional fields that your plugin might need... */
} }
...@@ -338,7 +338,7 @@ e.g. permission limitation. ...@@ -338,7 +338,7 @@ e.g. permission limitation.
# File API # File API
### POST /v3/$owner_key/upload_file/$databaseKey/$sha256hashOfTheFile ### POST /v4/$owner_key/upload_file/$databaseKey/$sha256hashOfTheFile
```text ```text
RAW-FILE-BINARY RAW-FILE-BINARY
``` ```
...@@ -355,7 +355,7 @@ with such `sha256` already exists in DB, Pod will accept the file and store it. ...@@ -355,7 +355,7 @@ with such `sha256` already exists in DB, Pod will accept the file and store it.
The properties `nonce` and `key` will be updated for this item. The properties `nonce` and `key` will be updated for this item.
### POST /v3/$owner_key/get_file ### POST /v4/$owner_key/get_file
```json ```json
{ {
"auth": $auth_json, "auth": $auth_json,
......
...@@ -18,7 +18,7 @@ During development, you can use the following script to make Pod start a new Plu ...@@ -18,7 +18,7 @@ During development, you can use the following script to make Pod start a new Plu
```sh ```sh
owner="$RANDOM$RANDOM$RANDOM$RANDOM" # replace with desired owner, or leave as-is for tests owner="$RANDOM$RANDOM$RANDOM$RANDOM" # replace with desired owner, or leave as-is for tests
dbkey= # note that the Plugin will not have access to this key, it'll only have `POD_AUTH_JSON` dbkey= # note that the Plugin will not have access to this key, it'll only have `POD_AUTH_JSON`
container="test" containerImage="test"
data=$(cat <<-END data=$(cat <<-END
{ {
...@@ -29,7 +29,7 @@ data=$(cat <<-END ...@@ -29,7 +29,7 @@ data=$(cat <<-END
"payload": { "payload": {
"createItems": [ "createItems": [
{"type": "Person", "id": "38583224e56e6d2385d36e05af9caa5e"}, {"type": "Person", "id": "38583224e56e6d2385d36e05af9caa5e"},
{"type": "StartPlugin", "container": "$container", "targetItemId": "38583224e56e6d2385d36e05af9caa5e"} {"type": "PluginRun", "containerImage": "$containerImage", "targetItemId": "38583224e56e6d2385d36e05af9caa5e"}
], ],
"updateItems": [], "updateItems": [],
"deleteItems": [] "deleteItems": []
...@@ -38,7 +38,7 @@ data=$(cat <<-END ...@@ -38,7 +38,7 @@ data=$(cat <<-END
END END
) )
curl -X POST -H "Content-Type: application/json" --insecure "http://localhost:3030/v3/$owner/bulk" -d "$data" curl -X POST -H "Content-Type: application/json" --insecure "http://localhost:3030/v4/$owner/bulk" -d "$data"
``` ```
This will start a container with the environment variables set as described below, This will start a container with the environment variables set as described below,
...@@ -65,10 +65,10 @@ For Plugins that means: ...@@ -65,10 +65,10 @@ For Plugins that means:
So regardless of which auth you use, the script below will give you an idea So regardless of which auth you use, the script below will give you an idea
of the basic structure of the docker command: of the basic structure of the docker command:
```sh ```sh
container="test" containerImage="test"
owner="e5c8f9a3d64f5394677fafb9cc1a63ea3f875dc391422e2f95e9f871d893b115" owner="e5c8f9a3d64f5394677fafb9cc1a63ea3f875dc391422e2f95e9f871d893b115"
target_item='{"type":"Person","id":"38583224e56e6d2385d36e05af9caa5e","dateCreated":1623241923508,"dateModified":1623241923508",dateServerModified":1623241923508,"deleted":false}' target_item='{"type":"Person","id":"38583224e56e6d2385d36e05af9caa5e","dateCreated":1623241923508,"dateModified":1623241923508",dateServerModified":1623241923508,"deleted":false}'
trigger_item_id="05abe8e2ef2d0fb4992239944a71bde5" # the id of the item that started the Plugin (the StartPlugin item) trigger_item_id="05abe8e2ef2d0fb4992239944a71bde5" # the id of the item that started the Plugin (the PluginRun item)
your_auth_json='{???}' # depends on whether you use test auth or real system auth your_auth_json='{???}' # depends on whether you use test auth or real system auth
network="localhost" # "localhost" on linux, "host.docker.internal" on Mac and Windows network="localhost" # "localhost" on linux, "host.docker.internal" on Mac and Windows
...@@ -78,10 +78,10 @@ docker run \ ...@@ -78,10 +78,10 @@ docker run \
--env=POD_TARGET_ITEM="$target_item" \ --env=POD_TARGET_ITEM="$target_item" \
--env=POD_OWNER="$owner" \ --env=POD_OWNER="$owner" \
--env=POD_AUTH_JSON="$your_auth" \ --env=POD_AUTH_JSON="$your_auth" \
--rm \ --rm \container
--name="$container-$trigger_item_id" \ --name="$containerImage-$trigger_item_id" \
-- \ -- \
"$container" "$containerImage"
``` ```
# How are plugins started # How are plugins started
...@@ -99,7 +99,7 @@ Pod will set the following environment variables for plugins: ...@@ -99,7 +99,7 @@ Pod will set the following environment variables for plugins:
``` ```
Where the `id` of this item is the same as the `targetItemId` from the request. Where the `id` of this item is the same as the `targetItemId` from the request.
* `POD_PLUGINRUN_ID` = The `id` of the StartPlugin item that made this Plugin start. * `POD_PLUGINRUN_ID` = The `id` of the PluginRun item that made this Plugin start.
* `POD_OWNER` = Pod owner key, 64-character hex string. * `POD_OWNER` = Pod owner key, 64-character hex string.
......
...@@ -95,7 +95,7 @@ This flow can be illustrated by the following diagram: ...@@ -95,7 +95,7 @@ This flow can be illustrated by the following diagram:
mermaid mermaid
sequenceDiagram sequenceDiagram
participant MCoP as Memri Client <br/> or Plugin participant MCoP as Memri Client <br/> or Plugin
MCoP->>Pod: 🔑 <br/>Create item <br/>{"type": "StartPlugin", ...} MCoP->>Pod: 🔑 <br/>Create item <br/>{"type": "PluginRun", ...}
Pod->>Pod: Encrypt key 🔑 -> 📦 <br/> Pod->>Pod: Encrypt key 🔑 -> 📦 <br/>
Pod->>Plugin: 📦 <br/> start plugin Pod->>Plugin: 📦 <br/> start plugin
Note over Pod: Immediately after starting the plugin,<br/>both keys are DROPPED/removed 🔑/📦 <br/> - <br/> Database is inaccessible to Pod. Note over Pod: Immediately after starting the plugin,<br/>both keys are DROPPED/removed 🔑/📦 <br/> - <br/> Database is inaccessible to Pod.
......
This diff is collapsed.
-- Items of type "StartPlugin" have property "container" (text) -- Items of type "PluginRun" have property "containerImage" (text)
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES( INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"8c914705a392432381561db94da76ad9", "8c914705a392432381561db94da76ad9",
"ItemPropertySchema", 0, 0, 0, 0 "ItemPropertySchema", 0, 0, 0, 0
); );
INSERT INTO strings(item, name, value) VALUES( INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"), (SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
"itemType", "StartPlugin" "itemType", "PluginRun"
); );
INSERT INTO strings(item, name, value) VALUES( INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"), (SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
"propertyName", "container" "propertyName", "containerImage"
); );
INSERT INTO strings(item, name, value) VALUES( INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"), (SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
"valueType", "text" "valueType", "text"
); );
-- Items of type "StartPlugin" have property "targetItemId" (text) -- Items of type "PluginRun" have property "targetItemId" (text)
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES( INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"5f43d3c0120c4f0cac3c98b8129c9cb7", "5f43d3c0120c4f0cac3c98b8129c9cb7",
"ItemPropertySchema", 0, 0, 0, 0 "ItemPropertySchema", 0, 0, 0, 0
); );
INSERT INTO strings(item, name, value) VALUES( INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"), (SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"),
"itemType", "StartPlugin" "itemType", "PluginRun"
); );
INSERT INTO strings(item, name, value) VALUES( INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"), (SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"),
......
// Constants used in the project. These are "convention over configuration" for now. // Constants used in the project. These are "convention over configuration" for now.
pub const DATABASE_DIR: &str = "./data/db"; pub const DATABASE_DIR: &str = "./data/db";
pub const DATABASE_SUFFIX: &str = ".db3"; pub const DATABASE_SUFFIX: &str = ".v4.sqlite";
pub const FILES_DIR: &str = "./data/files"; pub const FILES_DIR: &str = "./data/files";
/// Directory where fully uploaded and hash-checked files are stored /// Directory where fully uploaded and hash-checked files are stored
......
...@@ -19,7 +19,7 @@ use warp::http::status::StatusCode; ...@@ -19,7 +19,7 @@ use warp::http::status::StatusCode;
pub fn run_plugin_container( pub fn run_plugin_container(
tx: &Transaction, tx: &Transaction,
schema: &Schema, schema: &Schema,
container: String, container_image: String,
target_item_id: &str, target_item_id: &str,
triggered_by_item_id: &str, triggered_by_item_id: &str,
pod_owner: &str, pod_owner: &str,
...@@ -41,9 +41,17 @@ pub fn run_plugin_container( ...@@ -41,9 +41,17 @@ pub fn run_plugin_container(
let item = serde_json::to_string(&item)?; let item = serde_json::to_string(&item)?;
let auth = database_key.create_plugin_auth()?; let auth = database_key.create_plugin_auth()?;
let auth = serde_json::to_string(&auth)?; let auth = serde_json::to_string(&auth)?;
let container_id = format!(
"{}-{}-{}-{}",
pod_owner,
container_image,
item,
new_random_item_id()
);
if cli_options.use_kubernetes { if cli_options.use_kubernetes {
run_kubernetes_container( run_kubernetes_container(
&container, &container_image,
container_id,
&item, &item,
pod_owner, pod_owner,
&auth, &auth,
...@@ -52,7 +60,8 @@ pub fn run_plugin_container( ...@@ -52,7 +60,8 @@ pub fn run_plugin_container(
) )
} else { } else {
run_docker_container( run_docker_container(
&container, &container_image,
container_id,
&item, &item,
pod_owner, pod_owner,
&auth, &auth,
...@@ -69,12 +78,13 @@ pub fn run_plugin_container( ...@@ -69,12 +78,13 @@ pub fn run_plugin_container(
/// --env=POD_TARGET_ITEM="{...json...}" \ /// --env=POD_TARGET_ITEM="{...json...}" \
/// --env=POD_OWNER="...64-hex-chars..." \ /// --env=POD_OWNER="...64-hex-chars..." \
/// --env=POD_AUTH_JSON="{...json...}" \ /// --env=POD_AUTH_JSON="{...json...}" \
/// --name="$container-$trigger_item_id" \ /// --name="$containerImage-$trigger_item_id" \
/// --rm \ /// --rm \
/// -- \ /// -- \
/// "$container" /// "$containerImage"
fn run_docker_container( fn run_docker_container(
container: &str, container_image: &str,
container_id: String,
target_item: &str, target_item: &str,
pod_owner: &str, pod_owner: &str,
pod_auth: &str, pod_auth: &str,
...@@ -96,26 +106,23 @@ fn run_docker_container( ...@@ -96,26 +106,23 @@ fn run_docker_container(
args.push(format!("--env=POD_PLUGINRUN_ID={}", triggered_by_item_id)); args.push(format!("--env=POD_PLUGINRUN_ID={}", triggered_by_item_id));
args.push(format!("--env=POD_OWNER={}", pod_owner)); args.push(format!("--env=POD_OWNER={}", pod_owner));
args.push(format!("--env=POD_AUTH_JSON={}", pod_auth)); args.push(format!("--env=POD_AUTH_JSON={}", pod_auth));
args.push(format!( args.push(format!("--name={}", sanitize_docker_name(&container_id)));
"--name={}-{}",
sanitize_docker_name(container),
sanitize_docker_name(triggered_by_item_id)
));
args.push("--rm".to_string()); args.push("--rm".to_string());
args.push("--".to_string()); args.push("--".to_string());
args.push(container.to_string()); args.push(container_image.to_string());
run_any_command("docker", &args, triggered_by_item_id) run_any_command("docker", &args, triggered_by_item_id)
} }
/// Example: /// Example:
/// kubectl run $owner-$container-$targetItem-$randomHex /// kubectl run $owner-$containerImage-$targetItem-$randomHex
/// --image="$container" \ /// --image="$containerImage" \
/// --env=POD_FULL_ADDRESS="http://localhost:3030" \ /// --env=POD_FULL_ADDRESS="http://localhost:3030" \
/// --env=POD_TARGET_ITEM="{...json...}" \ /// --env=POD_TARGET_ITEM="{...json...}" \
/// --env=POD_OWNER="...64-hex-chars..." \ /// --env=POD_OWNER="...64-hex-chars..." \
/// --env=POD_AUTH_JSON="{...json...}" \ /// --env=POD_AUTH_JSON="{...json...}" \
fn run_kubernetes_container( fn run_kubernetes_container(
container: &str, container_image: &str,
container_id: String,
target_item: &str, target_item: &str,
pod_owner: &str, pod_owner: &str,
pod_auth: &str, pod_auth: &str,
...@@ -124,14 +131,8 @@ fn run_kubernetes_container( ...@@ -124,14 +131,8 @@ fn run_kubernetes_container(
) -> Result<()> { ) -> Result<()> {
let mut args: Vec<String> = Vec::with_capacity(7); let mut args: Vec<String> = Vec::with_capacity(7);
args.push("run".to_string()); args.push("run".to_string());
args.push(format!( args.push(container_id);
"{}-{}-{}-{}", args.push(format!("--image={}", container_image));
pod_owner,
container,
target_item,
new_random_item_id()
));
args.push(format!("--image={}", container));
args.push(format!( args.push(format!(
"--env=POD_FULL_ADDRESS={}", "--env=POD_FULL_ADDRESS={}",
callback_address(cli_options) callback_address(cli_options)
......
...@@ -28,8 +28,8 @@ pub struct SchemaItem { ...@@ -28,8 +28,8 @@ pub struct SchemaItem {
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct StartPluginItem { pub struct PluginRunItem {
pub container: String, pub container_image: String,
pub target_item_id: String, pub target_item_id: String,
} }
...@@ -71,14 +71,14 @@ pub fn trigger_after_item_create( ...@@ -71,14 +71,14 @@ pub fn trigger_after_item_create(
cli: &CliOptions, cli: &CliOptions,
database_key: &DatabaseKey, database_key: &DatabaseKey,
) -> Result<()> { ) -> Result<()> {
if item._type == "StartPlugin" { if item._type == "PluginRun" {
let json = internal_api::get_item_from_rowid(tx, schema, source_rowid)?; let json = internal_api::get_item_from_rowid(tx, schema, source_rowid)?;
let parsed: StartPluginItem = serde_json::from_value(json) let parsed: PluginRunItem = serde_json::from_value(json)
.context(|| format!("Parsing of item {:?}, {}:{}", item, file!(), line!()))?; .context(|| format!("Parsing of item {:?}, {}:{}", item, file!(), line!()))?;
plugin_run::run_plugin_container( plugin_run::run_plugin_container(
tx, tx,
schema, schema,
parsed.container, parsed.container_image,
&parsed.target_item_id, &parsed.target_item_id,
source_id, source_id,
pod_owner, pod_owner,
......
...@@ -43,10 +43,10 @@ pub async fn run_server(cli_options: CliOptions) { ...@@ -43,10 +43,10 @@ pub async fn run_server(cli_options: CliOptions) {
} }
let headers = warp::reply::with::headers(headers); let headers = warp::reply::with::headers(headers);
let items_api = warp::path("v3") let items_api = warp::path("v4")
.and(warp::body::content_length_limit(5 * 1024 * 1024)) .and(warp::body::content_length_limit(5 * 1024 * 1024))
.and(warp::post()); .and(warp::post());
let file_api = warp::path("v3") let file_api = warp::path("v4")
.and(warp::body::content_length_limit(500 * 1024 * 1024)) .and(warp::body::content_length_limit(500 * 1024 * 1024))
.and(warp::post()); .and(warp::post());
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment