Unverified Commit ab4c3ffa authored by Vasili Novikov's avatar Vasili Novikov
Browse files

Update PluginRun / StartPlugin item

* Rename StartPlugin -> PluginRun
* Rename PluginRun.container -> PluginRun.containerImage
  This terminology is technically better as docker runs
  container _images_ in _containers_.
* Reset database (via name change) due to breaking
  change in migration files content.
  (This could be avoided, but no need for now.)
* Update v3 -> v4 because a rename is a breaking change
parent a8c51bde
Showing with 60 additions and 59 deletions
+60 -59
......@@ -109,7 +109,7 @@ e.g. to what Plugin does the Schema addition belong to.
Get version of the Pod: the git commit and cargo version that it was built from.
### POST /v3/$owner_key/get_item
### POST /v4/$owner_key/get_item
```json
{
"auth": $auth_json,
......@@ -122,7 +122,7 @@ Returns an empty array if an item is not found,
or an array with 1 item if item exists.
### POST /v3/$owner_key/create_item
### POST /v4/$owner_key/create_item
```json
{
"auth": $auth_json,
......@@ -149,7 +149,7 @@ Returns an error if the new item doesn't conform to the Schema.
Returns `id` of the created item if the operation is successful.
### POST /v3/$owner_key/update_item
### POST /v4/$owner_key/update_item
```json
{
"auth": $auth_json,
......@@ -169,7 +169,7 @@ Update a single item.
Returns an empty object if the operation is successful.
### POST /v3/$owner_key/get_edges
### POST /v4/$owner_key/get_edges
```json5
{
"auth": $auth_json,
......@@ -207,7 +207,7 @@ Returns an array empty array if either the element does not exist or if it has n
⚠ WARNING: this endpoint is unstable, and it might be deprecated and removed in next releases of Pod.️
### POST /v3/$owner_key/create_edge
### POST /v4/$owner_key/create_edge
```json5
{
"auth": $auth_json,
......@@ -232,7 +232,7 @@ Returns an error if the new item doesn't conform to the Schema.
Returns `id` of the created item if the operation is successful.
### POST /v3/$owner_key/delete_item
### POST /v4/$owner_key/delete_item
```json
{
"auth": $auth_json,
......@@ -245,7 +245,7 @@ Mark an item as deleted:
* Update `dateServerModified`
### POST /v3/$owner_key/search
### POST /v4/$owner_key/search
```json5
{
"auth": $auth_json,
......@@ -277,7 +277,7 @@ For now only literally those two properties are supported, and all edges are ret
without filtering.
### POST /v3/$owner_key/bulk
### POST /v4/$owner_key/bulk
```json5
{
"auth": $auth_json,
......@@ -323,8 +323,8 @@ For now, plugins are started when items of a particular types are inserted into
Items of the following structure need to be inserted to Pod to start a plugin:
```json5
{
"type": "StartPlugin", /* exactly this, and nothing else */
"container": "your_docker_container:optional_version", /* any locally available docker container if you run Pod locally */
"type": "PluginRun", /* exactly this, and nothing else */
"containerImage": "your_docker_image:optional_version", /* any locally available docker container if you run Pod locally */
"targetItemId": "the item id this plugin needs to run against, NOT this item's id",
/* Any other optional fields that your plugin might need... */
}
......@@ -338,7 +338,7 @@ e.g. permission limitation.
# File API
### POST /v3/$owner_key/upload_file/$databaseKey/$sha256hashOfTheFile
### POST /v4/$owner_key/upload_file/$databaseKey/$sha256hashOfTheFile
```text
RAW-FILE-BINARY
```
......@@ -355,7 +355,7 @@ with such `sha256` already exists in DB, Pod will accept the file and store it.
The properties `nonce` and `key` will be updated for this item.
### POST /v3/$owner_key/get_file
### POST /v4/$owner_key/get_file
```json
{
"auth": $auth_json,
......
......@@ -18,7 +18,7 @@ During development, you can use the following script to make Pod start a new Plu
```sh
owner="$RANDOM$RANDOM$RANDOM$RANDOM" # replace with desired owner, or leave as-is for tests
dbkey= # note that the Plugin will not have access to this key, it'll only have `POD_AUTH_JSON`
container="test"
containerImage="test"
data=$(cat <<-END
{
......@@ -29,7 +29,7 @@ data=$(cat <<-END
"payload": {
"createItems": [
{"type": "Person", "id": "38583224e56e6d2385d36e05af9caa5e"},
{"type": "StartPlugin", "container": "$container", "targetItemId": "38583224e56e6d2385d36e05af9caa5e"}
{"type": "PluginRun", "containerImage": "$containerImage", "targetItemId": "38583224e56e6d2385d36e05af9caa5e"}
],
"updateItems": [],
"deleteItems": []
......@@ -38,7 +38,7 @@ data=$(cat <<-END
END
)
curl -X POST -H "Content-Type: application/json" --insecure "http://localhost:3030/v3/$owner/bulk" -d "$data"
curl -X POST -H "Content-Type: application/json" --insecure "http://localhost:3030/v4/$owner/bulk" -d "$data"
```
This will start a container with the environment variables set as described below,
......@@ -65,10 +65,10 @@ For Plugins that means:
So regardless of which auth you use, the script below will give you an idea
of the basic structure of the docker command:
```sh
container="test"
containerImage="test"
owner="e5c8f9a3d64f5394677fafb9cc1a63ea3f875dc391422e2f95e9f871d893b115"
target_item='{"type":"Person","id":"38583224e56e6d2385d36e05af9caa5e","dateCreated":1623241923508,"dateModified":1623241923508",dateServerModified":1623241923508,"deleted":false}'
trigger_item_id="05abe8e2ef2d0fb4992239944a71bde5" # the id of the item that started the Plugin (the StartPlugin item)
trigger_item_id="05abe8e2ef2d0fb4992239944a71bde5" # the id of the item that started the Plugin (the PluginRun item)
your_auth_json='{???}' # depends on whether you use test auth or real system auth
network="localhost" # "localhost" on linux, "host.docker.internal" on Mac and Windows
......@@ -78,10 +78,10 @@ docker run \
--env=POD_TARGET_ITEM="$target_item" \
--env=POD_OWNER="$owner" \
--env=POD_AUTH_JSON="$your_auth" \
--rm \
--name="$container-$trigger_item_id" \
--rm \container
--name="$containerImage-$trigger_item_id" \
-- \
"$container"
"$containerImage"
```
# How are plugins started
......@@ -99,7 +99,7 @@ Pod will set the following environment variables for plugins:
```
Where the `id` of this item is the same as the `targetItemId` from the request.
* `POD_PLUGINRUN_ID` = The `id` of the StartPlugin item that made this Plugin start.
* `POD_PLUGINRUN_ID` = The `id` of the PluginRun item that made this Plugin start.
* `POD_OWNER` = Pod owner key, 64-character hex string.
......
......@@ -95,7 +95,7 @@ This flow can be illustrated by the following diagram:
mermaid
sequenceDiagram
participant MCoP as Memri Client <br/> or Plugin
MCoP->>Pod: 🔑 <br/>Create item <br/>{"type": "StartPlugin", ...}
MCoP->>Pod: 🔑 <br/>Create item <br/>{"type": "PluginRun", ...}
Pod->>Pod: Encrypt key 🔑 -> 📦 <br/>
Pod->>Plugin: 📦 <br/> start plugin
Note over Pod: Immediately after starting the plugin,<br/>both keys are DROPPED/removed 🔑/📦 <br/> - <br/> Database is inaccessible to Pod.
......
This diff is collapsed.
-- Items of type "StartPlugin" have property "container" (text)
-- Items of type "PluginRun" have property "containerImage" (text)
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"8c914705a392432381561db94da76ad9",
"ItemPropertySchema", 0, 0, 0, 0
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
"itemType", "StartPlugin"
"itemType", "PluginRun"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
"propertyName", "container"
"propertyName", "containerImage"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "8c914705a392432381561db94da76ad9"),
"valueType", "text"
);
-- Items of type "StartPlugin" have property "targetItemId" (text)
-- Items of type "PluginRun" have property "targetItemId" (text)
INSERT INTO items(id, type, dateCreated, dateModified, dateServerModified, deleted) VALUES(
"5f43d3c0120c4f0cac3c98b8129c9cb7",
"ItemPropertySchema", 0, 0, 0, 0
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"),
"itemType", "StartPlugin"
"itemType", "PluginRun"
);
INSERT INTO strings(item, name, value) VALUES(
(SELECT rowid FROM items WHERE id = "5f43d3c0120c4f0cac3c98b8129c9cb7"),
......
// Constants used in the project. These are "convention over configuration" for now.
pub const DATABASE_DIR: &str = "./data/db";
pub const DATABASE_SUFFIX: &str = ".db3";
pub const DATABASE_SUFFIX: &str = ".v4.sqlite";
pub const FILES_DIR: &str = "./data/files";
/// Directory where fully uploaded and hash-checked files are stored
......
......@@ -19,7 +19,7 @@ use warp::http::status::StatusCode;
pub fn run_plugin_container(
tx: &Transaction,
schema: &Schema,
container: String,
container_image: String,
target_item_id: &str,
triggered_by_item_id: &str,
pod_owner: &str,
......@@ -41,9 +41,17 @@ pub fn run_plugin_container(
let item = serde_json::to_string(&item)?;
let auth = database_key.create_plugin_auth()?;
let auth = serde_json::to_string(&auth)?;
let container_id = format!(
"{}-{}-{}-{}",
pod_owner,
container_image,
item,
new_random_item_id()
);
if cli_options.use_kubernetes {
run_kubernetes_container(
&container,
&container_image,
container_id,
&item,
pod_owner,
&auth,
......@@ -52,7 +60,8 @@ pub fn run_plugin_container(
)
} else {
run_docker_container(
&container,
&container_image,
container_id,
&item,
pod_owner,
&auth,
......@@ -69,12 +78,13 @@ pub fn run_plugin_container(
/// --env=POD_TARGET_ITEM="{...json...}" \
/// --env=POD_OWNER="...64-hex-chars..." \
/// --env=POD_AUTH_JSON="{...json...}" \
/// --name="$container-$trigger_item_id" \
/// --name="$containerImage-$trigger_item_id" \
/// --rm \
/// -- \
/// "$container"
/// "$containerImage"
fn run_docker_container(
container: &str,
container_image: &str,
container_id: String,
target_item: &str,
pod_owner: &str,
pod_auth: &str,
......@@ -96,26 +106,23 @@ fn run_docker_container(
args.push(format!("--env=POD_PLUGINRUN_ID={}", triggered_by_item_id));
args.push(format!("--env=POD_OWNER={}", pod_owner));
args.push(format!("--env=POD_AUTH_JSON={}", pod_auth));
args.push(format!(
"--name={}-{}",
sanitize_docker_name(container),
sanitize_docker_name(triggered_by_item_id)
));
args.push(format!("--name={}", sanitize_docker_name(&container_id)));
args.push("--rm".to_string());
args.push("--".to_string());
args.push(container.to_string());
args.push(container_image.to_string());
run_any_command("docker", &args, triggered_by_item_id)
}
/// Example:
/// kubectl run $owner-$container-$targetItem-$randomHex
/// --image="$container" \
/// kubectl run $owner-$containerImage-$targetItem-$randomHex
/// --image="$containerImage" \
/// --env=POD_FULL_ADDRESS="http://localhost:3030" \
/// --env=POD_TARGET_ITEM="{...json...}" \
/// --env=POD_OWNER="...64-hex-chars..." \
/// --env=POD_AUTH_JSON="{...json...}" \
fn run_kubernetes_container(
container: &str,
container_image: &str,
container_id: String,
target_item: &str,
pod_owner: &str,
pod_auth: &str,
......@@ -124,14 +131,8 @@ fn run_kubernetes_container(
) -> Result<()> {
let mut args: Vec<String> = Vec::with_capacity(7);
args.push("run".to_string());
args.push(format!(
"{}-{}-{}-{}",
pod_owner,
container,
target_item,
new_random_item_id()
));
args.push(format!("--image={}", container));
args.push(container_id);
args.push(format!("--image={}", container_image));
args.push(format!(
"--env=POD_FULL_ADDRESS={}",
callback_address(cli_options)
......
......@@ -28,8 +28,8 @@ pub struct SchemaItem {
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct StartPluginItem {
pub container: String,
pub struct PluginRunItem {
pub container_image: String,
pub target_item_id: String,
}
......@@ -71,14 +71,14 @@ pub fn trigger_after_item_create(
cli: &CliOptions,
database_key: &DatabaseKey,
) -> Result<()> {
if item._type == "StartPlugin" {
if item._type == "PluginRun" {
let json = internal_api::get_item_from_rowid(tx, schema, source_rowid)?;
let parsed: StartPluginItem = serde_json::from_value(json)
let parsed: PluginRunItem = serde_json::from_value(json)
.context(|| format!("Parsing of item {:?}, {}:{}", item, file!(), line!()))?;
plugin_run::run_plugin_container(
tx,
schema,
parsed.container,
parsed.container_image,
&parsed.target_item_id,
source_id,
pod_owner,
......
......@@ -43,10 +43,10 @@ pub async fn run_server(cli_options: CliOptions) {
}
let headers = warp::reply::with::headers(headers);
let items_api = warp::path("v3")
let items_api = warp::path("v4")
.and(warp::body::content_length_limit(5 * 1024 * 1024))
.and(warp::post());
let file_api = warp::path("v3")
let file_api = warp::path("v4")
.and(warp::body::content_length_limit(500 * 1024 * 1024))
.and(warp::post());
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment