Skip to content
GitLab
Commit dfac2fe7 authored by Koen van der Veen's avatar Koen van der Veen
Browse files

-

parent 75812ca2
Hide whitespace changes
Inline Side-by-side
Showing
with 22 additions and 20 deletions
+22 -20
Dockerfile
+ 0
- 1
View file @ dfac2fe7
......@@ -27,6 +27,5 @@ COPY ./nbs ./nbs
# Build the final image
RUN pip3 install --editable .
# CMD ["python3", "tools/run_integrator.py"]
CMD ["run_plugin"]
nbs/plugin.pluginbase.ipynb
+ 13
- 13
View file @ dfac2fe7
%% Cell type:code id: tags:
``` python
%load_ext autoreload
%autoreload 2
# default_exp plugin.pluginbase
```
%% Cell type:code id: tags:
``` python
# export
from pymemri.data.schema import *
from pymemri.pod.client import PodClient, DEFAULT_POD_ADDRESS
from pymemri.imports import *
from os import environ
from abc import ABCMeta
import abc
import json
```
%% Cell type:code id: tags:
``` python
# hide
from nbdev.showdoc import *
```
%% Cell type:markdown id: tags:
# Plugins
%% Cell type:code id: tags:
``` python
# export
POD_FULL_ADDRESS_ENV = 'POD_FULL_ADDRESS'
POD_TARGET_ITEM_ENV = 'POD_TARGET_ITEM'
POD_OWNER_KEY_ENV = 'POD_OWNER'
POD_AUTH_JSON_ENV = 'POD_AUTH_JSON'
```
%% Cell type:code id: tags:
``` python
# export
# hide
class PluginBase(Item, metaclass=ABCMeta):
"""Base class for plugins"""
properties = Item.properties + ["name", "repository", "icon", "data_query", "bundleImage",
"runDestination", "pluginClass", "pluginPackage"]
edges = Item.edges + ["PluginRun"]
def __init__(self, name=None, repository=None, icon=None, query=None, bundleImage=None, runDestination=None,
pluginClass=None, indexerRun=None, **kwargs):
if pluginClass is None: pluginClass=self.__class__.__name__
self.pluginPackage=None
super().__init__(**kwargs)
self.name = name
self.repository = repository
self.icon = icon
self.query = query
self.bundleImage = bundleImage
self.runDestination = runDestination
self.pluginClass = pluginClass
self.indexerRun = indexerRun if indexerRun is not None else []
@abc.abstractmethod
def run(self):
raise NotImplementedError()
@abc.abstractmethod
def add_to_schema(self):
raise NotImplementedError()
```
%% Cell type:code id: tags:
``` python
# export
# hide
class PluginRun(Item):
properties = Item.properties
edges = Item.edges + ["plugin"]
def __init__(self, plugin=None, **kwargs):
super().__init__(**kwargs)
self.plugin=plugin if plugin is not None else []
```
%% Cell type:markdown id: tags:
## Creating a plugin
%% Cell type:markdown id: tags:
The memri [pod](https://gitlab.memri.io/memri/pod) uses a plugin system to add features to the backend memri backend. Plugins, can import your data (importers), change your data (indexers), or call other serivces. Users can define their own plugins to add new behaviour to their memri app. Let's use the following plugin as an example of how we can start plugins.
%% Cell type:code id: tags:
``` python
# export
# hide
class MyItem(Item):
properties = Item.properties + ["name", "age"]
edges = Item.edges
def __init__(self, name=None, age=None, **kwargs):
super().__init__(**kwargs)
self.name = name
self.age = age
class MyPlugin(PluginBase):
""""""
properties = PluginBase.properties
edges= PluginBase.edges
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.pluginPackage="pymemri.plugin.pluginbase"
def run(self, run, client):
print("running")
client.create(MyItem("some person", 20))
def add_to_schema(self, client):
client.add_to_schema(MyItem("my name", 10))
```
%% Cell type:markdown id: tags:
Memri plugins need to define at least 2 methods: `.run()` and `.add_to_schema()`. `.run()` defines the logic of the plugin. `.add_to_schema()` defines the schema for the plugin in the pod. Note that currently, `add_to_schema` requires all item to **have all properties defined that are used in the plugin**. In the future, we might replace add_to_schema, to be done automatically, based on a declarative schema defined in the plugin.
%% Cell type:code id: tags:
``` python
MyPlugin()
```
%% Output
MyPlugin (#None)
%% Cell type:markdown id: tags:
```python
class MyItem(Item):
properties = Item.properties + ["name", "age"]
edges = Item.edges
def __init__(self, name=None, age=None, **kwargs):
super().__init__(**kwargs)
self.name = name
self.age = age
class MyPlugin(PluginBase):
""""""
properties = PluginBase.properties
edges= PluginBase.edges
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.pluginPackage="pymemri.plugin.pluginbase"
def run(self, run, client):
print("running")
client.create(MyItem("some person", 20))
def add_to_schema(self, client):
client.add_to_schema(MyItem("my name", 10))
```
%% Cell type:code id: tags:
``` python
from pymemri.pod.client import PodClient
client = PodClient()
```
%% Cell type:code id: tags:
``` python
assert client.add_to_schema(MyPlugin(name="abc", data_query="abc"))
assert client.add_to_schema(PluginRun())
```
%% Cell type:code id: tags:
``` python
plugin = MyPlugin(name="abc", data_query="abc")
run = PluginRun()
run.add_edge("plugin", plugin)
```
%% Cell type:code id: tags:
``` python
client.create(run)
client.create(plugin)
client.create_edge(run.get_edges("plugin")[0]);
```
%% Cell type:code id: tags:
``` python
run = client.get(run.id)
```
%% Cell type:markdown id: tags:
## Running your plugin
%% Cell type:markdown id: tags:
Plugins can be started using the pymemri `run_plugin` CLI. To use the CLI, you can either pass your run arguments as parameters, or set them as environment variables. If both are set, the CLI will prefer the passed arguments.
%% Cell type:code id: tags:
``` python
# hide
# export
def run_plugin_from_run_id(run_id, client):
run = client.get(run_id)
plugins = run.plugin
if len(plugins) == 0:
raise ValueError(f"plugin run {run_id} has no plugin attached to it. Make sure there is a 'plugin' \
edge from your run to the actual plugin object.")
if len(plugins) > 1:
raise ValueError("Too many plugins attached to run")
plugin = plugins[0]
plugin.add_to_schema(client)
plugin.run(run, client)
```
%% Cell type:code id: tags:
``` python
# export
# hide
def register_base_classes(client):
try:
assert client.add_to_schema(PluginRun())
except Exception as e:
raise ValueError("Could not add base schema")
```
%% Cell type:code id: tags:
``` python
# hide
run_plugin_from_run_id(run.id, client)
```
%% Output
running
%% Cell type:code id: tags:
``` python
# export
def _run_plugin(client, plugin_run_id=None, verbose=False):
"""Runs an plugin, you can either provide the run settings as parameters to this function (for local testing)
or via environment variables (this is how the pod communicates with plugins)."""
register_base_classes(client)
run_plugin_from_run_id(plugin_run_id, client)
```
%% Cell type:code id: tags:
``` python
# hide
_run_plugin(client=client, plugin_run_id=run.id)
```
%% Output
running
%% Cell type:markdown id: tags:
### CLI
%% Cell type:code id: tags:
``` python
# export
# hide
def _parse_env(env):
try:
pod_full_address = env.get(POD_FULL_ADDRESS_ENV, DEFAULT_POD_ADDRESS)
plugin_run_json = json.loads(str(env[POD_TARGET_ITEM_ENV]))
print(plugin_run_json)
plugin_run_id = plugin_run_json["id"]
owner_key = env.get(POD_OWNER_KEY_ENV)
pod_auth_json = json.loads(str(env.get(POD_AUTH_JSON_ENV)))
# database_key = pod_service_payload[DATABASE_KEY_ENV]
# owner_key = pod_service_payload[OWNER_KEY_ENV]
return pod_full_address, plugin_run_id, pod_auth_json, owner_key
except KeyError as e:
raise Exception('Missing parameter: {}'.format(e)) from None
```
%% Cell type:code id: tags:
``` python
# export
from fastscript import *
import os
@call_parse
def run_plugin(pod_full_address:Param("The pod full address", str)=None,
plugin_run_id:Param("Run id of the plugin to be executed", str)=None,
database_key:Param("Database key of the pod", str)=None,
owner_key:Param("Owner key of the pod", str)=None,
from_pod:Param("Run by calling the pod", bool)=False,
container:Param("Pod container to run frod", str)=None):
env = os.environ
params = [pod_full_address, plugin_run_id, database_key, owner_key]
if all([p is None for p in params]):
print("Reading `run_plugin()` parameters from environment variables")
pod_full_address, plugin_run_id, pod_auth_json, owner_key = _parse_env(env)
database_key=None
else:
print("Used arguments passed to `run_plugin()` (ignoring environment)")
pod_auth_json=None
if (None in params):
raise ValueError(f"Defined some params to run indexer, but not all. Missing \
{[p for p in params if p is None]}")
client = PodClient(url=pod_full_address, database_key=database_key, owner_key=owner_key,
auth_json=pod_auth_json)
for name, val in [("pod_full_address", pod_full_address), ("plugin_run_id", plugin_run_id),
("owner_key", owner_key), ("auth_json", pod_auth_json)]:
print(f"{name}={val}")
print(f"{name} = {val}")
print()
if from_pod:
print(f"calling the `create` api on {pod_full_address} to make your Pod start "
f"a plugin with id {plugin_run_id}.")
print(f"*Check the pod log/console for debug output.*")
client.start_plugin("pymemri", plugin_run_id)
assert container is not None
client.start_plugin(container, plugin_run_id)
else:
_run_plugin(client=client, plugin_run_id=plugin_run_id)
```
%% Cell type:markdown id: tags:
To start a plugin on your local machine, you can use the CLI. This will create a client for you, and run the code defined in `<myplugin>.run()`
%% Cell type:code id: tags:
``` python
!run_plugin --pod_full_address=$DEFAULT_POD_ADDRESS --plugin_run_id=$run.id --owner_key=$client.owner_key \
--database_key=$client.database_key
```
%% Output
Used arguments passed to `run_plugin()` (ignoring environment)
pod_full_address=http://localhost:3030
plugin_run_id=0f32df2e21355877f6a7a5a52121ff9e
owner_key=1818730013390615608004006544815007352548160219130568937676646480
auth_json=None
pod_full_address = http://localhost:3030
plugin_run_id = d5a47f06079d1fc3a30465d975804a3c
owner_key = 1695144462728245497259821516084075751813751672211754961447730595
auth_json = None
running
%% Cell type:markdown id: tags:
## Run plugin from pod -
%% Cell type:code id: tags:
``` python
# export
# hide
class StartPlugin(Item):
properties = Item.properties + ["container", "targetItemId"]
edges = Item.edges
def __init__(self, container=None, targetItemId=None, **kwargs):
super().__init__(**kwargs)
self.container = container
self.targetItemId = targetItemId
```
%% Cell type:markdown id: tags:
In production, we start plugins by making an API call to the pod, which in turn creates an environment for the plugin and starts it (currently on docker is supported). We can start this process using the CLI by provding `--from_pod==True` and providing a `--container` (the docker container used by the pod). **Note that the provided docker container should be installed within the Pod environemnt (e.g. `docker build -t pymemri .` for this repo) in order to start it.**
%% Cell type:code id: tags:
``` python
!run_plugin --pod_full_address=$DEFAULT_POD_ADDRESS --plugin_run_id=$run.id --owner_key=$client.owner_key \
--database_key=$client.database_key --from_pod=True, --container="pymemri"
```
%% Output
Used arguments passed to `run_plugin()` (ignoring environment)
pod_full_address=http://localhost:3030
plugin_run_id=0f32df2e21355877f6a7a5a52121ff9e
owner_key=1818730013390615608004006544815007352548160219130568937676646480
auth_json=None
pod_full_address = http://localhost:3030
plugin_run_id = d5a47f06079d1fc3a30465d975804a3c
owner_key = 1695144462728245497259821516084075751813751672211754961447730595
auth_json = None
calling the `create` api on http://localhost:3030 to make your Pod start a plugin with id 0f32df2e21355877f6a7a5a52121ff9e.
*Check the pod log/console for debug output.*
calling the `create` api on http://localhost:3030 to make your Pod start a plugin with id d5a47f06079d1fc3a30465d975804a3c.
*Check the pod log/console for debug output.*
%% Cell type:markdown id: tags:
> Note: The data that was created earlier (PluginRun, plugin) should be in the pod in order for this to work
%% Cell type:code id: tags:
``` python
# hide
# client.start_plugin("pymemri", run.id)
```
%% Cell type:markdown id: tags:
## Running a Plugin by providing environment variables -
%% Cell type:code id: tags:
``` python
# hide
# # export
# def generate_test_env(client, indexer_run):
# payload = json.dumps({DATABASE_KEY_ENV: client.database_key, OWNER_KEY_ENV: client.owner_key})
# return {POD_FULL_ADDRESS_ENV: DEFAULT_POD_ADDRESS,
# POD_TARGET_ITEM: indexer_run.id,
# POD_SERVICE_PAYLOAD_ENV: payload}
```
%% Cell type:code id: tags:
``` python
# hide
# run_plugin(env=generate_test_env(client, run))
```
%% Cell type:markdown id: tags:
# Export -
%% Cell type:code id: tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted basic.ipynb.
Converted data.photo.ipynb.
Converted importers.Importer.ipynb.
Converted importers.util.ipynb.
Converted index.ipynb.
Converted indexers.indexer.ipynb.
Converted itembase.ipynb.
Converted plugin.pluginbase.ipynb.
Converted pod.client.ipynb.
%% Cell type:code id: tags:
``` python
```
......
pymemri/data/schema.py
+ 2
- 1
View file @ dfac2fe7
......@@ -23,7 +23,8 @@ def get_constructor(_type, plugin_class=None, plugin_package=None, extra=None):
mod = __import__(plugin_package, fromlist=[plugin_class])
dynamic = {plugin_class: getattr(mod, plugin_class)}
except Exception as e:
print(f"Could not import {plugin_class}.{plugin_package}")
print(e)
print(f"Could not import {plugin_package}.{plugin_class}")
else:
dynamic = dict()
......
pymemri/plugin/pluginbase.py
+ 3
- 3
View file @ dfac2fe7
......@@ -121,7 +121,6 @@ def _parse_env(env):
try:
pod_full_address = env.get(POD_FULL_ADDRESS_ENV, DEFAULT_POD_ADDRESS)
plugin_run_json = json.loads(str(env[POD_TARGET_ITEM_ENV]))
print(plugin_run_json)
plugin_run_id = plugin_run_json["id"]
owner_key = env.get(POD_OWNER_KEY_ENV)
pod_auth_json = json.loads(str(env.get(POD_AUTH_JSON_ENV)))
......@@ -161,13 +160,14 @@ def run_plugin(pod_full_address:Param("The pod full address", str)=None,
auth_json=pod_auth_json)
for name, val in [("pod_full_address", pod_full_address), ("plugin_run_id", plugin_run_id),
("owner_key", owner_key), ("auth_json", pod_auth_json)]:
print(f"{name}={val}")
print(f"{name} = {val}")
print()
if from_pod:
print(f"calling the `create` api on {pod_full_address} to make your Pod start "
f"a plugin with id {plugin_run_id}.")
print(f"*Check the pod log/console for debug output.*")
client.start_plugin("pymemri", plugin_run_id)
assert container is not None
client.start_plugin(container, plugin_run_id)
else:
_run_plugin(client=client, plugin_run_id=plugin_run_id)
......
settings.ini
+ 2
- 2
View file @ dfac2fe7
......@@ -16,8 +16,8 @@ custom_sidebar = True
license = apache2
status = 2
console_scripts = run_plugin=pymemri.plugin.pluginbase:run_plugin
requirements = requests tqdm ipdb fastprogress fastscript opencv-python nbdev==1.1.5 matplotlib
requirements = requests tqdm ipdb fastprogress fastscript opencv-python matplotlib nbdev==1.1.5
# dev_requirements = nbdev==1.1.5
nbs_path = nbs
doc_path = docs
......
setup.py
+ 2
- 0
View file @ dfac2fe7
......@@ -21,6 +21,7 @@ statuses = [ '1 - Planning', '2 - Pre-Alpha', '3 - Alpha',
py_versions = '2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8'.split()
requirements = cfg.get('requirements','').split()
dev_requirements = (cfg.get('dev_requirements') or '').split()
lic = licenses[cfg['license']]
min_python = cfg['min_python']
......@@ -37,6 +38,7 @@ setuptools.setup(
packages = setuptools.find_packages(),
include_package_data = True,
install_requires = requirements,
extras_require={ 'dev': dev_requirements },
python_requires = '>=' + cfg['min_python'],
long_description = open('README.md').read(),
long_description_content_type = 'text/markdown',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment