Skip to content
GitLab
Commit 29682997 authored by Koen van der Veen's avatar Koen van der Veen
Browse files

Merge branch 'rename_auth' into 'dev' 

update qr server

See merge request memri/pymemri!64
parents 881bf11b a0f8872d
Pipeline #3518 passed with stages
in 3 minutes and 52 seconds
Hide whitespace changes
Inline Side-by-side
Showing
with 20 additions and 22 deletions
+20 -22
nbs/plugin.pluginbase.ipynb
+ 4
- 10
View file @ 29682997
%% Cell type:code id: tags:
``` python
%load_ext autoreload
%autoreload 2
# default_exp plugin.pluginbase
```
%% Cell type:code id: tags:
``` python
# export
from pymemri.data.schema import *
from pymemri.pod.client import *
from pymemri.imports import *
from pymemri.pod.utils import *
from os import environ
from abc import ABCMeta
import abc
import json
import importlib
import string
import time
from enum import Enum
from fastscript import *
import os
from pymemri.plugin.schema import PluginRun, PersistentState, Account
from pymemri.data.basic import *
```
%% Cell type:code id: tags:
``` python
# hide
from nbdev.showdoc import *
```
%% Cell type:markdown id: tags:
# Plugins
%% Cell type:markdown id: tags:
PluginBase is the plugin class that the simplest plugin inherits.
Inheriting class should implement:
- run() that implements the logic of the plugin
- add_to_schema() for plugin specific item types
%% Cell type:code id: tags:
``` python
# export
POD_FULL_ADDRESS_ENV = 'POD_FULL_ADDRESS'
POD_TARGET_ITEM_ENV = 'POD_TARGET_ITEM'
POD_OWNER_KEY_ENV = 'POD_OWNER'
POD_AUTH_JSON_ENV = 'POD_AUTH_JSON'
```
%% Cell type:code id: tags:
``` python
# export
# hide
class PluginBase(Item, metaclass=ABCMeta):
"""Base class for plugins"""
properties = Item.properties + ["name", "repository", "icon", "data_query", "bundleImage",
"runDestination", "pluginClass"]
edges = Item.edges
def __init__(self, pluginRun=None, persistentState=None, name=None, repository=None, icon=None,
query=None, bundleImage=None, runDestination=None, pluginClass=None, **kwargs):
if pluginClass is None: pluginClass=self.__class__.__name__
super().__init__(**kwargs)
self.pluginRun = pluginRun
self.persistentState = persistentState
self.name = name
self.repository = repository
self.icon = icon
self.query = query
self.bundleImage = bundleImage
self.runDestination = runDestination
self.pluginClass = pluginClass
@abc.abstractmethod
def run(self, client):
raise NotImplementedError()
@abc.abstractmethod
def add_to_schema(self, client):
raise NotImplementedError()
```
%% Cell type:markdown id: tags:
## Creating a plugin
%% Cell type:markdown id: tags:
The memri [pod](https://gitlab.memri.io/memri/pod) uses a plugin system to add features to the backend memri backend. Plugins can import your data (importers), change your data (indexers), or call other serivces. Users can define their own plugins to add new behaviour to their memri app. Let's use the following plugin as an example of how we can start plugins.
%% Cell type:code id: tags:
``` python
# export
# hide
class MyItem(Item):
properties = Item.properties + ["name", "age"]
edges = Item.edges
def __init__(self, name=None, age=None, **kwargs):
super().__init__(**kwargs)
self.name = name
self.age = age
class MyPlugin(PluginBase):
""""""
properties = PluginBase.properties + ["containerImage"]
edges= PluginBase.edges
def __init__(self, **kwargs):
super().__init__(**kwargs)
def run(self, client):
print("running")
self.login()
client.create(MyItem("some person", 20))
def login(self):
account = self.pluginRun.account[0]
print(f"logging in with account {account.identifier} and password {account.secret}")
def add_to_schema(self, client):
client.add_to_schema(MyItem("my name", 10))
```
%% Cell type:code id: tags:
``` python
# hide
MyPlugin()
```
%% Output
MyPlugin (#None)
%% Cell type:markdown id: tags:
```python
class MyItem(Item):
properties = Item.properties + ["name", "age"]
edges = Item.edges
def __init__(self, name=None, age=None, **kwargs):
super().__init__(**kwargs)
self.name = name
self.age = age
class MyPlugin(PluginBase):
""""""
properties = PluginBase.properties
edges= PluginBase.edges
def __init__(self, **kwargs):
super().__init__(**kwargs)
def run(self, client):
print("running")
self.login()
client.create(MyItem("some person", 20))
def login(self):
account = self.pluginRun.account[0]
print(f"logging in with account {account.identifier} and password {account.secret}")
def add_to_schema(self, client):
client.add_to_schema(MyItem("my name", 10))
```
%% Cell type:markdown id: tags:
Memri plugins need to define at least 2 methods: `.run()` and `.add_to_schema()`. `.run()` defines the logic of the plugin. `.add_to_schema()` defines the schema for the plugin in the pod. Note that currently, `add_to_schema` requires all item to **have all properties defined that are used in the plugin**. In the future, we might replace add_to_schema, to be done automatically, based on a declarative schema defined in the plugin.
%% Cell type:markdown id: tags:
## Helper classes -
%% Cell type:code id: tags:
``` python
# hide
# export
def get_plugin_cls(plugin_module, plugin_name):
try:
module = importlib.import_module(plugin_module)
plugin_cls = getattr(module, plugin_name)
return plugin_cls
except (ImportError, AttributeError):
raise ImportError(f"Unknown plugin: {plugin_module}.{plugin_name}")
def get_plugin_state(run):
"""
Returns the PersistentState associated with the run.
Returns `None` if no state was found, and the `PersistentState` if a single state is found.
Raises `ValueError` if more than one `PersistentState` is registered in run.
"""
plugin_state = run.get_edges("persistentState")
if len(plugin_state) == 0:
return None
elif len(plugin_state) == 1:
return plugin_state[0]
else:
raise ValueError(f"Expected a single PersistentState for plugin, found {len(plugin_state)}.")
def run_plugin_from_run_id(run_id, client):
"""
Args:
run_id (int): id of the PluginRun
client (PodClient): client containing PluginRun
return_plugin (bool): Returns created plugin instance for testing purposes.
"""
run = client.get(run_id)
plugin_state = get_plugin_state(run)
plugin_cls = get_plugin_cls(run.pluginModule, run.pluginName)
plugin = plugin_cls(pluginRun=run, persistentState=plugin_state)
plugin.add_to_schema(client)
# TODO handle plugin status before run
plugin.run(client)
return plugin
```
%% Cell type:markdown id: tags:
## Run from id test -
%% Cell type:code id: tags:
``` python
# hide
# skip
client = PodClient()
run = PluginRun("pymemri", "pymemri.plugin.pluginbase", "MyPlugin", status="not started")
account = Account(identifier="login", secret="password")
run.add_edge("account", account)
assert client.add_to_schema(PluginRun("", "", "", "", ""))
assert client.create(run)
assert client.create(account)
assert client.create_edge(run.get_edges("account")[0])
run = client.get(run.id)
run_plugin_from_run_id(run.id, client);
```
%% Output
running
logging in with account login and password password
%% Cell type:code id: tags:
``` python
# export
# hide
def _parse_env():
env = os.environ
print("Reading `run_plugin()` parameters from environment variables")
try:
pod_full_address = env.get(POD_FULL_ADDRESS_ENV, DEFAULT_POD_ADDRESS)
plugin_run_json = json.loads(str(env[POD_TARGET_ITEM_ENV]))
print(plugin_run_json)
plugin_run_id = plugin_run_json["id"]
owner_key = env.get(POD_OWNER_KEY_ENV)
pod_auth_json = json.loads(str(env.get(POD_AUTH_JSON_ENV)))
return pod_full_address, plugin_run_id, pod_auth_json, owner_key
except KeyError as e:
raise Exception('Missing parameter: {}'.format(e)) from None
```
%% Cell type:markdown id: tags:
## Running your plugin using the CLI
%% Cell type:markdown id: tags:
Plugins can be started using the pymemri `run_plugin` or `simulate_run_plugin_from_frontend` CLI. With `run_plugin` the plugin is invoked directly by spawning a new python process, while `simulate_run_plugin_from_frontend` requests the pod to spawn a new process, docker container, or kubernetes container, which in calls `run_plugin` (for more info see `simulate_run_plugin_from_frontend`. When using `run_plugin`, you can either pass your run arguments as parameters, or set them as environment variables. If both are set, the CLI will use the passed arguments.
%% Cell type:code id: tags:
``` python
# export
# hide
@call_parse
def store_keys(path:Param("path to store the keys", str)=DEFAULT_POD_KEY_PATH,
database_key:Param("Database key of the pod", str)=None,
owner_key:Param("Owner key of the pod", str)=None):
if database_key is None: database_key = PodClient.generate_random_key()
if owner_key is None: owner_key = PodClient.generate_random_key()
obj = {"database_key": database_key,
"owner_key": owner_key}
Path(path).parent.mkdir(parents=True, exist_ok=True)
if path.exists():
timestr = time.strftime("%Y%m%d-%H%M%S")
path.rename(POD_KEYS_FULL_FOLDER / f"keys-{timestr}.json")
write_json(obj, path)
```
%% Cell type:code id: tags:
``` python
# hide
store_keys()
```
%% Cell type:code id: tags:
``` python
# export
# hide
def parse_config(file, remove_container=False):
json_dict = read_json(file)
account = Account.from_json(json_dict["account"])
del json_dict["account"]
settings = json.dumps(json_dict["settings"])
del json_dict["settings"]
run = PluginRun.from_json(json_dict)
run.settings = settings
run.add_edge("account", account)
if remove_container:
run.containerImage = "none"
return run
def create_run_expanded(client, run):
client.create(run)
accounts = run.account
if accounts:
account=accounts[0]
client.create(account)
client.create_edge(run.get_edges("account")[0])
```
%% Cell type:code id: tags:
``` python
# export
@call_parse
def run_plugin(pod_full_address:Param("The pod full address", str)=DEFAULT_POD_ADDRESS,
plugin_run_id:Param("Run id of the plugin to be executed", str)=None,
database_key:Param("Database key of the pod", str)=None,
owner_key:Param("Owner key of the pod", str)=None,
read_args_from_env:Param("Read the args from the environment", bool)=False,
config_file:Param("config file for the PluginRun", str)=None):
if read_args_from_env:
pod_full_address, plugin_run_id, pod_auth_json, owner_key = _parse_env()
database_key=None
else:
if database_key is None: database_key = read_pod_key("database_key")
if owner_key is None: owner_key = read_pod_key("owner_key")
pod_auth_json = None
client = PodClient(url=pod_full_address, database_key=database_key, owner_key=owner_key,
auth_json=pod_auth_json)
if config_file is not None:
run = parse_config(config_file, remove_container=True)
create_run_expanded(client, run)
plugin_run_id=run.id
print(f"pod_full_address={pod_full_address}\nowner_key={owner_key}\n")
run_plugin_from_run_id(run_id=plugin_run_id, client=client)
```
%% Cell type:markdown id: tags:
To start a plugin on your local machine, you can use the CLI. This will create a client for you, and run the code defined in `<myplugin>.run()`
%% Cell type:code id: tags:
``` python
run_plugin(config_file="../example_config.json")
```
%% Output
reading database_key from /Users/koen/.pymemri/pod_keys/keys.json
reading owner_key from /Users/koen/.pymemri/pod_keys/keys.json
pod_full_address=http://localhost:3030
owner_key=9293924815392284028926785483687081842013306769112315762668202298
owner_key=3443478669325419979439046514740261504070289229389778632922768792
running
logging in with account myusername and password mypassword
%% Cell type:markdown id: tags:
> Note: The data that is created here should be in the pod in order for this to work
%% Cell type:markdown id: tags:
## Run from pod
%% Cell type:markdown id: tags:
In production, we start plugins by making an API call to the pod, which in turn creates an environment for the plugin and starts it using docker containers, kubernetes containers or a shell script. We can start this process using the `simulate_run_plugin_from_frontend` CLI. **Note that when using docker, provided container name should be "installed" within the Pod environemnt (e.g. `docker build -t pymemri .` for this repo) in order to start it.**
%% Cell type:markdown id: tags:
![running a plugin](images/running_a_plugin.svg)
%% Cell type:code id: tags:
``` python
# export
from fastcore.script import call_parse, Param
import os
@call_parse
def simulate_run_plugin_from_frontend(pod_full_address:Param("The pod full address", str)=DEFAULT_POD_ADDRESS,
database_key:Param("Database key of the pod", str)=None,
owner_key:Param("Owner key of the pod", str)=None,
plugin_id:Param("Pod ID of the plugin", str)=None,
container:Param("Pod container to run frod", str)=None,
plugin_path:Param("Plugin path", str)=None,
settings_file:Param("Plugin settings (json)", str)=None,
config_file:Param("config file for the PluginRun", str)=None):
# TODO remove container, plugin_module, plugin_name and move to Plugin item.
# Open question: This presumes Plugin item is already in pod before simulate_run_plugin_from_frontend is called.
if database_key is None: database_key = read_pod_key("database_key")
if owner_key is None: owner_key = read_pod_key("owner_key")
params = [pod_full_address, database_key, owner_key]
if (None in params):
raise ValueError(f"Defined some params to run indexer, but not all")
client = PodClient(url=pod_full_address, database_key=database_key, owner_key=owner_key)
for name, val in [("pod_full_address", pod_full_address), ("owner_key", owner_key)]:
print(f"{name}={val}")
# if settings_file is not None:
# with open(settings_file, 'r') as f:
# settings = f.read()
# else:
# settings = None
if config_file is not None:
run = parse_config(config_file)
create_run_expanded(client, run)
else:
if container is None:
container = plugin_path.split(".", 1)[0]
print(f"Inferred '{container}' as plugin container name")
plugin_module, plugin_name = plugin_path.rsplit(".", 1)
run = PluginRun(container, plugin_module, plugin_name)
if plugin_id is not None:
persistent_state = client.get(plugin_id)
run.add_edge("persistentState", persistent_state)
client.create_edge(run.get_edges("persistentState")[0])
client.create(run)
print(f"\ncalling the `create` api on {pod_full_address} to make your Pod start "
f"a plugin with id {run.id}.")
print(f"*Check the pod log/console for debug output.*")
return run
```
%% Cell type:code id: tags:
``` python
client = PodClient()
```
%% Cell type:code id: tags:
``` python
!simulate_run_plugin_from_frontend --config_file="../example_config.json"
```
%% Output
reading database_key from /Users/koen/.pymemri/pod_keys/keys.json
reading owner_key from /Users/koen/.pymemri/pod_keys/keys.json
pod_full_address=http://localhost:3030
owner_key=9293924815392284028926785483687081842013306769112315762668202298
owner_key=3443478669325419979439046514740261504070289229389778632922768792
calling the `create` api on http://localhost:3030 to make your Pod start a plugin with id 5C00efeeaEBb9Cd256Cad5Bf433fa2Ef.
calling the `create` api on http://localhost:3030 to make your Pod start a plugin with id fdB73AfeCcA4DeF1F592c51Ac9391dF7.
*Check the pod log/console for debug output.*
%% Cell type:code id: tags:
``` python
# hide
# !simulate_run_plugin_from_frontend --plugin_path="pymemri.plugin.pluginbase.MyPlugin"
```
%% Cell type:markdown id: tags:
## Appendix -
%% Cell type:code id: tags:
``` python
# hide
# client.start_plugin("pymemri", run.id)
```
%% Cell type:code id: tags:
``` python
# hide
# class StartPlugin(Item):
# properties = Item.properties + ["container", "targetItemId"]
# edges = Item.edges
# def __init__(self, container=None, targetItemId=None, **kwargs):
# super().__init__(**kwargs)
# self.container = container
# self.targetItemId = targetItemId
```
%% Cell type:code id: tags:
``` python
# hide
# class PluginSettings(Item):
# def __init__(self, settings_dict):
# self.settings_dict=settings_dict
# def __getattr__(self, k):
# if k in self.settings_dict:
# return self.settings_dict[k]
# @classmethod
# def from_string(cls, s):
# objs = json.loads(s)
# cls(objs)
```
%% Cell type:code id: tags:
``` python
# hide
# # export
# def generate_test_env(client, indexer_run):
# payload = json.dumps({DATABASE_KEY_ENV: client.database_key, OWNER_KEY_ENV: client.owner_key})
# return {POD_FULL_ADDRESS_ENV: DEFAULT_POD_ADDRESS,
# POD_TARGET_ITEM: indexer_run.id,
# POD_SERVICE_PAYLOAD_ENV: payload}
```
%% Cell type:code id: tags:
``` python
# hide
# run_plugin(env=generate_test_env(client, run))
```
%% Cell type:markdown id: tags:
# Export -
%% Cell type:code id: tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted basic.ipynb.
Converted data.photo.ipynb.
Converted importers.Importer.ipynb.
Converted importers.util.ipynb.
Converted index.ipynb.
Converted indexers.indexer.ipynb.
Converted itembase.ipynb.
Converted plugin.pluginbase.ipynb.
Converted plugin.schema.ipynb.
Converted plugin.stateful.ipynb.
Converted plugin.states.ipynb.
Converted pod.client.ipynb.
Converted pod.db.ipynb.
Converted pod.utils.ipynb.
Converted test_utils.ipynb.
%% Cell type:code id: tags:
``` python
```
......
pymemri/client_simulator/qr_server.py
+ 16
- 6
View file @ 29682997
......@@ -3,24 +3,34 @@ import flask
import multiprocessing
from flask import render_template
from time import sleep
import os
app = flask.Flask(__name__, template_folder='template')
qr_code_data = None
qr_code_dict = None
QR_CODE_KEY = "qr_code"
@app.route('/qr')
def index():
global qr_code_data
global qr_code_dict
qr_code_data = qr_code_dict[QR_CODE_KEY]
return render_template('images.html', chart_output=qr_code_data)
def run_app(qr_dict, host="0.0.0.0", port=8000):
global qr_code_dict
qr_code_dict = qr_dict
app.run(host=host, port=port)
def run_qr_server(_qr_code_data):
global qr_code_data
qr_code_data = _qr_code_data
manager = multiprocessing.Manager()
process_dict = manager.dict()
process_dict["qr_code"] = _qr_code_data
print("GO TO http://localhost:8000/qr and scan the code")
process = multiprocessing.Process(target=app.run, kwargs={"host": "0.0.0.0", "port": 8000}, daemon=True)
process = multiprocessing.Process(target=run_app, args=(process_dict,),
kwargs={"host": "0.0.0.0", "port": 8000}, daemon=True)
process.start()
return process
return process, process_dict
if __name__ == "__main__":
# this is here for testing purposes, no function in production
......
pymemri/plugin/pluginbase.py
+ 0
- 6
View file @ 29682997
......@@ -244,12 +244,6 @@ def simulate_run_plugin_from_frontend(pod_full_address:Param("The pod full addre
for name, val in [("pod_full_address", pod_full_address), ("owner_key", owner_key)]:
print(f"{name}={val}")
# if settings_file is not None:
# with open(settings_file, 'r') as f:
# settings = f.read()
# else:
# settings = None
if config_file is not None:
run = parse_config(config_file)
create_run_expanded(client, run)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment