Commit 5aa7a457 authored by Alp Deniz Ogut's avatar Alp Deniz Ogut
Browse files

Initial commit

parents
Pipeline #2591 failed with stage
Showing with 565 additions and 0 deletions
+565 -0
__pycache__/
.pytest_cache/
*.py[cod]
*.egg-info/
*.egg
build/
builds/
dist/
\ No newline at end of file
default:
image: python:3.8
before_script:
- pip install .
run_tests:
stage: test
script:
- pytest ./tests/*
Dockerfile 0 → 100644
FROM python:3.8-slim as base
# Setup env
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONFAULTHANDLER 1
RUN apt-get update
RUN apt-get install ffmpeg libsm6 libxext6 python3 python-dev python-pip build-essential git -y
FROM base AS python-deps
# Install pipenv and compilation dependencies
RUN pip install poetry
RUN apt-get update && apt-get install -y --no-install-recommends gcc
# Required for pymemri
RUN pip install fastprogress fastscript ipdb matplotlib nbdev==1.1.5 opencv-python requests tqdm
# Get development branch of alpdeniz/pymemri
WORKDIR /tmp
RUN git clone --single-branch --branch dev https://gitlab.memri.io/alpdeniz/pymemri
FROM python-deps as project-base
# Setup app
RUN mkdir /tmp/plugin
WORKDIR /tmp/plugin
# Install itself
COPY . .
RUN pip install /tmp/pymemri
RUN pip install .
# Install this module
# RUN pip install .
ENTRYPOINT [ "python", "./scripts/main.py" ]
\ No newline at end of file
LICENSE 0 → 100644
This diff is collapsed.
README.md 0 → 100644
# Memri Plugin Template
This repository provides a boilerplate for those who want to create new plugins for [Memri](https://memri.io) per [Acceptance Criterias](https://gitlab.memri.io/memri/memri/-/wikis/Acceptance-criteria-for-plugins).
- [x] Standard folder structure
- [x] Has test setup
- [x] Has .gitlab-ci.yml
- [x] Has setup.py
- [x] Has Dockerfile
- [x] Has MPPL License
- [ ] Has icon/logo
Contributions are very welcome.
## Installation
Run below commands after cloning the repository.
Docker:
docker build -t plugin_template .
Local:
pip install .
## Usage
Create/rename files and folders, implement your own logic mostly inside submodules.
Add tests for every method and every case.
To run the plugin, after starting the POD and building the docker image, type:
python client-simulator.py
This will trigger a plugin run, starting the plugin's docker container. The script will listen the plugin and simulate a user authentication.
To test:
pytest ./tests/*
## Implementation
The main plugin flow is described in the submodule named 'plugin_flow'. It provides a set of methods for communication inbetween the user-client and the plugin.
I recommend that this functionality be moved into PluginBase.
## Data Structures and Flow
Used data types are:
- Plugin { .... }
- StartPlugin { ... }
- Credential { ... }
from plugin_template.plugin_flow.plugin_flow import PluginFlow
from pymemri.pod.client import PodClient
# SETUP
client = PodClient(database_key="a"*32, owner_key="plugin_test")
pf = PluginFlow(client)
# INSTALL
plugin_id = pf.install_plugin(name="Plugin Template", container="plugin_template")
# RUN
pf.run_plugin()
from pymemri.data.schema import Item, Edge
class Credential(Item):
def __init__(self, service, identifier=None, secret=None, id=None, deleted=None): #e.g. username, password or access key, secret key etc.
self.id = id
self.deleted = deleted
self.service = service
self.identifier = identifier
self.secret = secret
@classmethod
def from_json(cls, json):
id = json.get("id", None)
deleted = json.get("deleted", None)
service = json.get("service", None)
identifier = json.get("identifier", None)
secret = json.get("secret", None)
res = cls(service=service, identifier=identifier, secret=secret, id=id, deleted=deleted)
return res
class Plugin(Item):
def __init__(self, name, container, credential=None, id=None, deleted=None):
super().__init__(id=id, deleted=deleted)
self.name = name
self.container = container
self.credential = credential if credential else []
@classmethod
def from_json(cls, json):
id = json.get("id", None)
deleted = json.get("deleted", None)
name = json.get("name", None)
container = json.get("container", None)
all_edges = json.get("allEdges", None)
credential = []
if all_edges is not None:
for edge_json in all_edges:
edge = Edge.from_json(edge_json)
if edge._type == "credential" or edge._type == "~credential":
credential.append(edge)
res = cls(name=name, container=container, credential=credential, id=id, deleted=deleted)
for e in res.get_all_edges(): e.source = res
return res
class StartPlugin(Item):
def __init__(self, targetItemId, container, state=None, view=None, id=None, deleted=None):
super().__init__(id=id, deleted=deleted)
self.targetItemId = targetItemId
self.container = container
self.state = state
# CVUStoredDefinitions
self.view = view if view else None
@classmethod
def from_json(cls, json):
id = json.get("id", None)
deleted = json.get("deleted", None)
targetItemId = json.get("target_item_id", None)
container = json.get("container", None)
state = json.get("state", None)
all_edges = json.get("allEdges", None)
view = []
if all_edges is not None:
for edge_json in all_edges:
edge = Edge.from_json(edge_json)
if edge._type == "view" or edge._type == "~view":
view.append(edge)
res = cls(targetItemId=targetItemId, container=container, state=state, view=view, id=id, deleted=deleted)
return res
\ No newline at end of file
from time import sleep
from plugin_template.data.schema import Plugin, StartPlugin, Credential
RUN_USER_ACTION_NEEDED = 'userActionNeeded'
RUN_USER_ACTION_COMPLETED = 'ready'
RUN_IDLE = 'idle'
RUN_STARTED = 'start'
RUN_COMPLETED = 'done'
# A draft class to handle plugin flows like auth, state, progress...
# I think PluginFlow should be embeded in PluginBase.
class PluginFlow:
def __init__(self, client, run_id=None, plugin=None):
self.client = client
self.run_id = run_id
self.plugin = plugin
self.plugin_id = plugin.id if plugin else None
self._setup_schema()
# =======================================
# ---------- PLUGIN METHODS -------------
def get_credential(self, service):
# If user is already authenticated and plugin has the edges to keys
credential_to_use = self._get_credential_from_plugin(service)
if not credential_to_use:
credential_to_use = self._start_auth_flow(service)
return credential_to_use
def started(self):
self._set_run_state(RUN_STARTED)
def completed(self):
self._set_run_state(RUN_COMPLETED)
# =======================================
# --------- USER CLIENT METHODS ---------
def install_plugin(self, name, container):
plugin = Plugin(name=name, container=container)
self.client.create(plugin)
# set in instance
self.plugin_id = plugin.id
self.plugin = plugin
return plugin.id
def trigger_plugin(self):
starter = StartPlugin(targetItemId=self.plugin_id, container=self.plugin.container, state=RUN_IDLE)
# add cvus here
starter.update(self.client)
self.run_id = starter.id
print(f"Started plugin {self.plugin.name} - {self.plugin.id} and run id {self.run_id}")
def set_credential(self, identifier, secret, service):
# Create credential item
credential = Credential(identifier=identifier, secret=secret, service=service)
# Save credentials as an edge to the plugin
self.client.create(credential)
self.plugin.add_edge('credential', credential)
self.plugin.update(self.client)
def get_CVUs(self, run):
run = run.expand(self.client)
return run.get_edges('view')
# Dummy UI Client Simulation
def run_plugin(self):
self.trigger_plugin()
# START POLLING
while True:
# take a breath
sleep(0.6)
run = self._get_run()
if run.state == RUN_COMPLETED:
print("# Completed plugin run. Well done")
return True
if run.state == RUN_USER_ACTION_NEEDED:
# Extract views (CVUs) from the run. It should be added by the plugin.
views = self.get_CVUs(run)
## DISPLAYS CVU & AUTHENTICATES THE USER
sleep(3)
self.set_credential("username", "password", "template_service")
# Notify plugin
self._set_run_state(RUN_USER_ACTION_COMPLETED)
# =======================================
# --------- INTERNAL METHODS ------------
def _get_run(self):
return self.client.get(self.run_id, expanded=False)
def _get_run_state(self):
start_plugin = self._get_run()
return start_plugin.state
def _set_run_state(self, state):
start_plugin = self.client.get(self.run_id, expanded=False)
start_plugin.state = state
# start_plugin.view = createLoginCVU()
self.client.update_item(start_plugin)
def _get_credential_from_plugin(self, service):
credential_edges = self.plugin.get_edges('credential')
for credential_edge in credential_edges:
credential = credential_edge.traverse(self.plugin)
if credential.service == service:
print("Account to use", credential)
return credential
return None
def _start_auth_flow(self, service):
# start userActionNeeded flow
self._set_run_state(RUN_USER_ACTION_NEEDED)
# poll here
while True:
sleep(0.6)
run_state = self._get_run_state()
if run_state == RUN_USER_ACTION_COMPLETED:
# Now the client has set up the account as an edge to the plugin
self.plugin = self.client.get(self.plugin_id)
return self._get_credential_from_plugin(service)
def _setup_schema(self):
self.client.add_to_schema(Credential(identifier="", secret="", service=""))
self.client.add_to_schema(Plugin(name="", container=""))
self.client.add_to_schema(StartPlugin(targetItemId="", container="", state=""))
\ No newline at end of file
class PluginTemplate:
def __init__(self, client, credential):
self.client = client
self.credentials = credential
print("PLUGIN run is started", self.__dict__)
def run(self):
print("PLUGIN run is completed")
\ No newline at end of file
This diff is collapsed.
[tool.poetry]
name = "plugin_template"
version = "0.1.0"
description = "Memri Plugin Template"
authors = ["Alp Deniz Ogut <alpdeniz@protonmail.com>"]
[tool.poetry.dependencies]
python = "^3.8"
pytest = "^6.2.4"
pymemri = "^0.0.6"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
from plugin_template.plugin_template import PluginTemplate
from pymemri.pod.client import PodClient
from plugin_template.data.schema import Plugin # Remove this later, after moving json plugin parser into PluginFlow
from plugin_template.plugin_flow.plugin_flow import PluginFlow
import os, json
def main(pod_url=None, database_key=None, owner_key=None, plugin_item=None, start_plugin_id=None):
# setup client
client = PodClient(url=pod_url, database_key=database_key, owner_key=owner_key)
# Get plugin item from the POD.
# expand? we need credential edges. Also move into PluginFlow after getting plugin_id only.
plugin = Plugin.from_json(json.loads(plugin_item))
# start plugin flow.
flow = PluginFlow(client, start_plugin_id, plugin)
# get plugin credential. Either already saved or fresh acquisition
credential_to_use = flow.get_credential("template_service")
flow.started()
# ----- PLUGIN STARTS ------
plugin = PluginTemplate(client, credential_to_use)
plugin.run()
# ------ PLUGIN ENDS -------
# All done! Notify the POD
flow.completed()
def parse_environment_variables():
plugin_data = {}
env_variable_keys = {
'auth': 'POD_AUTH_JSON',
'owner_key': 'POD_OWNER',
'plugin_item': 'POD_TARGET_ITEM',
'pod_url': 'POD_FULL_ADDRESS',
'start_plugin_id': 'POD_PLUGINRUN_ID'
}
# extract
for k, v in env_variable_keys.items():
if not v in os.environ:
print(f"Missing environment variable: {v}. Exiting...")
exit()
plugin_data[k] = os.environ[v]
return plugin_data
# EXTRACT PARAMETERS FROM ENVIRONMENT
if __name__ == "__main__":
plugin_data = parse_environment_variables()
# Wait till PLUGIN_AUTH is complete. Authenticate with database_key instead
del plugin_data['auth']
plugin_data['database_key'] = "a"*32
# plugin_data['pod_url'] = 'http://172.17.0.1:3030'
main(**plugin_data)
setup.py 0 → 100644
# -*- coding: utf-8 -*-
from setuptools import setup
packages = \
['plugin_template', 'plugin_template.data', 'plugin_template.plugin_flow']
package_data = \
{'': ['*']}
install_requires = \
['pymemri>=0.0.6,<0.0.7', 'pytest>=6.2.4,<7.0.0']
setup_kwargs = {
'name': 'plugin_template',
'version': '0.1.0',
'description': 'Memri Plugin Boilerplate',
'long_description': None,
'author': 'Alp Deniz Ogut',
'author_email': 'alpdeniz@protonmail.com',
'maintainer': None,
'maintainer_email': None,
'url': None,
'packages': packages,
'package_data': package_data,
'install_requires': install_requires,
'python_requires': '>=3.8,<4.0',
}
setup(**setup_kwargs)
# This setup.py was autogenerated using poetry.
from pymemri.data.schema import Address, Event, Person
from pymemri.pod.client import PodClient
from datetime import datetime
def generate_test_data():
items = []
e1 = Event(itemDescription="New York Event", startTime=int(datetime.strptime("07/07/2022", "%d/%m/%Y").timestamp()))
a1 = Address(city="New York", country="USA")
e1.add_edge('location', a1)
e2 = Event(itemDescription="Berlin Event", startTime=int(datetime.strptime("09/07/2022", "%d/%m/%Y").timestamp()))
a2 = Address(city="Berlin", country="Germany")
e2.add_edge('location', a2)
items += [a1, a2, e1, e2]
p = Person(firstName="", lastName="")
a = Address(city="New York")
p.add_edge('address', a)
items += [a, p]
p = Person(firstName="", lastName="")
a = Address(city="Berlin")
p.add_edge('address', a)
items += [a, p]
p = Person(firstName="", lastName="")
a = Address(city="Bangkok")
p.add_edge('address', a)
items += [a, p]
p = Person(firstName="", lastName="")
a = Address(city="Tokyo")
p.add_edge('address', a)
items += [a, p]
return items
def setup_test_database(client):
## set schemas
client.add_to_schema(Event(itemDescription="", startTime=1))
client.add_to_schema(Person(firstName="", lastName="", ))
client.add_to_schema(Address(city="", country=""))
existing = client.search({'type': 'Event'})
if not existing or len(existing) == 0:
print("Setting up database")
items = generate_test_data()
for item in items:
client.create(item)
for item in items:
item.update(client, edges=True)
else:
print("Test database is already loaded")
\ No newline at end of file
from unittest import TestCase
class PluginFlowTestCase(TestCase):
def testPluginFlow(self):
#TODO:
# - Start a plugin
# - Authenticate (oauth, 2fa)
# - Get credentials
pass
from unittest import TestCase
from pymemri.pod.client import PodClient
from plugin_template.plugin_template import PluginTemplate
from plugin_template.data.schema import Credential
client = PodClient(url="172.17.0.1", database_key="abc", owner_key="plugin_template_test")
class PluginTemplateTestCase(TestCase):
def testPlugin(self):
plugin = PluginTemplate(client, Credential(id="100000000", service="test", identifier="test", secret="test"))
plugin.run()
assert True
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment