Commit a607dcc7 authored by Alp Deniz Ogut's avatar Alp Deniz Ogut
Browse files

Update to version 0.1.1

parent 50900d14
Showing with 293 additions and 183 deletions
+293 -183
......@@ -10,18 +10,11 @@ RUN apt-get install ffmpeg libsm6 libxext6 python3 python-dev python-pip build-e
FROM base AS python-deps
# Install pipenv and compilation dependencies
RUN pip install poetry
RUN apt-get update && apt-get install -y --no-install-recommends gcc
# Required for pymemri
RUN pip install fastprogress fastscript ipdb matplotlib nbdev==1.1.5 opencv-python requests tqdm
# Get development branch of alpdeniz/pymemri
WORKDIR /tmp
RUN git clone --single-branch --branch demo https://gitlab.memri.io/alpdeniz/pymemri
FROM python-deps as project-base
FROM python-deps as plugin-base
# Setup app
RUN mkdir /tmp/plugin
......
......@@ -16,17 +16,17 @@ Contributions are very welcome.
## Installation
First, make sure you have the latest dev branch of POD installed and running.
First, make sure you have the latest dev branch (v3 API) of POD installed and running.
Run below commands after cloning this repository.
Run below commands after cloning this repository and entering into its directory:
Docker:
docker build -t plugin_template .
docker build -t {PLUGIN_DOCKER_CONTAINER_NAME} .
Local:
pip install .
pip install poetry
poetry install
## Usage
......@@ -36,10 +36,9 @@ CVUStoredDefinition items should be created and connected to the Plugin item dur
Please create an issue or an MR if you find an issue or have a suggestion.
To run the plugin, after starting the POD (on dev branch) and building this docker image, type:
python client-simulator.py {POD_URL}
python client-simulator.py {POD_URL} {PLUGIN_DOCKER_CONTAINER_NAME}
This will trigger a plugin run, starting the plugin's docker container. The script will listen the plugin and simulate a user authentication.
......@@ -60,3 +59,8 @@ Used data types are:
- Plugin { .... }
- StartPlugin { ... }
- Account { ... }
## TODOs
- Refactor
- Add tests
\ No newline at end of file
from plugin_template.plugin_flow.plugin_flow import PluginFlow
from pymemri.pod.client import PodClient
import sys
# SETUP
client = PodClient(url=sys.argv[1], database_key="a"*64, owner_key="plugin_test")
pf = PluginFlow(client)
# INSTALL
plugin_id = pf.install_plugin(name="Plugin Template", container="plugin_template")
# pf.set_account("username","pass", "test-service")
# RUN
pf.run_plugin()
......@@ -2,13 +2,15 @@ from pymemri.data.schema import Item, Edge
class Account(Item):
def __init__(self, service, identifier=None, secret=None, code=None, id=None, deleted=None): #e.g. username, password or access key, secret key etc.
def __init__(self, service=None, identifier=None, secret=None, code=None, refreshToken=None, errorMessage=None, id=None, deleted=None): #e.g. username, password or access key, secret key etc.
self.id = id
self.deleted = deleted
self.service = service
self.identifier = identifier
self.secret = secret
self.refreshToken = refreshToken
self.code = code
self.errorMessage = errorMessage
@classmethod
def from_json(cls, json):
......@@ -18,8 +20,10 @@ class Account(Item):
identifier = json.get("identifier", None)
secret = json.get("secret", None)
code = json.get("code", None)
refreshToken = json.get("refreshToken", None)
errorMessage = json.get("errorMessage", None)
res = cls(service=service, identifier=identifier, secret=secret, code=code, id=id, deleted=deleted)
res = cls(service=service, identifier=identifier, secret=secret, code=code, refreshToken=refreshToken, errorMessage=errorMessage, id=id, deleted=deleted)
return res
......@@ -53,12 +57,13 @@ class Plugin(Item):
class StartPlugin(Item):
def __init__(self, targetItemId, container, state=None, oAuthUrl=None, view=None, id=None, deleted=None):
def __init__(self, targetItemId, container, state=None, oAuthUrl=None, interval=None, view=None, id=None, deleted=None):
super().__init__(id=id, deleted=deleted)
self.targetItemId = targetItemId
self.container = container
self.state = state
self.oAuthUrl = oAuthUrl
self.interval = interval
# CVUStoredDefinitions
self.view = view if view else None
......@@ -71,6 +76,7 @@ class StartPlugin(Item):
container = json.get("container", None)
state = json.get("state", None)
oAuthUrl = json.get("oAuthUrl", None)
interval = json.get("interval", None)
all_edges = json.get("allEdges", None)
view = []
......@@ -81,5 +87,5 @@ class StartPlugin(Item):
if edge._type == "view" or edge._type == "~view":
view.append(edge)
res = cls(targetItemId=targetItemId, container=container, state=state, view=view, oAuthUrl=oAuthUrl, id=id, deleted=deleted)
res = cls(targetItemId=targetItemId, container=container, state=state, view=view, oAuthUrl=oAuthUrl, interval=interval, id=id, deleted=deleted)
return res
\ No newline at end of file
from time import sleep
import logging
from time import time, sleep
from plugin_template.data.schema import Plugin, StartPlugin, Account
RUN_USER_ACTION_NEEDED = 'userActionNeeded'
RUN_USER_ACTION_COMPLETED = 'ready'
RUN_INITIALIZED = 'initilized'
RUN_IDLE = 'idle'
RUN_STARTED = 'start'
RUN_FAILED = 'error'
RUN_COMPLETED = 'done'
RUN_IDLE = 'idle' #1
RUN_INITIALIZED = 'initilized' #2
RUN_USER_ACTION_NEEDED = 'userActionNeeded' # 2-3
RUN_USER_ACTION_COMPLETED = 'ready' # 2-3
RUN_STARTED = 'start' #3
RUN_FAILED = 'error' # 3-4
RUN_COMPLETED = 'done' #4
AUTH_TYPES = ['userpass', 'token', 'oauth']
RUN_STATE_POLLING_INTERVAL = 0.6
RUN_USER_ACTION_TIMEOUT = 120
logging.basicConfig(format='%(asctime)s [%(levelname)s] - %(message)s')
# A draft class to handle plugin flows like auth, state, progress...
# I think PluginFlow should be embeded in PluginBase.
# A draft class to be inherited by the actual plugin class, to handle plugin flows like auth, state, progress...
# Requires the plugin class to have a "run" method.
class PluginFlow:
def __init__(self, client, run_id=None, plugin=None, app_id=None, app_secret=None):
......@@ -27,23 +31,44 @@ class PluginFlow:
self._setup_schema()
self.initialized()
def start(self):
""" Plugin run wrapper - sets status and allows daemon mode intervals """
logging.warning('Running')
self.started()
while True:
# Plugin class provides this method - plugin's main run logic
self.run()
# daemon run interval
run = self.get_run(expanded=False)
if not run.interval: # interval is falsy. To terminate, set run.interval = 0 or None
break
sleep(run.interval)
self.completed()
# =======================================
# ---------- PLUGIN METHODS -------------
def get_account_from_plugin(self, service):
def get_account_from_plugin(self, service=None):
# Update plugin
self.plugin = self.client.get(self.plugin_id)
# get its connected accounts
account_edges = self.plugin.get_edges('account')
for account_edge in account_edges:
print("ACCOUNTS OF PLUGIN", account_edge)
for account_edge in account_edges:
account = account_edge.traverse(self.plugin)
if account.service == service:
print("Account to use", account)
return account
return None
# if multiple accounts are used
if len(account_edges) > 1 and service:
for account_edge in account_edges:
account = account_edge.traverse(self.plugin)
if account.service == service:
return account
# assumes there is only one account
elif len(account_edges) == 1:
return account_edges[0].traverse(self.plugin)
def ask_user_for_accounts(self, service, view, oauth_url=None):
# start userActionNeeded flow
# TODO: Handle timeouts
vars = {
'state': RUN_USER_ACTION_NEEDED,
'oAuthUrl': oauth_url
......@@ -52,118 +77,120 @@ class PluginFlow:
self._set_run_view(view)
# poll here
while True:
sleep(0.6)
start_time = time()
# handle timeouts
while RUN_USER_ACTION_TIMEOUT > time() - start_time:
sleep(RUN_STATE_POLLING_INTERVAL)
run_state = self._get_run_state()
if run_state == RUN_USER_ACTION_COMPLETED:
# Now the client has set up the account as an edge to the plugin
self.plugin = self.client.get(self.plugin_id)
return self.get_account_from_plugin(service)
return self.get_account_from_plugin(service=service)
raise Exception("PluginFlow: User input timeout")
def set_account_vars(self, vars_dictionary, service=None):
account = self.get_account_from_plugin(service=service)
if account:
for k,v in vars_dictionary.items():
setattr(account, k, v)
self.client.update_item(account)
logging.warning(f"ACCOUNT updated: {account.__dict__}")
else:
# Create account item
account = Account(**vars_dictionary)
# Save accounts as an edge to the plugin
self.client.create(account)
logging.warning(f"ACCOUNT created: {account.__dict__}")
self.plugin.add_edge('account', account)
self.plugin.update(self.client)
# =======================================
# ---------- USER CLIENT METHODS -------------
def install_plugin(self, name, container):
plugin = Plugin(name=name, container=container)
self.client.create(plugin)
# set in instance
self.plugin_id = plugin.id
self.plugin = plugin
# TODO: Add an edge from the plugin to the desired CVUStoredDefinitions
# start_plugin.view = createLoginCVUs()
return plugin.id
def trigger_plugin(self, interval=None):
starter = StartPlugin(targetItemId=self.plugin_id, container=self.plugin.container, state=RUN_IDLE, interval=interval)
# add cvus here
self.client.create(starter)
self.run_id = starter.id
print(f"Started plugin {self.plugin.name} - {self.plugin_id} and run id {self.run_id}")
def terminate_run(self):
self._set_run_vars({'interval': None})
# =======================================
# ---------- COMMON METHODS -------------
def get_CVU(self, run):
try:
run = self.get_run(expanded=True)
return run.get_edges('view')[0]
except:
return None
def initialized(self):
logging.warning("PLUGIN run is initialized")
self.state = RUN_INITIALIZED
if self.run_id and self.plugin:
self._set_run_vars({'state':RUN_INITIALIZED})
def started(self):
logging.warning("PLUGIN run is started")
self.state = RUN_STARTED
if self.run_id and self.plugin:
self._set_run_vars({'state':RUN_STARTED})
def failed(self, error):
logging.error(f"PLUGIN run is failed: {error}")
print("Exception while running plugin:", error)
self.state = RUN_FAILED
if self.run_id and self.plugin:
self._set_run_vars({'state':RUN_FAILED, 'message': str(error)})
def completed(self):
logging.warning("PLUGIN run is completed")
self.state = RUN_COMPLETED
if self.run_id and self.plugin:
self._set_run_vars({'state':RUN_COMPLETED})
# =======================================
# --------- USER CLIENT METHODS ---------
def install_plugin(self, name, container):
plugin = Plugin(name=name, container=container)
self.client.create(plugin)
# set in instance
self.plugin_id = plugin.id
self.plugin = plugin
# TODO: Add an edge from the plugin to the desired CVUStoredDefinitions
# start_plugin.view = createLoginCVUs()
return plugin.id
def complete_user_action(self):
self._set_run_vars({'state': RUN_USER_ACTION_COMPLETED})
def trigger_plugin(self):
starter = StartPlugin(targetItemId=self.plugin_id, container=self.plugin.container, state=RUN_IDLE)
# add cvus here
self.client.create(starter)
self.run_id = starter.id
print(f"Started plugin {self.plugin.name} - {self.plugin.id} and run id {self.run_id}")
def set_account(self, identifier, secret, service, code=None):
# Create account item
account = Account(identifier=identifier, secret=secret, service=service, code=code)
# Save accounts as an edge to the plugin
self.client.create(account)
print('ACCOUNT', account.__dict__)
self.plugin.add_edge('account', account)
self.plugin.update(self.client)
def get_CVUs(self, run):
run = run.expand(self.client)
return run.get_edges('view')
# Dummy UI Client Simulation
def run_plugin(self):
self.trigger_plugin()
def is_user_action_needed(self):
return self._get_run_state() == RUN_USER_ACTION_NEEDED
# START POLLING
while True:
# take a breath
sleep(0.6)
run = self._get_run()
def is_completed(self):
return self._get_run_state() == RUN_COMPLETED
if run.state == RUN_COMPLETED:
print("# Completed plugin run. Well done")
return True
def is_daemon(self):
run = self.get_run(expanded=False)
return run.interval and run.interval > 0
if run.state == RUN_USER_ACTION_NEEDED:
# Extract views (CVUs) from the run. It should be added by the plugin.
print(run.state)
views = self.get_CVUs(run)
if views:
for view in views:
print(view.__dict__)
username = input("Username: ")
password = input("Password: ")
oauth_code = input("OAuth Code: ")
## DISPLAYS CVU & AUTHENTICATES THE USER
sleep(3)
self.set_account(username, password, 'demo-service', code=oauth_code)
# Notify plugin
self._set_run_vars({'state': RUN_USER_ACTION_COMPLETED})
def get_run(self, expanded=False):
return self.client.get(self.run_id, expanded=expanded)
# =======================================
# --------- INTERNAL METHODS ------------
def _get_run(self):
return self.client.get(self.run_id, expanded=False)
def _get_run_state(self):
start_plugin = self._get_run()
start_plugin = self.get_run()
return start_plugin.state
def _set_run_vars(self, vars):
start_plugin = self.client.get(self.run_id, expanded=False)
for k,v in vars.items():
if v:
setattr(start_plugin, k, v)
setattr(start_plugin, k, v)
self.client.update_item(start_plugin)
def _set_run_view(self, view_name):
......@@ -173,15 +200,23 @@ class PluginFlow:
if v.name == view_name:
found_cvu = v
if not found_cvu:
print("CVU NOT FOUND, Could not set CVU for user display")
# raise Exception("CVU NOT FOUND")
logging.error("CVU is NOT FOUND")
return
run = self._get_run()
run.add_edge('view', found_cvu)
run.update()
run = self.get_run()
bound_CVU_edge = self.get_CVU(run) # index error here if there is no already bound CVU
if bound_CVU_edge:
logging.warning(f"Plugin Run already has a view. Updating with {view_name}")
bound_CVU_edge.target = found_cvu # update CVU
bound_CVU_edge.update(self.client) # having doubts if this really updates the existing edge
else:
logging.warning(f"Plugin Run does not have a view. Creating {view_name}")
run.add_edge('view', found_cvu)
run.update()
def _setup_schema(self):
self.client.add_to_schema(Account(identifier="", secret="", service="", code=""))
self.client.add_to_schema(Account(identifier="", secret="", service="", code="", refreshToken="", errorMessage=""))
self.client.add_to_schema(Plugin(name="", container=""))
self.client.add_to_schema(StartPlugin(targetItemId="", container="", state="", oAuthUrl=""))
\ No newline at end of file
self.client.add_to_schema(StartPlugin(targetItemId="", container="", state="", oAuthUrl="", interval=1))
\ No newline at end of file
import logging
from time import sleep
from plugin_template.plugin_flow.plugin_flow import PluginFlow
from plugin_template.service_api.service_api import ServiceAPI
SERVICE_NAME = 'demo-service'
NUM_LOGIN_TRIES = 3
class PluginTemplate(PluginFlow):
def __init__(self, client, plugin=None, start_plugin_id=None):
super().__init__(client=client, plugin=plugin, run_id=start_plugin_id)
print("PLUGIN run is initialized")
self.dummy_service = ServiceAPI()
# activate plugin flow
if plugin and start_plugin_id:
# init login process
# self.start_auth_oauth()
self.start_auth_userpass_and_two_factor()
if self.dummy_service.authenticated:
logging.warning("Authtentication success")
else:
logging.warning("Authtentication failure")
def run(self):
# MAIN PLUGIN RUN LOGIC HERE
# items = self.dummy_service.getItems()
# for item in items:
# memri_item = Item(**item)
# self.client.create(item)
pass
#====================================
# Example authentication methods
# Existing account login check
def check_existing_auth(self):
account = self.get_account_from_plugin(SERVICE_NAME)
if account:
result = self.dummy_service.login(account.identifier, account.secret)
if result['success']:
logging.warning("Logged in with existing credentials")
return True
return False
# Example standard username password login flow
def start_auth_userpass_and_two_factor(self):
#
if self.check_existing_auth(): return True
for _ in range(3):
for _ in range(NUM_LOGIN_TRIES):
# we request username and password here
account = self.ask_user_for_accounts(SERVICE_NAME, 'username-password-view')
# check if they work
result = self.dummy_service.login(account.identifier, account.secret)
print("Checking login")
if result and result['success'] == True:
if result['2fa_required']:
for _ in range(3):
logging.warning("Authtentication requires 2FA")
for _ in range(NUM_LOGIN_TRIES):
account = self.ask_user_for_accounts(SERVICE_NAME, '2fa-view')
result = self.dummy_service.complete_2fa(account.code)
print("Checking 2fa")
if result['success']:
print("Access granted")
logging.warning("Authtentication 2FA success")
return True
# set error message for UI
self.set_account_vars({'errorMessage': 'Incorrect 2fa code'}, service=SERVICE_NAME)
else:
print("Access granted")
logging.warning("Authtentication success")
return True
# set error message for UI
self.set_account_vars({'errorMessage': 'Incorrect credentials'}, service=SERVICE_NAME)
# give up
return False
# Example OAuth flow
def start_auth_oauth(self):
if self.check_existing_auth(): return True
oauth_url = self.dummy_service.get_oauth_url(self.app_id, self.app_secret)
for i in range(3):
for i in range(NUM_LOGIN_TRIES):
account = self.ask_user_for_accounts(SERVICE_NAME, 'oauth-view', oauth_url)
result = self.dummy_service.get_oauth_token(account.code)
print("Checking login")
if result['success']:
print("Access granted")
self.set_account('oauth', result['token'], SERVICE_NAME) # or "username", "passord"
# set new token
self.set_account_vars({
'refreshToken': result['refreshToken'],
'secret': result['token']
})
return True
return False
def run(self):
print('Running')
self.started()
if self.dummy_service.authenticated:
print("Authtentication success")
# Import stuff
# items = self.dummy_service.getItems()
# for item in items:
# self.client.create(item)
print("API", self.dummy_service.__dict__)
items = self.client.search({'type': 'Plugin'})
for item in items:
print("PLUGIN ITEM", item.__dict__)
print("PLUGIN run is completed")
self.completed()
\ No newline at end of file
# set error message for UI
self.set_account_vars({'errorMessage': 'Incorrect credentials'}, SERVICE_NAME)
return False
\ No newline at end of file
from random import randint
# A dummy API class
# This is to be replaced with an actual API client for a service
class ServiceAPI:
def __init__(self, app_id=None, app_secret=None):
......@@ -37,7 +39,8 @@ class ServiceAPI:
self.authenticated = True
result = {
'success': success,
'token': f"service.com/access_token/for/corresponding/code/{code}" if success else ""
'token': f"service.com/access_token/for/corresponding/code/{code}" if success else "",
'refreshToken': f"service.com/refresh_token/for/corresponding/code/{code}" if success else ""
}
return result
......
......@@ -532,7 +532,7 @@ dev = ["pre-commit", "tox"]
[[package]]
name = "prompt-toolkit"
version = "3.0.18"
version = "3.0.19"
description = "Library for building powerful interactive command lines in Python"
category = "main"
optional = false
......@@ -575,7 +575,7 @@ python-versions = ">=3.5"
[[package]]
name = "pymemri"
version = "0.0.6"
version = "0.0.7"
description = "Python client for the memri data"
category = "main"
optional = false
......@@ -596,7 +596,7 @@ tqdm = "*"
type = "git"
url = "https://gitlab.memri.io/alpdeniz/pymemri"
reference = "demo"
resolved_reference = "d3a1a0ffec0e486b2d12e476d0bab75a88cb6788"
resolved_reference = "de23b528adb45b5913d3fecee9967c1acd2d2202"
[[package]]
name = "pyparsing"
......@@ -1166,8 +1166,8 @@ pluggy = [
{file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
]
prompt-toolkit = [
{file = "prompt_toolkit-3.0.18-py3-none-any.whl", hash = "sha256:bf00f22079f5fadc949f42ae8ff7f05702826a97059ffcc6281036ad40ac6f04"},
{file = "prompt_toolkit-3.0.18.tar.gz", hash = "sha256:e1b4f11b9336a28fa11810bc623c357420f69dfdb6d2dac41ca2c21a55c033bc"},
{file = "prompt_toolkit-3.0.19-py3-none-any.whl", hash = "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88"},
{file = "prompt_toolkit-3.0.19.tar.gz", hash = "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f"},
]
ptyprocess = [
{file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
......
[tool.poetry]
name = "plugin_template"
version = "0.1.0"
version = "0.1.1"
description = "Memri Plugin Template"
authors = ["Alp Deniz Ogut <alpdeniz@protonmail.com>"]
......
from plugin_template.plugin_flow.plugin_flow import PluginFlow
from pymemri.pod.client import PodClient
import sys, logging
from time import sleep
# SETUP
UI_POLLING_INTERVAL = 1
client = PodClient(url=sys.argv[1], database_key="a"*64, owner_key="plugin_test")
pf = PluginFlow(client)
# INSTALL
plugin_id = pf.install_plugin(name="Plugin Template", container=sys.argv[2])
# pf.set_account("username","pass", "test-service")
# RUN
# Dummy UI Client Simulation
def run_plugin(plugin_flow, interval=None):
service_name = 'demo-service' # or None if it is only one account is used
plugin_flow.trigger_plugin(interval=interval)
# START POLLING
while True:
# take a breath
sleep(UI_POLLING_INTERVAL)
run = plugin_flow.get_run()
if plugin_flow.is_daemon():
logging.warning("Started plugin in daemon mode.")
return True
if plugin_flow.is_completed():
logging.warning("Completed plugin run. Well done.")
return True
if plugin_flow.is_user_action_needed():
##==============================
## THIS IS THE UI PART
# Gets the active account
# Gets the active View (CVU)
# Displays the view to the user
# Gets user input
# Sets / Modifies the account
# Changes the run state
## ==== BEGIN USER INTERFACE ====
# get active account
account = plugin_flow.get_account_from_plugin(service=service_name)
## DISPLAYS CVU & GETS USER INPUT
view = plugin_flow.get_CVU(run)
# This error message is displayed inside UI if exists
if account and account.errorMessage: print(account.errorMessage)
# User input - normally via CVU (view)
username = input("Username: ")
password = input("Password: ")
oauth_code = input("OAuth Code: ")
sleep(3)
# Set account
plugin_flow.set_account_vars({
'service': service_name,
'identifier': username,
'secret': password,
'code': oauth_code,
'errorMessage': None # reset error on each input
}, service=service_name)
## ==== END USER INTERFACE ====
# Notify the plugin, continue the auth flow
plugin_flow.complete_user_action()
# start plugin
run_plugin(pf)
......@@ -18,7 +18,7 @@ def main(pod_url=None, database_key=None, owner_key=None, plugin_item=None, star
plugin = PluginTemplate(client, plugin, start_plugin_id)
try:
plugin.run()
plugin.start()
except Exception as e:
plugin.failed(e)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment