Merge branch 'plugin_base' into 'dev'

Plugin base

See merge request !16

docs/_data/sidebars/home_sidebar.yml

@@ -15,6 +15,9 @@ entries:
     - output: web,pdf
       title: ItemBase
       url: itembase.html
+    - output: web,pdf
+      title: Plugin
+      url: plugin.pluginbase.html
     output: web
     title: Getting Started
   output: web

docs/_includes/sidebar.html

@@ -17,7 +17,7 @@
   {% if folder.output contains "web" %}
   <li>
       <a href="#" class="firstLevelHeader">{{ folder.title }}</a>
-      <ul>
+      <ul class="sidebarlist" style="display:block;">
           {% for folderitem in folder.folderitems %}
           {% if folderitem.output contains "web" %}
           {% if folderitem.external_url %}

docs/assets/css/memri_styles.css

@@ -50,6 +50,8 @@ h4 code {
     /* background-color: white; */
 }
 
+
+
 #mysidebar .firstLevelHeader {
     text-transform: uppercase;
     font-style: normal;
@@ -70,6 +72,14 @@ h4 code {
     background-color: white !important;
 }
 
+.output_wrapper{
+    padding-top: 8px;
+}
+
+.sidebarlist{
+    display:block !important;
+}
+
 .nav ul li a{
     color: #554C63 !important;
     background-color: white !important;

docs/indexers.indexer.html

@@ -47,7 +47,7 @@ nb_path: "nbs/indexers.indexer.ipynb"
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h2 id="IndexerBase" class="doc_header"><code>class</code> <code>IndexerBase</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/indexers/indexer.py#L20" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>IndexerBase</code>(<strong><code>indexerClass</code></strong>=<em><code>None</code></em>, <strong>*<code>args</code></strong>, <strong>**<code>kwargs</code></strong>) :: <code>Indexer</code></p>
+<h2 id="IndexerBase" class="doc_header"><code>class</code> <code>IndexerBase</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/indexers/indexer.py#L20" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>IndexerBase</code>(<strong><code>pluginClass</code></strong>=<em><code>None</code></em>, <strong>*<code>args</code></strong>, <strong>**<code>kwargs</code></strong>) :: <code>Indexer</code></p>
 </blockquote>
 <p>Item is the baseclass for all of the data classes.</p>
 
@@ -173,7 +173,7 @@ when retrieving it from the database.</p>
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="run_integrator" class="doc_header"><code>run_integrator</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/indexers/indexer.py#L97" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>run_integrator</code>(<strong><code>environ</code></strong>=<em><code>None</code></em>, <strong><code>pod_full_address</code></strong>=<em><code>None</code></em>, <strong><code>integrator_run_id</code></strong>=<em><code>None</code></em>, <strong><code>database_key</code></strong>=<em><code>None</code></em>, <strong><code>owner_key</code></strong>=<em><code>None</code></em>, <strong><code>verbose</code></strong>=<em><code>False</code></em>)</p>
+<h4 id="run_integrator" class="doc_header"><code>run_integrator</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/indexers/indexer.py#L96" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>run_integrator</code>(<strong><code>environ</code></strong>=<em><code>None</code></em>, <strong><code>pod_full_address</code></strong>=<em><code>None</code></em>, <strong><code>integrator_run_id</code></strong>=<em><code>None</code></em>, <strong><code>database_key</code></strong>=<em><code>None</code></em>, <strong><code>owner_key</code></strong>=<em><code>None</code></em>, <strong><code>verbose</code></strong>=<em><code>False</code></em>)</p>
 </blockquote>
 <p>Runs an integrator, you can either provide the run settings as parameters to this function (for local testing)
 or via environment variables (this is how the pod communicates with integrators).</p>
@@ -200,7 +200,7 @@ or via environment variables (this is how the pod communicates with integrators)
 <span class="k">def</span> <span class="nf">create_toy_dataset</span><span class="p">(</span><span class="n">client</span><span class="p">):</span>
     <span class="n">location</span> <span class="o">=</span> <span class="n">Location</span><span class="o">.</span><span class="n">from_data</span><span class="p">(</span><span class="n">latitude</span><span class="o">=-</span><span class="mf">37.81</span><span class="p">,</span> <span class="n">longitude</span><span class="o">=</span><span class="mf">144.96</span><span class="p">)</span>
     <span class="n">address</span> <span class="o">=</span> <span class="n">Address</span><span class="o">.</span><span class="n">from_data</span><span class="p">()</span>
-    <span class="n">indexer</span> <span class="o">=</span> <span class="n">Indexer</span><span class="o">.</span><span class="n">from_data</span><span class="p">(</span><span class="n">indexerClass</span><span class="o">=</span><span class="s2">&quot;GeoIndexer&quot;</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;GeoIndexer&quot;</span><span class="p">)</span>
+    <span class="n">indexer</span> <span class="o">=</span> <span class="n">Indexer</span><span class="o">.</span><span class="n">from_data</span><span class="p">(</span><span class="n">pluginClass</span><span class="o">=</span><span class="s2">&quot;GeoIndexer&quot;</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;GeoIndexer&quot;</span><span class="p">)</span>
     <span class="n">indexer_run</span> <span class="o">=</span> <span class="n">IndexerRun</span><span class="o">.</span><span class="n">from_data</span><span class="p">(</span><span class="n">progress</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">targetDataType</span><span class="o">=</span><span class="s2">&quot;Address&quot;</span><span class="p">)</span>
     
     <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">location</span><span class="p">,</span> <span class="n">address</span><span class="p">,</span> <span class="n">indexer</span><span class="p">,</span> <span class="n">indexer_run</span><span class="p">]:</span> <span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
@@ -233,7 +233,7 @@ or via environment variables (this is how the pod communicates with integrators)
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="generate_test_env" class="doc_header"><code>generate_test_env</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/indexers/indexer.py#L129" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>generate_test_env</code>(<strong><code>client</code></strong>, <strong><code>indexer_run</code></strong>)</p>
+<h4 id="generate_test_env" class="doc_header"><code>generate_test_env</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/indexers/indexer.py#L128" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>generate_test_env</code>(<strong><code>client</code></strong>, <strong><code>indexer_run</code></strong>)</p>
 </blockquote>
 
 </div>

docs/itembase.html

@@ -47,7 +47,7 @@ nb_path: "nbs/itembase.ipynb"
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h2 id="Edge" class="doc_header"><code>class</code> <code>Edge</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L58" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>Edge</code>(<strong><code>source</code></strong>, <strong><code>target</code></strong>, <strong><code>_type</code></strong>, <strong><code>label</code></strong>=<em><code>None</code></em>, <strong><code>sequence</code></strong>=<em><code>None</code></em>, <strong><code>created</code></strong>=<em><code>False</code></em>, <strong><code>reverse</code></strong>=<em><code>True</code></em>)</p>
+<h2 id="Edge" class="doc_header"><code>class</code> <code>Edge</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L59" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>Edge</code>(<strong><code>source</code></strong>, <strong><code>target</code></strong>, <strong><code>_type</code></strong>, <strong><code>label</code></strong>=<em><code>None</code></em>, <strong><code>sequence</code></strong>=<em><code>None</code></em>, <strong><code>created</code></strong>=<em><code>False</code></em>, <strong><code>reverse</code></strong>=<em><code>True</code></em>)</p>
 </blockquote>
 <p>Edges makes a link between two <a href="/pymemri/itembase.html#ItemBase"><code>ItemBase</code></a> Items. You won't use this class a lot in practice, as edges are
 abstracted away for normal users. When items are retrieved from the database, the edges are parsed automatically.
@@ -81,7 +81,7 @@ When you add an edge between to items within pymemri, you will often use <a href
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="Edge.traverse" class="doc_header"><code>Edge.traverse</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L94" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>Edge.traverse</code>(<strong><code>start</code></strong>)</p>
+<h4 id="Edge.traverse" class="doc_header"><code>Edge.traverse</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L95" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>Edge.traverse</code>(<strong><code>start</code></strong>)</p>
 </blockquote>
 <p>We can traverse an edge starting from the source to the target or vice versa. In practice we often call
 item.some_edge_type, which calls item.traverse(edgetype), which in turn calls this function.</p>
@@ -93,6 +93,25 @@ item.some_edge_type, which calls item.traverse(edgetype), which in turn calls th
 </div>
 </div>
 
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="n">ITEMBASE_PROPERTIES</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;dateAccessed&quot;</span><span class="p">,</span> <span class="s2">&quot;dateCreated&quot;</span><span class="p">,</span> <span class="s2">&quot;dateModified&quot;</span><span class="p">,</span> <span class="s2">&quot;deleted&quot;</span><span class="p">,</span> <span class="s2">&quot;externalId&quot;</span><span class="p">,</span> <span class="s2">&quot;itemDescription&quot;</span><span class="p">,</span>
+                     <span class="s2">&quot;starred&quot;</span><span class="p">,</span> <span class="s2">&quot;version&quot;</span><span class="p">,</span> <span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="s2">&quot;importJson&quot;</span><span class="p">,</span> <span class="s2">&quot;name&quot;</span><span class="p">,</span> <span class="s2">&quot;repository&quot;</span><span class="p">,</span> <span class="s2">&quot;icon&quot;</span><span class="p">,</span> <span class="s2">&quot;bundleImage&quot;</span><span class="p">,</span>
+                     <span class="s2">&quot;runDestination&quot;</span><span class="p">,</span> <span class="s2">&quot;pluginClass&quot;</span><span class="p">]</span>
+</pre></div>
+
+    </div>
+</div>
+</div>
+
 </div>
     {% endraw %}
 
@@ -107,7 +126,7 @@ item.some_edge_type, which calls item.traverse(edgetype), which in turn calls th
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h2 id="ItemBase" class="doc_header"><code>class</code> <code>ItemBase</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L106" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>ItemBase</code>(<strong><code>id</code></strong>=<em><code>None</code></em>)</p>
+<h2 id="ItemBase" class="doc_header"><code>class</code> <code>ItemBase</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L107" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>ItemBase</code>(<strong><code>id</code></strong>=<em><code>None</code></em>)</p>
 </blockquote>
 <p>Provides a base class for all items. All items in the schema inherit from this class, and it provides some
 basic functionality for consistency and to enable easier usage.</p>
@@ -140,7 +159,7 @@ basic functionality for consistency and to enable easier usage.</p>
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h2 id="Item" class="doc_header"><code>class</code> <code>Item</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L228" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>Item</code>(<strong><code>dateAccessed</code></strong>=<em><code>None</code></em>, <strong><code>dateCreated</code></strong>=<em><code>None</code></em>, <strong><code>dateModified</code></strong>=<em><code>None</code></em>, <strong><code>deleted</code></strong>=<em><code>None</code></em>, <strong><code>externalId</code></strong>=<em><code>None</code></em>, <strong><code>itemDescription</code></strong>=<em><code>None</code></em>, <strong><code>starred</code></strong>=<em><code>None</code></em>, <strong><code>version</code></strong>=<em><code>None</code></em>, <strong><code>id</code></strong>=<em><code>None</code></em>, <strong><code>importJson</code></strong>=<em><code>None</code></em>, <strong><code>changelog</code></strong>=<em><code>None</code></em>, <strong><code>label</code></strong>=<em><code>None</code></em>, <strong><code>genericAttribute</code></strong>=<em><code>None</code></em>, <strong><code>measure</code></strong>=<em><code>None</code></em>, <strong><code>sharedWith</code></strong>=<em><code>None</code></em>) :: <a href="/pymemri/itembase.html#ItemBase"><code>ItemBase</code></a></p>
+<h2 id="Item" class="doc_header"><code>class</code> <code>Item</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L226" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>Item</code>(<strong>**<code>kwargs</code></strong>) :: <a href="/pymemri/itembase.html#ItemBase"><code>ItemBase</code></a></p>
 </blockquote>
 <p>Item is the baseclass for all of the data classes.</p>
 
@@ -172,7 +191,7 @@ basic functionality for consistency and to enable easier usage.</p>
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="ItemBase.add_edge" class="doc_header"><code>ItemBase.add_edge</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L135" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>ItemBase.add_edge</code>(<strong><code>name</code></strong>, <strong><code>val</code></strong>)</p>
+<h4 id="ItemBase.add_edge" class="doc_header"><code>ItemBase.add_edge</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L133" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>ItemBase.add_edge</code>(<strong><code>name</code></strong>, <strong><code>val</code></strong>)</p>
 </blockquote>
 <p>Creates an edge of type name and makes it point to val</p>
 
@@ -197,7 +216,7 @@ basic functionality for consistency and to enable easier usage.</p>
 
 
 <div class="output_markdown rendered_html output_subarea ">
-<h4 id="ItemBase.is_expanded" class="doc_header"><code>ItemBase.is_expanded</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L144" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>ItemBase.is_expanded</code>()</p>
+<h4 id="ItemBase.is_expanded" class="doc_header"><code>ItemBase.is_expanded</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L142" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>ItemBase.is_expanded</code>()</p>
 </blockquote>
 <p>returns whether the node is expanded. An expanded node retrieved nodes that are
 <em>directly</em> connected to it
@@ -213,25 +232,55 @@ from the pod, and stored their values via edges in the object.</p>
 </div>
     {% endraw %}
 
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<h1 id="Usage">Usage<a class="anchor-link" href="#Usage"> </a></h1>
+</div>
+</div>
+</div>
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<p>With the <a href="/pymemri/itembase.html#ItemBase"><code>ItemBase</code></a> and <a href="/pymemri/itembase.html#Edge"><code>Edge</code></a> classes we can create an item and its surrounding graph. The schema is defined in schema.py, in general we want to use the from_data staticmethod to generate new items, because it ensures that edges are linked from both the source and the target object. Let's make a new item and add it to the pod.</p>
+
+</div>
+</div>
+</div>
     {% raw %}
     
 <div class="cell border-box-sizing code_cell rendered">
+<div class="input">
 
-<div class="output_wrapper">
-<div class="output">
-
-<div class="output_area">
-
-
-<div class="output_markdown rendered_html output_subarea ">
-<h4 id="ItemBase.expand" class="doc_header"><code>ItemBase.expand</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L184" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>ItemBase.expand</code>(<strong><code>api</code></strong>)</p>
-</blockquote>
-<p>Expands a node (retrieves all directly connected nodes ands adds to object).</p>
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">MyItem</span><span class="p">(</span><span class="n">Item</span><span class="p">):</span>
+    <span class="n">properties</span> <span class="o">=</span> <span class="n">Item</span><span class="o">.</span><span class="n">properties</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">,</span> <span class="s2">&quot;age&quot;</span><span class="p">]</span>
+    <span class="n">edges</span> <span class="o">=</span> <span class="n">Item</span><span class="o">.</span><span class="n">edges</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;friend&quot;</span><span class="p">]</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span><span class="n">friend</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">age</span> <span class="o">=</span> <span class="n">age</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">friend</span> <span class="o">=</span> <span class="n">fried</span> <span class="k">if</span> <span class="n">friend</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[]</span>
+</pre></div>
 
+    </div>
+</div>
 </div>
 
 </div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
 
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">pymemri.pod.client</span> <span class="kn">import</span> <span class="n">PodClient</span>
+<span class="n">client</span> <span class="o">=</span> <span class="n">PodClient</span><span class="p">()</span>
+</pre></div>
+
+    </div>
 </div>
 </div>
 
@@ -241,43 +290,90 @@ from the pod, and stored their values via edges in the object.</p>
     {% raw %}
     
 <div class="cell border-box-sizing code_cell rendered">
+<div class="input">
 
-<div class="output_wrapper">
-<div class="output">
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="k">assert</span> <span class="n">client</span><span class="o">.</span><span class="n">add_to_schema</span><span class="p">(</span><span class="n">MyItem</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;abc&quot;</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
+</pre></div>
 
-<div class="output_area">
+    </div>
+</div>
+</div>
 
+</div>
+    {% endraw %}
 
-<div class="output_markdown rendered_html output_subarea ">
-<h4 id="ItemBase.inherit_funcs" class="doc_header"><code>ItemBase.inherit_funcs</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/data/itembase.py#L220" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>ItemBase.inherit_funcs</code>(<strong><code>other</code></strong>)</p>
-</blockquote>
-<p>This function can be used to inherit new functionalities from a subclass. This is a patch to solve
-the fact that python does provide extensions of classes that are defined in a different file that are
-dynamic enough for our use case.</p>
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="n">MyItem</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;me&quot;</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">30</span><span class="p">)</span>
+<span class="n">x</span><span class="o">.</span><span class="n">add_edge</span><span class="p">(</span><span class="s2">&quot;friend&quot;</span><span class="p">,</span> <span class="n">MyItem</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;my friend&quot;</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">31</span><span class="p">))</span>
+<span class="n">x</span><span class="o">.</span><span class="n">add_edge</span><span class="p">(</span><span class="s2">&quot;friend&quot;</span><span class="p">,</span> <span class="n">MyItem</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;my friend2&quot;</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">32</span><span class="p">))</span>
+</pre></div>
 
+    </div>
+</div>
 </div>
 
 </div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
 
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="k">assert</span> <span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+</pre></div>
+
+    </div>
 </div>
 </div>
 
 </div>
     {% endraw %}
 
-<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
-<div class="text_cell_render border-box-sizing rendered_html">
-<h1 id="Usage">Usage<a class="anchor-link" href="#Usage"> </a></h1>
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="n">y</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
+</pre></div>
+
+    </div>
 </div>
 </div>
+
 </div>
-<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
-<div class="text_cell_render border-box-sizing rendered_html">
-<p>With the <a href="/pymemri/itembase.html#ItemBase"><code>ItemBase</code></a> and <a href="/pymemri/itembase.html#Edge"><code>Edge</code></a> classes we can create an item and its surrounding graph. The schema is defined in schema.py, in general we want to use the from_data staticmethod to generate new items, because it ensures that edges are linked from both the source and the target object. Let's make an email item and create it in the pod.</p>
+    {% endraw %}
 
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="o">.</span><span class="n">friend</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span>
+</pre></div>
+
+    </div>
 </div>
 </div>
+
 </div>
+    {% endraw %}
+
     {% raw %}
     
 <div class="cell border-box-sizing code_cell rendered">
@@ -285,13 +381,29 @@ dynamic enough for our use case.</p>
 
 <div class="inner_cell">
     <div class="input_area">
-<div class=" highlight hl-ipython3"><pre><span></span><span class="n">item</span> <span class="o">=</span> <span class="n">EmailMessage</span><span class="o">.</span><span class="n">from_data</span><span class="p">(</span><span class="n">content</span><span class="o">=</span><span class="s2">&quot;example content field&quot;</span><span class="p">)</span>
+<div class=" highlight hl-ipython3"><pre><span></span><span class="n">y</span><span class="o">.</span><span class="n">friend</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">name</span>
 </pre></div>
 
     </div>
 </div>
 </div>
 
+<div class="output_wrapper">
+<div class="output">
+
+<div class="output_area">
+
+
+
+<div class="output_text output_subarea output_execute_result">
+<pre>&#39;my friend&#39;</pre>
+</div>
+
+</div>
+
+</div>
+</div>
+
 </div>
     {% endraw %}
 

docs/plugin.pluginbase.html

+---
+
+title: Plugins
+
+
+keywords: fastai
+sidebar: home_sidebar
+
+
+
+nb_path: "nbs/plugin.pluginbase.ipynb"
+---
+<!--
+
+#################################################
+### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
+#################################################
+# file to edit: nbs/plugin.pluginbase.ipynb
+# command to build the docs after a change: nbdev_build_docs
+
+-->
+
+<div class="container" id="notebook-container">
+        
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+
+</div>
+    {% endraw %}
+
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<p>Let's use the following plugin as an example of how we can start plugins.</p>
+
+</div>
+</div>
+</div>
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MyPlugin</span><span class="p">(</span><span class="n">PluginBase</span><span class="p">):</span>
+    <span class="n">properties</span> <span class="o">=</span> <span class="n">PluginBase</span><span class="o">.</span><span class="n">properties</span>
+    <span class="n">edges</span><span class="o">=</span> <span class="n">PluginBase</span><span class="o">.</span><span class="n">edges</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pluginPackage</span><span class="o">=</span><span class="s2">&quot;pymemri.plugin.pluginbase&quot;</span>
+
+    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">client</span><span class="p">):</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;running&quot;</span><span class="p">)</span>
+</pre></div>
+
+</div>
+</div>
+</div>
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">pymemri.pod.client</span> <span class="kn">import</span> <span class="n">PodClient</span>
+<span class="n">client</span> <span class="o">=</span> <span class="n">PodClient</span><span class="p">()</span>
+</pre></div>
+
+    </div>
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="k">assert</span> <span class="n">client</span><span class="o">.</span><span class="n">add_to_schema</span><span class="p">(</span><span class="n">MyPlugin</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;abc&quot;</span><span class="p">,</span> <span class="n">data_query</span><span class="o">=</span><span class="s2">&quot;abc&quot;</span><span class="p">))</span>
+<span class="k">assert</span> <span class="n">client</span><span class="o">.</span><span class="n">add_to_schema</span><span class="p">(</span><span class="n">PluginRun</span><span class="p">())</span>
+</pre></div>
+
+    </div>
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="n">plugin</span> <span class="o">=</span> <span class="n">MyPlugin</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;abc&quot;</span><span class="p">,</span> <span class="n">data_query</span><span class="o">=</span><span class="s2">&quot;abc&quot;</span><span class="p">)</span>
+<span class="n">run</span> <span class="o">=</span> <span class="n">PluginRun</span><span class="p">()</span>
+<span class="n">run</span><span class="o">.</span><span class="n">add_edge</span><span class="p">(</span><span class="s2">&quot;plugin&quot;</span><span class="p">,</span> <span class="n">plugin</span><span class="p">)</span>
+</pre></div>
+
+    </div>
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">run</span><span class="p">)</span>
+<span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">plugin</span><span class="p">)</span>
+<span class="n">client</span><span class="o">.</span><span class="n">create_edge</span><span class="p">(</span><span class="n">run</span><span class="o">.</span><span class="n">get_edges</span><span class="p">(</span><span class="s2">&quot;plugin&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
+</pre></div>
+
+    </div>
+</div>
+</div>
+
+<div class="output_wrapper">
+<div class="output">
+
+<div class="output_area">
+
+
+
+<div class="output_text output_subarea output_execute_result">
+<pre>True</pre>
+</div>
+
+</div>
+
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<h1 id="Running-your-plugin">Running your plugin<a class="anchor-link" href="#Running-your-plugin"> </a></h1>
+</div>
+</div>
+</div>
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<p>Plugins can be started using the pymemri <a href="/pymemri/plugin.pluginbase.html#run_plugin"><code>run_plugin</code></a> CLI. To use the CLI, you can either pass your run arguments as parameters, or set them as environment variables. If both are set, the CLI will prefer the passed arguments.</p>
+
+</div>
+</div>
+</div>
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+
+</div>
+    {% endraw %}
+
+<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
+<div class="text_cell_render border-box-sizing rendered_html">
+<h2 id="CLI">CLI<a class="anchor-link" href="#CLI"> </a></h2>
+</div>
+</div>
+</div>
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+
+<div class="output_wrapper">
+<div class="output">
+
+<div class="output_area">
+
+
+<div class="output_markdown rendered_html output_subarea ">
+<h4 id="run_plugin" class="doc_header"><code>run_plugin</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/plugin/pluginbase.py#L122" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>run_plugin</code>(<strong><code>pod_full_address</code></strong>:<code>Param object at 0x7fc036ab0b50&gt;</code>=<em><code>None</code></em>, <strong><code>plugin_run_id</code></strong>:<code>Param object at 0x7fc035b497d0&gt;</code>=<em><code>None</code></em>, <strong><code>database_key</code></strong>:<code>Param object at 0x7fc036cea550&gt;</code>=<em><code>None</code></em>, <strong><code>owner_key</code></strong>:<code>Param object at 0x7fc036cea5d0&gt;</code>=<em><code>None</code></em>)</p>
+</blockquote>
+
+</div>
+
+</div>
+
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+
+</div>
+    {% endraw %}
+
+    {% raw %}
+    
+<div class="cell border-box-sizing code_cell rendered">
+<div class="input">
+
+<div class="inner_cell">
+    <div class="input_area">
+<div class=" highlight hl-ipython3"><pre><span></span><span class="o">!</span>run_plugin --pod_full_address<span class="o">=</span><span class="nv">$DEFAULT_POD_ADDRESS</span> --plugin_run_id<span class="o">=</span><span class="nv">$run</span>.id --owner_key<span class="o">=</span><span class="nv">$client</span>.owner_key <span class="err">\</span>
+            <span class="o">--</span><span class="n">database_key</span><span class="o">=</span><span class="err">$</span><span class="n">client</span><span class="o">.</span><span class="n">database_key</span>
+</pre></div>
+
+    </div>
+</div>
+</div>
+
+<div class="output_wrapper">
+<div class="output">
+
+<div class="output_area">
+
+<div class="output_subarea output_stream output_stdout output_text">
+<pre>Used arguments passed to `run_plugin()` (ignoring environment)
+pod_full_address=http://localhost:3030
+plugin_run_id=61274342a226fe9ecb5d0efeb50ebc64
+database_key=2275799687202554321586156394256447011413538728384161840804653842
+owner_key=7877526860202989541720539875881419199482049862866439964933736900
+
+running
+</pre>
+</div>
+</div>
+
+</div>
+</div>
+
+</div>
+    {% endraw %}
+
+</div>
+ 
+

docs/pod.client.html

@@ -243,17 +243,12 @@ Property myAge not defined in Schema (attempted to use it for json value 20) -->
 <div class="inner_cell">
     <div class="input_area">
 <div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">Dog</span><span class="p">(</span><span class="n">Item</span><span class="p">):</span>
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">age</span><span class="p">,</span> <span class="nb">id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">deleted</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="nb">id</span><span class="o">=</span><span class="nb">id</span><span class="p">,</span> <span class="n">deleted</span><span class="o">=</span><span class="n">deleted</span><span class="p">)</span>
+    <span class="n">properties</span> <span class="o">=</span> <span class="n">Item</span><span class="o">.</span><span class="n">properties</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">,</span> <span class="s2">&quot;age&quot;</span><span class="p">]</span>
+    <span class="n">edges</span> <span class="o">=</span> <span class="n">Item</span><span class="o">.</span><span class="n">edges</span> 
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">age</span> <span class="o">=</span> <span class="n">age</span>
-    
-    <span class="nd">@classmethod</span>
-    <span class="k">def</span> <span class="nf">from_json</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
-        <span class="nb">id</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
-        <span class="n">name</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;name&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
-        <span class="n">age</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;age&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
-        <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="nb">id</span><span class="o">=</span><span class="nb">id</span><span class="p">,</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">,</span><span class="n">age</span><span class="o">=</span><span class="n">age</span><span class="p">)</span>
 </pre></div>
 
     </div>
@@ -369,7 +364,7 @@ Property myAge not defined in Schema (attempted to use it for json value 20) -->
 
 
 <div class="output_text output_subarea output_execute_result">
-<pre>[{&#39;item&#39;: Person (#a63b11ebe6996b2efb00b75c42ccd930), &#39;name&#39;: &#39;sender&#39;}]</pre>
+<pre>[{&#39;item&#39;: Person (#806a88787321a9a81054e63d17ad2fbb), &#39;name&#39;: &#39;sender&#39;}]</pre>
 </div>
 
 </div>
@@ -545,9 +540,9 @@ Property myAge not defined in Schema (attempted to use it for json value 20) -->
 
 
 <div class="output_text output_subarea output_execute_result">
-<pre>[Person (#a63b11ebe6996b2efb00b75c42ccd930),
- Person (#16560d3dd6550a6aa58d88405fab177a),
- Person (#fa5e4f210ec5b928e7eb263fcabf252f)]</pre>
+<pre>[Person (#806a88787321a9a81054e63d17ad2fbb),
+ Person (#1f8e823a9b9ddaedf8dd37c9682beb0b),
+ Person (#cba8e451efdfe7be87dfeb10444ee670)]</pre>
 </div>
 
 </div>
@@ -753,23 +748,6 @@ Property myAge not defined in Schema (attempted to use it for json value 20) -->
 </div>
 </div>
 
-<div class="output_wrapper">
-<div class="output">
-
-<div class="output_area">
-
-<div class="output_subarea output_stream output_stdout output_text">
-<pre>creating
-creating photo file
-creating
-Uploaded file
-</pre>
-</div>
-</div>
-
-</div>
-</div>
-
 </div>
     {% endraw %}
 
@@ -812,7 +790,7 @@ Uploaded file
 
 
 <div class="output_text output_subarea output_execute_result">
-<pre>IPhoto (#e49db579f6427af57651fc3ca5efafcf)</pre>
+<pre>IPhoto (#424cce3929a63f5849c40c181e0ce252)</pre>
 </div>
 
 </div>

docs/sidebar.json

@@ -2,6 +2,7 @@
   "Getting Started": {
     "Overview": "/",
     "Pod Client": "pod.client.html",
-    "ItemBase": "itembase.html"
+    "ItemBase": "itembase.html",
+    "Plugin": "plugin.pluginbase.html"
   }
 }

nbs/indexers.indexer.ipynb

@@ -65,9 +65,9 @@
     "\n",
     "class IndexerBase(Indexer):\n",
     "\n",
-    "    def __init__(self, indexerClass=None, *args, **kwargs):\n",
-    "        if indexerClass is None: indexerClass=self.__class__.__name__\n",
-    "        super().__init__(indexerClass=indexerClass, *args, **kwargs)    \n",
+    "    def __init__(self, pluginClass=None, *args, **kwargs):\n",
+    "        if pluginClass is None: pluginClass=self.__class__.__name__\n",
+    "        super().__init__(pluginClass=pluginClass, *args, **kwargs)    \n",
     "    \n",
     "    def populate(self, client, items, edges=False):\n",
     "        new_items = [x for x in items if x.id is None]\n",
@@ -167,7 +167,6 @@
    "outputs": [],
    "source": [
     "# export\n",
-    "\n",
     "def run_integrator(environ=None, pod_full_address=None, integrator_run_id=None, database_key=None, owner_key=None,\n",
     "                   verbose=False):\n",
     "    \"\"\"Runs an integrator, you can either provide the run settings as parameters to this function (for local testing)\n",
@@ -240,7 +239,7 @@
     "def create_toy_dataset(client):\n",
     "    location = Location.from_data(latitude=-37.81, longitude=144.96)\n",
     "    address = Address.from_data()\n",
-    "    indexer = Indexer.from_data(indexerClass=\"GeoIndexer\", name=\"GeoIndexer\")\n",
+    "    indexer = Indexer.from_data(pluginClass=\"GeoIndexer\", name=\"GeoIndexer\")\n",
     "    indexer_run = IndexerRun.from_data(progress=0, targetDataType=\"Address\")\n",
     "    \n",
     "    for x in [location, address, indexer, indexer_run]: client.create(x)\n",

nbs/itembase.ipynb

@@ -36,7 +36,8 @@
     "from pymemri.imports import *\n",
     "\n",
     "ALL_EDGES = \"allEdges\"\n",
-    "SOURCE, TARGET, TYPE, EDGE_TYPE, LABEL, SEQUENCE = \"_source\", \"_target\", \"_type\", \"_type\", \"label\", \"sequence\""
+    "SOURCE, TARGET, TYPE, EDGE_TYPE, LABEL, SEQUENCE = \"_source\", \"_target\", \"_type\", \"_type\", \"label\", \"sequence\"\n",
+    "EDGE_KEY = \"allEdges\""
    ]
   },
   {
@@ -128,8 +129,8 @@
     "        _type = json[EDGE_TYPE]\n",
     "        json_target = json[TARGET]\n",
     "        target_type = json_target[\"_type\"]\n",
-    "        indexer_class = json_target.get(\"indexerClass\", None)\n",
-    "        target_constructor = get_constructor(target_type, indexer_class)\n",
+    "        plugin_class = json_target.get(\"pluginClass\", None)\n",
+    "        target_constructor = get_constructor(target_type, plugin_class)\n",
     "        target = target_constructor.from_json(json_target)\n",
     "        return cls(source=None, target=target, _type=_type)\n",
     "\n",
@@ -182,6 +183,17 @@
     "show_doc(Edge.traverse)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ITEMBASE_PROPERTIES = [\"dateAccessed\", \"dateCreated\", \"dateModified\", \"deleted\", \"externalId\", \"itemDescription\",\n",
+    "                     \"starred\", \"version\", \"id\", \"importJson\", \"name\", \"repository\", \"icon\", \"bundleImage\",\n",
+    "                     \"runDestination\", \"pluginClass\"]"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -204,9 +216,6 @@
     "        existing = cls.global_db.get(node.id)\n",
     "        if existing is None and node.id is not None:\n",
     "            cls.global_db.add(node)\n",
-    "\n",
-    "    def replace_self(self, other):\n",
-    "        self.__dict__.update(other.__dict__)\n",
     "        \n",
     "    def __getattribute__(self, name):\n",
     "        val = object.__getattribute__(self, name)\n",
@@ -268,18 +277,18 @@
     "        if res is None: return False\n",
     "        return len(res) == 1\n",
     "\n",
-    "    def expand(self, api):\n",
-    "        \"\"\"Expands a node (retrieves all directly connected nodes ands adds to object).\"\"\"\n",
-    "        self._expanded = True\n",
-    "        res = api.get(self.id, expanded=True)\n",
-    "        for edge_name in res.get_all_edge_names():\n",
-    "            edges = res.get_edges(edge_name)\n",
-    "            for e in edges:\n",
-    "                e.source = self\n",
-    "            self.__setattr__(edge_name, edges)\n",
+    "#     def expand(self, api):\n",
+    "#         \"\"\"Expands a node (retrieves all directly connected nodes ands adds to object).\"\"\"\n",
+    "#         self._expanded = True\n",
+    "#         res = api.get(self.id, expanded=True)\n",
+    "#         for edge_name in res.get_all_edge_names():\n",
+    "#             edges = res.get_edges(edge_name)\n",
+    "#             for e in edges:\n",
+    "#                 e.source = self\n",
+    "#             self.__setattr__(edge_name, edges)\n",
     "\n",
-    "        # self.edges = res.edges\n",
-    "        return self\n",
+    "#         # self.edges = res.edges\n",
+    "#         return self\n",
     "    \n",
     "    def __repr__(self):\n",
     "        id = self.id\n",
@@ -304,12 +313,12 @@
     "            v.source = res\n",
     "        return res\n",
     "    \n",
-    "    def inherit_funcs(self, other):\n",
-    "        \"\"\"This function can be used to inherit new functionalities from a subclass. This is a patch to solve\n",
-    "        the fact that python does provide extensions of classes that are defined in a different file that are\n",
-    "        dynamic enough for our use case.\"\"\"\n",
-    "        assert issubclass(other, self.__class__)\n",
-    "        self.__class__ = other"
+    "#     def inherit_funcs(self, other):\n",
+    "#         \"\"\"This function can be used to inherit new functionalities from a subclass. This is a patch to solve\n",
+    "#         the fact that python does provide extensions of classes that are defined in a different file that are\n",
+    "#         dynamic enough for our use case.\"\"\"\n",
+    "#         assert issubclass(other, self.__class__)\n",
+    "#         self.__class__ = other"
    ]
   },
   {
@@ -321,24 +330,93 @@
     "# export\n",
     "class Item(ItemBase):\n",
     "    \"\"\"Item is the baseclass for all of the data classes.\"\"\"\n",
-    "    def __init__(self, dateAccessed=None, dateCreated=None, dateModified=None, deleted=None,\n",
-    "                 externalId=None, itemDescription=None, starred=None, version=None, id=None, importJson=None,\n",
-    "                 changelog=None, label=None, genericAttribute=None, measure=None, sharedWith=None):\n",
-    "        super().__init__(id)\n",
-    "        self.dateAccessed = dateAccessed\n",
-    "        self.dateCreated = dateCreated\n",
-    "        self.dateModified = dateModified\n",
-    "        self.deleted = deleted\n",
-    "        self.externalId = externalId\n",
-    "        self.itemDescription = itemDescription\n",
-    "        self.starred = starred\n",
-    "        self.version = version\n",
-    "        self.importJson = importJson\n",
-    "        self.changelog = changelog if changelog is not None else []\n",
-    "        self.label = label if label is not None else []\n",
-    "        self.genericAttribute = genericAttribute if genericAttribute is not None else []\n",
-    "        self.measure = measure if measure is not None else []\n",
-    "        self.sharedWith = sharedWith if sharedWith is not None else []"
+    "    properties = [\"dateAccessed\", \"dateCreated\", \"dateModified\", \"deleted\", \"externalId\", \"itemDescription\",\n",
+    "                  \"starred\", \"version\", \"id\", \"importJson\", \"pluginClass\"]\n",
+    "    edges = [\"changelog\", \"label\", \"genericAttribute\", \"measure\", \"sharedWith\"]\n",
+    "    def __init__(self, **kwargs):\n",
+    "#     def __init__(self, dateAccessed=None, dateCreated=None, dateModified=None, deleted=None,\n",
+    "#              externalId=None, itemDescription=None, starred=None, version=None, id=None, importJson=None,\n",
+    "#              changelog=None, label=None, genericAttribute=None, measure=None, sharedWith=None):\n",
+    "    \n",
+    "        super().__init__(kwargs.get(\"id\"))\n",
+    "        for p in self.properties:\n",
+    "            if p == \"id\":\n",
+    "                continue\n",
+    "            setattr(self, p, kwargs.get(p, None))\n",
+    "            \n",
+    "        for e in self.edges:\n",
+    "            setattr(self, e, kwargs.get(e, []))\n",
+    "            \n",
+    "    @classmethod\n",
+    "    def parse_json(self, cls, json):    \n",
+    "        property_kwargs = Item.parse_properties(cls, json)\n",
+    "        edge_kwargs = Item.parse_edges(cls, json)\n",
+    "        return {**property_kwargs, **edge_kwargs}\n",
+    "    \n",
+    "    @classmethod\n",
+    "    def parse_properties(self, cls, json):\n",
+    "        return {p: json.get(p, None) for p in cls.properties}\n",
+    "    \n",
+    "    @classmethod\n",
+    "    def parse_edges(self, cls, json):\n",
+    "        all_edges = json.get(EDGE_KEY, None)\n",
+    "        edge_kwargs = dict()\n",
+    "        reverse_edges = [f\"~{e}\" for e in cls.edges]\n",
+    "        if all_edges is not None:\n",
+    "            for edge_json in all_edges:\n",
+    "                edge = Edge.from_json(edge_json)\n",
+    "\n",
+    "                if edge.type in self.edges + reverse_edges:\n",
+    "                    edge_name = self.remove_prefix(edge.type)\n",
+    "                    if edge_name in edge_kwargs:\n",
+    "                        edge_kwargs[edge_name] += [edge]\n",
+    "                    else:\n",
+    "                        edge_kwargs[edge_name] = [edge]\n",
+    "                        \n",
+    "        return edge_kwargs\n",
+    "    \n",
+    "    @classmethod\n",
+    "    def remove_prefix(s, prefix=\"~\"):\n",
+    "        return s[1:] if s[0] == \"`\" else s\n",
+    "    \n",
+    "    @classmethod\n",
+    "    def from_json(cls, json):\n",
+    "        kwargs = Item.parse_json(cls, json)\n",
+    "        res = cls(**kwargs)\n",
+    "        for e in res.get_all_edges(): e.source = res\n",
+    "        return res\n",
+    "    \n",
+    "\n",
+    "                    \n",
+    "#                 if edge._type == \"changelog\" or edge._type == \"~changelog\": \n",
+    "#                     changelog.append(edge)\n",
+    "#                 elif edge._type == \"label\" or edge._type == \"~label\": \n",
+    "#                     label.append(edge)\n",
+    "#                 elif edge._type == \"genericAttribute\" or edge._type == \"~genericAttribute\": \n",
+    "#                     genericAttribute.append(edge)\n",
+    "#                 elif edge._type == \"measure\" or edge._type == \"~measure\": \n",
+    "#                     measure.append(edge)\n",
+    "#                 elif edge._type == \"sharedWith\" or edge._type == \"~sharedWith\": \n",
+    "#                     sharedWith.append(edge)\n",
+    "#                 elif edge._type == \"indexerRun\" or edge._type == \"~indexerRun\": \n",
+    "#                     indexerRun.append(edge)\n",
+    "\n",
+    "\n",
+    "        \n",
+    "#         self.dateAccessed = dateAccessed\n",
+    "#         self.dateCreated = dateCreated\n",
+    "#         self.dateModified = dateModified\n",
+    "#         self.deleted = deleted\n",
+    "#         self.externalId = externalId\n",
+    "#         self.itemDescription = itemDescription\n",
+    "#         self.starred = starred\n",
+    "#         self.version = version\n",
+    "#         self.importJson = importJson\n",
+    "#         self.changelog = changelog if changelog is not None else []\n",
+    "#         self.label = label if label is not None else []\n",
+    "#         self.genericAttribute = genericAttribute if genericAttribute is not None else []\n",
+    "#         self.measure = measure if measure is not None else []\n",
+    "#         self.sharedWith = sharedWith if sharedWith is not None else []"
    ]
   },
   {
@@ -349,7 +427,7 @@
     {
      "data": {
       "text/markdown": [
-       "<h4 id=\"ItemBase.add_edge\" class=\"doc_header\"><code>ItemBase.add_edge</code><a href=\"__main__.py#L32\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
+       "<h4 id=\"ItemBase.add_edge\" class=\"doc_header\"><code>ItemBase.add_edge</code><a href=\"__main__.py#L29\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
        "\n",
        "> <code>ItemBase.add_edge</code>(**`name`**, **`val`**)\n",
        "\n",
@@ -375,7 +453,7 @@
     {
      "data": {
       "text/markdown": [
-       "<h4 id=\"ItemBase.is_expanded\" class=\"doc_header\"><code>ItemBase.is_expanded</code><a href=\"__main__.py#L41\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
+       "<h4 id=\"ItemBase.is_expanded\" class=\"doc_header\"><code>ItemBase.is_expanded</code><a href=\"__main__.py#L38\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
        "\n",
        "> <code>ItemBase.is_expanded</code>()\n",
        "\n",
@@ -395,72 +473,64 @@
     "show_doc(ItemBase.is_expanded)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Usage"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "With the `ItemBase` and `Edge` classes we can create an item and its surrounding graph. The schema is defined in schema.py, in general we want to use the from_data staticmethod to generate new items, because it ensures that edges are linked from both the source and the target object. Let's make a new item and add it to the pod."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/markdown": [
-       "<h4 id=\"ItemBase.expand\" class=\"doc_header\"><code>ItemBase.expand</code><a href=\"__main__.py#L81\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
-       "\n",
-       "> <code>ItemBase.expand</code>(**`api`**)\n",
-       "\n",
-       "Expands a node (retrieves all directly connected nodes ands adds to object)."
-      ],
-      "text/plain": [
-       "<IPython.core.display.Markdown object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "show_doc(ItemBase.expand)"
+    "class MyItem(Item):\n",
+    "    properties = Item.properties + [\"name\", \"age\"]\n",
+    "    edges = Item.edges + [\"friend\"]\n",
+    "    def __init__(self, name=None, age=None,friend=None, **kwargs):\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.name = name\n",
+    "        self.age = age\n",
+    "        self.friend = fried if friend is not None else []"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/markdown": [
-       "<h4 id=\"ItemBase.inherit_funcs\" class=\"doc_header\"><code>ItemBase.inherit_funcs</code><a href=\"__main__.py#L117\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
-       "\n",
-       "> <code>ItemBase.inherit_funcs</code>(**`other`**)\n",
-       "\n",
-       "This function can be used to inherit new functionalities from a subclass. This is a patch to solve\n",
-       "the fact that python does provide extensions of classes that are defined in a different file that are\n",
-       "dynamic enough for our use case."
-      ],
-      "text/plain": [
-       "<IPython.core.display.Markdown object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "show_doc(ItemBase.inherit_funcs)"
+    "from pymemri.pod.client import PodClient\n",
+    "client = PodClient()"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "# Usage"
+    "assert client.add_to_schema(MyItem(name=\"abc\", age=1))"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "With the `ItemBase` and `Edge` classes we can create an item and its surrounding graph. The schema is defined in schema.py, in general we want to use the from_data staticmethod to generate new items, because it ensures that edges are linked from both the source and the target object. Let's make an email item and create it in the pod."
+    "x = MyItem(name=\"me\", age=30)\n",
+    "x.add_edge(\"friend\", MyItem(name=\"my friend\", age=31))\n",
+    "x.add_edge(\"friend\", MyItem(name=\"my friend2\", age=32))"
    ]
   },
   {
@@ -469,8 +539,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# hide\n",
-    "from pymemri.data.schema import *"
+    "assert client.create(x)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "y = client.get(x.id)"
    ]
   },
   {
@@ -479,7 +557,27 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "item = EmailMessage.from_data(content=\"example content field\")"
+    "assert len(y.friend) > 0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'my friend'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "y.friend[0].name"
    ]
   },
   {
@@ -505,6 +603,7 @@
       "Converted index.ipynb.\n",
       "Converted indexers.indexer.ipynb.\n",
       "Converted itembase.ipynb.\n",
+      "Converted plugin.pluginbase.ipynb.\n",
       "Converted pod.client.ipynb.\n"
      ]
     }

nbs/plugin.pluginbase.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "# default_exp plugin.pluginbase"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "from pymemri.data.schema import *\n",
+    "from pymemri.pod.client import PodClient, DEFAULT_POD_ADDRESS\n",
+    "from pymemri.imports import *\n",
+    "from os import environ"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hide\n",
+    "from nbdev.showdoc import *"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Plugins"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "POD_FULL_ADDRESS_ENV    = 'POD_FULL_ADDRESS'\n",
+    "RUN_UID_ENV             = 'RUN_ID'\n",
+    "POD_SERVICE_PAYLOAD_ENV = 'POD_SERVICE_PAYLOAD'\n",
+    "DATABASE_KEY_ENV        = 'databaseKey'\n",
+    "OWNER_KEY_ENV           = 'ownerKey'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "# hide\n",
+    "class PluginBase(Item):\n",
+    "    \"\"\"Base class for plugins\"\"\"\n",
+    "    properties = Item.properties + [\"name\", \"repository\", \"icon\", \"data_query\", \"bundleImage\",\n",
+    "                                    \"runDestination\", \"pluginClass\", \"pluginPackage\"]\n",
+    "    edges = Item.edges + [\"IndexerRun\"]\n",
+    "    \n",
+    "    def __init__(self, name=None, repository=None, icon=None, query=None, bundleImage=None, runDestination=None,\n",
+    "                 pluginClass=None, indexerRun=None, **kwargs):\n",
+    "        if pluginClass is None: pluginClass=self.__class__.__name__\n",
+    "        self.pluginPackage=None\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.name = name\n",
+    "        self.repository = repository\n",
+    "        self.icon = icon\n",
+    "        self.query = query\n",
+    "        self.bundleImage = bundleImage\n",
+    "        self.runDestination = runDestination\n",
+    "        self.pluginClass = pluginClass\n",
+    "        self.indexerRun = indexerRun if indexerRun is not None else []\n",
+    "        \n",
+    "    def run(self):\n",
+    "        raise NotImplementedError()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "# hide\n",
+    "class PluginRun(Item):\n",
+    "    properties = Item.properties\n",
+    "    edges = Item.edges + [\"plugin\"]\n",
+    "    \n",
+    "    def __init__(self, plugin=None, **kwargs):\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.plugin=plugin if plugin is not None else []"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's use the following plugin as an example of how we can start plugins."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "# hide\n",
+    "class MyPlugin(PluginBase):\n",
+    "    \"\"\"\"\"\"\n",
+    "    properties = PluginBase.properties\n",
+    "    edges= PluginBase.edges\n",
+    "    \n",
+    "    def __init__(self, **kwargs):\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.pluginPackage=\"pymemri.plugin.pluginbase\"\n",
+    "        \n",
+    "    def run(self, run, client):\n",
+    "        print(\"running\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "```python\n",
+    "class MyPlugin(PluginBase):\n",
+    "    properties = PluginBase.properties\n",
+    "    edges= PluginBase.edges\n",
+    "    \n",
+    "    def __init__(self, **kwargs):\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.pluginPackage=\"pymemri.plugin.pluginbase\"\n",
+    "        \n",
+    "    def run(self, run, client):\n",
+    "        print(\"running\")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymemri.pod.client import PodClient\n",
+    "client = PodClient()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert client.add_to_schema(MyPlugin(name=\"abc\", data_query=\"abc\"))\n",
+    "assert client.add_to_schema(PluginRun())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plugin = MyPlugin(name=\"abc\", data_query=\"abc\")\n",
+    "run = PluginRun()\n",
+    "run.add_edge(\"plugin\", plugin)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "client.create(run)\n",
+    "client.create(plugin)\n",
+    "client.create_edge(run.get_edges(\"plugin\")[0])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Running your plugin"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Plugins can be started using the pymemri `run_plugin` CLI. To use the CLI, you can either pass your run arguments as parameters, or set them as environment variables. If both are set, the CLI will prefer the passed arguments."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hide\n",
+    "# export\n",
+    "def run_plugin_from_run_id(run_id, client):\n",
+    "    run = client.get(run_id)\n",
+    "    plugins = run.plugin\n",
+    "    if len(plugins) == 0:\n",
+    "        raise ValueError(f\"plugin run {run_id} has no plugin attached to it. Make sure there is a 'plugin' \\\n",
+    "                           edge from your run to the actual plugin object.\")\n",
+    "    plugin = plugins[0]\n",
+    "    plugin.run(run, client)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "# hide\n",
+    "def register_base_classes(client):\n",
+    "    try:\n",
+    "        assert client.add_to_schema(PluginRun())\n",
+    "    except Exception as e:\n",
+    "        raise ValueError(\"Could not add base schema\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "running\n"
+     ]
+    }
+   ],
+   "source": [
+    "# hide\n",
+    "run_plugin_from_run_id(run.id, client)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "def _run_plugin(pod_full_address=None, plugin_run_id=None, database_key=None, owner_key=None,\n",
+    "                   verbose=False):\n",
+    "    \"\"\"Runs an plugin, you can either provide the run settings as parameters to this function (for local testing)\n",
+    "    or via environment variables (this is how the pod communicates with plugins).\"\"\"\n",
+    "    if verbose:\n",
+    "        for name, val in [(\"pod_full_address\", pod_full_address), (\"plugin_run_id\", plugin_run_id),\n",
+    "                  (\"database_key\", database_key), (\"owner_key\", owner_key)]:\n",
+    "            print(f\"{name}={val}\")\n",
+    "        print()\n",
+    "        \n",
+    "    client = PodClient(url=pod_full_address, database_key=database_key, owner_key=owner_key)\n",
+    "    register_base_classes(client)\n",
+    "    run_plugin_from_run_id(plugin_run_id, client)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "running\n"
+     ]
+    }
+   ],
+   "source": [
+    "# hide\n",
+    "_run_plugin(pod_full_address=DEFAULT_POD_ADDRESS, plugin_run_id=run.id,\n",
+    "           database_key=client.database_key, owner_key=client.owner_key)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## CLI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "# hide\n",
+    "def _parse_env(env):\n",
+    "    try:\n",
+    "        pod_full_address = env.get(POD_FULL_ADDRESS_ENV, DEFAULT_POD_ADDRESS)\n",
+    "        plugin_run_id  = str(env[RUN_UID_ENV])\n",
+    "        pod_service_payload = json.loads(env[POD_SERVICE_PAYLOAD_ENV])\n",
+    "\n",
+    "        database_key = pod_service_payload[DATABASE_KEY_ENV]\n",
+    "        owner_key    = pod_service_payload[OWNER_KEY_ENV]\n",
+    "        return pod_full_address, plugin_run_id, pod_service_payload, database_key, owner_key\n",
+    "    except KeyError as e:\n",
+    "        raise Exception('Missing parameter: {}'.format(e)) from None\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# export\n",
+    "from fastscript import *\n",
+    "import os\n",
+    "\n",
+    "@call_parse\n",
+    "def run_plugin(pod_full_address:Param(\"The pod full address\", str)=None,\n",
+    "               plugin_run_id:Param(\"Run id of the plugin to be executed\", str)=None,\n",
+    "               database_key:Param(\"Database key of the pod\", str)=None,\n",
+    "               owner_key:Param(\"Owner key of the pod\", str)=None):\n",
+    "\n",
+    "    env = os.environ\n",
+    "    params = [pod_full_address, plugin_run_id, database_key, owner_key]\n",
+    "    \n",
+    "    if all([p is None for p in params]):\n",
+    "        print(\"Reading `run_plugin()` parameters from environment variables\")\n",
+    "        pod_full_address, plugin_run_id, pod_service_payload, database_key, owner_key = _parse_env(env)\n",
+    "    else:\n",
+    "        print(\"Used arguments passed to `run_plugin()` (ignoring environment)\")\n",
+    "        if (None in params):\n",
+    "            raise ValueError(f\"Defined some params to run indexer, but not all. Missing \\\n",
+    "                             {[p for p in params if p is None]}\")\n",
+    "    \n",
+    "    _run_plugin(pod_full_address=pod_full_address, plugin_run_id=plugin_run_id,\n",
+    "                   database_key=database_key, owner_key=owner_key, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Used arguments passed to `run_plugin()` (ignoring environment)\n",
+      "pod_full_address=http://localhost:3030\n",
+      "plugin_run_id=61274342a226fe9ecb5d0efeb50ebc64\n",
+      "database_key=2275799687202554321586156394256447011413538728384161840804653842\n",
+      "owner_key=7877526860202989541720539875881419199482049862866439964933736900\n",
+      "\n",
+      "running\n"
+     ]
+    }
+   ],
+   "source": [
+    "!run_plugin --pod_full_address=$DEFAULT_POD_ADDRESS --plugin_run_id=$run.id --owner_key=$client.owner_key \\\n",
+    "            --database_key=$client.database_key"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Running a Plugin by providing environment variables -"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hide\n",
+    "# # export\n",
+    "# def generate_test_env(client, indexer_run):\n",
+    "#     payload = json.dumps({DATABASE_KEY_ENV: client.database_key, OWNER_KEY_ENV: client.owner_key})\n",
+    "              \n",
+    "#     return {POD_FULL_ADDRESS_ENV: DEFAULT_POD_ADDRESS,\n",
+    "#             RUN_UID_ENV: indexer_run.id,\n",
+    "#             POD_SERVICE_PAYLOAD_ENV: payload}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hide\n",
+    "# run_plugin(env=generate_test_env(client, run))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Export -"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Converted basic.ipynb.\n",
+      "Converted data.photo.ipynb.\n",
+      "Converted importers.Importer.ipynb.\n",
+      "Converted importers.util.ipynb.\n",
+      "Converted index.ipynb.\n",
+      "Converted indexers.indexer.ipynb.\n",
+      "Converted itembase.ipynb.\n",
+      "Converted plugin.pluginbase.ipynb.\n",
+      "Converted pod.client.ipynb.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# hide\n",
+    "from nbdev.export import *\n",
+    "notebook2script()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

nbs/pod.client.ipynb

@@ -25,9 +25,9 @@
    "outputs": [],
    "source": [
     "# export\n",
-    "from pymemri.data.itembase import Edge, ItemBase\n",
     "from pymemri.data.basic import *\n",
     "from pymemri.data.schema import *\n",
+    "from pymemri.data.itembase import Edge, ItemBase, Item\n",
     "from pymemri.data.photo import Photo\n",
     "from pymemri.imports import *\n",
     "from hashlib import sha256"
@@ -427,8 +427,11 @@
     "        return client.search(query)[0]\n",
     "        \n",
     "    def item_from_json(self, json):\n",
-    "        indexer_class = json.get(\"indexerClass\", None)\n",
-    "        constructor = get_constructor(json[\"type\"], indexer_class, extra=self.registered_classes)\n",
+    "        plugin_class = json.get(\"pluginClass\", None)\n",
+    "        plugin_package = json.get(\"pluginPackage\", None)\n",
+    "\n",
+    "        constructor = get_constructor(json[\"type\"], plugin_class, plugin_package=plugin_package,\n",
+    "                                      extra=self.registered_classes)\n",
     "        new_item = constructor.from_json(json)\n",
     "        existing = ItemBase.global_db.get(new_item.id)\n",
     "        # TODO: cleanup\n",
@@ -590,17 +593,12 @@
    "outputs": [],
    "source": [
     "class Dog(Item):\n",
-    "    def __init__(self, name, age, id=None, deleted=None):\n",
-    "        super().__init__(id=id, deleted=deleted)\n",
+    "    properties = Item.properties + [\"name\", \"age\"]\n",
+    "    edges = Item.edges \n",
+    "    def __init__(self, name=None, age=None, **kwargs):\n",
+    "        super().__init__(**kwargs)\n",
     "        self.name = name\n",
-    "        self.age = age\n",
-    "    \n",
-    "    @classmethod\n",
-    "    def from_json(cls, json):\n",
-    "        id = json.get(\"id\", None)\n",
-    "        name = json.get(\"name\", None)\n",
-    "        age = json.get(\"age\", None)\n",
-    "        return cls(id=id,name=name,age=age)"
+    "        self.age = age"
    ]
   },
   {
@@ -663,7 +661,7 @@
     {
      "data": {
       "text/plain": [
-       "[{'item': Person (#a63b11ebe6996b2efb00b75c42ccd930), 'name': 'sender'}]"
+       "[{'item': Person (#806a88787321a9a81054e63d17ad2fbb), 'name': 'sender'}]"
       ]
      },
      "execution_count": null,
@@ -784,9 +782,9 @@
     {
      "data": {
       "text/plain": [
-       "[Person (#a63b11ebe6996b2efb00b75c42ccd930),\n",
-       " Person (#16560d3dd6550a6aa58d88405fab177a),\n",
-       " Person (#fa5e4f210ec5b928e7eb263fcabf252f)]"
+       "[Person (#806a88787321a9a81054e63d17ad2fbb),\n",
+       " Person (#1f8e823a9b9ddaedf8dd37c9682beb0b),\n",
+       " Person (#cba8e451efdfe7be87dfeb10444ee670)]"
       ]
      },
      "execution_count": null,
@@ -936,18 +934,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "creating\n",
-      "creating photo file\n",
-      "creating\n",
-      "Uploaded file\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "assert client.create(photo)"
    ]
@@ -969,7 +956,7 @@
     {
      "data": {
       "text/plain": [
-       "IPhoto (#e49db579f6427af57651fc3ca5efafcf)"
+       "IPhoto (#424cce3929a63f5849c40c181e0ce252)"
       ]
      },
      "execution_count": null,
@@ -1061,6 +1048,7 @@
       "Converted index.ipynb.\n",
       "Converted indexers.indexer.ipynb.\n",
       "Converted itembase.ipynb.\n",
+      "Converted plugin.pluginbase.ipynb.\n",
       "Converted pod.client.ipynb.\n"
      ]
     }

pymemri/_nbdev.py

@@ -24,21 +24,28 @@ index = {"read_file": "basic.ipynb",
          "IndexerData": "indexers.indexer.ipynb",
          "get_indexer_run_data": "indexers.indexer.ipynb",
          "test_registration": "indexers.indexer.ipynb",
-         "POD_FULL_ADDRESS_ENV": "indexers.indexer.ipynb",
-         "RUN_UID_ENV": "indexers.indexer.ipynb",
-         "POD_SERVICE_PAYLOAD_ENV": "indexers.indexer.ipynb",
-         "DATABASE_KEY_ENV": "indexers.indexer.ipynb",
-         "OWNER_KEY_ENV": "indexers.indexer.ipynb",
+         "POD_FULL_ADDRESS_ENV": "plugin.pluginbase.ipynb",
+         "RUN_UID_ENV": "plugin.pluginbase.ipynb",
+         "POD_SERVICE_PAYLOAD_ENV": "plugin.pluginbase.ipynb",
+         "DATABASE_KEY_ENV": "plugin.pluginbase.ipynb",
+         "OWNER_KEY_ENV": "plugin.pluginbase.ipynb",
          "run_importer": "indexers.indexer.ipynb",
          "run_integrator_from_run_id": "indexers.indexer.ipynb",
          "run_integrator": "indexers.indexer.ipynb",
          "generate_test_env": "indexers.indexer.ipynb",
          "ALL_EDGES": "itembase.ipynb",
+         "EDGE_KEY": "itembase.ipynb",
          "DB": "itembase.ipynb",
          "parse_base_item_json": "itembase.ipynb",
          "Edge": "itembase.ipynb",
          "ItemBase": "itembase.ipynb",
          "Item": "itembase.ipynb",
+         "PluginBase": "plugin.pluginbase.ipynb",
+         "PluginRun": "plugin.pluginbase.ipynb",
+         "MyPlugin": "plugin.pluginbase.ipynb",
+         "run_plugin_from_run_id": "plugin.pluginbase.ipynb",
+         "register_base_classes": "plugin.pluginbase.ipynb",
+         "run_plugin": "plugin.pluginbase.ipynb",
          "DEFAULT_POD_ADDRESS": "pod.client.ipynb",
          "POD_VERSION": "pod.client.ipynb",
          "PodClient": "pod.client.ipynb"}
@@ -49,6 +56,7 @@ modules = ["data/basic.py",
            "importers/util.py",
            "indexers/indexer.py",
            "data/itembase.py",
+           "plugin/pluginbase.py",
            "pod/client.py"]
 
 doc_url = "http://memri.docs.memri.io/pymemri/"

pymemri/data/itembase.py

 # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/itembase.ipynb (unless otherwise specified).
 
-__all__ = ['ALL_EDGES', 'DB', 'parse_base_item_json', 'Edge', 'ItemBase', 'Item']
+__all__ = ['ALL_EDGES', 'EDGE_KEY', 'DB', 'parse_base_item_json', 'Edge', 'ItemBase', 'Item']
 
 # Cell
 # hide
@@ -8,6 +8,7 @@ from ..imports import *
 
 ALL_EDGES = "allEdges"
 SOURCE, TARGET, TYPE, EDGE_TYPE, LABEL, SEQUENCE = "_source", "_target", "_type", "_type", "label", "sequence"
+EDGE_KEY = "allEdges"
 
 # Cell
 # hide
@@ -75,8 +76,8 @@ class Edge():
         _type = json[EDGE_TYPE]
         json_target = json[TARGET]
         target_type = json_target["_type"]
-        indexer_class = json_target.get("indexerClass", None)
-        target_constructor = get_constructor(target_type, indexer_class)
+        plugin_class = json_target.get("pluginClass", None)
+        target_constructor = get_constructor(target_type, plugin_class)
         target = target_constructor.from_json(json_target)
         return cls(source=None, target=target, _type=_type)
 
@@ -118,9 +119,6 @@ class ItemBase():
         if existing is None and node.id is not None:
             cls.global_db.add(node)
 
-    def replace_self(self, other):
-        self.__dict__.update(other.__dict__)
-
     def __getattribute__(self, name):
         val = object.__getattribute__(self, name)
         if isinstance(val, Edge):
@@ -181,18 +179,18 @@ class ItemBase():
         if res is None: return False
         return len(res) == 1
 
-    def expand(self, api):
-        """Expands a node (retrieves all directly connected nodes ands adds to object)."""
-        self._expanded = True
-        res = api.get(self.id, expanded=True)
-        for edge_name in res.get_all_edge_names():
-            edges = res.get_edges(edge_name)
-            for e in edges:
-                e.source = self
-            self.__setattr__(edge_name, edges)
+#     def expand(self, api):
+#         """Expands a node (retrieves all directly connected nodes ands adds to object)."""
+#         self._expanded = True
+#         res = api.get(self.id, expanded=True)
+#         for edge_name in res.get_all_edge_names():
+#             edges = res.get_edges(edge_name)
+#             for e in edges:
+#                 e.source = self
+#             self.__setattr__(edge_name, edges)
 
-        # self.edges = res.edges
-        return self
+#         # self.edges = res.edges
+#         return self
 
     def __repr__(self):
         id = self.id
@@ -217,31 +215,100 @@ class ItemBase():
             v.source = res
         return res
 
-    def inherit_funcs(self, other):
-        """This function can be used to inherit new functionalities from a subclass. This is a patch to solve
-        the fact that python does provide extensions of classes that are defined in a different file that are
-        dynamic enough for our use case."""
-        assert issubclass(other, self.__class__)
-        self.__class__ = other
+#     def inherit_funcs(self, other):
+#         """This function can be used to inherit new functionalities from a subclass. This is a patch to solve
+#         the fact that python does provide extensions of classes that are defined in a different file that are
+#         dynamic enough for our use case."""
+#         assert issubclass(other, self.__class__)
+#         self.__class__ = other
 
 # Cell
 class Item(ItemBase):
     """Item is the baseclass for all of the data classes."""
-    def __init__(self, dateAccessed=None, dateCreated=None, dateModified=None, deleted=None,
-                 externalId=None, itemDescription=None, starred=None, version=None, id=None, importJson=None,
-                 changelog=None, label=None, genericAttribute=None, measure=None, sharedWith=None):
-        super().__init__(id)
-        self.dateAccessed = dateAccessed
-        self.dateCreated = dateCreated
-        self.dateModified = dateModified
-        self.deleted = deleted
-        self.externalId = externalId
-        self.itemDescription = itemDescription
-        self.starred = starred
-        self.version = version
-        self.importJson = importJson
-        self.changelog = changelog if changelog is not None else []
-        self.label = label if label is not None else []
-        self.genericAttribute = genericAttribute if genericAttribute is not None else []
-        self.measure = measure if measure is not None else []
-        self.sharedWith = sharedWith if sharedWith is not None else []
\ No newline at end of file
+    properties = ["dateAccessed", "dateCreated", "dateModified", "deleted", "externalId", "itemDescription",
+                  "starred", "version", "id", "importJson", "pluginClass"]
+    edges = ["changelog", "label", "genericAttribute", "measure", "sharedWith"]
+    def __init__(self, **kwargs):
+#     def __init__(self, dateAccessed=None, dateCreated=None, dateModified=None, deleted=None,
+#              externalId=None, itemDescription=None, starred=None, version=None, id=None, importJson=None,
+#              changelog=None, label=None, genericAttribute=None, measure=None, sharedWith=None):
+
+        super().__init__(kwargs.get("id"))
+        for p in self.properties:
+            if p == "id":
+                continue
+            setattr(self, p, kwargs.get(p, None))
+
+        for e in self.edges:
+            setattr(self, e, kwargs.get(e, []))
+
+    @classmethod
+    def parse_json(self, cls, json):
+        property_kwargs = Item.parse_properties(cls, json)
+        edge_kwargs = Item.parse_edges(cls, json)
+        return {**property_kwargs, **edge_kwargs}
+
+    @classmethod
+    def parse_properties(self, cls, json):
+        return {p: json.get(p, None) for p in cls.properties}
+
+    @classmethod
+    def parse_edges(self, cls, json):
+        all_edges = json.get(EDGE_KEY, None)
+        edge_kwargs = dict()
+        reverse_edges = [f"~{e}" for e in cls.edges]
+        if all_edges is not None:
+            for edge_json in all_edges:
+                edge = Edge.from_json(edge_json)
+
+                if edge.type in self.edges + reverse_edges:
+                    edge_name = self.remove_prefix(edge.type)
+                    if edge_name in edge_kwargs:
+                        edge_kwargs[edge_name] += [edge]
+                    else:
+                        edge_kwargs[edge_name] = [edge]
+
+        return edge_kwargs
+
+    @classmethod
+    def remove_prefix(s, prefix="~"):
+        return s[1:] if s[0] == "`" else s
+
+    @classmethod
+    def from_json(cls, json):
+        kwargs = Item.parse_json(cls, json)
+        res = cls(**kwargs)
+        for e in res.get_all_edges(): e.source = res
+        return res
+
+
+
+#                 if edge._type == "changelog" or edge._type == "~changelog":
+#                     changelog.append(edge)
+#                 elif edge._type == "label" or edge._type == "~label":
+#                     label.append(edge)
+#                 elif edge._type == "genericAttribute" or edge._type == "~genericAttribute":
+#                     genericAttribute.append(edge)
+#                 elif edge._type == "measure" or edge._type == "~measure":
+#                     measure.append(edge)
+#                 elif edge._type == "sharedWith" or edge._type == "~sharedWith":
+#                     sharedWith.append(edge)
+#                 elif edge._type == "indexerRun" or edge._type == "~indexerRun":
+#                     indexerRun.append(edge)
+
+
+
+#         self.dateAccessed = dateAccessed
+#         self.dateCreated = dateCreated
+#         self.dateModified = dateModified
+#         self.deleted = deleted
+#         self.externalId = externalId
+#         self.itemDescription = itemDescription
+#         self.starred = starred
+#         self.version = version
+#         self.importJson = importJson
+#         self.changelog = changelog if changelog is not None else []
+#         self.label = label if label is not None else []
+#         self.genericAttribute = genericAttribute if genericAttribute is not None else []
+#         self.measure = measure if measure is not None else []
+#         self.sharedWith = sharedWith if sharedWith is not None else []
\ No newline at end of file

pymemri/data/schema.py

@@ -10,19 +10,27 @@
 from .itembase import ItemBase, Edge, Item
 
 
-def get_constructor(_type, indexer_class=None, extra=None):
+def get_constructor(_type, plugin_class=None, plugin_package=None, extra=None):
     import pymemri.indexers as models
     from pymemri.data.photo import IPhoto
     from pymemri.indexers.indexer import IndexerBase
     import pymemri.integrator_registry
 
-    if _type == "Indexer" and indexer_class is not None and hasattr(pymemri.integrator_registry, indexer_class):
-        return getattr(pymemri.integrator_registry, indexer_class)
+    if _type == "Indexer" and plugin_class is not None and hasattr(pymemri.integrator_registry, plugin_class):
+        return getattr(pymemri.integrator_registry, plugin_class)
+    if plugin_class is not None and plugin_package is not None:
+        try:
+            mod = __import__(plugin_package, fromlist=[plugin_class])
+            dynamic = {plugin_class: getattr(mod, plugin_class)}
+        except Exception as e:
+            print(f"Could not import {plugin_class}.{plugin_package}")
+    else:
+        dynamic = dict()
 
-    classes = z = {**globals(), **locals(), **extra}
+    classes = z = {**globals(), **locals(), **extra, **dynamic}
     if _type in classes:
         if _type == "Indexer":
-            constructor = classes[indexer_class]
+            constructor = classes[plugin_class]
         else:
             i_class = "I" + _type
             if i_class in classes:
@@ -2062,7 +2070,7 @@ class Indexer(Item):
     def __init__(self, dateAccessed=None, dateCreated=None, dateModified=None, deleted=None,
                  externalId=None, itemDescription=None, starred=None, version=None, id=None, importJson=None,
                  name=None, repository=None, icon=None, query=None, bundleImage=None, runDestination=None,
-                 indexerClass=None, changelog=None, label=None, genericAttribute=None, measure=None, sharedWith=None,
+                 pluginClass=None, changelog=None, label=None, genericAttribute=None, measure=None, sharedWith=None,
                  indexerRun=None):
         super().__init__(dateAccessed=dateAccessed, dateCreated=dateCreated, dateModified=dateModified,
                          deleted=deleted, externalId=externalId, itemDescription=itemDescription, starred=starred,
@@ -2074,7 +2082,7 @@ class Indexer(Item):
         self.query = query
         self.bundleImage = bundleImage
         self.runDestination = runDestination
-        self.indexerClass = indexerClass
+        self.pluginClass = pluginClass
         self.indexerRun = indexerRun if indexerRun is not None else []
 
     @classmethod
@@ -2096,7 +2104,7 @@ class Indexer(Item):
         query = json.get("query", None)
         bundleImage = json.get("bundleImage", None)
         runDestination = json.get("runDestination", None)
-        indexerClass = json.get("indexerClass", None)
+        pluginClass = json.get("pluginClass", None)
        
         changelog = []
         label = []
@@ -2124,7 +2132,7 @@ class Indexer(Item):
         res = cls(dateAccessed=dateAccessed, dateCreated=dateCreated, dateModified=dateModified,
                   deleted=deleted, externalId=externalId, itemDescription=itemDescription, starred=starred,
                   version=version, id=id, importJson=importJson, name=name, repository=repository, icon=icon,
-                  query=query, bundleImage=bundleImage, runDestination=runDestination, indexerClass=indexerClass,
+                  query=query, bundleImage=bundleImage, runDestination=runDestination, pluginClass=pluginClass,
                   changelog=changelog, label=label, genericAttribute=genericAttribute, measure=measure,
                   sharedWith=sharedWith, indexerRun=indexerRun)
         for e in res.get_all_edges(): e.source = res

pymemri/indexers/indexer.py

@@ -19,9 +19,9 @@ OWNER_KEY_ENV           = 'ownerKey'
 
 class IndexerBase(Indexer):
 
-    def __init__(self, indexerClass=None, *args, **kwargs):
-        if indexerClass is None: indexerClass=self.__class__.__name__
-        super().__init__(indexerClass=indexerClass, *args, **kwargs)
+    def __init__(self, pluginClass=None, *args, **kwargs):
+        if pluginClass is None: pluginClass=self.__class__.__name__
+        super().__init__(pluginClass=pluginClass, *args, **kwargs)
 
     def populate(self, client, items, edges=False):
         new_items = [x for x in items if x.id is None]
@@ -93,7 +93,6 @@ def run_integrator_from_run_id(run_id, client):
         raise NotImplementedError(f"Cannot execute item of type {run}")
 
 # Cell
-
 def run_integrator(environ=None, pod_full_address=None, integrator_run_id=None, database_key=None, owner_key=None,
                    verbose=False):
     """Runs an integrator, you can either provide the run settings as parameters to this function (for local testing)

pymemri/plugin/pluginbase.py

+# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/plugin.pluginbase.ipynb (unless otherwise specified).
+
+__all__ = ['POD_FULL_ADDRESS_ENV', 'RUN_UID_ENV', 'POD_SERVICE_PAYLOAD_ENV', 'DATABASE_KEY_ENV', 'OWNER_KEY_ENV',
+           'PluginBase', 'PluginRun', 'MyPlugin', 'run_plugin_from_run_id', 'register_base_classes', 'run_plugin']
+
+# Cell
+from ..data.schema import *
+from ..pod.client import PodClient, DEFAULT_POD_ADDRESS
+from ..imports import *
+from os import environ
+
+# Cell
+POD_FULL_ADDRESS_ENV    = 'POD_FULL_ADDRESS'
+RUN_UID_ENV             = 'RUN_ID'
+POD_SERVICE_PAYLOAD_ENV = 'POD_SERVICE_PAYLOAD'
+DATABASE_KEY_ENV        = 'databaseKey'
+OWNER_KEY_ENV           = 'ownerKey'
+
+# Cell
+# hide
+class PluginBase(Item):
+    """Base class for plugins"""
+    properties = Item.properties + ["name", "repository", "icon", "data_query", "bundleImage",
+                                    "runDestination", "pluginClass", "pluginPackage"]
+    edges = Item.edges + ["IndexerRun"]
+
+    def __init__(self, name=None, repository=None, icon=None, query=None, bundleImage=None, runDestination=None,
+                 pluginClass=None, indexerRun=None, **kwargs):
+        if pluginClass is None: pluginClass=self.__class__.__name__
+        self.pluginPackage=None
+        super().__init__(**kwargs)
+        self.name = name
+        self.repository = repository
+        self.icon = icon
+        self.query = query
+        self.bundleImage = bundleImage
+        self.runDestination = runDestination
+        self.pluginClass = pluginClass
+        self.indexerRun = indexerRun if indexerRun is not None else []
+
+    def run(self):
+        raise NotImplementedError()
+
+# Cell
+# hide
+class PluginRun(Item):
+    properties = Item.properties
+    edges = Item.edges + ["plugin"]
+
+    def __init__(self, plugin=None, **kwargs):
+        super().__init__(**kwargs)
+        self.plugin=plugin if plugin is not None else []
+
+# Cell
+# hide
+class MyPlugin(PluginBase):
+    """"""
+    properties = PluginBase.properties
+    edges= PluginBase.edges
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.pluginPackage="pymemri.plugin.pluginbase"
+
+    def run(self, run, client):
+        print("running")
+
+# Cell
+# export
+def run_plugin_from_run_id(run_id, client):
+    run = client.get(run_id)
+    plugins = run.plugin
+    if len(plugins) == 0:
+        raise ValueError(f"plugin run {run_id} has no plugin attached to it. Make sure there is a 'plugin' \
+                           edge from your run to the actual plugin object.")
+    plugin = plugins[0]
+    plugin.run(run, client)
+
+# Cell
+# hide
+def register_base_classes(client):
+    try:
+        assert client.add_to_schema(PluginRun())
+    except Exception as e:
+        raise ValueError("Could not add base schema")
+
+# Cell
+def _run_plugin(pod_full_address=None, plugin_run_id=None, database_key=None, owner_key=None,
+                   verbose=False):
+    """Runs an plugin, you can either provide the run settings as parameters to this function (for local testing)
+    or via environment variables (this is how the pod communicates with plugins)."""
+    if verbose:
+        for name, val in [("pod_full_address", pod_full_address), ("plugin_run_id", plugin_run_id),
+                  ("database_key", database_key), ("owner_key", owner_key)]:
+            print(f"{name}={val}")
+        print()
+
+    client = PodClient(url=pod_full_address, database_key=database_key, owner_key=owner_key)
+    register_base_classes(client)
+    run_plugin_from_run_id(plugin_run_id, client)
+
+
+# Cell
+# hide
+def _parse_env(env):
+    try:
+        pod_full_address = env.get(POD_FULL_ADDRESS_ENV, DEFAULT_POD_ADDRESS)
+        plugin_run_id  = str(env[RUN_UID_ENV])
+        pod_service_payload = json.loads(env[POD_SERVICE_PAYLOAD_ENV])
+
+        database_key = pod_service_payload[DATABASE_KEY_ENV]
+        owner_key    = pod_service_payload[OWNER_KEY_ENV]
+        return pod_full_address, plugin_run_id, pod_service_payload, database_key, owner_key
+    except KeyError as e:
+        raise Exception('Missing parameter: {}'.format(e)) from None
+
+
+# Cell
+from fastscript import *
+import os
+
+@call_parse
+def run_plugin(pod_full_address:Param("The pod full address", str)=None,
+               plugin_run_id:Param("Run id of the plugin to be executed", str)=None,
+               database_key:Param("Database key of the pod", str)=None,
+               owner_key:Param("Owner key of the pod", str)=None):
+
+    env = os.environ
+    params = [pod_full_address, plugin_run_id, database_key, owner_key]
+
+    if all([p is None for p in params]):
+        print("Reading `run_plugin()` parameters from environment variables")
+        pod_full_address, plugin_run_id, pod_service_payload, database_key, owner_key = _parse_env(env)
+    else:
+        print("Used arguments passed to `run_plugin()` (ignoring environment)")
+        if (None in params):
+            raise ValueError(f"Defined some params to run indexer, but not all. Missing \
+                             {[p for p in params if p is None]}")
+
+    _run_plugin(pod_full_address=pod_full_address, plugin_run_id=plugin_run_id,
+                   database_key=database_key, owner_key=owner_key, verbose=True)
\ No newline at end of file

pymemri/pod/client.py

@@ -3,9 +3,9 @@
 __all__ = ['DEFAULT_POD_ADDRESS', 'POD_VERSION', 'PodClient']
 
 # Cell
-from ..data.itembase import Edge, ItemBase
 from ..data.basic import *
 from ..data.schema import *
+from ..data.itembase import Edge, ItemBase, Item
 from ..data.photo import Photo
 from ..imports import *
 from hashlib import sha256
@@ -391,8 +391,11 @@ class PodClient:
         return client.search(query)[0]
 
     def item_from_json(self, json):
-        indexer_class = json.get("indexerClass", None)
-        constructor = get_constructor(json["type"], indexer_class, extra=self.registered_classes)
+        plugin_class = json.get("pluginClass", None)
+        plugin_package = json.get("pluginPackage", None)
+
+        constructor = get_constructor(json["type"], plugin_class, plugin_package=plugin_package,
+                                      extra=self.registered_classes)
         new_item = constructor.from_json(json)
         existing = ItemBase.global_db.get(new_item.id)
         # TODO: cleanup

settings.ini

@@ -15,7 +15,7 @@ language = English
 custom_sidebar = True
 license = apache2
 status = 2
-
+console_scripts = run_plugin=pymemri.plugin.pluginbase:run_plugin
 requirements = requests tqdm ipdb fastprogress fastscript opencv-python nbdev==1.1.5 matplotlib
 
 nbs_path = nbs