Skip to content
GitLab
Commit f1679b61 authored by Koen van der Veen's avatar Koen van der Veen
Browse files

Merge branch 'gitlab_api' into 'dev' 

Gitlab api and template formatter

See merge request !209
parents e2e5214f 20c1fe58
Pipeline #8308 passed with stages
in 1 minute and 19 seconds
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1631 additions and 612 deletions
+1631 -612
README.md
+ 1
- 1
View file @ f1679b61
......@@ -65,7 +65,7 @@ This stores a random owner key and database key on your disk for future use, and
## Docs
[pymemri docs](https://memri.docs.memri.io/docs.memri.io/component-architectures/plugins/readme/)
[pymemri docs](https://docs.memri.io/component-architectures/plugins/readme/)
## Nbdev & Jupyter Notebooks
The Python integrators are written in [nbdev](https://nbdev.fast.ai/) ([video](https://www.youtube.com/watch?v=9Q6sLbz37gk&t=1301s)). With nbdev, it is encouraged to write code in
......
docs/data.loader.html
+ 110
- 178
View file @ f1679b61
This diff is collapsed.
docs/gitlab_api.html 0 → 100644
+ 622
- 0
View file @ f1679b61
---
title: - Downloading & Uploading functions for package registry
keywords: fastai
sidebar: home_sidebar
nb_path: "nbs/gitlab_api.ipynb"
---
<!--
#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: nbs/gitlab_api.ipynb
# command to build the docs after a change: nbdev_build_docs
-->
<div class="container" id="notebook-container">
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="find_git_repo" class="doc_header"><code>find_git_repo</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L33" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>find_git_repo</code>()</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="get_registry_api_key" class="doc_header"><code>get_registry_api_key</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L49" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>get_registry_api_key</code>()</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h2 id="upload_in_chunks" class="doc_header"><code>class</code> <code>upload_in_chunks</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L66" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>upload_in_chunks</code>(<strong><code>filename</code></strong>, <strong><code>chunksize</code></strong>=<em><code>16384</code></em>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h2 id="IterableToFileAdapter" class="doc_header"><code>class</code> <code>IterableToFileAdapter</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L98" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>IterableToFileAdapter</code>(<strong><code>iterable</code></strong>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="write_file_to_package_registry" class="doc_header"><code>write_file_to_package_registry</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L110" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>write_file_to_package_registry</code>(<strong><code>project_id</code></strong>, <strong><code>file_path</code></strong>, <strong><code>api_key</code></strong>, <strong><code>package_name</code></strong>, <strong><code>version</code></strong>=<em><code>'0.0.1'</code></em>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="project_id_from_name" class="doc_header"><code>project_id_from_name</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L126" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>project_id_from_name</code>(<strong><code>project_name</code></strong>, <strong><code>api_key</code></strong>, <strong><code>job_token</code></strong>=<em><code>None</code></em>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="get_project_id_from_project_path_unsafe" class="doc_header"><code>get_project_id_from_project_path_unsafe</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L145" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>get_project_id_from_project_path_unsafe</code>(<strong><code>project_path</code></strong>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="download_package_file" class="doc_header"><code>download_package_file</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L155" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>download_package_file</code>(<strong><code>filename</code></strong>, <strong><code>project_path</code></strong>, <strong><code>package_name</code></strong>, <strong><code>out_dir</code></strong>=<em><code>None</code></em>, <strong><code>package_version</code></strong>=<em><code>'0.0.1'</code></em>, <strong><code>download_if_exists</code></strong>=<em><code>False</code></em>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># todo: cleanup old package files during testing</span>
<span class="n">filename</span> <span class="o">=</span> <span class="s2">&quot;config.json&quot;</span>
<span class="n">out_file</span> <span class="o">=</span> <span class="n">download_package_file</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">&quot;memri/finetuning-example&quot;</span><span class="p">,</span> <span class="s2">&quot;plugin-model-package&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
<pre>/Users/koen/.memri/projects/finetuning-example/config.json
/Users/koen/.memri/projects/finetuning-example/config.json already exists, and `download_if_exists`==False, using cached version
</pre>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="create_repo" class="doc_header"><code>create_repo</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L183" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>create_repo</code>(<strong><code>repo_name</code></strong>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="get_current_username" class="doc_header"><code>get_current_username</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L194" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>get_current_username</code>()</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">delete_project</span><span class="p">(</span><span class="n">path_or_id</span><span class="p">):</span>
<span class="n">url_escape_id</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">path_or_id</span><span class="p">,</span> <span class="n">safe</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">GITLAB_API_BASE_URL</span><span class="si">}</span><span class="s2">/projects/</span><span class="si">{</span><span class="n">url_escape_id</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">api_key</span> <span class="o">=</span> <span class="n">get_registry_api_key</span><span class="p">()</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;PRIVATE-TOKEN&quot;</span><span class="p">:</span> <span class="n">api_key</span><span class="p">})</span>
<span class="k">if</span> <span class="n">res</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">201</span><span class="p">,</span> <span class="mi">202</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;failed to delete repo:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">res</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;deleted project </span><span class="si">{</span><span class="n">path_or_id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="commit_file" class="doc_header"><code>commit_file</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L205" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>commit_file</code>(<strong><code>project_name</code></strong>, <strong><code>path_in2out</code></strong>, <strong><code>branch</code></strong>=<em><code>'main'</code></em>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="rm_tree" class="doc_header"><code>rm_tree</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L232" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>rm_tree</code>(<strong><code>pth</code></strong>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="write_files_to_git" class="doc_header"><code>write_files_to_git</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L246" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>write_files_to_git</code>(<strong><code>repo</code></strong>, <strong><code>target_dir</code></strong>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_markdown rendered_html output_subarea ">
<h4 id="create_new_project" class="doc_header"><code>create_new_project</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/gitlab_api.py#L255" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>create_new_project</code>(<strong><code>project_name</code></strong>)</p>
</blockquote>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">repo</span> <span class="o">=</span> <span class="s2">&quot;test12345612345&quot;</span>
<span class="n">create_repo</span><span class="p">(</span><span class="n">repo</span><span class="p">)</span>
<span class="c1"># delete_project(f&quot;{get_current_username()}/{repo}&quot;)</span>
</pre></div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
<pre>created project test12345612345
</pre>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">create_new_project</span><span class="p">(</span><span class="n">repo</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
<pre>Created `Test12345612345` using the classifier_plugin template.
committed files [&#39;/tmp/test/test12345612345/Dockerfile&#39;, &#39;/tmp/test/test12345612345/metadata.json&#39;, &#39;/tmp/test/test12345612345/README.md&#39;, &#39;/tmp/test/test12345612345/setup.py&#39;, &#39;/tmp/test/test12345612345/.gitignore&#39;, &#39;/tmp/test/test12345612345/setup.cfg&#39;, &#39;/tmp/test/test12345612345/.gitlab-ci.yml&#39;, &#39;/tmp/test/test12345612345/LICENSE.txt&#39;, &#39;/tmp/test/test12345612345/tools/preload.py&#39;, &#39;/tmp/test/test12345612345/test12345612345/model.py&#39;, &#39;/tmp/test/test12345612345/test12345612345/utils.py&#39;, &#39;/tmp/test/test12345612345/test12345612345/plugin.py&#39;, &#39;/tmp/test/test12345612345/test12345612345/schema.py&#39;, &#39;/tmp/test/test12345612345/tests/test_plugin.py&#39;]
</pre>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
{% raw %}
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">delete_project</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">get_current_username</span><span class="p">()</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">repo</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
<pre>deleted project koenvanderveen/test12345612345
</pre>
</div>
</div>
</div>
</div>
</div>
{% endraw %}
</div>
docs/index.html
+ 1
- 1
View file @ f1679b61
......@@ -129,7 +129,7 @@ run_plugin --metadata <span class="s2">&quot;example_plugin.json&quot;</span>
</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Docs">Docs<a class="anchor-link" href="#Docs"> </a></h2><p><a href="https://memri.docs.memri.io/docs.memri.io/component-architectures/plugins/readme/">pymemri docs</a></p>
<h2 id="Docs">Docs<a class="anchor-link" href="#Docs"> </a></h2><p><a href="https://docs.memri.io/component-architectures/plugins/readme/">pymemri docs</a></p>
</div>
</div>
......
docs/pod.client.html
+ 1
- 1
View file @ f1679b61
......@@ -1093,7 +1093,7 @@ Completed Bulk action, written 3 items/edges
<div class="inner_cell">
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">dogs</span> <span class="o">=</span> <span class="p">[</span><span class="n">Dog</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;dog number </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">)]</span>
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">dogs</span> <span class="o">=</span> <span class="p">[</span><span class="n">Dog</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;dog number </span><span class="si">{i}</span><span class="s2">&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">)]</span>
<span class="n">person</span> <span class="o">=</span> <span class="n">Person</span><span class="p">(</span><span class="n">firstName</span><span class="o">=</span><span class="s2">&quot;Alice&quot;</span><span class="p">)</span>
<span class="n">edge1</span> <span class="o">=</span> <span class="n">Edge</span><span class="p">(</span><span class="n">dogs</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">person</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">)</span>
<span class="n">edge2</span> <span class="o">=</span> <span class="n">Edge</span><span class="p">(</span><span class="n">dogs</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">person</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">)</span>
......
docs/template.config.html
+ 4
- 2
View file @ f1679b61
......@@ -39,7 +39,7 @@ nb_path: "nbs/template.config.ipynb"
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<p>Often, plugins require some configuration before running. For example you might want to run plugin on data from a specific service, or run a specific version of your machine learning model. To configure your plugin when running it from the front-end, plugins require a <code>config.json</code> in the root of the plugin repository. This file contains a declarative definition, which our front-end app uses to display a configuration screen.</p>
<p>This module contains utilities to directly generate this config file from the plugin definition, by inferring all arguments from the plugin <code>__init__</code> method. All configuration fields are textboxes by default, which can be changed to different types in the future. For a full example, see our guide on <a href="https://memri.docs.memri.io/docs.memri.io/guides/build_and_deploy_your_model/">building plugins</a>.</p>
<p>This module contains utilities to directly generate this config file from the plugin definition, by inferring all arguments from the plugin <code>__init__</code> method. All configuration fields are textboxes by default, which can be changed to different types in the future. For a full example, see our guide on <a href="https://docs.memri.io/guides/build_and_deploy_your_model/">building plugins</a>.</p>
<p>Usage:</p>
<pre><code>create_plugin_config</code></pre>
......@@ -153,6 +153,7 @@ Args:
<span class="n">my_str</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">my_int</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">my_float</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">my_bool</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">unannotated</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">_private</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span>
......@@ -182,7 +183,7 @@ Args:
<div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">create_config</span><span class="p">(</span><span class="n">MyPlugin</span><span class="p">)</span>
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">config</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;display&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> (</span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;data_type&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">)&#39;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;</span><span class="si">{c[&quot;display&quot;]}</span><span class="s1"> (</span><span class="si">{c[&quot;data_type&quot;]}</span><span class="s1">)&#39;</span><span class="p">)</span>
</pre></div>
</div>
......@@ -200,6 +201,7 @@ My Arg (Text)
My Str (Text)
My Int (Integer)
My Float (Real)
My Bool (Bool)
</pre>
</div>
</div>
......
docs/template.formatter.html
+ 1
- 1
View file @ f1679b61
......@@ -73,7 +73,7 @@ nb_path: "nbs/template.formatter.ipynb"
<div class="output_markdown rendered_html output_subarea ">
<h4 id="plugin_from_template" class="doc_header"><code>plugin_from_template</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/template/formatter.py#L180" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>plugin_from_template</code>(<strong><code>list_templates</code></strong>:"List available plugin templates"=<em><code>False</code></em>, <strong><code>user</code></strong>:"Your Gitlab username"=<em><code>None</code></em>, <strong><code>repo_url</code></strong>:"The url of your empty Gitlab plugin repository"=<em><code>None</code></em>, <strong><code>plugin_name</code></strong>:"Display name of your plugin"=<em><code>None</code></em>, <strong><code>template_name</code></strong>:"Name of the template, use <code>list_templates</code> to see all available options"=<em><code>'basic'</code></em>, <strong><code>package_name</code></strong>:"Name of your plugin python package"=<em><code>None</code></em>, <strong><code>description</code></strong>:"Description of your plugin"=<em><code>None</code></em>, <strong><code>target_dir</code></strong>:"Directory to output the formatted template"=<em><code>'.'</code></em>)</p>
<h4 id="plugin_from_template" class="doc_header"><code>plugin_from_template</code><a href="https://gitlab.memri.io/memri/pymemri/tree/prod/pymemri/template/formatter.py#L203" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>plugin_from_template</code>(<strong><code>list_templates</code></strong>:"List available plugin templates"=<em><code>False</code></em>, <strong><code>user</code></strong>:"Your Gitlab username"=<em><code>None</code></em>, <strong><code>repo_url</code></strong>:"The url of your empty Gitlab plugin repository"=<em><code>None</code></em>, <strong><code>plugin_name</code></strong>:"Display name of your plugin"=<em><code>None</code></em>, <strong><code>template_name</code></strong>:"Name of the template, use <code>list_templates</code> to see all available options"=<em><code>'basic'</code></em>, <strong><code>package_name</code></strong>:"Name of your plugin python package"=<em><code>None</code></em>, <strong><code>description</code></strong>:"Description of your plugin"=<em><code>None</code></em>, <strong><code>target_dir</code></strong>:"Directory to output the formatted template"=<em><code>'.'</code></em>, <strong><code>verbose</code></strong>:"Should print out dir"=<em><code>True</code></em>, <strong><code>install_requires</code></strong>:"Extra packages to install, provided as comma separated, e.g. pymemri,requests"=<em><code>''</code></em>)</p>
</blockquote>
<pre><code>CLI that downloads and formats a plugin template according to the arguments, and local git repository.
......
nbs/data.loader.ipynb
+ 47
- 240
View file @ f1679b61
%% Cell type:code id: tags:
``` python
#default_exp data.loader
%load_ext autoreload
%autoreload 2
```
%% Cell type:code id: tags:
``` python
# export
from fastprogress.fastprogress import progress_bar
from pathlib import Path
import requests
import os, sys
from getpass import getpass
from datetime import datetime
from git import Repo
import re
from pymemri.gitlab_api import MEMRI_PATH, MEMRI_GITLAB_BASE_URL, ACCESS_TOKEN_PATH, GITLAB_API_BASE_URL, TIME_FORMAT_GITLAB, \
PROJET_ID_PATTERN, DEFAULT_PACKAGE_VERSION, download_package_file, project_id_from_name, write_file_to_package_registry, \
get_registry_api_key
```
%% Cell type:code id: tags:
``` python
# export
MEMRI_PATH = Path.home() / ".memri"
MEMRI_GITLAB_BASE_URL = "https://gitlab.memri.io"
ACCESS_TOKEN_PATH = Path.home() / ".memri/access_token/access_token.txt"
GITLAB_API_BASE_URL = "https://gitlab.memri.io/api/v4"
DEFAULT_PLUGIN_MODEL_PACKAGE_NAME = "plugin-model-package"
DEFAULT_PYTORCH_MODEL_NAME = "pytorch_model.bin"
DEFAULT_HUGGINFACE_CONFIG_NAME = "config.json"
DEFAULT_PACKAGE_VERSION = "0.0.1"
TIME_FORMAT_GITLAB = '%Y-%m-%dT%H:%M:%S.%fZ'
PROJET_ID_PATTERN = '(?<=<span class="gl-button-text">Project ID: )[0-9]+(?=</span>)'
```
%% Cell type:markdown id: tags:
# - Downloading & Uploading functions for package registry
%% Cell type:code id: tags:
``` python
# export
def find_git_repo():
path = "."
for i in range(10):
try:
repo = Repo(f"{path + ('.' * i)}/")
except:
pass
else:
break
if i == 9:
raise ValueError(f"could not fine git repo in {os.path.abspath('')}")
repo_name = repo.remotes.origin.url.split('.git')[0].split('/')[-1]
return repo_name
```
%% Cell type:code id: tags:
``` python
# export
def get_registry_api_key():
ACCESS_TOKEN_PATH.parent.mkdir(parents=True, exist_ok=True)
if ACCESS_TOKEN_PATH.is_file():
with open(ACCESS_TOKEN_PATH, "r") as f:
return f.read()
else:
print(f"""
The first time you are uploading a model you need to create an access_token
at https://gitlab.memri.io/-/profile/personal_access_tokens?name=Model+Access+token&scopes=api
Click at the blue button with 'Create personal access token'"
""")
access_token = getpass("Then copy your personal access token from 'Your new personal access token', and paste here: ")
with open(ACCESS_TOKEN_PATH, "w") as f:
f.write(access_token)
return access_token
```
%% Cell type:code id: tags:
``` python
# export
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 << 14):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
n = 100
pb = progress_bar(range(n))
pb_iter = iter(pb)
i = 1
delta = 1 / n
next(pb_iter, None)
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
while (percent / 100) > i * delta:
next(pb_iter, None)
i += 1
yield data
pb.update_bar(n)
def __len__(self):
return self.totalsize
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
return next(self.iterator, b'')
def __len__(self):
return self.length
```
%% Cell type:code id: tags:
``` python
# export
def write_file_to_package_registry(project_id, file_path, api_key, version=DEFAULT_PACKAGE_VERSION):
file_path = Path(file_path)
file_name = file_path.name
url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{DEFAULT_PLUGIN_MODEL_PACKAGE_NAME}/{version}/{file_name}"
print(f"uploading {file_path}")
it = upload_in_chunks(file_path)
res = requests.put(url=url, data=IterableToFileAdapter(it),
headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201]:
print(f"Failed to upload {file_path}: {res.content}")
else:
print(f"Succesfully uploaded {file_path}")
```
%% Cell type:code id: tags:
``` python
# export
def project_id_from_name(project_name, api_key, job_token=None):
if api_key:
headers = {"PRIVATE-TOKEN": api_key}
else:
headers = {"JOB-TOKEN": job_token}
res = requests.get(f"{GITLAB_API_BASE_URL}/projects",
headers=headers,
params={
"owned": True,
"search": project_name
})
# we need this extra filter (search is not exact match)
res = [x.get("id") for x in res.json() if x.get("path", None) == project_name]
if len(res) == 0:
raise ValueError(f"No plugin found with name {project_name}, make sure to enter the name as specified in the url of the repo")
else:
return res[0]
```
# - Downloading & Uploading functions for Models
%% Cell type:code id: tags:
``` python
# export
def get_project_id_from_project_path_unsafe(project_path):
try:
res = requests.get(f"{MEMRI_GITLAB_BASE_URL}/{project_path}")
html = str(res.content)
match = re.search(PROJET_ID_PATTERN, html)
return match.group()
except Exception:
raise ValueError(f"Could not find project with name {project_path}")
```
%% Cell type:code id: tags:
``` python
# export
def write_huggingface_model_to_package_registry(project_name, model):
def write_huggingface_model_to_package_registry(project_name, model, version=DEFAULT_PACKAGE_VERSION):
import torch
api_key = get_registry_api_key()
project_id = project_id_from_name(project_name, api_key)
local_save_dir = Path("/tmp")
torch.save(model.state_dict(), local_save_dir / DEFAULT_PYTORCH_MODEL_NAME)
model.config.to_json_file(local_save_dir / DEFAULT_HUGGINFACE_CONFIG_NAME)
for f in [DEFAULT_HUGGINFACE_CONFIG_NAME, DEFAULT_PYTORCH_MODEL_NAME]:
file_path = local_save_dir / f
print(f"writing {f} to package registry of {project_name} with project id {project_id}")
write_file_to_package_registry(project_id, file_path, api_key)
write_file_to_package_registry(project_id, file_path, api_key, package_name=DEFAULT_PLUGIN_MODEL_PACKAGE_NAME, version=version)
```
%% Cell type:code id: tags:
``` python
# export
def write_model_to_package_registry(model, project_name=None):
project_name = project_name if project_name is not None else find_git_repo()
if type(model).__module__.startswith("transformers"):
import transformers
import torch
if isinstance(model, transformers.PreTrainedModel):
write_huggingface_model_to_package_registry(project_name, model)
else:
raise ValueError(f"Model type not supported: {type(model)}")
```
%% Cell type:code id: tags:
``` python
# export
def download_package_file(filename, project_path=None, out_dir=None, package_name=DEFAULT_PLUGIN_MODEL_PACKAGE_NAME,
package_version=DEFAULT_PACKAGE_VERSION, download_if_exists=False):
project_name = str(project_path).split("/")[-1]
out_dir = out_dir if out_dir is not None else MEMRI_PATH / "projects" / project_name
out_dir.mkdir(parents=True, exist_ok=True)
project_id = get_project_id_from_project_path_unsafe(project_path)
file_path = out_dir / filename
print(file_path)
if file_path.exists() and not download_if_exists:
print(f"{file_path} already exists, and `download_if_exists`==False, using cached version")
return file_path
print(f"downloading {filename} from project {project_path}, package {package_name}")
res = requests.get(
url=f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{package_version}/{filename}"
)
res.raise_for_status()
with open(out_dir / filename, "wb") as f:
print(f"writing {filename} to {out_dir}")
f.write(res.content)
return file_path
```
%% Cell type:code id: tags:
``` python
# export
def download_huggingface_model_for_project(project_path=None, files=None, download_if_exists=False):
if files is None:
files = ["config.json", "pytorch_model.bin"]
for f in files:
out_file_path = download_package_file(f, project_path=project_path)
out_file_path = download_package_file(f, project_path=project_path, package_name=DEFAULT_PLUGIN_MODEL_PACKAGE_NAME)
return out_file_path.parent
```
%% Cell type:code id: tags:
``` python
# export
def load_huggingface_model_for_project(project_path=None, files=None, download_if_exists=False):
out_dir = download_huggingface_model_for_project(project_path, files, download_if_exists)
from transformers import AutoModelForSequenceClassification
model = AutoModelForSequenceClassification.from_pretrained(out_dir)
return model
```
%% Cell type:code id: tags:
``` python
# skip
# todo: cleanup old package files during testing
filename = "config.json"
out_file = download_package_file(filename, "memri/finetuning-example")
```
%% Output
/Users/koen/.memri/projects/finetuning-example/config.json
/Users/koen/.memri/projects/finetuning-example/config.json already exists, and `download_if_exists`==False, using cached version
%% Cell type:markdown id: tags:
# - Transformers tests
%% Cell type:code id: tags:
``` python
# skip
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from transformers import AutoModel
model = AutoModelForSequenceClassification.from_pretrained("distilroberta-base", num_labels=10)
```
%% Output
Some weights of the model checkpoint at distilroberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.weight']
Some weights of the model checkpoint at distilroberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.bias', 'roberta.pooler.dense.bias', 'lm_head.bias', 'roberta.pooler.dense.weight', 'lm_head.decoder.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.bias']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at distilroberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight']
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at distilroberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
%% Cell type:code id: tags:
``` python
# skip
write_model_to_package_registry(model, project_name="test-1234")
```
%% Output
writing config.json to package registry of test-1234 with project id 190
uploading /tmp/config.json
Succesfully uploaded /tmp/config.json
writing pytorch_model.bin to package registry of test-1234 with project id 190
uploading /tmp/pytorch_model.bin
Succesfully uploaded /tmp/pytorch_model.bin
%% Cell type:code id: tags:
``` python
# skip
model = load_huggingface_model_for_project(project_path="memri/finetuning-example")
```
%% Output
/Users/koen/.memri/projects/finetuning-example/config.json
/Users/koen/.memri/projects/finetuning-example/config.json already exists, and `download_if_exists`==False, using cached version
downloading config.json from project memri/finetuning-example, package plugin-model-package
writing config.json to /Users/koen/.memri/projects/finetuning-example
/Users/koen/.memri/projects/finetuning-example/pytorch_model.bin
/Users/koen/.memri/projects/finetuning-example/pytorch_model.bin already exists, and `download_if_exists`==False, using cached version
downloading pytorch_model.bin from project memri/finetuning-example, package plugin-model-package
writing pytorch_model.bin to /Users/koen/.memri/projects/finetuning-example
%% Cell type:code id: tags:
``` python
# skip
out_dir = download_huggingface_model_for_project(project_path="memri/finetuning-example")
model = AutoModelForSequenceClassification.from_pretrained(out_dir, num_labels=20)
```
%% Output
/Users/koen/.memri/projects/finetuning-example/config.json
/Users/koen/.memri/projects/finetuning-example/config.json already exists, and `download_if_exists`==False, using cached version
/Users/koen/.memri/projects/finetuning-example/pytorch_model.bin
/Users/koen/.memri/projects/finetuning-example/pytorch_model.bin already exists, and `download_if_exists`==False, using cached version
%% Cell type:markdown id: tags:
# Export -
%% Cell type:code id: tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted basic.ipynb.
Converted cvu.utils.ipynb.
Converted data.dataset.ipynb.
Converted data.loader.ipynb.
Converted data.photo.ipynb.
Converted exporters.exporters.ipynb.
Converted gitlab_api.ipynb.
Converted index.ipynb.
Converted itembase.ipynb.
Converted plugin.authenticators.credentials.ipynb.
Converted plugin.authenticators.oauth.ipynb.
Converted plugin.listeners.ipynb.
Converted plugin.pluginbase.ipynb.
Converted plugin.states.ipynb.
Converted plugins.authenticators.password.ipynb.
Converted pod.api.ipynb.
Converted pod.client.ipynb.
Converted pod.db.ipynb.
Converted pod.utils.ipynb.
Converted template.config.ipynb.
Converted template.formatter.ipynb.
Converted test_schema.ipynb.
Converted test_utils.ipynb.
%% Cell type:code id: tags:
``` python
```
......
nbs/gitlab_api.ipynb 0 → 100644
+ 537
- 0
View file @ f1679b61
%% Cell type:code id: tags:
``` python
#default_exp gitlab_api
%load_ext autoreload
%autoreload 2
```
%% Cell type:code id: tags:
``` python
# export
from fastprogress.fastprogress import progress_bar
from pathlib import Path
import requests
import os, sys
from getpass import getpass
from datetime import datetime
from git import Repo
import re
from pymemri.data.basic import *
from pymemri.template.formatter import plugin_from_template
import urllib
```
%% Cell type:code id: tags:
``` python
# export
MEMRI_PATH = Path.home() / ".memri"
MEMRI_GITLAB_BASE_URL = "https://gitlab.memri.io"
ACCESS_TOKEN_PATH = Path.home() / ".memri/access_token/access_token.txt"
GITLAB_API_BASE_URL = "https://gitlab.memri.io/api/v4"
DEFAULT_PACKAGE_VERSION = "0.0.1"
TIME_FORMAT_GITLAB = '%Y-%m-%dT%H:%M:%S.%fZ'
PROJET_ID_PATTERN = '(?<=<span class="gl-button-text">Project ID: )[0-9]+(?=</span>)'
```
%% Cell type:markdown id: tags:
# - Downloading & Uploading functions for package registry
%% Cell type:code id: tags:
``` python
# export
def find_git_repo():
path = "."
for i in range(10):
try:
repo = Repo(f"{path + ('.' * i)}/")
except:
pass
else:
break
if i == 9:
raise ValueError(f"could not fine git repo in {os.path.abspath('')}")
repo_name = repo.remotes.origin.url.split('.git')[0].split('/')[-1]
return repo_name
```
%% Cell type:code id: tags:
``` python
# export
def get_registry_api_key():
ACCESS_TOKEN_PATH.parent.mkdir(parents=True, exist_ok=True)
if ACCESS_TOKEN_PATH.is_file():
with open(ACCESS_TOKEN_PATH, "r") as f:
return f.read()
else:
print(f"""
The first time you are uploading a model you need to create an access_token
at https://gitlab.memri.io/-/profile/personal_access_tokens?name=Model+Access+token&scopes=api
Click at the blue button with 'Create personal access token'"
""")
access_token = getpass("Then copy your personal access token from 'Your new personal access token', and paste here: ")
with open(ACCESS_TOKEN_PATH, "w") as f:
f.write(access_token)
return access_token
```
%% Cell type:code id: tags:
``` python
# export
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 << 14):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
n = 100
pb = progress_bar(range(n))
pb_iter = iter(pb)
i = 1
delta = 1 / n
next(pb_iter, None)
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
while (percent / 100) > i * delta:
next(pb_iter, None)
i += 1
yield data
pb.update_bar(n)
def __len__(self):
return self.totalsize
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
return next(self.iterator, b'')
def __len__(self):
return self.length
```
%% Cell type:code id: tags:
``` python
# export
def write_file_to_package_registry(project_id, file_path, api_key, package_name, version=DEFAULT_PACKAGE_VERSION):
file_path = Path(file_path)
file_name = file_path.name
url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{version}/{file_name}"
print(f"uploading {file_path}")
it = upload_in_chunks(file_path)
res = requests.put(url=url, data=IterableToFileAdapter(it),
headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201]:
print(f"Failed to upload {file_path}: {res.content}")
else:
print(f"Succesfully uploaded {file_path}")
```
%% Cell type:code id: tags:
``` python
# export
def project_id_from_name(project_name, api_key, job_token=None):
if api_key:
headers = {"PRIVATE-TOKEN": api_key}
else:
headers = {"JOB-TOKEN": job_token}
res = requests.get(f"{GITLAB_API_BASE_URL}/projects",
headers=headers,
params={
"owned": True,
"search": project_name
})
# we need this extra filter (search is not exact match)
res = [x.get("id") for x in res.json() if x.get("path", None) == project_name]
if len(res) == 0:
raise ValueError(f"No plugin found with name {project_name}, make sure to enter the name as specified in the url of the repo")
else:
return res[0]
```
%% Cell type:code id: tags:
``` python
# export
def get_project_id_from_project_path_unsafe(project_path):
try:
res = requests.get(f"{MEMRI_GITLAB_BASE_URL}/{project_path}")
html = str(res.content)
match = re.search(PROJET_ID_PATTERN, html)
return match.group()
except Exception:
raise ValueError(f"Could not find project with name {project_path}")
```
%% Cell type:code id: tags:
``` python
# export
def download_package_file(filename, project_path, package_name, out_dir=None,
package_version=DEFAULT_PACKAGE_VERSION, download_if_exists=False):
project_name = str(project_path).split("/")[-1]
out_dir = out_dir if out_dir is not None else MEMRI_PATH / "projects" / project_name
out_dir.mkdir(parents=True, exist_ok=True)
project_id = get_project_id_from_project_path_unsafe(project_path)
file_path = out_dir / filename
print(file_path)
if file_path.exists() and not download_if_exists:
print(f"{file_path} already exists, and `download_if_exists`==False, using cached version")
return file_path
print(f"downloading {filename} from project {project_path}, package {package_name}")
res = requests.get(
url=f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{package_version}/{filename}"
)
res.raise_for_status()
with open(out_dir / filename, "wb") as f:
print(f"writing {filename} to {out_dir}")
f.write(res.content)
return file_path
```
%% Cell type:code id: tags:
``` python
# skip
# todo: cleanup old package files during testing
filename = "config.json"
out_file = download_package_file(filename, "memri/finetuning-example", "plugin-model-package")
```
%% Output
/Users/koen/.memri/projects/finetuning-example/config.json
/Users/koen/.memri/projects/finetuning-example/config.json already exists, and `download_if_exists`==False, using cached version
%% Cell type:code id: tags:
``` python
# export
def create_repo(repo_name):
url = f"{GITLAB_API_BASE_URL}/projects/"
api_key = get_registry_api_key()
payload = {"name": repo_name}
res = requests.post(url=url, json=payload,
headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201]:
raise ValueError(f"failed to create repo:\n {res.text}")
print(f"created project {repo_name}")
```
%% Cell type:code id: tags:
``` python
# export
def get_current_username():
url = f"{GITLAB_API_BASE_URL}/user/"
api_key = get_registry_api_key()
res = requests.get(url=url, headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200,201]:
raise ValueError(f"Could not find current user {res.content}")
else:
username = res.json()["username"]
return username
```
%% Cell type:code id: tags:
``` python
# NEVER EXPORT THIS
def delete_project(path_or_id):
url_escape_id = urllib.parse.quote(path_or_id, safe='')
url = f"{GITLAB_API_BASE_URL}/projects/{url_escape_id}"
api_key = get_registry_api_key()
res = requests.delete(url=url, headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201, 202]:
raise ValueError(f"failed to delete repo:\n {res.text}")
print(f"deleted project {path_or_id}")
```
%% Cell type:code id: tags:
``` python
# export
def commit_file(project_name, path_in2out, branch="main"):
api_key = get_registry_api_key()
project_id = project_id_from_name(repo, api_key)
# file_out_path_escaped = urllib.parse.quote(file_out_path, safe='')
actions = []
for file_in_path, file_out_path in path_in2out.items():
content = read_file(file_in_path)
action_payload = {"action": "create", "file_path": file_out_path, "content": content}
actions.append(action_payload)
url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/repository/commits"
payload = {"branch": branch, "commit_message": "automated commit", "content": content, "actions": actions}
res = requests.post(url=url, json=payload,
headers={"PRIVATE-TOKEN": api_key})
files_in = list(path_in2out.keys())
if res.status_code not in [200, 201, 202]:
raise ValueError(f"failed to make commit with files {files_in}:\n {res.text}")
print(f"committed files {files_in}")
# --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname",
# "content": "some content", "commit_message": "create a new file"}' \
```
%% Cell type:code id: tags:
``` python
# export
def rm_tree(pth):
pth = Path(pth)
for child in pth.glob('*'):
if child.is_file():
child.unlink()
else:
rm_tree(child)
try:
pth.rmdir()
except FileNotFoundError as e:
pass
```
%% Cell type:code id: tags:
``` python
# export
def write_files_to_git(repo, target_dir):
path_in2out = dict()
for p in target_dir.rglob("*"):
if p.is_file():
path_in_repo = p.relative_to(target_dir)
path_in2out[str(p)] = str(path_in_repo)
commit_file(str(repo), path_in2out)
def create_new_project(project_name):
tmp_dir = Path("/tmp/test") / project_name
rm_tree(tmp_dir)
plugin_from_template(
template_name="classifier_plugin",
description="A transformer based sentiment analyis plugin",
install_requires="transformers,sentencepiece,protobuf,torch==1.10.0",
target_dir=str(tmp_dir),
repo_url=f"{MEMRI_GITLAB_BASE_URL}/plugins/{project_name}",
verbose=False
)
write_files_to_git(project_name, tmp_dir)
```
%% Cell type:code id: tags:
``` python
# skip
repo = "test12345612345"
create_repo(repo)
# delete_project(f"{get_current_username()}/{repo}")
```
%% Output
created project test12345612345
%% Cell type:code id: tags:
``` python
create_new_project(repo)
```
%% Output
Created `Test12345612345` using the classifier_plugin template.
committed files ['/tmp/test/test12345612345/Dockerfile', '/tmp/test/test12345612345/metadata.json', '/tmp/test/test12345612345/README.md', '/tmp/test/test12345612345/setup.py', '/tmp/test/test12345612345/.gitignore', '/tmp/test/test12345612345/setup.cfg', '/tmp/test/test12345612345/.gitlab-ci.yml', '/tmp/test/test12345612345/LICENSE.txt', '/tmp/test/test12345612345/tools/preload.py', '/tmp/test/test12345612345/test12345612345/model.py', '/tmp/test/test12345612345/test12345612345/utils.py', '/tmp/test/test12345612345/test12345612345/plugin.py', '/tmp/test/test12345612345/test12345612345/schema.py', '/tmp/test/test12345612345/tests/test_plugin.py']
%% Cell type:code id: tags:
``` python
# skip
delete_project(f"{get_current_username()}/{repo}")
```
%% Output
deleted project koenvanderveen/test12345612345
%% Cell type:markdown id: tags:
# Export -
%% Cell type:code id: tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted basic.ipynb.
Converted cvu.utils.ipynb.
Converted data.dataset.ipynb.
Converted data.loader.ipynb.
Converted data.photo.ipynb.
Converted exporters.exporters.ipynb.
Converted gitlab_api.ipynb.
Converted index.ipynb.
Converted itembase.ipynb.
Converted plugin.authenticators.credentials.ipynb.
Converted plugin.authenticators.oauth.ipynb.
Converted plugin.listeners.ipynb.
Converted plugin.pluginbase.ipynb.
Converted plugin.states.ipynb.
Converted plugins.authenticators.password.ipynb.
Converted pod.api.ipynb.
Converted pod.client.ipynb.
Converted pod.db.ipynb.
Converted pod.utils.ipynb.
Converted template.config.ipynb.
Converted template.formatter.ipynb.
Converted test_schema.ipynb.
Converted test_utils.ipynb.
%% Cell type:code id: tags:
``` python
```
nbs/index.ipynb
+ 1
- 1
View file @ f1679b61
%% Cell type:code id: tags:
``` python
# hide
from pymemri import *
from pymemri.plugin.pluginbase import run_plugin
```
%% Cell type:markdown id: tags:
# Pymemri
> Pymemri is a python library for creating <b>Plugins</b> for the Memri Personal online datastore [(pod)](https://gitlab.memri.io/memri/pod). Pymemri has a PodClient to communicate with the pod, and tools to build and test plugins.
%% Cell type:markdown id: tags:
[![Gitlab pipeline status (self-hosted)](https://img.shields.io/gitlab/pipeline/memri/pymemri/dev?gitlab_url=https%3A%2F%2Fgitlab.memri.io&label=CI&logo=gitlab&style=plastic)](https://gitlab.memri.io/memri/pymemri/-/pipelines/latest)
[![Discord](https://img.shields.io/discord/799216875480678430?color=blue&label=Discord&logo=discord&style=plastic)](https://discord.gg/BcRfajJk4k)
[![Twitter URL](https://img.shields.io/twitter/url?label=%40YourMemri&logo=twitter&style=plastic&url=https%3A%2F%2Ftwitter.com%2FYourMemri)](https://twitter.com/YourMemri)
<a href="https://pypi.org/project/pymemri/"><img src="https://pepy.tech/badge/pymemri" /></a>
%% Cell type:markdown id: tags:
Plugins connect and add the information to your Pod. Plugins that <b>import your data from external services</b> are called **Importers** (Gmail, WhatsApp, etc.). Plugins that <b>connect new data to the existing data</b> are called **indexers** (face recognition, spam detection, object detection, etc.). Lastly there are plugins that <b>execute actions</b> (sending messages, uploading files). This repository is built with [nbdev](https://github.com/fastai/nbdev), which means that the repo structure has a few differences compared to a standard python repo.
%% Cell type:markdown id: tags:
## Installing
%% Cell type:markdown id: tags:
### As a package
```bash
pip install pymemri
```
%% Cell type:markdown id: tags:
### Development
To install the Python package, and correctly setup nbdev for development run:
```bash
pip install -e . && nbdev_install_git_hooks
```
The last command configures git to automatically clean metadata from your notebooks before a commit.
%% Cell type:markdown id: tags:
## Quickstart: Pod Client
All interaction between plugins and the pod goes via the Pymemri `PodClient`. To use this client in development, we first need to have a pod running locally. The quickest way to do this is to install from the [pod repo](https://gitlab.memri.io/memri/pod), and run `./examples/run_development.sh`.
If you have a running pod, you can define and add your own item definitions:
```python
from pymemri.data.itembase import Item
from pymemri.pod.client import PodClient
class Dog(Item):
properties = Item.properties + ["name", "age"]
def __init__(self, name: str = None, age: int = None, **kwargs):
super().__init__(**kwargs)
self.name = name
self.age = age
# Connect to the pod and add the Dog item definition
client = PodClient()
client.add_to_schema(Dog)
# Add a Dog to the pod
dog = Dog("bob", 3)
client.create(dog)
```
%% Cell type:markdown id: tags:
## Quickstart: Running a plugin
%% Cell type:markdown id: tags:
After installation, users can use the plugin CLI to manually run a plugin. For more information on how to build a plugin, see `run_plugin`.
<b>With the pod running, run in your terminal|: </b>
%% Cell type:markdown id: tags:
```bash
store_keys
run_plugin --metadata "example_plugin.json"
```
%% Cell type:markdown id: tags:
This stores a random owner key and database key on your disk for future use, and runs the pymemri example plugin. If everything works correctly, the output should read `Plugin run success.`
%% Cell type:markdown id: tags:
## Docs
[pymemri docs](https://memri.docs.memri.io/docs.memri.io/component-architectures/plugins/readme/)
[pymemri docs](https://docs.memri.io/component-architectures/plugins/readme/)
%% Cell type:markdown id: tags:
## Nbdev & Jupyter Notebooks
The Python integrators are written in [nbdev](https://nbdev.fast.ai/) ([video](https://www.youtube.com/watch?v=9Q6sLbz37gk&t=1301s)). With nbdev, it is encouraged to write code in
[Jupyter Notebooks](https://jupyter.readthedocs.io/en/latest/install/notebook-classic.html). Nbdev syncs all the notebooks in `/nbs` with the python code in `/pymemri`. Tests are written side by side with the code in the notebooks, and documentation is automatically generated from the code and markdown in the notebooks and exported into the `/docs` folder. Check out the [nbdev quickstart](wiki/nbdev_quickstart.md) for an introduction, **watch the video linked above**, or see the [nbdev documentation](https://nbdev.fast.ai/) for a all functionalities and tutorials.
%% Cell type:code id: tags:
``` python
```
......
nbs/template.config.ipynb
+ 1
- 1
View file @ f1679b61
%% Cell type:code id:9dc1dd55 tags:
``` python
%load_ext autoreload
%autoreload 2
# default_exp template.config
```
%% Cell type:code id:ba7f7b8e tags:
``` python
# export
from typing import List
from fastcore.script import call_parse, Param
import inspect
import os
import importlib
import json
from pathlib import Path
from pymemri.plugin.pluginbase import get_plugin_cls
from pymemri.pod.client import PodClient
```
%% Cell type:code id:7726d3b4 tags:
``` python
# hide
# test imports
from pymemri.plugin.pluginbase import PluginBase
from pprint import pprint
import os
```
%% Cell type:markdown id:d9a80467 tags:
# Plugin Configuration
%% Cell type:markdown id:a554cfac tags:
Often, plugins require some configuration before running. For example you might want to run plugin on data from a specific service, or run a specific version of your machine learning model. To configure your plugin when running it from the front-end, plugins require a `config.json` in the root of the plugin repository. This file contains a declarative definition, which our front-end app uses to display a configuration screen.
This module contains utilities to directly generate this config file from the plugin definition, by inferring all arguments from the plugin `__init__` method. All configuration fields are textboxes by default, which can be changed to different types in the future. For a full example, see our guide on [building plugins](https://memri.docs.memri.io/docs.memri.io/guides/build_and_deploy_your_model/).
This module contains utilities to directly generate this config file from the plugin definition, by inferring all arguments from the plugin `__init__` method. All configuration fields are textboxes by default, which can be changed to different types in the future. For a full example, see our guide on [building plugins](https://docs.memri.io/guides/build_and_deploy_your_model/).
Usage:
```
create_plugin_config
```
%% Cell type:code id:faa994ca tags:
``` python
# export
# hide
ALLOWED_TYPES = [int, str, float, bool]
# export
def get_params(cls):
params = inspect.signature(cls.__init__).parameters
return {k: v for k, v in list(params.items()) if k not in {"self", "args", "kwargs"}}
def identifier_to_displayname(identifier: str) -> str:
return identifier.replace("_", " ").title()
def get_param_config(name, dtype, is_optional, default):
return {
"name": name,
"display": identifier_to_displayname(name),
"data_type": dtype,
"type": "textbox",
"default": default,
"optional": is_optional,
}
```
%% Cell type:code id:b9eb78f4 tags:
``` python
# export
def create_config(plugin_cls: type) -> List[dict]:
"""
Returns a declarative plugin configuration, inferred from the `__init__` method signature of `plugin_cls`.
This function is used internally by the `create_plugin_config` CLI. For general use, use the CLI instead.
Arguments that start with `_`, untyped arguments or arguments that are not in `ALLOWED_TYPES` are skipped.
Args:
plugin_cls (type): A plugin class, inherited from PluginBase.
Returns:
List[dict]: A declarative configuration definition as list of dictionaries.
"""
config = list()
for param_name, param in get_params(plugin_cls).items():
if param_name.startswith("_"):
continue
if param.annotation == inspect._empty:
print(f"Skipping unannotated parameter `{param_name}`")
continue
if param.annotation not in ALLOWED_TYPES:
print(f"Skipping parameter with unknown type: `{param_name}: {param.annotation}`")
continue
is_optional = param.default != inspect._empty
dtype = PodClient.TYPE_TO_SCHEMA[param.annotation]
default = param.default if is_optional else None
param_config = get_param_config(param_name, dtype, is_optional, default)
config.append(param_config)
return config
```
%% Cell type:code id:cbd682fb tags:
``` python
# export
@call_parse
def create_plugin_config(
metadata: Param("metadata.json of the plugin", str) = "./metadata.json",
tgt_file: Param("Filename of config file", str) = "config.json",
schema_file: Param("Filename of exported plugin schema", str) = "schema.json"
):
"""
Creates a plugin configuration definition from the arguments of your plugin class.
Configuration arguments are inferred from the arguments of your plugin `__init__` method.
Arguments that start with `_`, untyped arguments or arguments that are not in `ALLOWED_TYPES` are skipped.
All generated fields are "textbox" by default, in the future our front-end will support more
types of fields.
Args:
metadata (Param, optional): Location of the "metadata.json" file,
Defaults to "./metadata.json"
tgt_file (Param, optional): File the config definition is saved to.
Defaults to "config.json".
"""
if metadata is None:
if os.path.exists("./metadata.json"):
metadata = "./metadata.json"
else:
print("Define a metadata file with --metadata <filename>")
return
with open(metadata, "r") as f:
metadata = json.load(f)
plugin_module = metadata["pluginModule"]
plugin_name = metadata["pluginName"]
try:
plugin_cls = get_plugin_cls(plugin_module, plugin_name)
except Exception as e:
print(e)
return
config = create_config(plugin_cls)
with open(tgt_file, "w") as f:
json.dump(config, f, indent=2)
print(f"Config saved to {Path(tgt_file)}")
plugin_schema = plugin_cls.get_schema()
with open(schema_file, "w") as f:
json.dump(plugin_schema, f, indent=2)
```
%% Cell type:markdown id:6a142f19 tags:
### Example
As example, we create a test plugin with various configuration arguments. The `create_config` method is called on the plugin, and generates a declarative configuration field for each plugin argument. Note that arguments that start with a `_`, and untyped arguments are skipped in the annotation.
%% Cell type:code id:f885a624 tags:
``` python
class MyPlugin(PluginBase):
def __init__(
self,
my_arg: str,
my_str: str = None,
my_int: int = None,
my_float: float = None,
my_bool: bool = None,
unannotated=None,
_private=None,
**kwargs
):
super().__init__(**kwargs)
def run(self):
pass
def add_to_schema(self):
pass
```
%% Cell type:code id:ae222d40 tags:
``` python
config = create_config(MyPlugin)
for c in config:
print(f'{c["display"]} ({c["data_type"]})')
```
%% Output
Skipping unannotated parameter `unannotated`
My Arg (Text)
My Str (Text)
My Int (Integer)
My Float (Real)
My Bool (Bool)
%% Cell type:code id:f64fa90e tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted basic.ipynb.
Converted cvu.utils.ipynb.
Converted data.dataset.ipynb.
Converted data.loader.ipynb.
Converted data.photo.ipynb.
Converted exporters.exporters.ipynb.
Converted index.ipynb.
Converted itembase.ipynb.
Converted plugin.authenticators.credentials.ipynb.
Converted plugin.authenticators.oauth.ipynb.
Converted plugin.listeners.ipynb.
Converted plugin.pluginbase.ipynb.
Converted plugin.states.ipynb.
Converted plugins.authenticators.password.ipynb.
Converted pod.api.ipynb.
Converted pod.client.ipynb.
Converted pod.db.ipynb.
Converted pod.utils.ipynb.
Converted template.config.ipynb.
Converted template.formatter.ipynb.
Converted test_schema.ipynb.
Converted test_utils.ipynb.
%% Cell type:code id:7d8d6100 tags:
``` python
```
......
nbs/template.formatter.ipynb
+ 5
- 3
View file @ f1679b61
%% Cell type:code id:f380ac34 tags:
``` python
%load_ext autoreload
%autoreload 2
# default_exp template.formatter
```
%% Cell type:code id:a8a7e4c9 tags:
``` python
# export
# hide
from pathlib import Path
from typing import Dict, Union, List
from fastcore.script import call_parse, Param, store_true
import zipfile
from string import Template
import re
import giturlparse
import subprocess
import pymemri
import urllib
from pathlib import PurePosixPath
import requests
```
%% Cell type:code id:74de84a3 tags:
``` python
# hide
# test imports
from pprint import pprint
import os
```
%% Cell type:markdown id:77d47d51 tags:
# Creating plugins from a template
%% Cell type:code id:9898bd14 tags:
``` python
# export
# hide
TEMPLATE_URL = "https://gitlab.memri.io/memri/plugin-templates/-/archive/dev/plugin-templates-dev.zip"
TEMPLATE_BASE_PATH = "plugin-templates-dev"
```
%% Cell type:markdown id:475c3c1b tags:
Pymemri offers a range of plugin templates to set up testing, docker and CI for you. This way, you can focus on building your plugin, and be sure it works within the Memri ecosystem.
All plugins are hosted on our [GitLab](https://gitlab.memri.io/). In order to make your own plugin from a template,
1. Create an account on [GitLab](https://gitlab.memri.io/)
2. Create a _public_ [blank repository](https://gitlab.memri.io/projects/new#blank_project)
3. Clone the repository
4. run the `plugin_from_template` CLI inside the repository folder.
The CLI will infer most settings for you from your git account and repository name, only a template name and optional description are required.
```
plugin_from_template --template classifier_plugin --description "My Classifier Plugin"
```
To make sure all settings are correct, you can inspect `metadata.json`, which holds all information like your plugin name, and python package name.
-----------------
You can list the available templates with. All plugin templates are hosted [here](https://gitlab.memri.io/memri/plugin-templates).
```
plugin_from_template --list
```
The CLI has options to customize the plugin name, package name and other aspects of your plugin. For advanced use, run:
```
plugin_from_template --help
```
%% Cell type:markdown id:2c9883e9 tags:
## Utility functions -
%% Cell type:code id:2933c056 tags:
``` python
# export
# hide
# If the owner of the repository is one of these groups, the CLI requires an additional `user` argument
GITLAB_GROUPS = ["memri", "plugins"]
def get_remote_url():
path = Path(".")
url = subprocess.getoutput(f'git config --get remote.origin.url')
if not url:
raise ValueError(f"You can only run this from a initialized gitlab repository, and '{path}' is not an initialized git repository")
parsed = giturlparse.parse(url)
repo_url = parsed.url2https
if repo_url.endswith(".git"):
repo_url = repo_url[:-4]
return repo_url
def infer_git_info(url):
parsed = giturlparse.parse(url)
return parsed.owner, parsed.repo
```
%% Cell type:code id:16dfc0d2 tags:
``` python
# hide
remote_url = get_remote_url()
repo_owner, repo_name = infer_git_info(remote_url)
assert repo_owner == "memri"
assert repo_name == "pymemri"
```
%% Cell type:code id:42378645 tags:
``` python
# export
# hide
def download_file(url, fname=None):
cert_path = Path(pymemri.__file__).parent / "cert" / "gitlab.memri.io.pem"
r = requests.get(url, stream=True, verify=cert_path)
fname = url.rsplit('/', 1)[1] if fname is None else fname
with open(fname, 'wb') as fd:
for chunk in r.iter_content(chunk_size=128):
fd.write(chunk)
return fname
```
%% Cell type:code id:9a72f504 tags:
``` python
# export
# hide
def str_to_identifier(s, lower=True):
result = re.sub("\W|^(?=\d)", "_", s)
if lower:
result = result.lower()
return result
def reponame_to_displayname(reponame: str) -> str:
return re.sub("[-_]+", " ", reponame).title()
def download_plugin_template(
template_name: str, url: str = TEMPLATE_URL, base_path: str = TEMPLATE_BASE_PATH
):
base_path = Path(base_path) / template_name
zip_path = download_file(url)
with zipfile.ZipFile(zip_path, "r") as f:
result = {name: f.read(name) for name in f.namelist() if base_path in Path(name).parents}
if len(result) == 0:
raise ValueError(f"Could not find template: {template_name}")
result = {str(PurePosixPath(k).relative_to(PurePosixPath(base_path))): v.decode("utf-8") for k, v in result.items() if v}
Path(zip_path).unlink()
return result
def get_templates(url: str = TEMPLATE_URL) -> List[str]:
zip_path = download_file(url)
with zipfile.ZipFile(zip_path, "r") as f:
files_split = [name.split("/") for name in f.namelist()]
result = [fn[1] for fn in files_split if fn[-1] == '' and len(fn) == 3]
return result
```
%% Cell type:code id:becd3e18 tags:
``` python
# hide
assert len(get_templates())
```
%% Cell type:code id:f99827cd tags:
``` python
# hide
assert str_to_identifier("My Plugin") == "my_plugin"
template = download_plugin_template("classifier_plugin")
assert len(template)
pprint(list(template.keys()))
```
%% Output
['$package_name/model.py',
'$package_name/plugin.py',
'$package_name/schema.py',
'$package_name/utils.py',
'.gitignore',
'.gitlab-ci.yml',
'Dockerfile',
'LICENSE.txt',
'README.md',
'metadata.json',
'setup.cfg',
'setup.py',
'tests/test_plugin.py',
'tools/preload.py']
%% Cell type:code id:2707a14d tags:
``` python
# export
# hide
class TemplateFormatter:
def __init__(
self,
template_dict: Dict[str, str],
replace_dict: Dict[str, str],
tgt_path: Union[str, Path],
verbose: bool = False,
):
self.template_dict = template_dict
self.tgt_path = Path(tgt_path)
self.replace_dict = replace_dict
self.verbose = verbose
def format_content(self, content):
return Template(content).safe_substitute(self.replace_dict)
def format_path(self, path):
new_path = Template(path).safe_substitute(self.replace_dict)
return self.tgt_path / new_path
def format_file(self, filename, content):
new_path = self.format_path(filename)
new_content = self.format_content(content)
new_path.parent.mkdir(exist_ok=True, parents=True)
if self.verbose:
print(f"Formatting {filename} -> {new_path}")
with open(new_path, "w", encoding="utf-8") as f:
f.write(new_content)
def get_files(self):
return [self.format_path(filename) for filename in self.template_dict.keys()]
def format(self):
for filename, content in self.template_dict.items():
self.format_file(filename, content)
def print_filetree(self):
previous_prefix = None
res = "Created the following files"
for path in sorted([x.relative_to(self.tgt_path) for x in self.get_files()],
key=lambda item: 100 * str(item).count("/")):
n_slashes = str(path).count("/")
new_prefix = path.parent
if previous_prefix != new_prefix and str(new_prefix) != ".":
res = f"{res}\n├── {new_prefix}"
if n_slashes == 0:
res = f"{res}\n├── {path}"
elif n_slashes == 1:
res = f"{res}\n│ ├── {path.name}"
previous_prefix=new_prefix
print(res.strip() + "\n")
```
%% Cell type:markdown id:30c2cdd4 tags:
### Plugin Template CLI
With the `plugin_from_template` CLI, you can easily create a plugin where all CI pipelines, docker files, and test setups are configured for you. Multiple templates are available, to see the complete list use:
`plugin_from_template --list_templates`
%% Cell type:code id:9ef65986 tags:
``` python
# export
# hide
def get_template_replace_dict(
repo_url=None, user=None, plugin_name=None, package_name=None, description=None, install_requires=None
):
if repo_url is None:
repo_url = get_remote_url()
try:
repo_owner, repo_name = infer_git_info(repo_url)
except ValueError:
url_inf, owner_inf, name_inf = None, None, None
print("Could not infer git information from current directory, no initialized repository found.")
if repo_url is None:
repo_url = url_inf
if user is None:
if repo_owner in GITLAB_GROUPS:
user = None
else:
user = repo_owner
if plugin_name is None:
if repo_name is None:
plugin_name = None
else:
plugin_name = reponame_to_displayname(repo_name)
if package_name is None:
if repo_name is None:
package_name = None
else:
package_name = str_to_identifier(repo_name)
if install_requires is None:
install_requires = ""
else:
install_requires = "\n ".join([x.strip() for x in install_requires.split(",")
if x.strip() != "" and x.strip() not in ["pymemri", "pytest"]])
return {
"user": user,
"package_name": package_name,
"plugin_name": plugin_name,
"repo_name": repo_name,
"repo_url": repo_url,
"description": str(description),
"install_requires": install_requires
}
```
%% Cell type:code id:5c4df6b0 tags:
``` python
# export
@call_parse
def plugin_from_template(
list_templates: Param("List available plugin templates", store_true) = False,
user: Param("Your Gitlab username", str) = None,
repo_url: Param("The url of your empty Gitlab plugin repository", str) = None,
plugin_name: Param("Display name of your plugin", str) = None,
template_name: Param(
"Name of the template, use `list_templates` to see all available options"
) = "basic",
package_name: Param("Name of your plugin python package", str) = None,
description: Param("Description of your plugin", str) = None,
target_dir: Param("Directory to output the formatted template", str) = ".",
verbose: Param("Should print out dir", bool) = True,
install_requires: Param("Extra packages to install, provided as comma separated, e.g. pymemri,requests", str)=""
):
"""
CLI that downloads and formats a plugin template according to the arguments, and local git repository.
Args:
list_templates (Param, optional): If True, only list available templates. Defaults to False.
user (Param, optional): Your GitLab username. Defaults to None.
repo_url (Param, optional): The url of your gitlab plugin repository. Defaults to None.
plugin_name (Param, optional): The name of your plugin. Defaults to None.
template_name (Param, optional): The name of the template used. To list all options, see `list_templates`.
Defaults to "basic".
package_name (Param, optional): The name of the python package of your plugin. Inferred if left blank. Defaults to None.
description (Param, optional): An optional plugin description. Defaults to None.
target_dir (Param, optional): Directory where the plugin template is generated. Defaults to ".".
"""
if list_templates:
print("Available templates:")
for template in get_templates():
print(template)
return
template = download_plugin_template(template_name)
tgt_path = Path(target_dir)
replace_dict = get_template_replace_dict(
repo_url=repo_url,
user=user,
plugin_name=plugin_name,
package_name=package_name,
description=description,
install_requires=install_requires
)
formatter = TemplateFormatter(template, replace_dict, tgt_path)
formatter.format()
formatter.print_filetree()
if verbose:
formatter.print_filetree()
print(f"Created `{replace_dict['plugin_name']}` using the {template_name} template.")
```
%% Cell type:code id:e7597558 tags:
``` python
!plugin_from_template --list_templates
```
%% Output
Available templates:
plugin-templates-dev.zip
basic
classifier_plugin
%% Cell type:code id:3d54e18e tags:
``` python
# hide
import tempfile
template = download_plugin_template("classifier_plugin")
replace_dict = {
"user": "eelcovdw",
"repo_name": "sentiment-plugin",
"package_name": "sentiment_plugin",
"plugin_name": "Sentiment Plugin",
"description": "Predict sentiment on text messages"
}
with tempfile.TemporaryDirectory() as result_path:
print(result_path)
result_path = Path(result_path)
formatter = TemplateFormatter(template, replace_dict, result_path)
formatter.format()
created_files = [f for f in result_path.rglob("*") if not os.path.isdir(f)]
contents = {}
for fn in created_files:
with open(fn, "r") as f:
contents[str(fn)] = f.read()
formatter.print_filetree()
# print("Created files:")
# pprint(created_files)
assert len(template) == len(created_files)
```
%% Output
/var/folders/q1/ryq93kwj055dlbpngxv1c7z40000gn/T/tmpwc3gkq_t
/var/folders/q1/ryq93kwj055dlbpngxv1c7z40000gn/T/tmpli6b4mju
Created the following files
├── .gitignore
├── .gitlab-ci.yml
├── Dockerfile
├── LICENSE.txt
├── README.md
├── metadata.json
├── setup.cfg
├── setup.py
├── sentiment_plugin
│ ├── model.py
│ ├── plugin.py
│ ├── schema.py
│ ├── utils.py
├── tests
│ ├── test_plugin.py
├── tools
│ ├── preload.py
%% Cell type:code id:fc4993b2 tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted basic.ipynb.
Converted cvu.utils.ipynb.
Converted data.dataset.ipynb.
Converted data.loader.ipynb.
Converted data.photo.ipynb.
Converted exporters.exporters.ipynb.
Converted gitlab_api.ipynb.
Converted index.ipynb.
Converted itembase.ipynb.
Converted plugin.authenticators.credentials.ipynb.
Converted plugin.authenticators.oauth.ipynb.
Converted plugin.listeners.ipynb.
Converted plugin.pluginbase.ipynb.
Converted plugin.states.ipynb.
Converted plugins.authenticators.password.ipynb.
Converted pod.api.ipynb.
Converted pod.client.ipynb.
Converted pod.db.ipynb.
Converted pod.utils.ipynb.
Converted template.config.ipynb.
Converted template.formatter.ipynb.
Converted test_schema.ipynb.
Converted test_utils.ipynb.
%% Cell type:code id:9d9cfe6f tags:
``` python
```
......
pymemri/_nbdev.py
+ 22
- 15
View file @ f1679b61
......@@ -17,32 +17,38 @@ index = {"read_file": "basic.ipynb",
"list_default_cvus": "cvu.utils.ipynb",
"filter_rows": "data.dataset.ipynb",
"Dataset": "data.dataset.ipynb",
"MEMRI_PATH": "data.loader.ipynb",
"MEMRI_GITLAB_BASE_URL": "data.loader.ipynb",
"ACCESS_TOKEN_PATH": "data.loader.ipynb",
"GITLAB_API_BASE_URL": "data.loader.ipynb",
"DEFAULT_PLUGIN_MODEL_PACKAGE_NAME": "data.loader.ipynb",
"DEFAULT_PYTORCH_MODEL_NAME": "data.loader.ipynb",
"DEFAULT_HUGGINFACE_CONFIG_NAME": "data.loader.ipynb",
"DEFAULT_PACKAGE_VERSION": "data.loader.ipynb",
"TIME_FORMAT_GITLAB": "data.loader.ipynb",
"PROJET_ID_PATTERN": "data.loader.ipynb",
"find_git_repo": "data.loader.ipynb",
"get_registry_api_key": "data.loader.ipynb",
"upload_in_chunks": "data.loader.ipynb",
"IterableToFileAdapter": "data.loader.ipynb",
"write_file_to_package_registry": "data.loader.ipynb",
"project_id_from_name": "data.loader.ipynb",
"get_project_id_from_project_path_unsafe": "data.loader.ipynb",
"write_huggingface_model_to_package_registry": "data.loader.ipynb",
"write_model_to_package_registry": "data.loader.ipynb",
"download_package_file": "data.loader.ipynb",
"download_huggingface_model_for_project": "data.loader.ipynb",
"load_huggingface_model_for_project": "data.loader.ipynb",
"DEFAULT_ENCODING": "data.photo.ipynb",
"show_images": "data.photo.ipynb",
"Photo": "data.photo.ipynb",
"Query": "exporters.exporters.ipynb",
"MEMRI_PATH": "gitlab_api.ipynb",
"MEMRI_GITLAB_BASE_URL": "gitlab_api.ipynb",
"ACCESS_TOKEN_PATH": "gitlab_api.ipynb",
"GITLAB_API_BASE_URL": "gitlab_api.ipynb",
"DEFAULT_PACKAGE_VERSION": "gitlab_api.ipynb",
"TIME_FORMAT_GITLAB": "gitlab_api.ipynb",
"PROJET_ID_PATTERN": "gitlab_api.ipynb",
"find_git_repo": "gitlab_api.ipynb",
"get_registry_api_key": "gitlab_api.ipynb",
"upload_in_chunks": "gitlab_api.ipynb",
"IterableToFileAdapter": "gitlab_api.ipynb",
"write_file_to_package_registry": "gitlab_api.ipynb",
"project_id_from_name": "gitlab_api.ipynb",
"get_project_id_from_project_path_unsafe": "gitlab_api.ipynb",
"download_package_file": "gitlab_api.ipynb",
"create_repo": "gitlab_api.ipynb",
"get_current_username": "gitlab_api.ipynb",
"commit_file": "gitlab_api.ipynb",
"rm_tree": "gitlab_api.ipynb",
"write_files_to_git": "gitlab_api.ipynb",
"create_new_project": "gitlab_api.ipynb",
"ALL_EDGES": "itembase.ipynb",
"Edge": "itembase.ipynb",
"EdgeList": "itembase.ipynb",
......@@ -126,6 +132,7 @@ modules = ["data/basic.py",
"data/loader.py",
"data/photo.py",
"exporters/exporters.py",
"gitlab_api.py",
"data/itembase.py",
"plugin/authenticators/credentials.py",
"plugin/authenticators/oauth.py",
......
pymemri/data/loader.py
+ 8
- 167
View file @ f1679b61
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/data.loader.ipynb (unless otherwise specified).
__all__ = ['MEMRI_PATH', 'MEMRI_GITLAB_BASE_URL', 'ACCESS_TOKEN_PATH', 'GITLAB_API_BASE_URL',
'DEFAULT_PLUGIN_MODEL_PACKAGE_NAME', 'DEFAULT_PYTORCH_MODEL_NAME', 'DEFAULT_HUGGINFACE_CONFIG_NAME',
'DEFAULT_PACKAGE_VERSION', 'TIME_FORMAT_GITLAB', 'PROJET_ID_PATTERN', 'find_git_repo',
'get_registry_api_key', 'upload_in_chunks', 'IterableToFileAdapter', 'write_file_to_package_registry',
'project_id_from_name', 'get_project_id_from_project_path_unsafe',
'write_huggingface_model_to_package_registry', 'write_model_to_package_registry', 'download_package_file',
__all__ = ['DEFAULT_PLUGIN_MODEL_PACKAGE_NAME', 'DEFAULT_PYTORCH_MODEL_NAME', 'DEFAULT_HUGGINFACE_CONFIG_NAME',
'write_huggingface_model_to_package_registry', 'write_model_to_package_registry',
'download_huggingface_model_for_project', 'load_huggingface_model_for_project']
# Cell
......@@ -17,144 +13,17 @@ from getpass import getpass
from datetime import datetime
from git import Repo
import re
from ..gitlab_api import MEMRI_PATH, MEMRI_GITLAB_BASE_URL, ACCESS_TOKEN_PATH, GITLAB_API_BASE_URL, TIME_FORMAT_GITLAB, \
PROJET_ID_PATTERN, DEFAULT_PACKAGE_VERSION, download_package_file, project_id_from_name, write_file_to_package_registry, \
get_registry_api_key
# Cell
MEMRI_PATH = Path.home() / ".memri"
MEMRI_GITLAB_BASE_URL = "https://gitlab.memri.io"
ACCESS_TOKEN_PATH = Path.home() / ".memri/access_token/access_token.txt"
GITLAB_API_BASE_URL = "https://gitlab.memri.io/api/v4"
DEFAULT_PLUGIN_MODEL_PACKAGE_NAME = "plugin-model-package"
DEFAULT_PYTORCH_MODEL_NAME = "pytorch_model.bin"
DEFAULT_HUGGINFACE_CONFIG_NAME = "config.json"
DEFAULT_PACKAGE_VERSION = "0.0.1"
TIME_FORMAT_GITLAB = '%Y-%m-%dT%H:%M:%S.%fZ'
PROJET_ID_PATTERN = '(?<=<span class="gl-button-text">Project ID: )[0-9]+(?=</span>)'
# Cell
def find_git_repo():
path = "."
for i in range(10):
try:
repo = Repo(f"{path + ('.' * i)}/")
except:
pass
else:
break
if i == 9:
raise ValueError(f"could not fine git repo in {os.path.abspath('')}")
repo_name = repo.remotes.origin.url.split('.git')[0].split('/')[-1]
return repo_name
# Cell
def get_registry_api_key():
ACCESS_TOKEN_PATH.parent.mkdir(parents=True, exist_ok=True)
if ACCESS_TOKEN_PATH.is_file():
with open(ACCESS_TOKEN_PATH, "r") as f:
return f.read()
else:
print(f"""
The first time you are uploading a model you need to create an access_token
at https://gitlab.memri.io/-/profile/personal_access_tokens?name=Model+Access+token&scopes=api
Click at the blue button with 'Create personal access token'"
""")
access_token = getpass("Then copy your personal access token from 'Your new personal access token', and paste here: ")
with open(ACCESS_TOKEN_PATH, "w") as f:
f.write(access_token)
return access_token
# Cell
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 << 14):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
n = 100
pb = progress_bar(range(n))
pb_iter = iter(pb)
i = 1
delta = 1 / n
next(pb_iter, None)
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
while (percent / 100) > i * delta:
next(pb_iter, None)
i += 1
yield data
pb.update_bar(n)
def __len__(self):
return self.totalsize
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
return next(self.iterator, b'')
def __len__(self):
return self.length
# Cell
def write_file_to_package_registry(project_id, file_path, api_key, version=DEFAULT_PACKAGE_VERSION):
file_path = Path(file_path)
file_name = file_path.name
url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{DEFAULT_PLUGIN_MODEL_PACKAGE_NAME}/{version}/{file_name}"
print(f"uploading {file_path}")
it = upload_in_chunks(file_path)
res = requests.put(url=url, data=IterableToFileAdapter(it),
headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201]:
print(f"Failed to upload {file_path}: {res.content}")
else:
print(f"Succesfully uploaded {file_path}")
# Cell
def project_id_from_name(project_name, api_key, job_token=None):
if api_key:
headers = {"PRIVATE-TOKEN": api_key}
else:
headers = {"JOB-TOKEN": job_token}
res = requests.get(f"{GITLAB_API_BASE_URL}/projects",
headers=headers,
params={
"owned": True,
"search": project_name
})
# we need this extra filter (search is not exact match)
res = [x.get("id") for x in res.json() if x.get("path", None) == project_name]
if len(res) == 0:
raise ValueError(f"No plugin found with name {project_name}, make sure to enter the name as specified in the url of the repo")
else:
return res[0]
# Cell
def get_project_id_from_project_path_unsafe(project_path):
try:
res = requests.get(f"{MEMRI_GITLAB_BASE_URL}/{project_path}")
html = str(res.content)
match = re.search(PROJET_ID_PATTERN, html)
return match.group()
except Exception:
raise ValueError(f"Could not find project with name {project_path}")
# Cell
def write_huggingface_model_to_package_registry(project_name, model):
def write_huggingface_model_to_package_registry(project_name, model, version=DEFAULT_PACKAGE_VERSION):
import torch
api_key = get_registry_api_key()
project_id = project_id_from_name(project_name, api_key)
......@@ -165,7 +34,7 @@ def write_huggingface_model_to_package_registry(project_name, model):
for f in [DEFAULT_HUGGINFACE_CONFIG_NAME, DEFAULT_PYTORCH_MODEL_NAME]:
file_path = local_save_dir / f
print(f"writing {f} to package registry of {project_name} with project id {project_id}")
write_file_to_package_registry(project_id, file_path, api_key)
write_file_to_package_registry(project_id, file_path, api_key, package_name=DEFAULT_PLUGIN_MODEL_PACKAGE_NAME, version=version)
# Cell
def write_model_to_package_registry(model, project_name=None):
......@@ -178,40 +47,12 @@ def write_model_to_package_registry(model, project_name=None):
else:
raise ValueError(f"Model type not supported: {type(model)}")
# Cell
def download_package_file(filename, project_path=None, out_dir=None, package_name=DEFAULT_PLUGIN_MODEL_PACKAGE_NAME,
package_version=DEFAULT_PACKAGE_VERSION, download_if_exists=False):
project_name = str(project_path).split("/")[-1]
out_dir = out_dir if out_dir is not None else MEMRI_PATH / "projects" / project_name
out_dir.mkdir(parents=True, exist_ok=True)
project_id = get_project_id_from_project_path_unsafe(project_path)
file_path = out_dir / filename
print(file_path)
if file_path.exists() and not download_if_exists:
print(f"{file_path} already exists, and `download_if_exists`==False, using cached version")
return file_path
print(f"downloading {filename} from project {project_path}, package {package_name}")
res = requests.get(
url=f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{package_version}/{filename}"
)
res.raise_for_status()
with open(out_dir / filename, "wb") as f:
print(f"writing {filename} to {out_dir}")
f.write(res.content)
return file_path
# Cell
def download_huggingface_model_for_project(project_path=None, files=None, download_if_exists=False):
if files is None:
files = ["config.json", "pytorch_model.bin"]
for f in files:
out_file_path = download_package_file(f, project_path=project_path)
out_file_path = download_package_file(f, project_path=project_path, package_name=DEFAULT_PLUGIN_MODEL_PACKAGE_NAME)
return out_file_path.parent
# Cell
......
pymemri/gitlab_api.py 0 → 100644
+ 267
- 0
View file @ f1679b61
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/gitlab_api.ipynb (unless otherwise specified).
__all__ = ['MEMRI_PATH', 'MEMRI_GITLAB_BASE_URL', 'ACCESS_TOKEN_PATH', 'GITLAB_API_BASE_URL', 'DEFAULT_PACKAGE_VERSION',
'TIME_FORMAT_GITLAB', 'PROJET_ID_PATTERN', 'find_git_repo', 'get_registry_api_key', 'upload_in_chunks',
'IterableToFileAdapter', 'write_file_to_package_registry', 'project_id_from_name',
'get_project_id_from_project_path_unsafe', 'download_package_file', 'create_repo', 'get_current_username',
'commit_file', 'rm_tree', 'write_files_to_git', 'create_new_project']
# Cell
from fastprogress.fastprogress import progress_bar
from pathlib import Path
import requests
import os, sys
from getpass import getpass
from datetime import datetime
from git import Repo
import re
from .data.basic import *
from .template.formatter import plugin_from_template
import urllib
# Cell
MEMRI_PATH = Path.home() / ".memri"
MEMRI_GITLAB_BASE_URL = "https://gitlab.memri.io"
ACCESS_TOKEN_PATH = Path.home() / ".memri/access_token/access_token.txt"
GITLAB_API_BASE_URL = "https://gitlab.memri.io/api/v4"
DEFAULT_PACKAGE_VERSION = "0.0.1"
TIME_FORMAT_GITLAB = '%Y-%m-%dT%H:%M:%S.%fZ'
PROJET_ID_PATTERN = '(?<=<span class="gl-button-text">Project ID: )[0-9]+(?=</span>)'
# Cell
def find_git_repo():
path = "."
for i in range(10):
try:
repo = Repo(f"{path + ('.' * i)}/")
except:
pass
else:
break
if i == 9:
raise ValueError(f"could not fine git repo in {os.path.abspath('')}")
repo_name = repo.remotes.origin.url.split('.git')[0].split('/')[-1]
return repo_name
# Cell
def get_registry_api_key():
ACCESS_TOKEN_PATH.parent.mkdir(parents=True, exist_ok=True)
if ACCESS_TOKEN_PATH.is_file():
with open(ACCESS_TOKEN_PATH, "r") as f:
return f.read()
else:
print(f"""
The first time you are uploading a model you need to create an access_token
at https://gitlab.memri.io/-/profile/personal_access_tokens?name=Model+Access+token&scopes=api
Click at the blue button with 'Create personal access token'"
""")
access_token = getpass("Then copy your personal access token from 'Your new personal access token', and paste here: ")
with open(ACCESS_TOKEN_PATH, "w") as f:
f.write(access_token)
return access_token
# Cell
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 << 14):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
n = 100
pb = progress_bar(range(n))
pb_iter = iter(pb)
i = 1
delta = 1 / n
next(pb_iter, None)
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
while (percent / 100) > i * delta:
next(pb_iter, None)
i += 1
yield data
pb.update_bar(n)
def __len__(self):
return self.totalsize
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
return next(self.iterator, b'')
def __len__(self):
return self.length
# Cell
def write_file_to_package_registry(project_id, file_path, api_key, package_name, version=DEFAULT_PACKAGE_VERSION):
file_path = Path(file_path)
file_name = file_path.name
url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{version}/{file_name}"
print(f"uploading {file_path}")
it = upload_in_chunks(file_path)
res = requests.put(url=url, data=IterableToFileAdapter(it),
headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201]:
print(f"Failed to upload {file_path}: {res.content}")
else:
print(f"Succesfully uploaded {file_path}")
# Cell
def project_id_from_name(project_name, api_key, job_token=None):
if api_key:
headers = {"PRIVATE-TOKEN": api_key}
else:
headers = {"JOB-TOKEN": job_token}
res = requests.get(f"{GITLAB_API_BASE_URL}/projects",
headers=headers,
params={
"owned": True,
"search": project_name
})
# we need this extra filter (search is not exact match)
res = [x.get("id") for x in res.json() if x.get("path", None) == project_name]
if len(res) == 0:
raise ValueError(f"No plugin found with name {project_name}, make sure to enter the name as specified in the url of the repo")
else:
return res[0]
# Cell
def get_project_id_from_project_path_unsafe(project_path):
try:
res = requests.get(f"{MEMRI_GITLAB_BASE_URL}/{project_path}")
html = str(res.content)
match = re.search(PROJET_ID_PATTERN, html)
return match.group()
except Exception:
raise ValueError(f"Could not find project with name {project_path}")
# Cell
def download_package_file(filename, project_path, package_name, out_dir=None,
package_version=DEFAULT_PACKAGE_VERSION, download_if_exists=False):
project_name = str(project_path).split("/")[-1]
out_dir = out_dir if out_dir is not None else MEMRI_PATH / "projects" / project_name
out_dir.mkdir(parents=True, exist_ok=True)
project_id = get_project_id_from_project_path_unsafe(project_path)
file_path = out_dir / filename
print(file_path)
if file_path.exists() and not download_if_exists:
print(f"{file_path} already exists, and `download_if_exists`==False, using cached version")
return file_path
print(f"downloading {filename} from project {project_path}, package {package_name}")
res = requests.get(
url=f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{package_version}/{filename}"
)
res.raise_for_status()
with open(out_dir / filename, "wb") as f:
print(f"writing {filename} to {out_dir}")
f.write(res.content)
return file_path
# Cell
def create_repo(repo_name):
url = f"{GITLAB_API_BASE_URL}/projects/"
api_key = get_registry_api_key()
payload = {"name": repo_name}
res = requests.post(url=url, json=payload,
headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200, 201]:
raise ValueError(f"failed to create repo:\n {res.text}")
print(f"created project {repo_name}")
# Cell
def get_current_username():
url = f"{GITLAB_API_BASE_URL}/user/"
api_key = get_registry_api_key()
res = requests.get(url=url, headers={"PRIVATE-TOKEN": api_key})
if res.status_code not in [200,201]:
raise ValueError(f"Could not find current user {res.content}")
else:
username = res.json()["username"]
return username
# Cell
def commit_file(project_name, path_in2out, branch="main"):
api_key = get_registry_api_key()
project_id = project_id_from_name(repo, api_key)
# file_out_path_escaped = urllib.parse.quote(file_out_path, safe='')
actions = []
for file_in_path, file_out_path in path_in2out.items():
content = read_file(file_in_path)
action_payload = {"action": "create", "file_path": file_out_path, "content": content}
actions.append(action_payload)
url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/repository/commits"
payload = {"branch": branch, "commit_message": "automated commit", "content": content, "actions": actions}
res = requests.post(url=url, json=payload,
headers={"PRIVATE-TOKEN": api_key})
files_in = list(path_in2out.keys())
if res.status_code not in [200, 201, 202]:
raise ValueError(f"failed to make commit with files {files_in}:\n {res.text}")
print(f"committed files {files_in}")
# --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname",
# "content": "some content", "commit_message": "create a new file"}' \
# Cell
def rm_tree(pth):
pth = Path(pth)
for child in pth.glob('*'):
if child.is_file():
child.unlink()
else:
rm_tree(child)
try:
pth.rmdir()
except FileNotFoundError as e:
pass
# Cell
def write_files_to_git(repo, target_dir):
path_in2out = dict()
for p in target_dir.rglob("*"):
if p.is_file():
path_in_repo = p.relative_to(target_dir)
path_in2out[str(p)] = str(path_in_repo)
commit_file(str(repo), path_in2out)
def create_new_project(project_name):
tmp_dir = Path("/tmp/test") / project_name
rm_tree(tmp_dir)
plugin_from_template(
template_name="classifier_plugin",
description="A transformer based sentiment analyis plugin",
install_requires="transformers,sentencepiece,protobuf,torch==1.10.0",
target_dir=str(tmp_dir),
repo_url=f"{MEMRI_GITLAB_BASE_URL}/plugins/{project_name}",
verbose=False
)
write_files_to_git(project_name, tmp_dir)
pymemri/template/formatter.py
+ 3
- 1
View file @ f1679b61
......@@ -212,6 +212,7 @@ def plugin_from_template(
package_name: Param("Name of your plugin python package", str) = None,
description: Param("Description of your plugin", str) = None,
target_dir: Param("Directory to output the formatted template", str) = ".",
verbose: Param("Should print out dir", bool) = True,
install_requires: Param("Extra packages to install, provided as comma separated, e.g. pymemri,requests", str)=""
):
"""
......@@ -249,7 +250,8 @@ def plugin_from_template(
formatter = TemplateFormatter(template, replace_dict, tgt_path)
formatter.format()
formatter.print_filetree()
if verbose:
formatter.print_filetree()
print(f"Created `{replace_dict['plugin_name']}` using the {template_name} template.")
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment