Skip to content
GitLab
Commit 824702ca authored by Eelco van der Wel's avatar Eelco van der Wel :speech_balloon:
Browse files

update formatter url

parent e1d38682
Hide whitespace changes
Inline Side-by-side
Showing
with 16 additions and 15 deletions
+16 -15
nbs/template.formatter.ipynb
+ 15
- 14
View file @ 824702ca
%% Cell type:code id:f380ac34 tags:
``` python
%load_ext autoreload
%autoreload 2
# default_exp template.formatter
```
%% Cell type:code id:a8a7e4c9 tags:
``` python
# export
from pathlib import Path
from typing import Dict, Union, List
from fastcore.script import call_parse, Param, store_true
import zipfile
import tempfile
import urllib
from string import Template
import re
```
%% Cell type:code id:74de84a3 tags:
``` python
# test imports
from pprint import pprint
import os
```
%% Cell type:markdown id:77d47d51 tags:
# Template Formatter
This formatter makes getting started with plugins easier, by generating a plugin template with an easy to use CLI. setup files, docker files and the plugin structure are generated for you.
Example usage:
```
plugin_from_template --name <gitlab-username> --plugin_name "My Indexer Plugin"
```
%% Cell type:code id:9898bd14 tags:
``` python
# export
# hide
TEMPLATE_URL = "https://gitlab.memri.io/plugins/plugin-templates/-/archive/dev/plugin-templates-dev.zip"
TEMPLATE_URL = "https://gitlab.memri.io/memri/plugin-templates/-/archive/dev/plugin-templates-dev.zip"
TEMPLATE_BASE_PATH = "plugin-templates-dev"
```
%% Cell type:markdown id:2c9883e9 tags:
## Utility functions -
%% Cell type:code id:9a72f504 tags:
``` python
# export
# hide
def str_to_identifier(s, lower=True):
result = re.sub("\W|^(?=\d)", "_", s)
if lower:
result = result.lower()
return result
def download_plugin_template(
template_name: str, url: str = TEMPLATE_URL, base_path: str = TEMPLATE_BASE_PATH
):
base_path = str(Path(base_path) / template_name)
zip_path, _ = urllib.request.urlretrieve(url)
with zipfile.ZipFile(zip_path, "r") as f:
result = {name: f.read(name) for name in f.namelist() if base_path in name}
if len(result) == 0:
raise ValueError(f"Could not find template: {template_name}")
result = {k.replace(base_path, "").strip("/"): v.decode("utf-8") for k, v in result.items() if v}
return result
def get_templates(url: str = TEMPLATE_URL) -> List[str]:
zip_path, _ = urllib.request.urlretrieve(url)
with zipfile.ZipFile(zip_path, "r") as f:
files_split = [name.split("/") for name in f.namelist()]
result = [fn[1] for fn in files_split if fn[-1] == '' and len(fn) == 3]
return result
```
%% Cell type:code id:becd3e18 tags:
``` python
assert len(get_templates())
```
%% Cell type:code id:f99827cd tags:
``` python
assert str_to_identifier("My Plugin") == "my_plugin"
template = download_plugin_template("classifier_plugin")
assert len(template)
pprint(list(template.keys()))
```
%% Output
['$package_name/model.py',
'$package_name/plugin.py',
'$package_name/schema.py',
'$package_name/utils.py',
'.gitignore',
'.gitlab-ci.yml',
'Dockerfile',
'README.md',
'metadata.json',
'setup.cfg',
'setup.py',
'tests/test_plugin.py',
'tools/preload.py']
%% Cell type:markdown id:940800d7 tags:
## Template Formatter
%% Cell type:code id:2707a14d tags:
``` python
# export
class TemplateFormatter:
def __init__(
self,
template_dict: Dict[str, str],
replace_dict: Dict[str, str],
tgt_path: Union[str, Path],
verbose: bool = False,
):
self.template_dict = template_dict
self.tgt_path = Path(tgt_path)
self.replace_dict = replace_dict
self.verbose = verbose
def format_content(self, content):
return Template(content).safe_substitute(self.replace_dict)
def format_path(self, path):
new_path = Template(path).safe_substitute(self.replace_dict)
return self.tgt_path / new_path
def format_file(self, filename, content):
new_path = self.format_path(filename)
new_content = self.format_content(content)
new_path.parent.mkdir(exist_ok=True, parents=True)
if self.verbose:
print(f"Formatting {filename} -> {new_path}")
with open(new_path, "w", encoding="utf-8") as f:
f.write(new_content)
def format(self):
for filename, content in self.template_dict.items():
self.format_file(filename, content)
```
%% Cell type:markdown id:30c2cdd4 tags:
## Template CLI
With the `plugin_from_template` CLI, you can easily create a plugin where all CI pipelines, docker files, and test setups are configured for you. Multiple templates are available, to see the complete list use:
`plugin_from_template --list_templates`
%% Cell type:code id:5c4df6b0 tags:
``` python
# export
@call_parse
def plugin_from_template(
list_templates: Param("List available plugin templates", store_true) = False,
user: Param("Your Gitlab username", str) = None,
repo_url: Param("The url of your empty Gitlab plugin repository", str) = None,
plugin_name: Param("Display name of your plugin", str) = None,
template_name: Param("Name of the template, see the Plugin Templates repository.") = None,
package_name: Param("Name of your plugin python package", str) = None,
description: Param("Description of your plugin", str) = None,
target_dir: Param("Directory to output the formatted template", str) = ".",
):
if list_templates:
print("Available templates:")
for template in get_templates():
print(template)
return
if template_name is None:
print("template name not defined, using the classifier_plugin template.")
template_name = "classifier_plugin"
if user is None:
print("Define your gitlab user name with `--user <username>`")
return
if plugin_name is None:
print("Define your gitlab user name with `--plugin_name <name>`")
return
if repo_url is None:
print("Define your gitlab repository url with `--repo_url <url>`")
return
repo_name = repo_url.strip("/").split("/")[-1]
if package_name is None:
package_name = str_to_identifier(plugin_name, lower=True)
template = download_plugin_template(template_name)
tgt_path = Path(target_dir)
replace_dict = {
"user": user,
"package_name": package_name,
"plugin_name": plugin_name,
"repo_name": repo_name,
"repo_url": repo_url,
"description": str(description),
}
formatter = TemplateFormatter(template, replace_dict, tgt_path)
formatter.format()
print("Created template with:")
for k, v in replace_dict.items():
print("{:<15} {:<15}".format(k, v))
```
%% Cell type:code id:e7597558 tags:
``` python
!plugin_from_template --list_templates
```
%% Output
Available templates:
basic
classifier_plugin
%% Cell type:code id:3d54e18e tags:
``` python
template = download_plugin_template("classifier_plugin")
replace_dict = {
"user": "eelcovdw",
"repo_name": "sentiment-plugin",
"package_name": "sentiment_plugin",
"plugin_name": "Sentiment Plugin",
"description": "Predict sentiment on text messages"
}
with tempfile.TemporaryDirectory() as result_path:
result_path = Path(result_path)
formatter = TemplateFormatter(template, replace_dict, result_path)
formatter.format()
created_files = [f for f in result_path.rglob("*") if not os.path.isdir(f)]
contents = {}
for fn in created_files:
with open(fn, "r") as f:
contents[str(fn)] = f.read()
print("Created files:")
pprint(created_files)
assert len(template) == len(created_files)
```
%% Output
Created files:
[Path('/tmp/tmpskokzmh4/setup.cfg'),
Path('/tmp/tmpskokzmh4/.gitlab-ci.yml'),
Path('/tmp/tmpskokzmh4/README.md'),
Path('/tmp/tmpskokzmh4/Dockerfile'),
Path('/tmp/tmpskokzmh4/metadata.json'),
Path('/tmp/tmpskokzmh4/setup.py'),
Path('/tmp/tmpskokzmh4/.gitignore'),
Path('/tmp/tmpskokzmh4/sentiment_plugin/utils.py'),
Path('/tmp/tmpskokzmh4/sentiment_plugin/model.py'),
Path('/tmp/tmpskokzmh4/sentiment_plugin/plugin.py'),
Path('/tmp/tmpskokzmh4/sentiment_plugin/schema.py'),
Path('/tmp/tmpskokzmh4/tools/preload.py'),
Path('/tmp/tmpskokzmh4/tests/test_plugin.py')]
[Path('/tmp/tmp05r53_jq/setup.py'),
Path('/tmp/tmp05r53_jq/Dockerfile'),
Path('/tmp/tmp05r53_jq/.gitignore'),
Path('/tmp/tmp05r53_jq/README.md'),
Path('/tmp/tmp05r53_jq/.gitlab-ci.yml'),
Path('/tmp/tmp05r53_jq/metadata.json'),
Path('/tmp/tmp05r53_jq/setup.cfg'),
Path('/tmp/tmp05r53_jq/sentiment_plugin/model.py'),
Path('/tmp/tmp05r53_jq/sentiment_plugin/schema.py'),
Path('/tmp/tmp05r53_jq/sentiment_plugin/utils.py'),
Path('/tmp/tmp05r53_jq/sentiment_plugin/plugin.py'),
Path('/tmp/tmp05r53_jq/tools/preload.py'),
Path('/tmp/tmp05r53_jq/tests/test_plugin.py')]
%% Cell type:code id:fc4993b2 tags:
``` python
# hide
from nbdev.export import *
notebook2script()
```
%% Output
Converted Untitled.ipynb.
Converted basic.ipynb.
Converted cvu.utils.ipynb.
Converted data.photo.ipynb.
Converted index.ipynb.
Converted itembase.ipynb.
Converted plugin.authenticators.credentials.ipynb.
Converted plugin.authenticators.oauth.ipynb.
Converted plugin.listeners.ipynb.
Converted plugin.pluginbase.ipynb.
Converted plugin.states.ipynb.
Converted plugins.authenticators.password.ipynb.
Converted pod.api.ipynb.
Converted pod.client.ipynb.
Converted pod.db.ipynb.
Converted pod.utils.ipynb.
Converted template.config.ipynb.
Converted template.formatter.ipynb.
Converted test_schema.ipynb.
Converted test_utils.ipynb.
......
pymemri/template/formatter.py
+ 1
- 1
View file @ 824702ca
......@@ -15,7 +15,7 @@ import re
# Cell
# hide
TEMPLATE_URL = "https://gitlab.memri.io/plugins/plugin-templates/-/archive/dev/plugin-templates-dev.zip"
TEMPLATE_URL = "https://gitlab.memri.io/memri/plugin-templates/-/archive/dev/plugin-templates-dev.zip"
TEMPLATE_BASE_PATH = "plugin-templates-dev"
# Cell
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment