fix nbdev

298e8527 · Aziz Berkay Yesilyurt · a8ad5765 · 298e8527 · 298e8527 · 298e8527
Commit 298e8527 authored 2 years ago by Aziz Berkay Yesilyurt
Hide whitespace changes
Inline Side-by-side

Showing

with 13 additions and 13 deletions
+13 -13
--- a/nbs/gitlab_api.ipynb
+++ b/nbs/gitlab_api.ipynb
@@ -61,6 +61,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
+    "\n",
    "\n",
    "class GitlabAPI():\n",
    "\n",
@@ -149,7 +150,7 @@
    "        if res.status_code not in [200, 201]:\n",
    "            print(res.content)\n",
    "            raise RuntimeError(f\"Failed to get project id for {project_name}\")\n",
-    "        \n",
+    "\n",
    "        project_id = res.json().get(\"id\", None)\n",
    "        if project_id:\n",
    "            return project_id\n",

 %% Cell type:code id: tags:

 ``` python
 #default_exp gitlab_api
 %load_ext autoreload
 %autoreload 2
 ```

 %% Cell type:code id: tags:

 ``` python
 # export
 from fastprogress.fastprogress import progress_bar
 from pathlib import Path
 import requests
 import os, sys
 from getpass import getpass
 from datetime import datetime
 from git import Repo
 import re
 from pymemri.data.basic import *
 from pymemri.template.formatter import _plugin_from_template, gitlab_slugify
 import urllib
 ```

 %% Cell type:code id: tags:

 ``` python
 # export
 MEMRI_PATH = Path.home() / ".memri"
 MEMRI_GITLAB_BASE_URL = "https://gitlab.memri.io"
 ACCESS_TOKEN_PATH = Path.home() / ".memri/access_token/access_token.txt"
 GITLAB_API_BASE_URL = "https://gitlab.memri.io/api/v4"
 DEFAULT_PACKAGE_VERSION = "0.0.1"

 TIME_FORMAT_GITLAB = '%Y-%m-%dT%H:%M:%S.%fZ'
 PROJET_ID_PATTERN = '(?<=<span class="gl-button-text">Project ID: )[0-9]+(?=</span>)'
 ```

 %% Cell type:markdown id: tags:

 # - Downloading & Uploading functions for package registry

 %% Cell type:code id: tags:

 ``` python

+
 class GitlabAPI():

    def __init__(self, client=None, request_auth_if_needed=False):
        self.client=client
        self.auth_headers=dict()
        self.auth_params=dict()
        self.auth_initialized=False
        self.request_auth_if_needed = request_auth_if_needed
        self.get_registry_params_headers()


    # export
    def get_registry_params_headers(self):
        job_token = os.environ.get("CI_JOB_TOKEN", None)
        if job_token is not None:
            self.auth_initialized = True
            self.auth_headers = {"JOB-TOKEN": job_token}
        elif self.client is not None:
            self.auth_params = {"access_token": self.client.get_oauth_item().accessToken}
            self.auth_initialized = True
        else:
            ACCESS_TOKEN_PATH.parent.mkdir(parents=True, exist_ok=True)
            if ACCESS_TOKEN_PATH.is_file():
                self.auth_initialized = True
                with open(ACCESS_TOKEN_PATH, "r") as f:
                    self.auth_headers = {"PRIVATE-TOKEN": f.read()}
            else:
                if self.request_auth_if_needed:
                    print(f"""
                    The first time you are uploading a model you need to create an access_token
                    at https://gitlab.memri.io/-/profile/personal_access_tokens?name=Model+Access+token&scopes=api
                    Click at the blue button with 'Create personal access token'"
                    """)
                    access_token = getpass("Then copy your personal access token from 'Your new personal access token', and paste here: ")
                    with open(ACCESS_TOKEN_PATH, "w") as f:
                        f.write(access_token)
                    self.auth_headers = {"PRIVATE-TOKEN": access_token}
                    self.auth_initialized = True


        # test if we are authenticated
        res = requests.get(f"{GITLAB_API_BASE_URL}/projects", headers=self.auth_headers, params=self.auth_params)
        if res.status_code not in [200, 201]:
            print(res.content)
            raise RuntimeError(res.content)

    def write_file_to_package_registry(
        self,
        project_id,
        file_path,
        package_name,
        version=DEFAULT_PACKAGE_VERSION,
        trigger_pipeline=True
    ):
        file_path = Path(file_path)
        file_name = file_path.name

        url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{version}/{file_name}"
        print(f"uploading {file_path}")
        it = upload_in_chunks(file_path)
        res = requests.put(url=url, data=IterableToFileAdapter(it),
                         headers=self.auth_headers, params=self.auth_params)

        if res.status_code not in [200, 201]:
            print(f"Failed to upload {file_path}: {res.content}")
        else:
            print(f"Succesfully uploaded {file_path}")
            if trigger_pipeline:
                self.trigger_pipeline(project_id)

    def trigger_pipeline(self, project_id):
        url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/pipeline?ref=main"
        res = requests.post(url, headers=self.auth_headers, params=self.auth_params)
        if res.status_code not in [200, 201]:
            print(f"Failed to trigger pipeline")

    # export
    def project_id_from_name(self, project_name):
        iden = gitlab_slugify(project_name)
        username = self.get_current_username()
        uri_encoded_name = urllib.parse.quote_plus(f"{username}/{iden}")
        res = requests.get(f"{GITLAB_API_BASE_URL}/projects/{uri_encoded_name}",
                            headers=self.auth_headers, params=self.auth_params)

        if res.status_code not in [200, 201]:
            print(res.content)
            raise RuntimeError(f"Failed to get project id for {project_name}")

        project_id = res.json().get("id", None)
        if project_id:
            return project_id
        else:
            raise ValueError(f"No plugin found with name {project_name}, make sure to enter the name as specified in the url of the repo")

    # export
    def get_project_id_from_project_path_unsafe(self, project_path):
        try:
            res = requests.get(f"{MEMRI_GITLAB_BASE_URL}/{project_path}")
            html = str(res.content)
            match = re.search(PROJET_ID_PATTERN, html)
            return match.group()
        except Exception:
            raise ValueError(f"Could not find project with name {project_path}")

    # export
    def download_package_file(self, filename, project_path, package_name, out_dir=None,
                              package_version=DEFAULT_PACKAGE_VERSION, download_if_exists=False):

        project_name = str(project_path).split("/")[-1]
        out_dir = out_dir if out_dir is not None else MEMRI_PATH / "projects" / project_name
        out_dir.mkdir(parents=True, exist_ok=True)

        project_id = self.get_project_id_from_project_path_unsafe(project_path)

        file_path = out_dir / filename
        print(file_path)

        if file_path.exists() and not download_if_exists:
            print(f"{file_path} already exists, and `download_if_exists`==False, using cached version")
            return file_path

        print(f"downloading {filename} from project {project_path}, package {package_name}")

        res = requests.get(
            url=f"{GITLAB_API_BASE_URL}/projects/{project_id}/packages/generic/{package_name}/{package_version}/{filename}"
        )
        res.raise_for_status()
        with open(out_dir / filename, "wb") as f:
            print(f"writing {filename} to {out_dir}")
            f.write(res.content)
        return file_path

    def create_repo(self, repo_name, client=None):
        url = f"{GITLAB_API_BASE_URL}/projects/"
        payload = {"name": repo_name}
        res = requests.post(url=url, json=payload, headers=self.auth_headers, params=self.auth_params)

        if res.status_code not in [200, 201]:
            raise ValueError(f"failed to create repo:\n {res.text}")
        print(f"created project {repo_name}")

    # export
    def get_current_username(self, client=None):
        url = f"{GITLAB_API_BASE_URL}/user/"
        res = requests.get(url=url, headers=self.auth_headers, params=self.auth_params)
        if res.status_code not in [200,201]:
            raise ValueError(f"Could not find current user {res.content}")
        else:
            username = res.json()["username"]
            return username

    # NEVER EXPORT THIS
    def delete_project(self, path_or_id, client=None):
        url_escape_id = urllib.parse.quote(str(path_or_id), safe='')
        url = f"{GITLAB_API_BASE_URL}/projects/{url_escape_id}"
        res = requests.delete(url=url, headers=self.auth_headers, params=self.auth_params)
        if res.status_code not in [200, 201, 202]:
            raise ValueError(f"failed to delete repo:\n {res.text}")
        print(f"deleted project {path_or_id}")

    # export
    def commit_file(self, project_name, path_in2out, branch="main", client=None):
        project_id = self.project_id_from_name(project_name)
    #     file_out_path_escaped = urllib.parse.quote(file_out_path, safe='')

        actions = []

        for file_in_path, file_out_path in path_in2out.items():
            content = read_file(file_in_path)
            action_payload = {"action": "create", "file_path": file_out_path, "content": content}
            actions.append(action_payload)

        url = f"{GITLAB_API_BASE_URL}/projects/{project_id}/repository/commits"
        payload = {"branch": branch, "commit_message": "automated commit", "content": content, "actions": actions}

        res = requests.post(url=url, json=payload, headers=self.auth_headers, params=self.auth_params)
        files_in = list(path_in2out.keys())
        if res.status_code not in [200, 201, 202]:
            raise ValueError(f"failed to make commit with files {files_in}:\n {res.text}")
        print(f"committed files {files_in}")

    # export
    def write_files_to_git(self, repo, target_dir, **kwargs):
        path_in2out = dict()
        for p in target_dir.rglob("*"):
            if p.is_file():
                path_in_repo = p.relative_to(target_dir)
                path_in2out[str(p)] = str(path_in_repo)
        self.commit_file(str(repo), path_in2out, **kwargs)


    def create_new_project(self, project_name, user=None):
        tmp_dir = Path("/tmp/test") / project_name
        rm_tree(tmp_dir)
        repo_url = f"{MEMRI_GITLAB_BASE_URL}/{user}/{project_name}" if user is not None else f"{MEMRI_GITLAB_BASE_URL}/plugins/{project_name}"

        _plugin_from_template(
            template_name="classifier_plugin",
            description="A transformer based sentiment analyis plugin",
            install_requires="transformers,sentencepiece,protobuf,torch==1.10.0",
            target_dir=str(tmp_dir),
            repo_url=repo_url,
            verbose=False,
            user=user
        )
        self.write_files_to_git(project_name, tmp_dir)
 ```

 %% Cell type:code id: tags:

 ``` python
 # export
 def find_git_repo():
    path = "."
    for i in range(10):
        try:
            repo = Repo(f"{path + ('.' * i)}/")
        except:
            pass
        else:
            break
    if i == 9:
        raise ValueError(f"could not fine git repo in {os.path.abspath('')}")

    repo_name = repo.remotes.origin.url.split('.git')[0].split('/')[-1]
    return repo_name
 ```

 %% Cell type:code id: tags:

 ``` python
 # export
 class upload_in_chunks(object):
    def __init__(self, filename, chunksize=1 << 14):
        self.filename = filename
        self.chunksize = chunksize
        self.totalsize = os.path.getsize(filename)
        self.readsofar = 0

    def __iter__(self):
        n = 100
        pb = progress_bar(range(n))
        pb_iter = iter(pb)
        i = 1
        delta = 1 / n
        next(pb_iter, None)

        with open(self.filename, 'rb') as file:
            while True:
                data = file.read(self.chunksize)
                if not data:
                    sys.stderr.write("\n")
                    break
                self.readsofar += len(data)
                percent = self.readsofar * 1e2 / self.totalsize
                while (percent / 100) > i * delta:
                    next(pb_iter, None)
                    i += 1
                yield data
        pb.update_bar(n)

    def __len__(self):
        return self.totalsize

 class IterableToFileAdapter(object):
    def __init__(self, iterable):
        self.iterator = iter(iterable)
        self.length = len(iterable)

    def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
        return next(self.iterator, b'')

    def __len__(self):
        return self.length
 ```

 %% Cell type:code id: tags:

 ``` python
 # skip
 # todo: cleanup old package files during testing
 filename = "config.json"
 out_file = GitlabAPI().download_package_file(filename, "memri/finetuning-example", "plugin-model-package")
 ```

 %% Output

    /Users/koen/.memri/projects/finetuning-example/config.json
    /Users/koen/.memri/projects/finetuning-example/config.json already exists, and `download_if_exists`==False, using cached version

 %% Cell type:code id: tags:

 ``` python
 # export
 def rm_tree(pth):
    pth = Path(pth)
    for child in pth.glob('*'):
        if child.is_file():
            child.unlink()
        else:
            rm_tree(child)

    try:
        pth.rmdir()
    except FileNotFoundError as e:
        pass
 ```

 %% Cell type:code id: tags:

 ``` python
 from pymemri.pod.client import PodClient
 from pymemri.data.oauth import OauthFlow
 ```

 %% Cell type:code id: tags:

 ``` python
 client = PodClient()
 ```

 %% Cell type:code id: tags:

 ``` python
 client.owner_key
 ```

 %% Output

    '2838381004062553896151966760618629120788438647868531550721520507'

 %% Cell type:code id: tags:

 ``` python
 oauth_item = OauthFlow(accessToken="d1e83b04f28e90670e8f615b558a83252344f4d7181e68699a6033bebac64884", refreshToken="")
 client.add_to_schema(oauth_item)
 client.create(oauth_item)
 ```

 %% Output

    True

 %% Cell type:code id: tags:

 ``` python
 client.get_oauth_item()
 ```

 %% Output

    OauthFlow (#b581bb20bafe452a8c839fb004791025)

 %% Cell type:code id: tags:

 ``` python
 api = GitlabAPI(client)
 ```

 %% Cell type:code id: tags:

 ``` python
 # skip
 repo = "test2131"
 api.create_repo(repo)
 # delete_project(f"{get_current_username()}/{repo}")
 ```

 %% Output

    created project test2131

 %% Cell type:code id: tags:

 ``` python
 # skip
 api.create_new_project(repo)
 ```

 %% Output

    Created `Test2131` using the classifier_plugin template.
    committed files ['/tmp/test/test2131/Dockerfile', '/tmp/test/test2131/metadata.json', '/tmp/test/test2131/README.md', '/tmp/test/test2131/setup.py', '/tmp/test/test2131/.gitignore', '/tmp/test/test2131/setup.cfg', '/tmp/test/test2131/.gitlab-ci.yml', '/tmp/test/test2131/LICENSE.txt', '/tmp/test/test2131/tools/preload.py', '/tmp/test/test2131/tests/test_plugin.py', '/tmp/test/test2131/test2131/model.py', '/tmp/test/test2131/test2131/utils.py', '/tmp/test/test2131/test2131/plugin.py', '/tmp/test/test2131/test2131/schema.py']

 %% Cell type:code id: tags:

 ``` python
 # skip
 api.delete_project(f"{api.get_current_username()}/{repo}")
 ```

 %% Output

    deleted project koenvanderveen/test2131

 %% Cell type:markdown id: tags:

 # Export -

 %% Cell type:code id: tags:

 ``` python
 # hide
 from nbdev.export import *
 notebook2script()
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

--- a/nbs/template.formatter.ipynb
+++ b/nbs/template.formatter.ipynb
@@ -190,11 +190,12 @@
    "        result = result.lower()\n",
    "    return result\n",
    "\n",
-    "def str_to_gitlab_identifier(s, lower=True):\n",
-    "    result = re.sub(\"\\W|^(?=\\d)\", \"-\", s)\n",
-    "    if lower:\n",
-    "        result = result.lower()\n",
-    "    return result\n",
+    "# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/utils.rb#L92\n",
+    "def gitlab_slugify(s):\n",
+    "    s = s.lower()\n",
+    "    s = re.sub(r'[^a-z0-9]', '-', s)[:63]\n",
+    "    s = re.sub(r'(^-+|-+$)', '', s)\n",
+    "    return s\n",
    "\n",
    "def reponame_to_displayname(reponame: str) -> str:\n",
    "    return re.sub(\"[-_]+\", \" \", reponame).title()\n",

 %% Cell type:code id:f380ac34 tags:

 ``` python
 %load_ext autoreload
 %autoreload 2
 # default_exp template.formatter
 ```

 %% Cell type:code id:a8a7e4c9 tags:

 ``` python
 # export
 # hide
 from pathlib import Path
 from typing import Dict, Union, List
 from fastcore.script import call_parse, Param, store_true
 import zipfile
 from string import Template
 import re
 import giturlparse
 import subprocess
 import pymemri
 import urllib
 from pathlib import PurePosixPath
 import requests
 ```

 %% Cell type:code id:74de84a3 tags:

 ``` python
 # hide
 # test imports
 from pprint import pprint
 import os
 ```

 %% Cell type:markdown id:77d47d51 tags:

 # Creating plugins from a template

 %% Cell type:code id:9898bd14 tags:

 ``` python
 # export
 # hide
 TEMPLATE_URL = "https://gitlab.memri.io/memri/plugin-templates/-/archive/dev/plugin-templates-dev.zip"
 TEMPLATE_BASE_PATH = "plugin-templates-dev"
 ```

 %% Cell type:markdown id:475c3c1b tags:

 Pymemri offers a range of plugin templates to set up testing, docker and CI for you. This way, you can focus on building your plugin, and be sure it works within the Memri ecosystem.

 All plugins are hosted on our [GitLab](https://gitlab.memri.io/). In order to make your own plugin from a template,

 1. Create an account on [GitLab](https://gitlab.memri.io/)
 2. Create a _public_ [blank repository](https://gitlab.memri.io/projects/new#blank_project)
 3. Clone the repository
 4. run the `plugin_from_template` CLI inside the repository folder.

 The CLI will infer most settings for you from your git account and repository name, only a template name and optional description are required.
 ```
 plugin_from_template --template classifier_plugin --description "My Classifier Plugin"
 ```

 To make sure all settings are correct, you can inspect `metadata.json`, which holds all information like your plugin name, and python package name.

 -----------------
 You can list the available templates with. All plugin templates are hosted [here](https://gitlab.memri.io/memri/plugin-templates).

 ```
 plugin_from_template --list
 ```

 The CLI has options to customize the plugin name, package name and other aspects of your plugin. For advanced use, run:
 ```
 plugin_from_template --help
 ```

 %% Cell type:markdown id:2c9883e9 tags:

 ## Utility functions -

 %% Cell type:code id:2933c056 tags:

 ``` python
 # export
 # hide

 # If the owner of the repository is one of these groups, the CLI requires an additional `user` argument
 GITLAB_GROUPS = ["memri", "plugins"]

 def get_remote_url():
    path = Path(".")
    url = subprocess.getoutput(f'git config --get remote.origin.url')
    if not url:
        raise ValueError(f"You can only run this from a initialized gitlab repository, and '{path}' is not an initialized git repository")
    parsed = giturlparse.parse(url)
    repo_url = parsed.url2https
    if repo_url.endswith(".git"):
        repo_url = repo_url[:-4]
    return repo_url

 def infer_git_info(url):
    parsed = giturlparse.parse(url)
    return parsed.owner, parsed.repo
 ```

 %% Cell type:code id:16dfc0d2 tags:

 ``` python
 # hide
 remote_url = get_remote_url()
 repo_owner, repo_name = infer_git_info(remote_url)

 assert repo_owner == "memri"
 assert repo_name == "pymemri"
 ```

 %% Cell type:code id:42378645 tags:

 ``` python
 # export
 # hide

 def download_file(url, fname=None):
    cert_path = Path(pymemri.__file__).parent / "cert" / "gitlab.memri.io.pem"
    r = requests.get(url, stream=True, verify=cert_path)
    fname = url.rsplit('/', 1)[1] if fname is None else fname
    with open(fname, 'wb') as fd:
        for chunk in r.iter_content(chunk_size=128):
            fd.write(chunk)
    return fname
 ```

 %% Cell type:code id:9a72f504 tags:

 ``` python
 # export
 # hide
 def str_to_identifier(s, lower=True):
    result = re.sub("\W|^(?=\d)", "_", s)
    if lower:
        result = result.lower()
    return result

-def str_to_gitlab_identifier(s, lower=True):
-    result = re.sub("\W|^(?=\d)", "-", s)
-    if lower:
-        result = result.lower()
-    return result
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/utils.rb#L92
+def gitlab_slugify(s):
+    s = s.lower()
+    s = re.sub(r'[^a-z0-9]', '-', s)[:63]
+    s = re.sub(r'(^-+|-+$)', '', s)
+    return s

 def reponame_to_displayname(reponame: str) -> str:
    return re.sub("[-_]+", " ", reponame).title()

 def download_plugin_template(
    template_name: str, url: str = TEMPLATE_URL, base_path: str = TEMPLATE_BASE_PATH
 ):
    base_path = Path(base_path) / template_name
    zip_path = download_file(url)
    with zipfile.ZipFile(zip_path, "r") as f:
        result = {name: f.read(name) for name in f.namelist() if base_path in Path(name).parents}

    if len(result) == 0:
        raise ValueError(f"Could not find template: {template_name}")

    result = {str(PurePosixPath(k).relative_to(PurePosixPath(base_path))): v.decode("utf-8") for k, v in result.items() if v}
    Path(zip_path).unlink()
    return result


 def get_templates(url: str = TEMPLATE_URL) -> List[str]:
    zip_path = download_file(url)

    with zipfile.ZipFile(zip_path, "r") as f:
        files_split = [name.split("/") for name in f.namelist()]
        result = [fn[1] for fn in files_split if fn[-1] == '' and len(fn) == 3]
    return result
 ```

 %% Cell type:code id:becd3e18 tags:

 ``` python
 # hide
 assert len(get_templates())
 ```

 %% Cell type:code id:f99827cd tags:

 ``` python
 # hide
 assert str_to_identifier("My Plugin") == "my_plugin"

 template = download_plugin_template("classifier_plugin")
 assert len(template)
 pprint(list(template.keys()))
 ```

 %% Output

    ['$package_name/model.py',
     '$package_name/plugin.py',
     '$package_name/schema.py',
     '$package_name/utils.py',
     '.gitignore',
     '.gitlab-ci.yml',
     'Dockerfile',
     'LICENSE.txt',
     'README.md',
     'metadata.json',
     'setup.cfg',
     'setup.py',
     'tests/test_plugin.py',
     'tools/preload.py']

 %% Cell type:code id:2707a14d tags:

 ``` python
 # export
 # hide
 class TemplateFormatter:
    def __init__(
        self,
        template_dict: Dict[str, str],
        replace_dict: Dict[str, str],
        tgt_path: Union[str, Path],
        verbose: bool = False,
    ):
        self.template_dict = template_dict
        self.tgt_path = Path(tgt_path)
        self.replace_dict = replace_dict
        self.verbose = verbose

    def format_content(self, content):
        return Template(content).safe_substitute(self.replace_dict)

    def format_path(self, path):
        new_path = Template(path).safe_substitute(self.replace_dict)
        return self.tgt_path / new_path

    def format_file(self, filename, content):
        new_path = self.format_path(filename)
        new_content = self.format_content(content)
        new_path.parent.mkdir(exist_ok=True, parents=True)
        if self.verbose:
            print(f"Formatting {filename} -> {new_path}")
        with open(new_path, "w", encoding="utf-8") as f:
            f.write(new_content)

    def get_files(self):
        return [self.format_path(filename) for filename in self.template_dict.keys()]

    def format(self):
        for filename, content in self.template_dict.items():
            self.format_file(filename, content)

    def print_filetree(self):
        previous_prefix = None
        res = "Created the following files"

        for path in sorted([x.relative_to(self.tgt_path) for x in self.get_files()],
                           key=lambda item: 100 * str(item).count("/")):
            n_slashes = str(path).count("/")
            new_prefix = path.parent
            if previous_prefix != new_prefix and str(new_prefix) != ".":
                res = f"{res}\n├── {new_prefix}"
            if n_slashes == 0:
                res = f"{res}\n├── {path}"
            elif n_slashes == 1:
                res = f"{res}\n│   ├── {path.name}"

            previous_prefix=new_prefix

        print(res.strip() + "\n")
 ```

 %% Cell type:markdown id:30c2cdd4 tags:

 ### Plugin Template CLI

 With the `plugin_from_template` CLI, you can easily create a plugin where all CI pipelines, docker files, and test setups are configured for you. Multiple templates are available, to see the complete list use:

 `plugin_from_template --list_templates`

 %% Cell type:code id:9ef65986 tags:

 ``` python
 # export
 # hide
 def get_template_replace_dict(
    repo_url=None, user=None, plugin_name=None, package_name=None, description=None, install_requires=None, template_name=None
 ):
    if repo_url is None:
        repo_url = get_remote_url()

    try:
        repo_owner, repo_name = infer_git_info(repo_url)
    except ValueError:
        url_inf, owner_inf, name_inf = None, None, None
        print("Could not infer git information from current directory, no initialized repository found.")

    if repo_url is None:
        repo_url = url_inf

    if user is None:
        if repo_owner in GITLAB_GROUPS:
            user = None
        else:
            user = repo_owner

    if plugin_name is None:
        if repo_name is None:
            plugin_name = None
        else:
            plugin_name = reponame_to_displayname(repo_name)

    if package_name is None:
        if repo_name is None:
            package_name = None
        else:
            package_name = str_to_identifier(repo_name)
    if install_requires is None:
        install_requires = ""
    else:
        install_requires = "\n    ".join([x.strip() for x in install_requires.split(",")
                                if x.strip() != "" and x.strip() not in ["pymemri", "pytest"]])

    if template_name == "classifier_plugin":
        assert package_name is not None
        assert user is not None
        repo_name_gitlab = str_to_gitlab_identifier(repo_name)

        # hacky, dont change!
        model_imports_ = f"""
 from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
 from """ +"pymemri.data.loader import load_huggingface_model_for_project"
        model_init = f"""
        model = load_huggingface_model_for_project(project_path="{user}/{repo_name_gitlab}", client=client)
        tokenizer = AutoTokenizer.from_pretrained("distilroberta-base", model_max_length=512)
        self.pipeline = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer, return_all_scores=True, truncation=True)
        """
        model_predict = """
        return self.pipeline(x)
        """
    else:
        model_imports_ = ""
        model_init = "raise NotImplementedError()"
        model_predict = "raise NotImplementedError()"

    return {
        "user": user,
        "package_name": package_name,
        "plugin_name": plugin_name,
        "repo_name": repo_name,
        "repo_url": repo_url,
        "description": str(description),
        "install_requires": install_requires,
        "model_imports": model_imports_,
        "model_init": model_init,
        "model_predict": model_predict
    }
 ```

 %% Cell type:code id:074d003b tags:

 ``` python
 # export
 def _plugin_from_template(list_templates=False, user=None,repo_url=None,plugin_name=None,template_name="basic",
                          package_name=None,description=None,target_dir=".",verbose=True,install_requires=""):
    if list_templates:
        print("Available templates:")
        for template in get_templates():
            print(template)
        return

    template = download_plugin_template(template_name)

    tgt_path = Path(target_dir)
    replace_dict = get_template_replace_dict(
        repo_url=repo_url,
        user=user,
        plugin_name=plugin_name,
        package_name=package_name,
        description=description,
        install_requires=install_requires,
        template_name=template_name
    )
    print(replace_dict)

    formatter = TemplateFormatter(template, replace_dict, tgt_path)
    formatter.format()
    if verbose:
        formatter.print_filetree()


    print(f"Created `{replace_dict['plugin_name']}` using the {template_name} template.")
 ```

 %% Cell type:code id:5c4df6b0 tags:

 ``` python
 # export
 @call_parse
 def plugin_from_template(
    list_templates: Param("List available plugin templates", store_true) = False,
    user: Param("Your Gitlab username", str) = None,
    repo_url: Param("The url of your empty Gitlab plugin repository", str) = None,
    plugin_name: Param("Display name of your plugin", str) = None,
    template_name: Param(
        "Name of the template, use `list_templates` to see all available options"
    ) = "basic",
    package_name: Param("Name of your plugin python package", str) = None,
    description: Param("Description of your plugin", str) = None,
    target_dir: Param("Directory to output the formatted template", str) = ".",
    verbose: Param("Should print out dir", bool) = True,
    install_requires: Param("Extra packages to install, provided as comma separated, e.g. pymemri,requests", str)=""
 ):
    """
    CLI that downloads and formats a plugin template according to the arguments, and local git repository.

    Args:
        list_templates (Param, optional): If True, only list available templates. Defaults to False.
        user (Param, optional): Your GitLab username. Defaults to None.
        repo_url (Param, optional): The url of your gitlab plugin repository. Defaults to None.
        plugin_name (Param, optional): The name of your plugin. Defaults to None.
        template_name (Param, optional): The name of the template used. To list all options, see `list_templates`.
            Defaults to "basic".
        package_name (Param, optional): The name of the python package of your plugin. Inferred if left blank. Defaults to None.
        description (Param, optional): An optional plugin description. Defaults to None.
        target_dir (Param, optional): Directory where the plugin template is generated. Defaults to ".".
    """
    _plugin_from_template(list_templates, user,repo_url,plugin_name,template_name,package_name,description,target_dir,
                          verbose,install_requires)
 ```

 %% Cell type:code id:e7597558 tags:

 ``` python
 !plugin_from_template --list_templates
 ```

 %% Output

    Available templates:
    basic
    classifier_plugin

 %% Cell type:code id:3d54e18e tags:

 ``` python
 # hide
 import tempfile
 template = download_plugin_template("classifier_plugin")
 replace_dict = {
    "user": "eelcovdw",
    "repo_name": "sentiment-plugin",
    "package_name": "sentiment_plugin",
    "plugin_name": "Sentiment Plugin",
    "description": "Predict sentiment on text messages",
    "model_imports": """from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
 from pymemri.data.loader import load_huggingface_model_for_project""",
    "model_init": """
        model = load_huggingface_model_for_project(project_path="koenvanderveen/sentiment-plugin")
        tokenizer = AutoTokenizer.from_pretrained("distilroberta-base", model_max_length=512)
        self.pipeline = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer, return_all_scores=True, truncation=True)
        """,
    "model_predict": """
        return self.pipeline(x)
    """

 }
 #        raise NotImplementedError()

 with tempfile.TemporaryDirectory() as result_path:
    result_path = Path(result_path)
    formatter = TemplateFormatter(template, replace_dict, result_path)
    formatter.format()
    created_files = [f for f in result_path.rglob("*") if not os.path.isdir(f)]

    contents = {}
    for fn in created_files:
        with open(fn, "r") as f:
            contents[str(fn)] = f.read()

 formatter.print_filetree()
 # print("Created files:")
 # pprint(created_files)
 assert len(template) == len(created_files)
 ```

 %% Output

    /var/folders/q1/ryq93kwj055dlbpngxv1c7z40000gn/T/tmp5flm_y6x
    Created the following files
    ├── .gitignore
    ├── .gitlab-ci.yml
    ├── Dockerfile
    ├── LICENSE.txt
    ├── README.md
    ├── metadata.json
    ├── setup.cfg
    ├── setup.py
    ├── sentiment_plugin
    │   ├── model.py
    │   ├── plugin.py
    │   ├── schema.py
    │   ├── utils.py
    ├── tests
    │   ├── test_plugin.py
    ├── tools
    │   ├── preload.py
    

 %% Cell type:code id:8e008696 tags:

 ``` python
 # hide
 key = [k for k in contents.keys() if k.endswith("model.py")][0]
 print(contents[key])
 ```

 %% Cell type:code id:fc4993b2 tags:

 ``` python
 # hide
 from nbdev.export import *
 notebook2script()
 ```

 %% Output

    Converted Untitled.ipynb.
    Converted Untitled1.ipynb.
    Converted Untitled2.ipynb.
    Converted basic.ipynb.
    Converted cvu.utils.ipynb.
    Converted data.dataset.ipynb.
    Converted data.loader.ipynb.
    Converted data.oauth.ipynb.
    Converted data.photo.ipynb.
    Converted exporters.exporters.ipynb.
    Converted gitlab_api.ipynb.
    Converted index.ipynb.
    Converted itembase.ipynb.
    Converted plugin.authenticators.credentials.ipynb.
    Converted plugin.authenticators.oauth.ipynb.
    Converted plugin.listeners.ipynb.
    Converted plugin.pluginbase.ipynb.
    Converted plugin.states.ipynb.
    Converted plugins.authenticators.password.ipynb.
    Converted pod.api.ipynb.
    Converted pod.client.ipynb.
    Converted pod.db.ipynb.
    Converted pod.utils.ipynb.
    Converted template.config.ipynb.
    Converted template.formatter.ipynb.
    Converted test_owner_key.ipynb.
    Converted test_schema.ipynb.
    Converted test_utils.ipynb.
    Converted wa_dummy_data.ipynb.

 %% Cell type:code id:9d9cfe6f tags:

 ``` python
 ```

--- a/pymemri/_nbdev.py
+++ b/pymemri/_nbdev.py
@@ -113,7 +113,7 @@ index = {"read_file": "basic.ipynb",
         "GITLAB_GROUPS": "template.formatter.ipynb",
         "download_file": "template.formatter.ipynb",
         "str_to_identifier": "template.formatter.ipynb",
-         "str_to_gitlab_identifier": "template.formatter.ipynb",
+         "gitlab_slugify": "template.formatter.ipynb",
         "reponame_to_displayname": "template.formatter.ipynb",
         "download_plugin_template": "template.formatter.ipynb",
         "get_templates": "template.formatter.ipynb",

--- a/pymemri/gitlab_api.py
+++ b/pymemri/gitlab_api.py
@@ -28,6 +28,8 @@ TIME_FORMAT_GITLAB = '%Y-%m-%dT%H:%M:%S.%fZ'
 PROJET_ID_PATTERN = '(?<=<span class="gl-button-text">Project ID: )[0-9]+(?=</span>)'

 # Cell
+
+
 class GitlabAPI():

    def __init__(self, client=None, request_auth_if_needed=False):
@@ -115,7 +117,7 @@ class GitlabAPI():
        if res.status_code not in [200, 201]:
            print(res.content)
            raise RuntimeError(f"Failed to get project id for {project_name}")
-        
+
        project_id = res.json().get("id", None)
        if project_id:
            return project_id
@@ -235,9 +237,6 @@ class GitlabAPI():
        )
        self.write_files_to_git(project_name, tmp_dir)

-
-
-
 # Cell
 def find_git_repo():
    path = "."
@@ -254,7 +253,6 @@ def find_git_repo():
    repo_name = repo.remotes.origin.url.split('.git')[0].split('/')[-1]
    return repo_name

-
 # Cell
 class upload_in_chunks(object):
    def __init__(self, filename, chunksize=1 << 14):

--- a/pymemri/template/formatter.py
+++ b/pymemri/template/formatter.py
 # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/template.formatter.ipynb (unless otherwise specified).

 __all__ = ['TEMPLATE_URL', 'TEMPLATE_BASE_PATH', 'get_remote_url', 'infer_git_info', 'GITLAB_GROUPS', 'download_file',
-           'str_to_identifier', 'str_to_gitlab_identifier', 'reponame_to_displayname', 'download_plugin_template',
+           'str_to_identifier', 'gitlab_slugify', 'reponame_to_displayname', 'download_plugin_template',
           'get_templates', 'TemplateFormatter', 'get_template_replace_dict', 'plugin_from_template']

 # Cell