From 1a0664621d8e683446a56d8987307844f400571c Mon Sep 17 00:00:00 2001 From: rubenseggers <rubenseggers@gmail.com> Date: Wed, 29 Jul 2020 15:07:15 +0200 Subject: [PATCH 1/4] export schema for indexers (python) --- .gitignore | 1 + tools/export_schema_indexers.js | 106 ++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tools/export_schema_indexers.js diff --git a/.gitignore b/.gitignore index 9d37f45..75b6a75 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *schema_target.ts *autogenerated_database_schema.json *autogenerated_database_schema_target.json +*schema.py # IDE / editor *.swp diff --git a/tools/export_schema_indexers.js b/tools/export_schema_indexers.js new file mode 100644 index 0000000..3ce527b --- /dev/null +++ b/tools/export_schema_indexers.js @@ -0,0 +1,106 @@ +const fs = require('fs'); +const helpers = require('./helpers'); +const path = require('path'); + +const entityHierarchyPath = path.resolve('../TypeHierarchy/Item'); +const predicateHierarchyPath = path.resolve('../EdgeAndPropertyHierarchy'); +const outputFile = './schema.py'; + +function getItemClasses() { + let attributesItem = entityHierarchy['Item']['properties'].concat(Object.keys(entityHierarchy['Item']['relations'])); + let itemArguments = ""; + let itemClasses = []; + for (const item of Object.keys(entityHierarchy)) { + if (['SyncableItem', 'Edge', 'Datasource', 'UserState', 'ViewArguments', 'CVUStateDefinition'].includes(item)) continue; + + let classDescription = `\n# ${entityHierarchy[item]['description']}\n`; + classDescription = helpers.wrapText(`# ${entityHierarchy[item]['description']}`, 100, '\n# '); + + let ancestry = helpers.getAncestry(entityHierarchy[item]['path'].split('/')); + let properties = [], edges = []; + for (const _item in ancestry) { + properties = properties.concat(entityHierarchy[_item]['properties']); + edges = edges.concat(Object.keys(entityHierarchy[_item]['relations'])); + } + + let arguments = "", edgesAttribute = ""; + let attributes = [], jsonGets = [], fromJsons = []; + for (const attribute of properties.concat(edges)) { + if (['genericType', 'functions', 'updatedFields'].includes(attribute)) continue; + arguments += `${arguments === '' ? '' : ', '}${attribute}`; + if (item === 'Item') itemArguments += `${itemArguments === '' ? '' : ', '}${attribute}`; + + if (properties.includes(attribute)) { + jsonGets.push(`${attribute} = json.get("${attribute}", None)`); + } else { + fromJsons.push(`if json.get("${attribute}", None) is not None: ${attribute} = Edge.from_json(json.get("${attribute}", None))`) + edgesAttribute += `${edgesAttribute === '' ? '' : ' + '}${attribute}`; + } + + if (attributesItem.includes(attribute) && item !== 'Item') continue; + attributes.push(`self.${attribute} = ${attribute}`); + } + let dataItemClass; + if (item === 'Item') { + dataItemClass = ` + +${classDescription} +class Item: + ${helpers.wrapText(`def __init__(self, ${arguments})`, 100, '\n' + ' '.repeat(17))}: + ${helpers.insertList(attributes, 8)} + self.edges = ${edgesAttribute} + + def __getattribute__(self, name, edge): + res = self._meta[name] + if isinstance(res, edge): + return res.traverse() + else: + return res + + def get_edge(self, name): + return self._meta[name]`; + } else { + dataItemClass = ` + +${classDescription} +class ${item}(Item): + ${helpers.wrapText(`def __init__(self, ${arguments})`, 100, '\n' + ' '.repeat(17))}: + ${helpers.wrapText(`super().__init__(self, ${itemArguments})`, 100, '\n' + ' '.repeat(25))} + ${helpers.insertList(attributes, 8)} + ${helpers.wrapText(`self.edges = ${edgesAttribute}`, 100, ' \\\n' + ' '.repeat(12))} + + @classmethod + def from_json(cls, json): + ${helpers.insertList(jsonGets, 8)} + + ${helpers.insertList(fromJsons,8)} + + ${helpers.wrapText(`cls(${arguments}`, 100, '\n' + ' '.repeat(12))})`; + } + itemClasses.push(dataItemClass); + } + return itemClasses; +} + +let entityHierarchy = {}; +let predicateHierarchy = {}; +(async () => { + await helpers.getHierarchy(entityHierarchyPath, entityHierarchy, entityHierarchyPath, 'Item'); + await helpers.getHierarchy(predicateHierarchyPath, predicateHierarchy, predicateHierarchyPath, 'EdgeOrProperty'); + + const itemClasses = getItemClasses(); + const output = `# +# WARNING: THIS FILE IS AUTOGENERATED; DO NOT CHANGE. +# Visit https://gitlab.memri.io/memri/schema to learn more. +# +# schema.py +# +# Copyright © 2020 memri. All rights reserved. +# +${helpers.insertList(itemClasses, 0)}`; + + fs.writeFile(outputFile, output, (err) => { + if (err) throw err; + console.log('File saved as ' + outputFile); + }); +})(); \ No newline at end of file -- GitLab From 914c34a86ba129e6823784422248894b78c6c7c3 Mon Sep 17 00:00:00 2001 From: rubenseggers <rubenseggers@gmail.com> Date: Wed, 29 Jul 2020 18:24:47 +0200 Subject: [PATCH 2/4] export schema for indexers (python) pt.2 --- tools/export_schema_indexers.js | 60 +++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tools/export_schema_indexers.js b/tools/export_schema_indexers.js index 3ce527b..ae3bdca 100644 --- a/tools/export_schema_indexers.js +++ b/tools/export_schema_indexers.js @@ -11,6 +11,7 @@ function getItemClasses() { let itemArguments = ""; let itemClasses = []; for (const item of Object.keys(entityHierarchy)) { + // if (!['Item', 'Location'].includes(item)) continue if (['SyncableItem', 'Edge', 'Datasource', 'UserState', 'ViewArguments', 'CVUStateDefinition'].includes(item)) continue; let classDescription = `\n# ${entityHierarchy[item]['description']}\n`; @@ -23,18 +24,26 @@ function getItemClasses() { edges = edges.concat(Object.keys(entityHierarchy[_item]['relations'])); } - let arguments = "", edgesAttribute = ""; - let attributes = [], jsonGets = [], fromJsons = []; + let arguments = "", fromJsonEdgeLoop = []; + let attributes = [], fromJsonEdges = [], fromJsonProperties = []; for (const attribute of properties.concat(edges)) { - if (['genericType', 'functions', 'updatedFields'].includes(attribute)) continue; - arguments += `${arguments === '' ? '' : ', '}${attribute}`; - if (item === 'Item') itemArguments += `${itemArguments === '' ? '' : ', '}${attribute}`; + if (['genericType', 'functions', 'updatedFields', 'allEdges'].includes(attribute)) continue; + arguments += `${arguments === '' ? '' : ', '}${attribute}=None`; + if (item === 'Item' && attribute === 'uid') continue + if (item === 'Item') itemArguments += `${itemArguments === '' ? '' : ', '}${attribute}=${attribute}`; if (properties.includes(attribute)) { - jsonGets.push(`${attribute} = json.get("${attribute}", None)`); + fromJsonProperties.push(`${attribute} = json.get("${attribute}", None)`); } else { - fromJsons.push(`if json.get("${attribute}", None) is not None: ${attribute} = Edge.from_json(json.get("${attribute}", None))`) - edgesAttribute += `${edgesAttribute === '' ? '' : ' + '}${attribute}`; + fromJsonEdges.push(`${attribute} = []`) + let singular; + for (const _item of Object.keys(entityHierarchy)) { + if (entityHierarchy[_item]['relations'][attribute]) singular = entityHierarchy[_item]['relations'][attribute]['singular']; + } + let isOrAppend = singular ? '= edge' : '.append(edge)' + let ifOrElif = fromJsonEdgeLoop.length === 0 ? 'if' : 'elif'; + fromJsonEdgeLoop.push(`${ifOrElif} edge._type == "${attribute}" or edge._type == "~${attribute}": + ${attribute}${isOrAppend}`) } if (attributesItem.includes(attribute) && item !== 'Item') continue; @@ -45,37 +54,34 @@ function getItemClasses() { dataItemClass = ` ${classDescription} -class Item: +class Item(ItemBase): ${helpers.wrapText(`def __init__(self, ${arguments})`, 100, '\n' + ' '.repeat(17))}: - ${helpers.insertList(attributes, 8)} - self.edges = ${edgesAttribute} - - def __getattribute__(self, name, edge): - res = self._meta[name] - if isinstance(res, edge): - return res.traverse() - else: - return res - - def get_edge(self, name): - return self._meta[name]`; + super().__init__(uid) + ${helpers.insertList(attributes, 8)}`; } else { dataItemClass = ` ${classDescription} class ${item}(Item): ${helpers.wrapText(`def __init__(self, ${arguments})`, 100, '\n' + ' '.repeat(17))}: - ${helpers.wrapText(`super().__init__(self, ${itemArguments})`, 100, '\n' + ' '.repeat(25))} + ${helpers.wrapText(`super().__init__(${itemArguments})`, 100, '\n' + ' '.repeat(25))} ${helpers.insertList(attributes, 8)} - ${helpers.wrapText(`self.edges = ${edgesAttribute}`, 100, ' \\\n' + ' '.repeat(12))} @classmethod def from_json(cls, json): - ${helpers.insertList(jsonGets, 8)} + all_edges = json.get("allEdges", None) + ${helpers.insertList(fromJsonProperties, 8)} - ${helpers.insertList(fromJsons,8)} + ${helpers.insertList(fromJsonEdges,8)} - ${helpers.wrapText(`cls(${arguments}`, 100, '\n' + ' '.repeat(12))})`; + if all_edges is not None: + for edge_json in all_edges: + edge = Edge.from_json(edge_json) + ${helpers.insertList(fromJsonEdgeLoop,16)} + + ${helpers.wrapText(`res = cls(${arguments}`, 100, '\n' + ' '.repeat(18))}) + + return res`; } itemClasses.push(dataItemClass); } @@ -97,6 +103,8 @@ let predicateHierarchy = {}; # # Copyright © 2020 memri. All rights reserved. # + +from .itembase import ItemBase ${helpers.insertList(itemClasses, 0)}`; fs.writeFile(outputFile, output, (err) => { -- GitLab From 062d051da1f87830f61f8f150b28a34e4bd701f7 Mon Sep 17 00:00:00 2001 From: rubenseggers <rubenseggers@gmail.com> Date: Wed, 29 Jul 2020 18:55:44 +0200 Subject: [PATCH 3/4] bugfix super init uid --- tools/export_schema_indexers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/export_schema_indexers.js b/tools/export_schema_indexers.js index ae3bdca..5d7f7af 100644 --- a/tools/export_schema_indexers.js +++ b/tools/export_schema_indexers.js @@ -29,8 +29,8 @@ function getItemClasses() { for (const attribute of properties.concat(edges)) { if (['genericType', 'functions', 'updatedFields', 'allEdges'].includes(attribute)) continue; arguments += `${arguments === '' ? '' : ', '}${attribute}=None`; - if (item === 'Item' && attribute === 'uid') continue if (item === 'Item') itemArguments += `${itemArguments === '' ? '' : ', '}${attribute}=${attribute}`; + if (item === 'Item' && attribute === 'uid') continue if (properties.includes(attribute)) { fromJsonProperties.push(`${attribute} = json.get("${attribute}", None)`); -- GitLab From 2168301b5a61d87fb845fc3c91740f6041c64af1 Mon Sep 17 00:00:00 2001 From: rubenseggers <rubenseggers@gmail.com> Date: Thu, 30 Jul 2020 11:29:19 +0200 Subject: [PATCH 4/4] removed scaffolding --- tools/export_schema_indexers.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/export_schema_indexers.js b/tools/export_schema_indexers.js index 5d7f7af..872be50 100644 --- a/tools/export_schema_indexers.js +++ b/tools/export_schema_indexers.js @@ -11,7 +11,6 @@ function getItemClasses() { let itemArguments = ""; let itemClasses = []; for (const item of Object.keys(entityHierarchy)) { - // if (!['Item', 'Location'].includes(item)) continue if (['SyncableItem', 'Edge', 'Datasource', 'UserState', 'ViewArguments', 'CVUStateDefinition'].includes(item)) continue; let classDescription = `\n# ${entityHierarchy[item]['description']}\n`; -- GitLab